add separate PreCache step for GcReferenceChecker (#578)

- Improvement: GCv2: Use separate PreCache step to improve concurrency when checking references
- Improvement: GCv2: Improved verbose logging
- Improvement: GCv2: Sort chunks to read by block/offset when finding references
- Improvement: GCv2: Exit as soon as no more unreferenced items are left

2 files changed, 6 insertions, 0 deletions

diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 82e1c71c6..dcd4b5e87 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -180,6 +180,8 @@ public:
 
 	inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); }
 
+	Ref<BlockStoreFile> GetBlockFile(uint32_t BlockIndex);
+
 private:
 	uint32_t GetFreeBlockIndex(uint32_t StartProbeIndex, RwLock::ExclusiveLockScope&, std::filesystem::path& OutBlockPath) const;
 
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 7a6249970..698b0d4e8 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -87,6 +87,7 @@ struct GcReferencerStats
 	GcCompactStoreStats CompactStoreStats;
 
 	std::chrono::milliseconds CreateReferenceCheckersMS = {};
+	std::chrono::milliseconds PreCacheStateMS			= {};
 	std::chrono::milliseconds LockStateMS				= {};
 	std::chrono::milliseconds ElapsedMS					= {};
 };
@@ -112,6 +113,7 @@ struct GcResult
 	// Wall times, not sum of each
 	std::chrono::milliseconds RemoveExpiredDataMS		= {};
 	std::chrono::milliseconds CreateReferenceCheckersMS = {};
+	std::chrono::milliseconds PreCacheStateMS			= {};
 	std::chrono::milliseconds LockStateMS				= {};
 
 	std::chrono::milliseconds CreateReferencePrunersMS = {};
@@ -171,6 +173,8 @@ public:
 	// Destructor should unlock what was locked in LockState
 	virtual ~GcReferenceChecker() = default;
 
+	virtual void PreCache(GcCtx& Ctx) = 0;
+
 	// Lock the state and make sure no references changes, usually a read-lock is taken until the destruction
 	// of the instance. Called once before any calls to RemoveUsedReferencesFromSet
 	// The implementation should be as fast as possible as LockState is part of a stop the world (from changes)