gc v2 tests (#512)

* set MaxBlockCount at init 
* properly calculate total size
* basic blockstore compact blocks test
* correct detection of block swap
* Use one implementation for CreateRandomBlob
* reduce some data sets to increase speed of tests
* reduce test time
* rename BlockStoreCompactState::AddBlock -> BlockStoreCompactState::IncludeBlock

12 files changed, 389 insertions, 171 deletions

diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h
index 04648c6de..f5526945a 100644
--- a/src/zencore/include/zencore/testutils.h
+++ b/src/zencore/include/zencore/testutils.h
@@ -2,7 +2,10 @@
 
 #pragma once
 
-#include <filesystem>
+#if ZEN_WITH_TESTS
+
+#	include <zencore/iobuffer.h>
+#	include <filesystem>
 
 namespace zen {
 
@@ -29,4 +32,8 @@ struct ScopedCurrentDirectoryChange
 	~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); }
 };
 
+IoBuffer CreateRandomBlob(uint64_t Size);
+
 }  // namespace zen
+
+#endif	// ZEN_WITH_TESTS
diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp
index dbc3ab5af..d4c8aeaef 100644
--- a/src/zencore/testutils.cpp
+++ b/src/zencore/testutils.cpp
@@ -1,10 +1,13 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include "zencore/testutils.h"
-#include <zencore/session.h>
-#include "zencore/string.h"
 
-#include <atomic>
+#if ZEN_WITH_TESTS
+
+#	include <zencore/session.h>
+#	include "zencore/string.h"
+
+#	include <atomic>
 
 namespace zen {
 
@@ -39,4 +42,35 @@ ScopedTemporaryDirectory::~ScopedTemporaryDirectory()
 	std::filesystem::remove_all(m_RootPath, Ec);
 }
 
-}  // namespace zen
\ No newline at end of file
+IoBuffer
+CreateRandomBlob(uint64_t Size)
+{
+	static uint64_t Seed{0x7CEBF54E45B9F5D1};
+	auto			Next = [](uint64_t& seed) {
+		   uint64_t z = (seed += UINT64_C(0x9E3779B97F4A7C15));
+		   z		  = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9);
+		   z		  = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
+		   return z ^ (z >> 31);
+	};
+
+	IoBuffer  Data(Size);
+	uint64_t* DataPtr = reinterpret_cast<uint64_t*>(Data.MutableData());
+	while (Size > sizeof(uint64_t))
+	{
+		*DataPtr++ = Next(Seed);
+		Size -= sizeof(uint64_t);
+	}
+	uint64_t ByteNext	 = Next(Seed);
+	uint8_t* ByteDataPtr = reinterpret_cast<uint8_t*>(DataPtr);
+	while (Size > 0)
+	{
+		*ByteDataPtr++ = static_cast<uint8_t>(ByteNext & 0xff);
+		ByteNext >>= 8;
+		Size--;
+	}
+	return Data;
+};
+
+}  // namespace zen
+
+#endif	// ZEN_WITH_TESTS
diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp
index e25ef7460..1f1b1b8f5 100644
--- a/src/zencore/thread.cpp
+++ b/src/zencore/thread.cpp
@@ -1209,7 +1209,7 @@ TEST_CASE("NamedEvent")
 	// Timeout test
 	for (uint32_t i = 0; i < 8; ++i)
 	{
-		bool bEventSet = TestEvent.Wait(100);
+		bool bEventSet = TestEvent.Wait(10);
 		CHECK(!bEventSet);
 	}
 
@@ -1219,13 +1219,13 @@ TEST_CASE("NamedEvent")
 		ReadyEvent.Set();
 
 		NamedEvent TestEvent(Name);
-		TestEvent.Wait(1000);
+		TestEvent.Wait(100);
 	});
 
 	NamedEvent ReadyEvent(Name + "_ready");
 	ReadyEvent.Wait();
 
-	zen::Sleep(500);
+	zen::Sleep(50);
 	TestEvent.Set();
 
 	Waiter.join();
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 3fae3235f..056bf82cd 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -2650,18 +2650,7 @@ CreateAttachments(const std::span<const size_t>& Sizes)
 	Result.reserve(Sizes.size());
 	for (size_t Size : Sizes)
 	{
-		std::vector<uint8_t> Data;
-		Data.resize(Size);
-		uint16_t* DataPtr = reinterpret_cast<uint16_t*>(Data.data());
-		for (size_t Idx = 0; Idx < Size / 2; ++Idx)
-		{
-			DataPtr[Idx] = static_cast<uint16_t>(Idx % 0xffffu);
-		}
-		if (Size & 1)
-		{
-			Data[Size - 1] = static_cast<uint8_t>((Size - 1) & 0xff);
-		}
-		CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size()));
+		CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)));
 		Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
 	}
 	return Result;
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index c670d5088..f937f7d4d 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2374,7 +2374,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
 				bool	 IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
 				if (!IsActiveWriteBlock)
 				{
-					BlockCompactState.AddBlock(BlockIndex);
+					BlockCompactState.IncludeBlock(BlockIndex);
 				}
 				ExpiredEntries.push_back(ExpiredEntry);
 			}
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 09c9f1468..db6156ea1 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -774,24 +774,6 @@ using namespace std::literals;
 namespace testutils {
 	IoHash CreateKey(size_t KeyValue) { return IoHash::HashBuffer(&KeyValue, sizeof(size_t)); }
 
-	IoBuffer CreateBinaryCacheValue(uint64_t Size)
-	{
-		static std::random_device rd;
-		static std::mt19937		  g(rd());
-
-		std::vector<uint8_t> Values;
-		Values.resize(Size);
-		for (size_t Idx = 0; Idx < Size; ++Idx)
-		{
-			Values[Idx] = static_cast<uint8_t>(Idx);
-		}
-		std::shuffle(Values.begin(), Values.end(), g);
-
-		IoBuffer Buf(IoBuffer::Clone, Values.data(), Values.size());
-		Buf.SetContentType(ZenContentType::kBinary);
-		return Buf;
-	};
-
 	IoHash ToIoHash(const Oid& Id)
 	{
 		char OIdString[24 + 1];
@@ -800,33 +782,7 @@ namespace testutils {
 		return Key;
 	}
 
-	std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size)
-	{
-		uint64_t seed{Size};
-		auto	 next = [](uint64_t& seed) {
-			uint64_t z = (seed += UINT64_C(0x9E3779B97F4A7C15));
-			z		   = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9);
-			z		   = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
-			return z ^ (z >> 31);
-		};
-
-		IoBuffer  Data(Size);
-		uint64_t* DataPtr = reinterpret_cast<uint64_t*>(Data.MutableData());
-		while (Size > sizeof(uint64_t))
-		{
-			*DataPtr++ = next(seed);
-			Size -= sizeof(uint64_t);
-		}
-		uint64_t ByteNext	 = next(seed);
-		uint8_t* ByteDataPtr = reinterpret_cast<uint8_t*>(DataPtr);
-		while (Size > 0)
-		{
-			*ByteDataPtr++ = static_cast<uint8_t>(ByteNext & 0xff);
-			ByteNext >>= 8;
-			Size--;
-		}
-		return {Oid::NewOid(), Data};
-	}
+	std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size) { return {Oid::NewOid(), CreateRandomBlob(Size)}; }
 
 	std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
 	{
@@ -1105,7 +1061,7 @@ TEST_CASE("z$.gc")
 
 		for (const auto& Key : Keys)
 		{
-			IoBuffer Value = testutils::CreateBinaryCacheValue(128 << 10);
+			IoBuffer Value = CreateRandomBlob(128 << 10);
 			Zcs.Put(Bucket, Key, {.Value = Value}, {});
 		}
 
@@ -1153,7 +1109,7 @@ TEST_CASE("z$.gc")
 
 			for (const auto& Key : Keys)
 			{
-				IoBuffer Value = testutils::CreateBinaryCacheValue(128);
+				IoBuffer Value = CreateRandomBlob(128);
 				Zcs.Put(Bucket, Key, {.Value = Value}, {});
 			}
 
@@ -1224,7 +1180,7 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 		{
 			while (true)
 			{
-				IoBuffer Chunk = testutils::CreateBinaryCacheValue(kChunkSize);
+				IoBuffer Chunk = CreateRandomBlob(kChunkSize);
 				IoHash	 Hash  = HashBuffer(Chunk);
 				if (Chunks.contains(Hash))
 				{
@@ -1235,7 +1191,7 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 			}
 			while (true)
 			{
-				IoBuffer Chunk = testutils::CreateBinaryCacheValue(kChunkSize);
+				IoBuffer Chunk = CreateRandomBlob(kChunkSize);
 				IoHash	 Hash  = HashBuffer(Chunk);
 				if (Chunks.contains(Hash))
 				{
@@ -1303,12 +1259,12 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 			for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
 			{
 				{
-					IoBuffer Chunk	= testutils::CreateBinaryCacheValue(kChunkSize);
+					IoBuffer Chunk	= CreateRandomBlob(kChunkSize);
 					IoHash	 Hash	= HashBuffer(Chunk);
 					NewChunks[Hash] = {.Bucket = Bucket1, .Buffer = Chunk};
 				}
 				{
-					IoBuffer Chunk	= testutils::CreateBinaryCacheValue(kChunkSize);
+					IoBuffer Chunk	= CreateRandomBlob(kChunkSize);
 					IoHash	 Hash	= HashBuffer(Chunk);
 					NewChunks[Hash] = {.Bucket = Bucket2, .Buffer = Chunk};
 				}
@@ -1759,7 +1715,7 @@ TEST_CASE("z$.scrub")
 			Record.EndObject();
 			for (size_t Index = 0; Index < AttachmentSizes.size(); Index++)
 			{
-				IoBuffer		 AttachmentData			  = CreateBinaryCacheValue(AttachmentSizes[Index]);
+				IoBuffer		 AttachmentData			  = CreateRandomBlob(AttachmentSizes[Index]);
 				CompressedBuffer CompressedAttachmentData = CompressedBuffer::Compress(SharedBuffer(AttachmentData));
 				Record.AddBinaryAttachment(fmt::format("attachment-{}", Index), CompressedAttachmentData.DecodeRawHash());
 				Result.Attachments[Index] = CompressedAttachmentData;
@@ -1781,7 +1737,7 @@ TEST_CASE("z$.scrub")
 			DataPtr += RecordData.length() + 1;
 			for (size_t AttachmentSize : AttachmentSizes)
 			{
-				IoBuffer AttachmentData = CreateBinaryCacheValue(AttachmentSize);
+				IoBuffer AttachmentData = CreateRandomBlob(AttachmentSize);
 				memcpy(DataPtr, AttachmentData.GetData(), AttachmentData.GetSize());
 				DataPtr += AttachmentData.GetSize();
 			}
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 7da2f7695..ba30dcd7b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3347,18 +3347,7 @@ namespace testutils {
 		Result.reserve(Sizes.size());
 		for (size_t Size : Sizes)
 		{
-			std::vector<uint8_t> Data;
-			Data.resize(Size);
-			uint16_t* DataPtr = reinterpret_cast<uint16_t*>(Data.data());
-			for (size_t Idx = 0; Idx < Size / 2; ++Idx)
-			{
-				DataPtr[Idx] = static_cast<uint16_t>(Idx % 0xffffu);
-			}
-			if (Size & 1)
-			{
-				Data[Size - 1] = static_cast<uint8_t>((Size - 1) & 0xff);
-			}
-			CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size()));
+			CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)));
 			Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
 		}
 		return Result;
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 837185201..30a659784 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -173,6 +173,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 	m_TotalSize		 = 0;
 	m_BlocksBasePath = BlocksBasePath;
 	m_MaxBlockSize	 = MaxBlockSize;
+	m_MaxBlockCount	 = MaxBlockCount;
 
 	if (std::filesystem::is_directory(m_BlocksBasePath))
 	{
@@ -322,9 +323,10 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, cons
 
 	RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
 
-	uint32_t WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
-	bool	 IsWriting		 = !!m_WriteBlock;
-	if (!IsWriting || (m_CurrentInsertOffset + Size) > m_MaxBlockSize)
+	uint32_t WriteBlockIndex	 = m_WriteBlockIndex.load(std::memory_order_acquire);
+	bool	 IsWriting			 = !!m_WriteBlock;
+	uint64_t AlignedInsertOffset = RoundUp(m_CurrentInsertOffset, Alignment);
+	if (!IsWriting || (AlignedInsertOffset + Size) > m_MaxBlockSize)
 	{
 		if (m_WriteBlock)
 		{
@@ -347,18 +349,18 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, cons
 		m_WriteBlock				   = NewBlockFile;
 		m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
 		m_CurrentInsertOffset = 0;
+		AlignedInsertOffset	  = 0;
 	}
-	uint64_t InsertOffset				 = m_CurrentInsertOffset;
-	m_CurrentInsertOffset				 = RoundUp(InsertOffset + Size, Alignment);
-	uint64_t			AlignedWriteSize = m_CurrentInsertOffset - InsertOffset;
-	Ref<BlockStoreFile> WriteBlock		 = m_WriteBlock;
+	uint64_t AlignedWriteSize	   = AlignedInsertOffset - m_CurrentInsertOffset + Size;
+	m_CurrentInsertOffset		   = AlignedInsertOffset + Size;
+	Ref<BlockStoreFile> WriteBlock = m_WriteBlock;
 	m_ActiveWriteBlocks.push_back(WriteBlockIndex);
 	InsertLock.ReleaseNow();
 
-	WriteBlock->Write(Data, Size, InsertOffset);
+	WriteBlock->Write(Data, Size, AlignedInsertOffset);
 	m_TotalSize.fetch_add(AlignedWriteSize, std::memory_order::relaxed);
 
-	Callback({.BlockIndex = WriteBlockIndex, .Offset = InsertOffset, .Size = Size});
+	Callback({.BlockIndex = WriteBlockIndex, .Offset = AlignedInsertOffset, .Size = Size});
 
 	{
 		RwLock::ExclusiveLockScope _(m_InsertLock);
@@ -1003,12 +1005,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 	CompactState.IterateBlocks(
 		[&](uint32_t BlockIndex, const std::vector<size_t>& KeepChunkIndexes, const std::vector<BlockStoreLocation>& ChunkLocations) {
-			ZEN_ASSERT(BlockIndex != m_WriteBlockIndex.load());
-
 			Ref<BlockStoreFile> OldBlockFile;
 			{
 				RwLock::SharedLockScope _(m_InsertLock);
-				auto					It = m_ChunkBlocks.find(BlockIndex);
+				if ((BlockIndex == m_WriteBlockIndex.load()) && m_WriteBlock)
+				{
+					// You are trying to collect the currently writing block, Report error?
+					return;
+				}
+				auto It = m_ChunkBlocks.find(BlockIndex);
 				if (It == m_ChunkBlocks.end())
 				{
 					// This block has unknown, we can't move anything. Report error?
@@ -1320,21 +1325,6 @@ namespace blockstore::impl {
 		return Result;
 	};
 
-	static IoBuffer CreateChunk(uint64_t Size)
-	{
-		static std::random_device rd;
-		static std::mt19937		  g(rd());
-
-		std::vector<uint8_t> Values;
-		Values.resize(Size);
-		for (size_t Idx = 0; Idx < Size; ++Idx)
-		{
-			Values[Idx] = static_cast<uint8_t>(Idx);
-		}
-		std::shuffle(Values.begin(), Values.end(), g);
-
-		return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size());
-	}
 }  // namespace blockstore::impl
 
 TEST_CASE("blockstore.chunks")
@@ -1546,7 +1536,7 @@ TEST_CASE("blockstore.reclaim.space")
 	ChunkHashes.reserve(ChunkCount);
 	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
 	{
-		IoBuffer Chunk = CreateChunk(57 + ChunkIndex);
+		IoBuffer Chunk = CreateRandomBlob(57 + ChunkIndex);
 
 		Store.WriteChunk(Chunk.Data(), Chunk.Size(), Alignment, [&](const BlockStoreLocation& L) { ChunkLocations.push_back(L); });
 		ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
@@ -1665,7 +1655,7 @@ TEST_CASE("blockstore.thread.read.write")
 	ChunkHashes.reserve(ChunkCount);
 	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
 	{
-		IoBuffer Chunk = CreateChunk(57 + ChunkIndex / 2);
+		IoBuffer Chunk = CreateRandomBlob(57 + ChunkIndex / 2);
 		Chunks.push_back(Chunk);
 		ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
 	}
@@ -1730,6 +1720,293 @@ TEST_CASE("blockstore.thread.read.write")
 	}
 }
 
+TEST_CASE("blockstore.compact.blocks")
+{
+	using namespace blockstore::impl;
+
+	ScopedTemporaryDirectory TempDir;
+	auto					 RootDirectory = TempDir.Path();
+
+	BlockStore Store;
+	Store.Initialize(RootDirectory / "store", 1088, 1024);
+
+	constexpr size_t	  ChunkCount = 200;
+	constexpr size_t	  Alignment	 = 8;
+	std::vector<IoBuffer> Chunks;
+	std::vector<IoHash>	  ChunkHashes;
+	Chunks.reserve(ChunkCount);
+	ChunkHashes.reserve(ChunkCount);
+	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
+	{
+		IoBuffer Chunk = CreateRandomBlob(57 + ChunkIndex / 2);
+		Chunks.push_back(Chunk);
+		ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
+	}
+
+	std::vector<BlockStoreLocation> ChunkLocations;
+	ChunkLocations.resize(ChunkCount);
+
+	for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex)
+	{
+		IoBuffer& Chunk = Chunks[ChunkIndex];
+		Store.WriteChunk(Chunk.Data(), Chunk.Size(), Alignment, [&](const BlockStoreLocation& L) { ChunkLocations[ChunkIndex] = L; });
+	}
+
+	SUBCASE("touch nothing")
+	{
+		uint64_t PreSize = Store.TotalSize();
+		CHECK(PreSize > 0);
+		BlockStoreCompactState State;
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray&, uint64_t) { CHECK(false); },
+			[]() {
+				CHECK(false);
+				return 0;
+			});
+		CHECK_EQ(PreSize, Store.TotalSize());
+	}
+	SUBCASE("keep nothing")
+	{
+		Store.Flush(true);
+
+		uint64_t PreSize = Store.TotalSize();
+		CHECK(PreSize > 0);
+		BlockStoreCompactState State;
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			State.IncludeBlock(Location.BlockIndex);
+		}
+		uint64_t RemovedSize = 0;
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+				RemovedSize += Removed;
+				CHECK(Moved.empty());
+			},
+			[]() { return 0; });
+		CHECK_EQ(RemovedSize, PreSize);
+		CHECK_EQ(0u, Store.TotalSize());
+	}
+	SUBCASE("keep current write block")
+	{
+		uint64_t						 PreSize = Store.TotalSize();
+		BlockStoreCompactState			 State;
+		BlockStore::ReclaimSnapshotState SnapshotState = Store.GetReclaimSnapshotState();
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			if (SnapshotState.m_ActiveWriteBlocks.contains(Location.BlockIndex))
+			{
+				continue;
+			}
+			State.IncludeBlock(Location.BlockIndex);
+		}
+		uint64_t RemovedSize = 0;
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+				RemovedSize += Removed;
+				CHECK(Moved.empty());
+			},
+			[]() { return 0; });
+		CHECK_EQ(Store.TotalSize() + RemovedSize, PreSize);
+		CHECK_LE(Store.TotalSize(), 1088);
+		CHECK_GT(Store.TotalSize(), 0);
+	}
+	SUBCASE("keep everthing")
+	{
+		Store.Flush(true);
+
+		uint64_t						 PreSize = Store.TotalSize();
+		BlockStoreCompactState			 State;
+		BlockStore::ReclaimSnapshotState SnapshotState = Store.GetReclaimSnapshotState();
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			State.AddKeepLocation(Location);
+		}
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray&, uint64_t) { CHECK(false); },
+			[]() {
+				CHECK(false);
+				return 0;
+			});
+		CHECK_EQ(Store.TotalSize(), PreSize);
+	}
+	SUBCASE("drop first block")
+	{
+		uint64_t						 PreSize = Store.TotalSize();
+		BlockStoreCompactState			 State;
+		BlockStore::ReclaimSnapshotState SnapshotState = Store.GetReclaimSnapshotState();
+
+		CHECK(!SnapshotState.m_ActiveWriteBlocks.contains(0));
+		State.IncludeBlock(0);
+
+		uint64_t FirstBlockSize = 0;
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			if (Location.BlockIndex == 0)
+			{
+				FirstBlockSize = Max<uint64_t>(FirstBlockSize, Location.Offset + Location.Size);
+			}
+		}
+
+		uint64_t RemovedSize = 0;
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+				CHECK(Moved.empty());
+				RemovedSize += Removed;
+			},
+			[]() {
+				CHECK(false);
+				return 0;
+			});
+		CHECK_EQ(FirstBlockSize, RemovedSize);
+		CHECK_EQ(Store.TotalSize(), PreSize - FirstBlockSize);
+	}
+	SUBCASE("compact first block")
+	{
+		uint64_t						 PreSize = Store.TotalSize();
+		BlockStoreCompactState			 State;
+		BlockStore::ReclaimSnapshotState SnapshotState = Store.GetReclaimSnapshotState();
+
+		CHECK(!SnapshotState.m_ActiveWriteBlocks.contains(0));
+		State.IncludeBlock(0);
+
+		uint64_t						SkipChunkCount = 2;
+		std::vector<BlockStoreLocation> DroppedLocations(ChunkLocations.begin(), ChunkLocations.begin() + 2);
+		for (auto It = ChunkLocations.begin() + 2; It != ChunkLocations.end(); It++)
+		{
+			const BlockStoreLocation& Location = *It;
+			if (Location.BlockIndex != 0)
+			{
+				continue;
+			}
+			State.AddKeepLocation(Location);
+		}
+		uint64_t RemovedSize = 0;
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+				for (const auto& Move : Moved)
+				{
+					const BlockStoreLocation& OldLocation = State.GetLocation(Move.first);
+					CHECK(OldLocation.BlockIndex == 0);	 // Only move from block 0
+					CHECK(std::find(DroppedLocations.begin(), DroppedLocations.end(), OldLocation) == DroppedLocations.end());
+					auto It = std::find(ChunkLocations.begin(), ChunkLocations.end(), OldLocation);
+					CHECK(It != ChunkLocations.end());
+					(*It) = Move.second;
+				}
+				RemovedSize += Removed;
+			},
+			[]() {
+				CHECK(false);
+				return 0;
+			});
+
+		SkipChunkCount = 2;
+
+		for (size_t Index = 0; Index < ChunkLocations.size(); Index++)
+		{
+			const BlockStoreLocation& Location = ChunkLocations[Index];
+			if (Location.BlockIndex == 0 && SkipChunkCount > 0)
+			{
+				CHECK(!Store.TryGetChunk(Location));
+				continue;
+			}
+			IoBuffer Buffer = Store.TryGetChunk(Location);
+			CHECK(Buffer);
+			IoHash RawHash = IoHash::HashBuffer(Buffer.Data(), Buffer.Size());
+			CHECK_EQ(ChunkHashes[Index], RawHash);
+		}
+		CHECK_LT(Store.TotalSize(), PreSize);
+	}
+	SUBCASE("compact every other item")
+	{
+		uint64_t						 PreSize = Store.TotalSize();
+		BlockStoreCompactState			 State;
+		BlockStore::ReclaimSnapshotState SnapshotState = Store.GetReclaimSnapshotState();
+		bool							 SkipFlag	   = false;
+
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			if (SnapshotState.m_ActiveWriteBlocks.contains(Location.BlockIndex))
+			{
+				continue;
+			}
+			if (SkipFlag)
+			{
+				State.IncludeBlock(Location.BlockIndex);
+				SkipFlag = false;
+				continue;
+			}
+			SkipFlag = true;
+		}
+
+		SkipFlag = false;
+		std::vector<BlockStoreLocation> DroppedLocations;
+		for (const BlockStoreLocation& Location : ChunkLocations)
+		{
+			if (SnapshotState.m_ActiveWriteBlocks.contains(Location.BlockIndex))
+			{
+				continue;
+			}
+			if (SkipFlag)
+			{
+				DroppedLocations.push_back(Location);
+				SkipFlag = false;
+				continue;
+			}
+			State.AddKeepLocation(Location);
+			SkipFlag = true;
+		}
+		uint64_t RemovedSize = 0;
+		Store.CompactBlocks(
+			State,
+			Alignment,
+			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
+				for (const auto& Move : Moved)
+				{
+					const BlockStoreLocation& OldLocation = State.GetLocation(Move.first);
+					CHECK(std::find(DroppedLocations.begin(), DroppedLocations.end(), OldLocation) == DroppedLocations.end());
+					auto It = std::find(ChunkLocations.begin(), ChunkLocations.end(), OldLocation);
+					CHECK(It != ChunkLocations.end());
+					(*It) = Move.second;
+				}
+				RemovedSize += Removed;
+			},
+			[]() {
+				CHECK(false);
+				return 0;
+			});
+		SkipFlag = false;
+		for (size_t Index = 0; Index < ChunkLocations.size(); Index++)
+		{
+			const BlockStoreLocation& Location = ChunkLocations[Index];
+			if (SkipFlag && !SnapshotState.m_ActiveWriteBlocks.contains(Location.BlockIndex))
+			{
+				CHECK(std::find(DroppedLocations.begin(), DroppedLocations.end(), Location) != DroppedLocations.end());
+				CHECK(!Store.TryGetChunk(Location));
+				SkipFlag = false;
+				continue;
+			}
+			IoBuffer Buffer = Store.TryGetChunk(Location);
+			CHECK(Buffer);
+			IoHash RawHash = IoHash::HashBuffer(Buffer.Data(), Buffer.Size());
+			CHECK_EQ(ChunkHashes[Index], RawHash);
+			SkipFlag = true;
+		}
+		CHECK_LT(Store.TotalSize(), PreSize);
+	}
+}
+
 #endif
 
 void
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 655c0558d..de2800895 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -701,7 +701,7 @@ public:
 					bool	 IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
 					if (!IsActiveWriteBlock)
 					{
-						CompactState.AddBlock(BlockIndex);
+						CompactState.IncludeBlock(BlockIndex);
 					}
 					ExpiredEntries.push_back(ExpiredEntry);
 				}
@@ -1099,24 +1099,6 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 
 #if ZEN_WITH_TESTS
 
-namespace {
-	static IoBuffer CreateRandomChunk(uint64_t Size)
-	{
-		static std::random_device rd;
-		static std::mt19937		  g(rd());
-
-		std::vector<uint8_t> Values;
-		Values.resize(Size);
-		for (size_t Idx = 0; Idx < Size; ++Idx)
-		{
-			Values[Idx] = static_cast<uint8_t>(Idx);
-		}
-		std::shuffle(Values.begin(), Values.end(), g);
-
-		return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size());
-	}
-}  // namespace
-
 TEST_CASE("compactcas.hex")
 {
 	uint32_t	Value;
@@ -1233,7 +1215,7 @@ TEST_CASE("compactcas.compact.totalsize")
 
 			for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
 			{
-				IoBuffer			   Chunk		= CreateRandomChunk(kChunkSize);
+				IoBuffer			   Chunk		= CreateRandomBlob(kChunkSize);
 				const IoHash		   Hash			= HashBuffer(Chunk);
 				CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, Hash);
 				ZEN_ASSERT(InsertResult.New);
@@ -1272,7 +1254,7 @@ TEST_CASE("compactcas.gc.basic")
 	CasContainerStrategy Cas(Gc);
 	Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, true);
 
-	IoBuffer Chunk	   = CreateRandomChunk(128);
+	IoBuffer Chunk	   = CreateRandomBlob(128);
 	IoHash	 ChunkHash = IoHash::HashBuffer(Chunk);
 
 	const CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, ChunkHash);
@@ -1291,7 +1273,7 @@ TEST_CASE("compactcas.gc.removefile")
 {
 	ScopedTemporaryDirectory TempDir;
 
-	IoBuffer Chunk	   = CreateRandomChunk(128);
+	IoBuffer Chunk	   = CreateRandomBlob(128);
 	IoHash	 ChunkHash = IoHash::HashBuffer(Chunk);
 	{
 		GcManager			 Gc;
@@ -1331,7 +1313,7 @@ TEST_CASE("compactcas.gc.compact")
 		Chunks.reserve(9);
 		for (uint64_t Size : ChunkSizes)
 		{
-			Chunks.push_back(CreateRandomChunk(Size));
+			Chunks.push_back(CreateRandomBlob(Size));
 		}
 
 		std::vector<IoHash> ChunkHashes;
@@ -1568,7 +1550,7 @@ TEST_CASE("compactcas.gc.deleteblockonopen")
 	Chunks.reserve(20);
 	for (uint64_t Size : ChunkSizes)
 	{
-		Chunks.push_back(CreateRandomChunk(Size));
+		Chunks.push_back(CreateRandomBlob(Size));
 	}
 
 	std::vector<IoHash> ChunkHashes;
@@ -1634,7 +1616,7 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
 	Chunks.reserve(20);
 	for (const uint64_t& Size : ChunkSizes)
 	{
-		Chunks.push_back(CreateRandomChunk(Size));
+		Chunks.push_back(CreateRandomBlob(Size));
 	}
 
 	std::vector<IoHash> ChunkHashes;
@@ -1686,7 +1668,7 @@ TEST_CASE("compactcas.threadedinsert")
 		{
 			while (true)
 			{
-				IoBuffer Chunk = CreateRandomChunk(kChunkSize);
+				IoBuffer Chunk = CreateRandomBlob(kChunkSize);
 				IoHash	 Hash  = HashBuffer(Chunk);
 				if (Chunks.contains(Hash))
 				{
@@ -1755,7 +1737,7 @@ TEST_CASE("compactcas.threadedinsert")
 
 			for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
 			{
-				IoBuffer Chunk	= CreateRandomChunk(kChunkSize);
+				IoBuffer Chunk	= CreateRandomBlob(kChunkSize);
 				IoHash	 Hash	= HashBuffer(Chunk);
 				NewChunks[Hash] = Chunk;
 				GcChunkHashes.insert(Hash);
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index c021e0e21..a72619e4b 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1612,7 +1612,7 @@ TEST_CASE("cas.file.gc")
 	FileCasStrategy FileCas(Gc);
 	FileCas.Initialize(TempDir.Path() / "cas", /* IsNewStore */ true);
 
-	const int			kIterationCount = 1000;
+	const int			kIterationCount = 100;
 	std::vector<IoHash> Keys{kIterationCount};
 
 	auto InsertChunks = [&] {
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index f319475f3..365eb10ab 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1652,22 +1652,6 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 #if ZEN_WITH_TESTS
 
 namespace gc::impl {
-	static IoBuffer CreateChunk(uint64_t Size)
-	{
-		static std::random_device rd;
-		static std::mt19937		  g(rd());
-
-		std::vector<uint8_t> Values;
-		Values.resize(Size);
-		for (size_t Idx = 0; Idx < Size; ++Idx)
-		{
-			Values[Idx] = static_cast<uint8_t>(Idx);
-		}
-		std::shuffle(Values.begin(), Values.end(), g);
-
-		return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size());
-	}
-
 	static CompressedBuffer Compress(IoBuffer Buffer)
 	{
 		return CompressedBuffer::Compress(SharedBuffer::MakeView(Buffer.GetData(), Buffer.GetSize()));
@@ -1688,7 +1672,7 @@ TEST_CASE("gc.basic")
 
 	CidStore.Initialize(CasConfig);
 
-	IoBuffer Chunk			 = CreateChunk(128);
+	IoBuffer Chunk			 = CreateRandomBlob(128);
 	auto	 CompressedChunk = Compress(Chunk);
 
 	const auto InsertResult = CidStore.AddChunk(CompressedChunk.GetCompressed().Flatten().AsIoBuffer(), CompressedChunk.DecodeRawHash());
@@ -1718,15 +1702,15 @@ TEST_CASE("gc.full")
 	CasStore->Initialize(CasConfig);
 
 	uint64_t ChunkSizes[9]	= {128, 541, 1023, 781, 218, 37, 4, 997, 5};
-	IoBuffer Chunks[9]		= {CreateChunk(ChunkSizes[0]),
-						   CreateChunk(ChunkSizes[1]),
-						   CreateChunk(ChunkSizes[2]),
-						   CreateChunk(ChunkSizes[3]),
-						   CreateChunk(ChunkSizes[4]),
-						   CreateChunk(ChunkSizes[5]),
-						   CreateChunk(ChunkSizes[6]),
-						   CreateChunk(ChunkSizes[7]),
-						   CreateChunk(ChunkSizes[8])};
+	IoBuffer Chunks[9]		= {CreateRandomBlob(ChunkSizes[0]),
+						   CreateRandomBlob(ChunkSizes[1]),
+						   CreateRandomBlob(ChunkSizes[2]),
+						   CreateRandomBlob(ChunkSizes[3]),
+						   CreateRandomBlob(ChunkSizes[4]),
+						   CreateRandomBlob(ChunkSizes[5]),
+						   CreateRandomBlob(ChunkSizes[6]),
+						   CreateRandomBlob(ChunkSizes[7]),
+						   CreateRandomBlob(ChunkSizes[8])};
 	IoHash	 ChunkHashes[9] = {
 		  IoHash::HashBuffer(Chunks[0].Data(), Chunks[0].Size()),
 		  IoHash::HashBuffer(Chunks[1].Data(), Chunks[1].Size()),
@@ -2080,7 +2064,7 @@ TEST_CASE("scrub.basic")
 
 	CidStore.Initialize(CasConfig);
 
-	IoBuffer Chunk			 = CreateChunk(128);
+	IoBuffer Chunk			 = CreateRandomBlob(128);
 	auto	 CompressedChunk = Compress(Chunk);
 
 	const auto InsertResult = CidStore.AddChunk(CompressedChunk.GetCompressed().Flatten().AsIoBuffer(), CompressedChunk.DecodeRawHash());
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index cd475cd8b..75accd9b8 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -193,7 +193,7 @@ class BlockStoreCompactState
 public:
 	BlockStoreCompactState() = default;
 
-	void AddBlock(uint32_t BlockIndex)
+	void IncludeBlock(uint32_t BlockIndex)
 	{
 		auto It = m_BlockIndexToChunkMapIndex.find(BlockIndex);
 		if (It == m_BlockIndexToChunkMapIndex.end())