optimized index snapshot reading/writing (#561)

the previous implementation of in-memory index snapshots serialise data to memory before writing to disk and vice versa when reading. This leads to some memory spikes which end up pushing useful data out of system cache and also cause stalls on I/O operations.

this change moves more code to a streaming serialisation approach which scales better from a memory usage perspective and also performs much better

1 files changed, 25 insertions, 6 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index e4a66daf4..89774f26d 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -227,7 +227,17 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 }
 
 void
-BlockStore::SyncExistingBlocksOnDisk(const std::vector<BlockStoreLocation>& KnownLocations)
+BlockStore::BlockIndexSet::Add(uint32_t BlockIndex)
+{
+	if (!std::binary_search(begin(BlockIndexes), end(BlockIndexes), BlockIndex))
+	{
+		auto It = std::lower_bound(begin(BlockIndexes), end(BlockIndexes), BlockIndex);
+		BlockIndexes.insert(It, BlockIndex);
+	}
+}
+
+void
+BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownLocations)
 {
 	ZEN_TRACE_CPU("BlockStore::SyncExistingBlocksOnDisk");
 
@@ -240,14 +250,18 @@ BlockStore::SyncExistingBlocksOnDisk(const std::vector<BlockStoreLocation>& Know
 	{
 		DeleteBlocks.insert(It.first);
 	}
-	for (const auto& Entry : KnownLocations)
+
+	for (const uint32_t BlockIndex : KnownLocations.GetBlockIndices())
 	{
-		DeleteBlocks.erase(Entry.BlockIndex);
-		if (auto It = m_ChunkBlocks.find(Entry.BlockIndex); It != m_ChunkBlocks.end() && !It->second.IsNull())
+		DeleteBlocks.erase(BlockIndex);
+		if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && !It->second.IsNull())
 		{
 			continue;
 		}
-		MissingBlocks.insert(Entry.BlockIndex);
+		else
+		{
+			MissingBlocks.insert(BlockIndex);
+		}
 	}
 	for (std::uint32_t BlockIndex : MissingBlocks)
 	{
@@ -1473,7 +1487,12 @@ TEST_CASE("blockstore.clean.stray.blocks")
 	CHECK(!ThirdChunk);
 
 	// Recreate a fake block for a missing chunk location
-	Store.SyncExistingBlocksOnDisk({FirstChunkLocation, SecondChunkLocation, ThirdChunkLocation});
+	BlockStore::BlockIndexSet KnownBlocks;
+	KnownBlocks.Add(FirstChunkLocation.BlockIndex);
+	KnownBlocks.Add(SecondChunkLocation.BlockIndex);
+	KnownBlocks.Add(ThirdChunkLocation.BlockIndex);
+	Store.SyncExistingBlocksOnDisk(KnownBlocks);
+
 	// We create a fake block for the location - we should still not be able to get the chunk
 	CHECK(GetDirectoryContent(RootDirectory / "store", true, false).size() == 2);
 	ThirdChunk = Store.TryGetChunk(ThirdChunkLocation);