Merge branch 'main' of https://github.com/EpicGames/zen

1 files changed, 198 insertions, 155 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 00a018948..b21f9f8d8 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -25,6 +25,9 @@
 #	include <zenstore/cidstore.h>
 #	include <algorithm>
 #	include <random>
+ZEN_THIRD_PARTY_INCLUDES_START
+#	include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 #endif
 
 //////////////////////////////////////////////////////////////////////////
@@ -114,8 +117,14 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get("containercas")), m_Gc(Gc)
 {
+	m_LocationMap.min_load_factor(IndexMinLoadFactor);
+	m_LocationMap.max_load_factor(IndexMaxLoadFactor);
+
 	m_Gc.AddGcStorage(this);
 	m_Gc.AddGcReferenceStore(*this);
 }
@@ -130,7 +139,7 @@ void
 CasContainerStrategy::Initialize(const std::filesystem::path& RootDirectory,
 								 const std::string_view		  ContainerBaseName,
 								 uint32_t					  MaxBlockSize,
-								 uint64_t					  Alignment,
+								 uint32_t					  Alignment,
 								 bool						  IsNewStore)
 {
 	ZEN_ASSERT(IsPow2(Alignment));
@@ -245,6 +254,12 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 {
 	ZEN_TRACE_CPU("CasContainer::ScrubStorage");
 
+	if (Ctx.IsSkipCas())
+	{
+		ZEN_INFO("SKIPPED scrubbing: '{}'", m_BlocksBasePath);
+		return;
+	}
+
 	ZEN_INFO("scrubbing '{}'", m_BlocksBasePath);
 
 	std::vector<IoHash>				BadKeys;
@@ -288,21 +303,12 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 			uint64_t RawSize;
 			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				if (RawHash != Hash)
+				if (RawHash == Hash)
 				{
-					// Hash mismatch
-					BadKeys.push_back(Hash);
+					// TODO: this should also hash the (decompressed) contents
 					return;
 				}
-				return;
-			}
-#if ZEN_WITH_TESTS
-			IoHash ComputedHash = IoHash::HashBuffer(Data, Size);
-			if (ComputedHash == Hash)
-			{
-				return;
 			}
-#endif
 			BadKeys.push_back(Hash);
 		};
 
@@ -317,26 +323,15 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 
 			IoHash	 RawHash;
 			uint64_t RawSize;
-			// TODO: Add API to verify compressed buffer without having to memorymap the whole file
+			// TODO: Add API to verify compressed buffer without having to memory-map the whole file
 			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				if (RawHash != Hash)
+				if (RawHash == Hash)
 				{
-					// Hash mismatch
-					BadKeys.push_back(Hash);
+					// TODO: this should also hash the (decompressed) contents
 					return;
 				}
-				return;
-			}
-#if ZEN_WITH_TESTS
-			IoHashStream Hasher;
-			File.StreamByteRange(Offset, Size, [&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
-			IoHash ComputedHash = Hasher.GetHash();
-			if (ComputedHash == Hash)
-			{
-				return;
 			}
-#endif
 			BadKeys.push_back(Hash);
 		};
 
@@ -389,7 +384,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 		Ctx.ReportBadCidChunks(BadKeys);
 	}
 
-	ZEN_INFO("compact cas scrubbed: {} chunks ({})", ChunkCount, NiceBytes(ChunkBytes));
+	ZEN_INFO("scrubbed {} chunks ({}) in '{}'", ChunkCount, NiceBytes(ChunkBytes), m_RootDirectory / m_ContainerBaseName);
 }
 
 void
@@ -553,82 +548,143 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 	GcCtx.AddDeletedCids(DeletedChunks);
 }
 
-class CasContainerStoreCompactor : public GcReferenceStoreCompactor
+class CasContainerStoreCompactor : public GcStoreCompactor
 {
 public:
-	CasContainerStoreCompactor(CasContainerStrategy&	Owner,
-							   BlockStoreCompactState&& CompactState,
-							   std::vector<IoHash>&&	CompactStateKeys,
-							   std::vector<IoHash>&&	PrunedKeys)
-	: m_CasContainerStrategy(Owner)
-	, m_CompactState(std::move(CompactState))
-	, m_CompactStateKeys(std::move(CompactStateKeys))
-	, m_PrunedKeys(std::move(PrunedKeys))
-	{
-	}
+	CasContainerStoreCompactor(CasContainerStrategy& Owner) : m_CasContainerStrategy(Owner) {}
 
-	virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override
 	{
+		ZEN_TRACE_CPU("CasContainer::CompactStore");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
 			{
 				return;
 			}
-			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': RemovedDisk: {} in {}",
 					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
-					 Stats.Count,
-					 Stats.Pruned,
-					 Stats.Compacted,
 					 NiceBytes(Stats.RemovedDisk),
-					 NiceBytes(Stats.RemovedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		if (Ctx.Settings.IsDeleteMode && Ctx.Settings.CollectSmallObjects)
+		if (Ctx.Settings.CollectSmallObjects)
 		{
-			// Compact block store
-			m_CasContainerStrategy.m_BlockStore.CompactBlocks(
-				m_CompactState,
-				m_CasContainerStrategy.m_PayloadAlignment,
-				[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
-					std::vector<CasDiskIndexEntry> MovedEntries;
-					RwLock::ExclusiveLockScope	   _(m_CasContainerStrategy.m_LocationMapLock);
-					for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+			BlockStore::BlockUsageMap BlockUsage;
+			{
+				RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
+				if (Ctx.IsCancelledFlag.load())
+				{
+					return;
+				}
+
+				for (const auto& Entry : m_CasContainerStrategy.m_LocationMap)
+				{
+					size_t						  Index = Entry.second;
+					const BlockStoreDiskLocation& Loc	= m_CasContainerStrategy.m_Locations[Index];
+
+					uint32_t BlockIndex = Loc.GetBlockIndex();
+					uint64_t ChunkSize	= RoundUp(Loc.GetSize(), m_CasContainerStrategy.m_PayloadAlignment);
+					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
+					{
+						It->second.EntryCount++;
+						It->second.DiskUsage += ChunkSize;
+					}
+					else
 					{
-						size_t		  ChunkIndex = Moved.first;
-						const IoHash& Key		 = m_CompactStateKeys[ChunkIndex];
+						BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
+					}
+				}
+			}
+
+			{
+				BlockStoreCompactState BlockCompactState;
+				std::vector<IoHash>	   BlockCompactStateKeys;
 
-						if (auto It = m_CasContainerStrategy.m_LocationMap.find(Key); It != m_CasContainerStrategy.m_LocationMap.end())
+				BlockStore::BlockEntryCountMap BlocksToCompact =
+					m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
+				BlockCompactState.IncludeBlocks(BlocksToCompact);
+
+				if (BlocksToCompact.size() > 0)
+				{
+					{
+						RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
+						for (const auto& Entry : m_CasContainerStrategy.m_LocationMap)
 						{
-							BlockStoreDiskLocation&	  Location	  = m_CasContainerStrategy.m_Locations[It->second];
-							const BlockStoreLocation& OldLocation = m_CompactState.GetLocation(ChunkIndex);
-							if (Location.Get(m_CasContainerStrategy.m_PayloadAlignment) != OldLocation)
+							size_t						  Index = Entry.second;
+							const BlockStoreDiskLocation& Loc	= m_CasContainerStrategy.m_Locations[Index];
+
+							if (!BlockCompactState.AddKeepLocation(Loc.Get(m_CasContainerStrategy.m_PayloadAlignment)))
 							{
-								// Someone has moved our chunk so lets just skip the new location we were provided, it will be GC:d at a
-								// later time
 								continue;
 							}
-
-							const BlockStoreLocation& NewLocation = Moved.second;
-							Location = BlockStoreDiskLocation(NewLocation, m_CasContainerStrategy.m_PayloadAlignment);
-							MovedEntries.push_back(CasDiskIndexEntry{.Key = Key, .Location = Location});
+							BlockCompactStateKeys.push_back(Entry.first);
 						}
 					}
-					m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
-					Stats.RemovedDisk += FreedDiskSpace;
-				},
-				[&]() { return 0; });
 
-			Stats.Compacted +=
-				m_PrunedKeys.size();  // Slightly missleading, it might not be compacted if the block is the currently writing block
+					if (Ctx.Settings.IsDeleteMode)
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': compacting {} blocks",
+									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 BlocksToCompact.size());
+						}
+
+						m_CasContainerStrategy.m_BlockStore.CompactBlocks(
+							BlockCompactState,
+							m_CasContainerStrategy.m_PayloadAlignment,
+							[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+								std::vector<CasDiskIndexEntry> MovedEntries;
+								RwLock::ExclusiveLockScope	   _(m_CasContainerStrategy.m_LocationMapLock);
+								for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+								{
+									size_t		  ChunkIndex = Moved.first;
+									const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
+
+									if (auto It = m_CasContainerStrategy.m_LocationMap.find(Key);
+										It != m_CasContainerStrategy.m_LocationMap.end())
+									{
+										BlockStoreDiskLocation&	  Location	  = m_CasContainerStrategy.m_Locations[It->second];
+										const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
+										if (Location.Get(m_CasContainerStrategy.m_PayloadAlignment) != OldLocation)
+										{
+											// Someone has moved our chunk so lets just skip the new location we were provided, it will be
+											// GC:d at a later time
+											continue;
+										}
+										const BlockStoreLocation& NewLocation = Moved.second;
+
+										Location = BlockStoreDiskLocation(NewLocation, m_CasContainerStrategy.m_PayloadAlignment);
+										MovedEntries.push_back(CasDiskIndexEntry{.Key = Key, .Location = Location});
+									}
+								}
+								m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
+								Stats.RemovedDisk += FreedDiskSpace;
+								if (Ctx.IsCancelledFlag.load())
+								{
+									return false;
+								}
+								return true;
+							},
+							ClaimDiskReserveCallback);
+					}
+					else
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': skipped compacting of {} eligible blocks",
+									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 BlocksToCompact.size());
+						}
+					}
+				}
+			}
 		}
 	}
 
-	CasContainerStrategy&  m_CasContainerStrategy;
-	BlockStoreCompactState m_CompactState;
-	std::vector<IoHash>	   m_CompactStateKeys;
-	std::vector<IoHash>	   m_PrunedKeys;
+	CasContainerStrategy& m_CasContainerStrategy;
 };
 
 class CasContainerReferencePruner : public GcReferencePruner
@@ -640,27 +696,27 @@ public:
 	{
 	}
 
-	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx&						 Ctx,
-															  GcReferenceStoreStats&		 Stats,
-															  const GetUnusedReferencesFunc& GetUnusedReferences)
+	virtual GcStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, GcStats& Stats, const GetUnusedReferencesFunc& GetUnusedReferences)
 	{
+		ZEN_TRACE_CPU("CasContainer::RemoveUnreferencedData");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
 			{
 				return;
 			}
-			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Checked: {}, Deleted: {}, FreedMemory: {} in {}",
 					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
-					 Stats.Count,
-					 Stats.Pruned,
-					 Stats.Compacted,
-					 NiceBytes(Stats.RemovedDisk),
-					 NiceBytes(Stats.RemovedMemory),
+					 Stats.CheckedCount,
+					 Stats.DeletedCount,
+					 NiceBytes(Stats.FreedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
+		Stats.CheckedCount			   = m_Cids.size();
+		Stats.FoundCount			   = UnusedCids.size();
 
 		if (UnusedCids.empty())
 		{
@@ -668,18 +724,14 @@ public:
 			return nullptr;
 		}
 
-		BlockStoreCompactState			 CompactState;
-		BlockStore::ReclaimSnapshotState BlockSnapshotState;
-		std::vector<IoHash>				 CompactStateKeys;
-		std::vector<CasDiskIndexEntry>	 ExpiredEntries;
+		std::vector<CasDiskIndexEntry> ExpiredEntries;
 		ExpiredEntries.reserve(UnusedCids.size());
-		tsl::robin_set<IoHash, IoHash::Hasher> UnusedKeys;
 
 		{
 			RwLock::ExclusiveLockScope __(m_CasContainerStrategy.m_LocationMapLock);
-			if (Ctx.Settings.CollectSmallObjects)
+			if (Ctx.IsCancelledFlag.load())
 			{
-				BlockSnapshotState = m_CasContainerStrategy.m_BlockStore.GetReclaimSnapshotState();
+				return nullptr;
 			}
 
 			for (const IoHash& Cid : UnusedCids)
@@ -689,59 +741,28 @@ public:
 				{
 					continue;
 				}
-				CasDiskIndexEntry			  ExpiredEntry	= {.Key		 = Cid,
-															   .Location = m_CasContainerStrategy.m_Locations[It->second],
-															   .Flags	 = CasDiskIndexEntry::kTombstone};
-				const BlockStoreDiskLocation& Location		= m_CasContainerStrategy.m_Locations[It->second];
-				BlockStoreLocation			  BlockLocation = Location.Get(m_CasContainerStrategy.m_PayloadAlignment);
 				if (Ctx.Settings.CollectSmallObjects)
 				{
-					UnusedKeys.insert(Cid);
-					uint32_t BlockIndex			= BlockLocation.BlockIndex;
-					bool	 IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
-					if (!IsActiveWriteBlock)
-					{
-						CompactState.IncludeBlock(BlockIndex);
-					}
+					CasDiskIndexEntry ExpiredEntry = {.Key		= Cid,
+													  .Location = m_CasContainerStrategy.m_Locations[It->second],
+													  .Flags	= CasDiskIndexEntry::kTombstone};
 					ExpiredEntries.push_back(ExpiredEntry);
 				}
 			}
 
-			// Get all locations we need to keep for affected blocks
-			if (Ctx.Settings.CollectSmallObjects && !UnusedKeys.empty())
-			{
-				for (const auto& Entry : m_CasContainerStrategy.m_LocationMap)
-				{
-					const IoHash& Key = Entry.first;
-					if (UnusedKeys.contains(Key))
-					{
-						continue;
-					}
-					const BlockStoreDiskLocation& Location		= m_CasContainerStrategy.m_Locations[Entry.second];
-					BlockStoreLocation			  BlockLocation = Location.Get(m_CasContainerStrategy.m_PayloadAlignment);
-					if (CompactState.AddKeepLocation(BlockLocation))
-					{
-						CompactStateKeys.push_back(Key);
-					}
-				}
-			}
-
 			if (Ctx.Settings.IsDeleteMode)
 			{
 				for (const CasDiskIndexEntry& Entry : ExpiredEntries)
 				{
 					m_CasContainerStrategy.m_LocationMap.erase(Entry.Key);
+					Stats.DeletedCount++;
 				}
 				m_CasContainerStrategy.m_CasLog.Append(ExpiredEntries);
 				m_CasContainerStrategy.m_CasLog.Flush();
 			}
 		}
 
-		Stats.Pruned += UnusedKeys.size();
-		return new CasContainerStoreCompactor(m_CasContainerStrategy,
-											  std::move(CompactState),
-											  std::move(CompactStateKeys),
-											  std::vector<IoHash>(UnusedKeys.begin(), UnusedKeys.end()));
+		return new CasContainerStoreCompactor(m_CasContainerStrategy);
 	}
 
 private:
@@ -756,21 +777,18 @@ CasContainerStrategy::GetGcName(GcCtx&)
 }
 
 GcReferencePruner*
-CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
+CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&)
 {
+	ZEN_TRACE_CPU("CasContainer::CreateReferencePruner");
+
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
 		if (!Ctx.Settings.Verbose)
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: compactcas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+		ZEN_INFO("GCV2: compactcas [CREATE PRUNER] '{}' in {}",
 				 m_RootDirectory / m_ContainerBaseName,
-				 Stats.Count,
-				 Stats.Pruned,
-				 Stats.Compacted,
-				 NiceBytes(Stats.RemovedDisk),
-				 NiceBytes(Stats.RemovedMemory),
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
@@ -781,13 +799,17 @@ CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& S
 		{
 			return {};
 		}
+		if (Ctx.IsCancelledFlag.load())
+		{
+			return nullptr;
+		}
+
 		CidsToCheck.reserve(m_LocationMap.size());
 		for (const auto& It : m_LocationMap)
 		{
 			CidsToCheck.push_back(It.first);
 		}
 	}
-	Stats.Count += CidsToCheck.size();
 	return new CasContainerReferencePruner(*this, std::move(CidsToCheck));
 }
 
@@ -863,9 +885,11 @@ CasContainerStrategy::MakeIndexSnapshot()
 
 		// Write the current state of the location map to a new index state
 		std::vector<CasDiskIndexEntry> Entries;
+		uint64_t					   IndexLogPosition = 0;
 
 		{
 			RwLock::SharedLockScope ___(m_LocationMapLock);
+			IndexLogPosition = m_CasLog.GetLogCount();
 			Entries.resize(m_LocationMap.size());
 
 			uint64_t EntryIndex = 0;
@@ -880,7 +904,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		BasicFile ObjectIndexFile;
 		ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kTruncate);
 		CasDiskIndexHeader Header = {.EntryCount	   = Entries.size(),
-									 .LogPosition	   = LogCount,
+									 .LogPosition	   = IndexLogPosition,
 									 .PayloadAlignment = gsl::narrow<uint32_t>(m_PayloadAlignment)};
 
 		Header.Checksum = CasDiskIndexHeader::ComputeChecksum(Header);
@@ -890,7 +914,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.Close();
 		EntryCount		   = Entries.size();
-		m_LogFlushPosition = LogCount;
+		m_LogFlushPosition = IndexLogPosition;
 	}
 	catch (std::exception& Err)
 	{
@@ -924,10 +948,10 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 {
 	ZEN_TRACE_CPU("CasContainer::ReadIndexFile");
 
-	std::vector<CasDiskIndexEntry> Entries;
-	Stopwatch					   Timer;
-	const auto					   _ = MakeGuard([&] {
-		ZEN_INFO("read store '{}' index containing {} entries in {}", IndexPath, Entries.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	uint64_t   EntryCount = 0;
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_INFO("read store '{}' index containing {} entries in {}", IndexPath, EntryCount, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	BasicFile ObjectIndexFile;
@@ -942,21 +966,40 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 			(Header.Checksum == CasDiskIndexHeader::ComputeChecksum(Header)) && (Header.PayloadAlignment > 0) &&
 			(Header.EntryCount <= ExpectedEntryCount))
 		{
-			Entries.resize(Header.EntryCount);
-			ObjectIndexFile.Read(Entries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
 			m_PayloadAlignment = Header.PayloadAlignment;
 
-			std::string InvalidEntryReason;
-			for (const CasDiskIndexEntry& Entry : Entries)
+			m_Locations.reserve(ExpectedEntryCount);
+			m_LocationMap.reserve(ExpectedEntryCount);
+
+			std::vector<CasDiskIndexEntry> Entries;
+			Entries.resize(128 * 1024 / sizeof(CasDiskIndexEntry));
+
+			uint64_t RemainingEntries = Header.EntryCount;
+			uint64_t ReadOffset		  = sizeof(CasDiskIndexHeader);
+
+			do
 			{
-				if (!ValidateEntry(Entry, InvalidEntryReason))
+				const uint64_t NumToRead = Min(RemainingEntries, Entries.size());
+				Entries.resize(NumToRead);
+
+				ObjectIndexFile.Read(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), ReadOffset);
+
+				std::string InvalidEntryReason;
+				for (const CasDiskIndexEntry& Entry : Entries)
 				{
-					ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", IndexPath, InvalidEntryReason);
-					continue;
+					if (!ValidateEntry(Entry, InvalidEntryReason))
+					{
+						ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", IndexPath, InvalidEntryReason);
+						continue;
+					}
+					m_LocationMap[Entry.Key] = m_Locations.size();
+					m_Locations.push_back(Entry.Location);
+					++EntryCount;
 				}
-				m_LocationMap[Entry.Key] = m_Locations.size();
-				m_Locations.push_back(Entry.Location);
-			}
+
+				RemainingEntries -= NumToRead;
+				ReadOffset += NumToRead * sizeof(CasDiskIndexEntry);
+			} while (RemainingEntries);
 
 			OutVersion = CasDiskIndexHeader::CurrentVersion;
 			return Header.LogPosition;
@@ -1076,16 +1119,16 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 
 	m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
 
-	std::vector<BlockStoreLocation> KnownLocations;
-	KnownLocations.reserve(m_LocationMap.size());
+	BlockStore::BlockIndexSet KnownBlocks;
+
 	for (const auto& Entry : m_LocationMap)
 	{
 		const BlockStoreDiskLocation& DiskLocation	= m_Locations[Entry.second];
 		BlockStoreLocation			  BlockLocation = DiskLocation.Get(m_PayloadAlignment);
-		KnownLocations.emplace_back(std::move(BlockLocation));
+		KnownBlocks.Add(BlockLocation.BlockIndex);
 	}
 
-	m_BlockStore.SyncExistingBlocksOnDisk(KnownLocations);
+	m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
 
 	if (IsNewStore || (LogEntryCount > 0))
 	{