From 12058b79d7ba17902b90c3e777b10a8c4403fe5e Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Tue, 9 May 2023 14:50:41 +0200
Subject: Validate that entries points inside valid blocks at startup (#280)

* Separate initialization of block store from pruning of unknown blocks
* Validate that entries points inside valid blocks
---
 src/zenstore/compactcas.cpp | 47 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 42 insertions(+), 5 deletions(-)

(limited to 'src/zenstore/compactcas.cpp')

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 7b2c21b0f..2974570e5 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -701,23 +701,60 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		std::filesystem::remove_all(BasePath);
 	}
 
+	CreateDirectories(BasePath);
+
+	std::unordered_map<uint32_t, uint64_t> BlockSizes =
+		m_BlockStore.Initialize(m_BlocksBasePath, m_MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1);
+
 	m_LogFlushPosition	   = ReadIndexFile();
 	uint64_t LogEntryCount = ReadLog(m_LogFlushPosition);
 
-	CreateDirectories(BasePath);
-
 	std::filesystem::path LogPath = GetLogPath(m_RootDirectory, m_ContainerBaseName);
 	m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
 
 	std::vector<BlockStoreLocation> KnownLocations;
 	KnownLocations.reserve(m_LocationMap.size());
+	std::vector<CasDiskIndexEntry> BadEntries;
 	for (const auto& Entry : m_LocationMap)
 	{
-		const BlockStoreDiskLocation& Location = Entry.second;
-		KnownLocations.push_back(Location.Get(m_PayloadAlignment));
+		const BlockStoreDiskLocation& DiskLocation = Entry.second;
+		auto						  BlockIt	   = BlockSizes.find(DiskLocation.GetBlockIndex());
+		if (BlockIt == BlockSizes.end())
+		{
+			ZEN_WARN("Unknown block {} for entry {}", DiskLocation.GetBlockIndex(), Entry.first.ToHexString());
+		}
+		else
+		{
+			BlockStoreLocation BlockLocation = DiskLocation.Get(m_PayloadAlignment);
+
+			uint64_t BlockSize = BlockIt->second;
+			if (BlockLocation.Offset + BlockLocation.Size > BlockSize)
+			{
+				ZEN_WARN("Range is outside of block {} for entry {}", BlockLocation.BlockIndex, Entry.first.ToHexString());
+			}
+			else
+			{
+				KnownLocations.emplace_back(std::move(BlockLocation));
+				continue;
+			}
+			BadEntries.push_back({.Key = Entry.first, .Location = DiskLocation, .Flags = CasDiskIndexEntry::kTombstone});
+		}
+	}
+
+	if (!BadEntries.empty())
+	{
+		m_CasLog.Append(BadEntries);
+		m_CasLog.Flush();
+
+		LogEntryCount += BadEntries.size();
+
+		for (const CasDiskIndexEntry& BadEntry : BadEntries)
+		{
+			m_LocationMap.erase(BadEntry.Key);
+		}
 	}
 
-	m_BlockStore.Initialize(m_BlocksBasePath, m_MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1, KnownLocations);
+	m_BlockStore.Prune(KnownLocations);
 
 	if (IsNewStore || (LogEntryCount > 0))
 	{
-- 
cgit v1.2.3