merged from main

11 files changed, 341 insertions, 149 deletions

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 76cc863a4..0d6baa193 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -2,18 +2,14 @@ name: Create Release
 
 on:
   push:
-    # Sequence of patterns matched against refs/tags
-    tags:
-      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+      paths:
+      - 'VERSION.txt'
 
 jobs:
   bundle-windows:
-
     runs-on: [self-hosted, windows, x64]
-
     env:
       VCPKG_VERSION: 2022.03.10
-      ZEN_VERSION: ${{github.ref.name}}
 
     steps:
     - uses: actions/checkout@v2
@@ -43,7 +39,6 @@ jobs:
         xmake bundle -v -y
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
-        ZEN_VERSION: ${{ github.ref_name }}
 
     - name: Upload zenserver-win64
       uses: actions/upload-artifact@v3
@@ -53,7 +48,6 @@ jobs:
 
   bundle-linux:
     runs-on: [self-hosted, linux, x64]
-
     env:
       VCPKG_VERSION: 2022.03.10
 
@@ -90,7 +84,6 @@ jobs:
         xmake bundle -v -y
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
-        ZEN_VERSION: ${{ github.ref_name }}
 
     - name: Upload zenserver-linux
       uses: actions/upload-artifact@v3
@@ -101,6 +94,7 @@ jobs:
   create-release:
     runs-on: [self-hosted, linux, x64]
     needs: [bundle-linux, bundle-windows]
+
     steps:
     - uses: actions/checkout@v2
 
@@ -134,13 +128,19 @@ jobs:
       with:
         path: "CHANGELOG.tmp"
 
+    - name: Read VERSION.txt
+      id: read_version
+      uses: andstor/file-reader-action@v1
+      with:
+        path: "VERSION.txt"
+
     - name: Create Release
       id: create_release
       uses: softprops/action-gh-release@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
-        tag_name: ${{github.ref.name}}
+        tag_name: v${{steps.read_version.outputs.contents}}
         body: |
           ${{steps.read_changelog.outputs.contents}}
         draft: false
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 3988a9dfb..7ceec353b 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -4,8 +4,14 @@ on:
   pull_request:
     types: [opened, reopened, synchronize]
     branches: [ main ]
+    paths-ignore:
+    - 'VERSION.txt'
+    - 'CHANGELOG.md'
   push:
       branches: [ main ]
+      paths-ignore:
+      - 'VERSION.txt'
+      - 'CHANGELOG.md'
 
 jobs:
   cancel-old-build:
diff --git a/VERSION.txt b/VERSION.txt
new file mode 100644
index 000000000..0da163cd7
--- /dev/null
+++ b/VERSION.txt
@@ -0,0 +1 @@
+0.1.4-pre2
diff --git a/xmake.lua b/xmake.lua
index de63f205b..3683c5344 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -1,9 +1,6 @@
 -- Copyright Epic Games, Inc. All Rights Reserved.
 
 set_configvar("ZEN_SCHEMA_VERSION", 3) -- changed cas oplog format (p3rl)
-local zenversion = os.getenv("ZEN_VERSION") or "0.0.0"
-zenversion = string.gsub(zenversion, "^v", "")
-set_version(zenversion, { build = "%Y%m%d%H%M" })
 
 add_requires(
     "vcpkg::asio",
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index 56b05d86d..37ef57c2d 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -274,6 +274,8 @@ IoBufferExtendedCore::Materialize() const
 	const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset;
 	const uint64_t MapSize					= m_DataBytes + MappedOffsetDisplacement;
 
+	ZEN_ASSERT(MapSize > 0);
+
 #if ZEN_PLATFORM_WINDOWS
 	NewMmapHandle = CreateFileMapping(m_FileHandle,
 									  /* lpFileMappingAttributes */ nullptr,
diff --git a/zencore/xmake.lua b/zencore/xmake.lua
index 63e874ac5..3f78be99c 100644
--- a/zencore/xmake.lua
+++ b/zencore/xmake.lua
@@ -4,6 +4,10 @@ target('zencore')
     set_kind("static")
     add_headerfiles("**.h")
     add_configfiles("include/zencore/config.h.in")
+    on_load(function (target)
+        local version = io.readfile("VERSION.txt")
+        target:set("version", version:trim(), {build = "%Y%m%d%H%M"})
+    end)
     set_configdir("include/zencore")
     add_files("**.cpp")
     add_includedirs("include", {public=true})
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 0ccd5d52a..4be33170c 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1308,56 +1308,148 @@ ZenCacheDiskLayer::CacheBucket::SaveManifest()
 void
 ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 {
-	std::vector<IoHash> BadKeys;
+	std::vector<IoHash>				BadKeys;
+	std::vector<BlockStoreLocation> ChunkLocations;
+	std::vector<IoHash>				ChunkIndexToChunkHash;
+
+	auto ValidateEntry = [](ZenContentType ContentType, IoBuffer Buffer) {
+		if (ContentType == ZenContentType::kCbObject)
+		{
+			CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
+			return Error == CbValidateError::None;
+		}
+		if (ContentType == ZenContentType::kCompressedBinary)
+		{
+			if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); !Compressed)
+			{
+				return false;
+			}
+		}
+		return true;
+	};
+
+	RwLock::SharedLockScope _(m_IndexLock);
+
+	const size_t BlockChunkInitialCount = m_Index.size() / 4;
+	ChunkLocations.reserve(BlockChunkInitialCount);
+	ChunkIndexToChunkHash.reserve(BlockChunkInitialCount);
 
+	for (auto& Kv : m_Index)
 	{
-		RwLock::SharedLockScope _(m_IndexLock);
+		const IoHash&		HashKey = Kv.first;
+		const DiskLocation& Loc		= Kv.second.Location;
 
-		for (auto& Kv : m_Index)
+		if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 		{
-			const IoHash&		HashKey = Kv.first;
-			const DiskLocation& Loc		= Kv.second.Location;
+			if (Loc.GetContentType() == ZenContentType::kBinary)
+			{
+				ExtendablePathBuilder<256> DataFilePath;
+				BuildPath(DataFilePath, HashKey);
 
-			ZenCacheValue Value;
+				RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
 
-			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
-			{
-				if (GetStandaloneCacheValue(Loc, HashKey, Value))
+				std::error_code Ec;
+				uintmax_t		size = std::filesystem::file_size(DataFilePath.ToPath(), Ec);
+				if (Ec)
 				{
-					// Note: we cannot currently validate contents since we don't
-					// have a content hash!
-					continue;
+					BadKeys.push_back(HashKey);
+				}
+				if (size != Loc.Size())
+				{
+					BadKeys.push_back(HashKey);
 				}
+				continue;
 			}
-			else if (GetInlineCacheValue(Loc, Value))
+			ZenCacheValue Value;
+			if (!GetStandaloneCacheValue(Loc, HashKey, Value))
 			{
-				// Validate contents
+				BadKeys.push_back(HashKey);
+				continue;
+			}
+			if (!ValidateEntry(Loc.GetContentType(), Value.Value))
+			{
+				BadKeys.push_back(HashKey);
 				continue;
 			}
-			// Value not found
-			BadKeys.push_back(HashKey);
+		}
+		else
+		{
+			ChunkLocations.emplace_back(Loc.GetBlockLocation(m_PayloadAlignment));
+			ChunkIndexToChunkHash.push_back(HashKey);
+			continue;
 		}
 	}
 
+	const auto ValidateSmallChunk = [&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+		if (!Data)
+		{
+			// ChunkLocation out of range of stored blocks
+			BadKeys.push_back(Hash);
+			return;
+		}
+		IoBuffer Buffer(IoBuffer::Wrap, Data, Size);
+		if (!Buffer)
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+		ZenContentType ContentType = m_Index.at(Hash).Location.GetContentType();
+		if (!ValidateEntry(ContentType, Buffer))
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+	};
+
+	const auto ValidateLargeChunk = [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+		IoBuffer	  Buffer(IoBuffer::BorrowedFile, File.GetBasicFile().Handle(), Offset, Size);
+		if (!Buffer)
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+		ZenContentType ContentType = m_Index.at(Hash).Location.GetContentType();
+		if (!ValidateEntry(ContentType, Buffer))
+		{
+			BadKeys.push_back(Hash);
+			return;
+		}
+	};
+
+	m_BlockStore.IterateChunks(ChunkLocations, ValidateSmallChunk, ValidateLargeChunk);
+
+	_.ReleaseNow();
+
 	if (BadKeys.empty())
 	{
 		return;
 	}
 
+	ZEN_ERROR("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_BucketDir / m_BucketName);
+
 	if (Ctx.RunRecovery())
 	{
-		RwLock::ExclusiveLockScope _(m_IndexLock);
+		// Deal with bad chunks by removing them from our lookup map
+
+		std::vector<DiskIndexEntry> LogEntries;
+		LogEntries.reserve(BadKeys.size());
 
-		for (const IoHash& BadKey : BadKeys)
 		{
-			// Log a tombstone and delete the in-memory index for the bad entry
+			RwLock::ExclusiveLockScope __(m_IndexLock);
+			for (const IoHash& BadKey : BadKeys)
+			{
+				// Log a tombstone and delete the in-memory index for the bad entry
 
-			const auto	 It		  = m_Index.find(BadKey);
-			DiskLocation Location = It->second.Location;
-			Location.Flags |= DiskLocation::kTombStone;
-			m_SlogFile.Append(DiskIndexEntry{.Key = BadKey, .Location = Location});
-			m_Index.erase(BadKey);
+				const auto	 It		  = m_Index.find(BadKey);
+				DiskLocation Location = It->second.Location;
+				Location.Flags |= DiskLocation::kTombStone;
+				LogEntries.push_back(DiskIndexEntry{.Key = BadKey, .Location = Location});
+				m_Index.erase(BadKey);
+			}
 		}
+		m_SlogFile.Append(LogEntries);
 	}
 
 	// Let whomever it concerns know about the bad chunks. This could
diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp
index 842587708..e203e0631 100644
--- a/zenserver/frontend/frontend.cpp
+++ b/zenserver/frontend/frontend.cpp
@@ -26,7 +26,7 @@ FindZipFsInBinary(const IoBuffer& BinBuffer)
 	uintptr_t Cursor  = uintptr_t(BinBuffer.GetData());
 	size_t	  BinSize = 0;
 
-	uint32_t Magic = *(uint32_t*)(BinBuffer.GetData());
+	uint32_t Magic = *(uint32_t*)(Cursor);
 #if ZEN_PLATFORM_LINUX
 	if (Magic == 0x464c457f)
 	{
@@ -134,6 +134,12 @@ FindZipFsInBinary(const IoBuffer& BinBuffer)
 		return {};
 	}
 
+	size_t ZipFsSize = BinBuffer.Size() - BinSize;
+	if (!ZipFsSize)
+	{
+		return {};
+	}
+
 	return IoBuffer(BinBuffer, BinSize);
 }
 
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 4e61c23cf..88592d785 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -7,12 +7,13 @@
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
 
+#include <algorithm>
+
 #if ZEN_WITH_TESTS
 #	include <zencore/compactbinarybuilder.h>
 #	include <zencore/testing.h>
 #	include <zencore/testutils.h>
 #	include <zencore/workthreadpool.h>
-#	include <algorithm>
 #	include <random>
 #endif
 
@@ -208,7 +209,7 @@ BlockStore::Close()
 }
 
 void
-BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, WriteChunkCallback Callback)
+BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, const WriteChunkCallback& Callback)
 {
 	ZEN_ASSERT(Data != nullptr);
 	ZEN_ASSERT(Size > 0u);
@@ -612,74 +613,108 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 
 void
 BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
-						  IterateChunksSmallSizeCallback		 SmallSizeCallback,
-						  IterateChunksLargeSizeCallback		 LargeSizeCallback)
+						  const IterateChunksSmallSizeCallback&	 SmallSizeCallback,
+						  const IterateChunksLargeSizeCallback&	 LargeSizeCallback)
 {
-	// We do a read sweep through the payloads file and validate
-	// any entries that are contained within each segment, with
-	// the assumption that most entries will be checked in this
-	// pass. An alternative strategy would be to use memory mapping.
-
+	std::vector<size_t> LocationIndexes;
+	LocationIndexes.reserve(ChunkLocations.size());
+	for (size_t ChunkIndex = 0; ChunkIndex < ChunkLocations.size(); ++ChunkIndex)
 	{
-		ChunkIndexArray BigChunks;
-		IoBuffer		ReadBuffer{ScrubSmallChunkWindowSize};
-		void*			BufferBase = ReadBuffer.MutableData();
-
-		RwLock::SharedLockScope _(m_InsertLock);
-
-		for (const auto& Block : m_ChunkBlocks)
+		LocationIndexes.push_back(ChunkIndex);
+	}
+	std::sort(LocationIndexes.begin(), LocationIndexes.end(), [&](size_t IndexA, size_t IndexB) -> bool {
+		const BlockStoreLocation& LocationA = ChunkLocations[IndexA];
+		const BlockStoreLocation& LocationB = ChunkLocations[IndexB];
+		if (LocationA.BlockIndex < LocationB.BlockIndex)
 		{
-			uint64_t				   WindowStart = 0;
-			uint64_t				   WindowEnd   = ScrubSmallChunkWindowSize;
-			uint32_t				   BlockIndex  = Block.first;
-			const Ref<BlockStoreFile>& BlockFile   = Block.second;
-			const uint64_t			   FileSize	   = BlockFile->FileSize();
-
-			do
-			{
-				const uint64_t ChunkSize = Min(ScrubSmallChunkWindowSize, FileSize - WindowStart);
-				BlockFile->Read(BufferBase, ChunkSize, WindowStart);
+			return true;
+		}
+		else if (LocationA.BlockIndex > LocationB.BlockIndex)
+		{
+			return false;
+		}
+		return LocationA.Offset < LocationB.Offset;
+	});
 
-				// TODO: We could be smarter here if the ChunkLocations were sorted on block index - we could
-				// then only scan a subset of ChunkLocations instead of scanning through them all...
-				for (size_t ChunkIndex = 0; ChunkIndex < ChunkLocations.size(); ++ChunkIndex)
-				{
-					const BlockStoreLocation Location = ChunkLocations[ChunkIndex];
-					if (BlockIndex != Location.BlockIndex)
-					{
-						continue;
-					}
+	IoBuffer ReadBuffer{ScrubSmallChunkWindowSize};
+	void*	 BufferBase = ReadBuffer.MutableData();
 
-					const uint64_t EntryOffset = Location.Offset;
-					if ((EntryOffset >= WindowStart) && (EntryOffset < WindowEnd))
-					{
-						const uint64_t EntryEnd = EntryOffset + Location.Size;
+	RwLock::SharedLockScope _(m_InsertLock);
 
-						if (EntryEnd >= WindowEnd)
-						{
-							BigChunks.push_back(ChunkIndex);
+	auto GetNextRange = [&](size_t StartIndexOffset) {
+		size_t					  ChunkCount	= 0;
+		size_t					  StartIndex	= LocationIndexes[StartIndexOffset];
+		const BlockStoreLocation& StartLocation = ChunkLocations[StartIndex];
+		uint64_t				  StartOffset	= StartLocation.Offset;
+		while (StartIndexOffset + ChunkCount < LocationIndexes.size())
+		{
+			size_t					  NextIndex = LocationIndexes[StartIndexOffset + ChunkCount];
+			const BlockStoreLocation& Location	= ChunkLocations[NextIndex];
+			if (Location.BlockIndex != StartLocation.BlockIndex)
+			{
+				break;
+			}
+			if ((Location.Offset + Location.Size) - StartOffset > ScrubSmallChunkWindowSize)
+			{
+				break;
+			}
+			++ChunkCount;
+		}
+		return ChunkCount;
+	};
 
-							continue;
-						}
+	size_t LocationIndexOffset = 0;
+	while (LocationIndexOffset < LocationIndexes.size())
+	{
+		size_t					  ChunkIndex	= LocationIndexes[LocationIndexOffset];
+		const BlockStoreLocation& FirstLocation = ChunkLocations[ChunkIndex];
 
-						SmallSizeCallback(ChunkIndex,
-										  reinterpret_cast<uint8_t*>(BufferBase) + Location.Offset - WindowStart,
-										  Location.Size);
-					}
+		const Ref<BlockStoreFile>& BlockFile = m_ChunkBlocks[FirstLocation.BlockIndex];
+		if (!BlockFile)
+		{
+			while (ChunkLocations[ChunkIndex].BlockIndex == FirstLocation.BlockIndex)
+			{
+				SmallSizeCallback(ChunkIndex, nullptr, 0);
+				LocationIndexOffset++;
+				if (LocationIndexOffset == LocationIndexes.size())
+				{
+					break;
 				}
-
-				WindowStart += ScrubSmallChunkWindowSize;
-				WindowEnd += ScrubSmallChunkWindowSize;
-			} while (WindowStart < FileSize);
+				ChunkIndex = LocationIndexes[LocationIndexOffset];
+			}
+			continue;
 		}
-
-		// Deal with large chunks and chunks that extend over a ScrubSmallChunkWindowSize border
-		for (size_t ChunkIndex : BigChunks)
+		size_t BlockSize  = BlockFile->FileSize();
+		size_t RangeCount = GetNextRange(LocationIndexOffset);
+		if (RangeCount > 0)
 		{
-			const BlockStoreLocation   Location	 = ChunkLocations[ChunkIndex];
-			const Ref<BlockStoreFile>& BlockFile = m_ChunkBlocks[Location.BlockIndex];
-			LargeSizeCallback(ChunkIndex, BlockFile, Location.Offset, Location.Size);
+			size_t					  LastChunkIndex = LocationIndexes[LocationIndexOffset + RangeCount - 1];
+			const BlockStoreLocation& LastLocation	 = ChunkLocations[LastChunkIndex];
+			uint64_t				  Size			 = LastLocation.Offset + LastLocation.Size - FirstLocation.Offset;
+			BlockFile->Read(BufferBase, Size, FirstLocation.Offset);
+			for (size_t RangeIndex = 0; RangeIndex < RangeCount; ++RangeIndex)
+			{
+				size_t					  NextChunkIndex = LocationIndexes[LocationIndexOffset + RangeIndex];
+				const BlockStoreLocation& ChunkLocation	 = ChunkLocations[NextChunkIndex];
+				if (ChunkLocation.Size == 0 || (ChunkLocation.Offset + ChunkLocation.Size > BlockSize))
+				{
+					SmallSizeCallback(NextChunkIndex, nullptr, 0);
+					continue;
+				}
+				void* BufferPtr = &((char*)BufferBase)[ChunkLocation.Offset - FirstLocation.Offset];
+				SmallSizeCallback(NextChunkIndex, BufferPtr, ChunkLocation.Size);
+			}
+			LocationIndexOffset += RangeCount;
+			continue;
+		}
+		if (FirstLocation.Size == 0 || (FirstLocation.Offset + FirstLocation.Size > BlockSize))
+		{
+			SmallSizeCallback(ChunkIndex, nullptr, 0);
+			LocationIndexOffset++;
+			continue;
 		}
+		LargeSizeCallback(ChunkIndex, *BlockFile.Get(), FirstLocation.Offset, FirstLocation.Size);
+		LocationIndexOffset++;
 	}
 }
 
@@ -1176,35 +1211,77 @@ TEST_CASE("blockstore.iterate.chunks")
 	std::string		   VeryLargeChunk(ScrubSmallChunkWindowSize * 2, 'L');
 	BlockStoreLocation VeryLargeChunkLocation = WriteStringAsChunk(Store, VeryLargeChunk, 4);
 
+	BlockStoreLocation BadLocationZeroSize	 = {.BlockIndex = 0, .Offset = 0, .Size = 0};
+	BlockStoreLocation BadLocationOutOfRange = {.BlockIndex = 0,
+												.Offset		= ScrubSmallChunkWindowSize,
+												.Size		= ScrubSmallChunkWindowSize * 2};
+	BlockStoreLocation BadBlockIndex		 = {.BlockIndex = 0xfffff, .Offset = 1024, .Size = 1024};
+
 	Store.IterateChunks(
-		{FirstChunkLocation, SecondChunkLocation, VeryLargeChunkLocation},
+		{FirstChunkLocation, SecondChunkLocation, VeryLargeChunkLocation, BadLocationZeroSize, BadLocationOutOfRange, BadBlockIndex},
 		[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
-			CHECK(Data);
-			CHECK(Size > 0);
-			std::string AsString((const char*)Data, Size);
 			switch (ChunkIndex)
 			{
 				case 0:
-					CHECK(AsString == FirstChunkData);
+					CHECK(Data);
+					CHECK(Size == FirstChunkData.size());
+					CHECK(std::string((const char*)Data, Size) == FirstChunkData);
 					break;
 				case 1:
-					CHECK(AsString == SecondChunkData);
+					CHECK(Data);
+					CHECK(Size == SecondChunkData.size());
+					CHECK(std::string((const char*)Data, Size) == SecondChunkData);
+					break;
+				case 2:
+					CHECK(false);
+					break;
+				case 3:
+					CHECK(!Data);
+					break;
+				case 4:
+					CHECK(!Data);
+					break;
+				case 5:
+					CHECK(!Data);
 					break;
 				default:
 					CHECK(false);
 					break;
 			}
 		},
-		[&](size_t ChunkIndex, Ref<BlockStoreFile> BlockFile, uint64_t Offset, uint64_t Size) {
-			CHECK(BlockFile);
-			CHECK(ChunkIndex == 2);
-			CHECK(Offset == VeryLargeChunkLocation.Offset);
-			CHECK(Size == VeryLargeChunkLocation.Size);
-			size_t StreamOffset = 0;
-			BlockFile->StreamByteRange(Offset, Size, [&](const void* Data, size_t Size) {
-				const char* VeryLargeChunkSection = &(VeryLargeChunk.data()[StreamOffset]);
-				CHECK(memcmp(VeryLargeChunkSection, Data, Size) == 0);
-			});
+		[&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+			switch (ChunkIndex)
+			{
+				case 0:
+				case 1:
+					CHECK(false);
+					break;
+				case 2:
+					{
+						CHECK(Size == VeryLargeChunk.size());
+						char*  Buffer	  = new char[Size];
+						size_t HashOffset = 0;
+						File.StreamByteRange(Offset, Size, [&](const void* Data, uint64_t Size) {
+							memcpy(&Buffer[HashOffset], Data, Size);
+							HashOffset += Size;
+						});
+						CHECK(memcmp(Buffer, VeryLargeChunk.data(), Size) == 0);
+						delete[] Buffer;
+					}
+					break;
+				case 3:
+					CHECK(false);
+					break;
+				case 4:
+					CHECK(false);
+					break;
+				case 5:
+					CHECK(false);
+					break;
+				default:
+					CHECK(false);
+					break;
+			}
 		});
 }
 
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 65f959a0e..5aed02e7f 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -315,11 +315,13 @@ CasContainerStrategy::Flush()
 void
 CasContainerStrategy::Scrub(ScrubContext& Ctx)
 {
-	RwLock::SharedLockScope _(m_LocationMapLock);
-
-	uint64_t						TotalChunkCount = m_LocationMap.size();
+	std::vector<IoHash>				BadKeys;
 	std::vector<BlockStoreLocation> ChunkLocations;
 	std::vector<IoHash>				ChunkIndexToChunkHash;
+
+	RwLock::SharedLockScope _(m_LocationMapLock);
+
+	uint64_t TotalChunkCount = m_LocationMap.size();
 	ChunkLocations.reserve(TotalChunkCount);
 	ChunkIndexToChunkHash.reserve(TotalChunkCount);
 	{
@@ -328,37 +330,45 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
 			const IoHash&				  ChunkHash	   = Entry.first;
 			const BlockStoreDiskLocation& DiskLocation = Entry.second;
 			BlockStoreLocation			  Location	   = DiskLocation.Get(m_PayloadAlignment);
-			size_t						  ChunkIndex   = ChunkLocations.size();
 
 			ChunkLocations.push_back(Location);
-			ChunkIndexToChunkHash[ChunkIndex] = ChunkHash;
+			ChunkIndexToChunkHash.push_back(ChunkHash);
 		}
 	}
 
-	std::vector<IoHash> BadKeys;
+	const auto ValidateSmallChunk = [&](size_t ChunkIndex, const void* Data, uint64_t Size) {
+		const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
+		if (!Data)
+		{
+			// ChunkLocation out of range of stored blocks
+			BadKeys.push_back(Hash);
+			return;
+		}
+		const IoHash ComputedHash = IoHash::HashBuffer(Data, Size);
+		if (ComputedHash != Hash)
+		{
+			// Hash mismatch
+			BadKeys.push_back(Hash);
+			return;
+		}
+	};
 
-	m_BlockStore.IterateChunks(
-		ChunkLocations,
-		[&](size_t ChunkIndex, const void* Data, uint64_t Size) {
-			const IoHash  ComputedHash = IoHash::HashBuffer(Data, Size);
-			const IoHash& ExpectedHash = ChunkIndexToChunkHash[ChunkIndex];
-			if (ComputedHash != ExpectedHash)
-			{
-				// Hash mismatch
-				BadKeys.push_back(ExpectedHash);
-			}
-		},
-		[&](size_t ChunkIndex, Ref<BlockStoreFile> BlockFile, uint64_t Offset, uint64_t Size) {
-			IoHashStream Hasher;
-			BlockFile->StreamByteRange(Offset, Size, [&](const void* Data, uint64_t Size) { Hasher.Append(Data, Size); });
-			IoHash		  ComputedHash = Hasher.GetHash();
-			const IoHash& ExpectedHash = ChunkIndexToChunkHash[ChunkIndex];
-			if (ComputedHash != ExpectedHash)
-			{
-				// Hash mismatch
-				BadKeys.push_back(ExpectedHash);
-			}
-		});
+	const auto ValidateLargeChunk = [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) {
+		IoHashStream Hasher;
+		File.StreamByteRange(Offset, Size, [&](const void* Data, uint64_t Size) { Hasher.Append(Data, Size); });
+		IoHash		  ComputedHash = Hasher.GetHash();
+		const IoHash& Hash		   = ChunkIndexToChunkHash[ChunkIndex];
+		if (ComputedHash != Hash)
+		{
+			// Hash mismatch
+			BadKeys.push_back(Hash);
+			return;
+		}
+	};
+
+	m_BlockStore.IterateChunks(ChunkLocations, ValidateSmallChunk, ValidateLargeChunk);
+
+	_.ReleaseNow();
 
 	if (BadKeys.empty())
 	{
@@ -367,8 +377,6 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
 
 	ZEN_ERROR("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_Config.RootDirectory / m_ContainerBaseName);
 
-	_.ReleaseNow();
-
 	if (Ctx.RunRecovery())
 	{
 		// Deal with bad chunks by removing them from our lookup map
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
index 000395fb9..fe435cdff 100644
--- a/zenstore/include/zenstore/blockstore.h
+++ b/zenstore/include/zenstore/blockstore.h
@@ -122,10 +122,9 @@ public:
 	typedef std::function<void(const MovedChunksArray& MovedChunks, const ChunkIndexArray& RemovedChunks)> ReclaimCallback;
 	typedef std::function<uint64_t()>																	   ClaimDiskReserveCallback;
 	typedef std::function<void(size_t ChunkIndex, const void* Data, uint64_t Size)>						   IterateChunksSmallSizeCallback;
-	typedef std::function<void(size_t ChunkIndex, Ref<BlockStoreFile> BlockFile, uint64_t Offset, uint64_t Size)>
-																	 IterateChunksLargeSizeCallback;
-	typedef std::function<void(const MovedChunksArray& MovedChunks)> SplitCallback;
-	typedef std::function<void(const BlockStoreLocation& Location)>	 WriteChunkCallback;
+	typedef std::function<void(size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size)>   IterateChunksLargeSizeCallback;
+	typedef std::function<void(const MovedChunksArray& MovedChunks)>									   SplitCallback;
+	typedef std::function<void(const BlockStoreLocation& Location)>										   WriteChunkCallback;
 
 	void Initialize(const std::filesystem::path&		   BlocksBasePath,
 					uint64_t							   MaxBlockSize,
@@ -133,7 +132,7 @@ public:
 					const std::vector<BlockStoreLocation>& KnownLocations);
 	void Close();
 
-	void WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, WriteChunkCallback Callback);
+	void WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, const WriteChunkCallback& Callback);
 
 	IoBuffer TryGetChunk(const BlockStoreLocation& Location);
 	void	 Flush();
@@ -149,8 +148,8 @@ public:
 						const ClaimDiskReserveCallback&		   DiskReserveCallback = []() { return 0; });
 
 	void IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
-					   IterateChunksSmallSizeCallback		  SmallSizeCallback,
-					   IterateChunksLargeSizeCallback		  LargeSizeCallback);
+					   const IterateChunksSmallSizeCallback&  SmallSizeCallback,
+					   const IterateChunksLargeSizeCallback&  LargeSizeCallback);
 
 	static bool Split(const std::vector<BlockStoreLocation>& ChunkLocations,
 					  const std::filesystem::path&			 SourceBlockFilePath,