path: root/zenstore/compactcas.cpp

// Copyright Epic Games, Inc. All Rights Reserved.

#include <zenstore/cas.h>

#include "compactcas.h"

#include <zencore/compactbinarybuilder.h>
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/memory.h>
#include <zencore/string.h>
#include <zencore/testing.h>
#include <zencore/testutils.h>
#include <zencore/thread.h>
#include <zencore/uid.h>

#include <zenstore/gc.h>

#include <filesystem>
#include <functional>
#include <gsl/gsl-lite.hpp>

#if ZEN_WITH_TESTS
#	include <algorithm>
#	include <random>
#endif

//////////////////////////////////////////////////////////////////////////

namespace zen {

static uint64_t
AlignPositon(uint64_t Offset, uint64_t Alignment)
{
	return (Offset + Alignment - 1) & ~(Alignment - 1);
}

CasContainerStrategy::CasContainerStrategy(const CasStoreConfiguration& Config, CasGc& Gc)
: GcStorage(Gc)
, m_Config(Config)
, m_Log(logging::Get("containercas"))
{
}

CasContainerStrategy::~CasContainerStrategy()
{
}

void
CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore)
{
	ZEN_ASSERT(IsPow2(Alignment));
	ZEN_ASSERT(!m_IsInitialized);

	m_ContainerBaseName = ContainerBaseName;
	m_PayloadAlignment	= Alignment;

	OpenContainer(IsNewStore);

	m_IsInitialized = true;
}

CasStore::InsertResult
CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash)
{
	RwLock::ExclusiveLockScope _i(m_InsertLock);

	{
		RwLock::SharedLockScope _l(m_LocationMapLock);
		auto					KeyIt = m_LocationMap.find(ChunkHash);

		if (KeyIt != m_LocationMap.end())
		{
			return CasStore::InsertResult{.New = false};
		}
	}

	// New entry

	const uint64_t InsertOffset = m_CurrentInsertOffset;
	m_SmallObjectFile.Write(ChunkData, ChunkSize, InsertOffset);
	m_CurrentInsertOffset = AlignPositon(m_CurrentInsertOffset + ChunkSize, m_PayloadAlignment);

	const CasDiskLocation Location{InsertOffset, ChunkSize};
	CasDiskIndexEntry	  IndexEntry{.Key = ChunkHash, .Location = Location};

	RwLock::ExclusiveLockScope __(m_LocationMapLock);
	m_LocationMap[ChunkHash] = Location;
	m_TotalSize.fetch_add(static_cast<uint64_t>(ChunkSize));
	m_CasLog.Append(IndexEntry);

	return CasStore::InsertResult{.New = true};
}

CasStore::InsertResult
CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
{
	return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash);
}

IoBuffer
CasContainerStrategy::FindChunk(const IoHash& ChunkHash)
{
	RwLock::SharedLockScope _(m_LocationMapLock);

	if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end())
	{
		const CasDiskLocation& Location = KeyIt->second;

		return IoBufferBuilder::MakeFromFileHandle(m_SmallObjectFile.Handle(), Location.GetOffset(), Location.GetSize());
	}

	// Not found

	return IoBuffer();
}

bool
CasContainerStrategy::HaveChunk(const IoHash& ChunkHash)
{
	RwLock::SharedLockScope _(m_LocationMapLock);

	if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end())
	{
		return true;
	}

	return false;
}

void
CasContainerStrategy::FilterChunks(CasChunkSet& InOutChunks)
{
	// This implementation is good enough for relatively small
	// chunk sets (in terms of chunk identifiers), but would
	// benefit from a better implementation which removes
	// items incrementally for large sets, especially when
	// we're likely to already have a large proportion of the
	// chunks in the set

	InOutChunks.RemoveChunksIf([&](const IoHash& Hash) { return HaveChunk(Hash); });
}

void
CasContainerStrategy::Flush()
{
	RwLock::SharedLockScope _(m_LocationMapLock);
	m_CasLog.Flush();
	m_SmallObjectFile.Flush();
}

void
CasContainerStrategy::Scrub(ScrubContext& Ctx)
{
	const uint64_t WindowSize  = 4 * 1024 * 1024;
	uint64_t	   WindowStart = 0;
	uint64_t	   WindowEnd   = WindowSize;
	const uint64_t FileSize	   = m_SmallObjectFile.FileSize();

	std::vector<CasDiskIndexEntry> BigChunks;
	std::vector<CasDiskIndexEntry> BadChunks;

	// We do a read sweep through the payloads file and validate
	// any entries that are contained within each segment, with
	// the assumption that most entries will be checked in this
	// pass. An alternative strategy would be to use memory mapping.

	{
		IoBuffer ReadBuffer{WindowSize};
		void*	 BufferBase = ReadBuffer.MutableData();

		RwLock::SharedLockScope _(m_LocationMapLock);

		do
		{
			const uint64_t ChunkSize = Min(WindowSize, FileSize - WindowStart);
			m_SmallObjectFile.Read(BufferBase, ChunkSize, WindowStart);

			for (auto& Entry : m_LocationMap)
			{
				const uint64_t EntryOffset = Entry.second.GetOffset();

				if ((EntryOffset >= WindowStart) && (EntryOffset < WindowEnd))
				{
					const uint64_t EntryEnd = EntryOffset + Entry.second.GetSize();

					if (EntryEnd >= WindowEnd)
					{
						BigChunks.push_back({.Key = Entry.first, .Location = Entry.second});

						continue;
					}

					const IoHash ComputedHash =
						IoHash::HashBuffer(reinterpret_cast<uint8_t*>(BufferBase) + Entry.second.GetOffset() - WindowStart,
										   Entry.second.GetSize());

					if (Entry.first != ComputedHash)
					{
						// Hash mismatch

						BadChunks.push_back({.Key = Entry.first, .Location = Entry.second});
					}
				}
			}

			WindowStart += WindowSize;
			WindowEnd += WindowSize;
		} while (WindowStart < FileSize);

		// Deal with large chunks

		for (const CasDiskIndexEntry& Entry : BigChunks)
		{
			IoHashStream Hasher;
			m_SmallObjectFile.StreamByteRange(Entry.Location.GetOffset(), Entry.Location.GetSize(), [&](const void* Data, uint64_t Size) {
				Hasher.Append(Data, Size);
			});
			IoHash ComputedHash = Hasher.GetHash();

			if (Entry.Key != ComputedHash)
			{
				BadChunks.push_back(Entry);
			}
		}
	}

	if (BadChunks.empty())
	{
		return;
	}

	ZEN_ERROR("Scrubbing found {} bad chunks in '{}'", BadChunks.size(), m_ContainerBaseName);

	// Deal with bad chunks by removing them from our lookup map

	std::vector<IoHash> BadChunkHashes;

	RwLock::ExclusiveLockScope _(m_LocationMapLock);
	for (const CasDiskIndexEntry& Entry : BadChunks)
	{
		BadChunkHashes.push_back(Entry.Key);
		m_CasLog.Append({.Key = Entry.Key, .Location = Entry.Location, .Flags = CasDiskIndexEntry::kTombstone});
		m_LocationMap.erase(Entry.Key);
	}

	// Let whomever it concerns know about the bad chunks. This could
	// be used to invalidate higher level data structures more efficiently
	// than a full validation pass might be able to do

	Ctx.ReportBadCasChunks(BadChunkHashes);
}

void
CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
{
	namespace fs = std::filesystem;

	// Garbage collection will first remove any chunks that are flushed from the index.
	// It then tries to compact the existing small object file, it does this by
	// collecting all chunks that should be kept and sort them in position order.
	// It then steps from chunk to chunk and checks if there is space to move the last
	// chunk before the current chunk. It repeats this until it can't fit the last chunk
	// or the last chunk is the current chunk.
	// After this it check to see if there is space to move the current chunk closer to
	// the preceeding chunk (or beginning of file if there is no preceeding chunk).
	// It updates the new write position for any new chunks and rewrites the cas log
	// to match the new content of the store.
	//
	// It currently grabs a full lock during the GC operation but the compacting is
	// done gradually and can be stopped after each chunk if the GC operation needs to
	// be time limited. This will leave holes in the small object file that will not
	// be reclaimed unless a GC operation is executed again, but the state of the
	// cas store is intact.
	//
	// It is also possible to more fine-grained locking of GC operation when moving
	// blocks but that requires more work and additional checking if new blocks are
	// added betwen each move of a block.
	ZEN_INFO("collecting garbage from '{}'", m_Config.RootDirectory / m_ContainerBaseName);

	{
		RwLock::ExclusiveLockScope _i(m_InsertLock);
		RwLock::ExclusiveLockScope _l(m_LocationMapLock);

		Flush();

		if (m_LocationMap.empty())
		{
			ZEN_INFO("garbage collect SKIPPED, for '{}', container is empty", m_Config.RootDirectory / m_ContainerBaseName);
			return;
		}

		const uint64_t TotalChunkCount = m_LocationMap.size();
		uint64_t	   TotalSize	   = m_TotalSize.load();

		std::vector<IoHash> TotalChunkHashes;
		TotalChunkHashes.reserve(m_LocationMap.size());
		for (auto& Entry : m_LocationMap)
		{
			TotalChunkHashes.push_back(Entry.first);
		}

		std::vector<IoHash> DeletedChunks;
		std::vector<IoHash> ChunkHashes;  // Same sort order as ChunkLocations
		ChunkHashes.reserve(m_LocationMap.size());

		const bool CollectSmallObjects = GcCtx.IsDeletionMode() && GcCtx.CollectSmallObjects();

		GcCtx.FilterCas(TotalChunkHashes, [&](const IoHash& ChunkHash, bool Keep) {
			if (Keep)
			{
				ChunkHashes.push_back(ChunkHash);
			}
			else
			{
				DeletedChunks.push_back(ChunkHash);
			}
		});

		if (ChunkHashes.size() == TotalChunkCount)
		{
			ZEN_INFO("garbage collect DONE, scanned #{} {} chunks from '{}', nothing to delete",
					 TotalChunkCount,
					 NiceBytes(TotalSize),
					 m_Config.RootDirectory / m_ContainerBaseName);
			return;
		}

		const uint64_t ChunkCount = ChunkHashes.size();

		std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
			auto LhsKeyIt = m_LocationMap.find(Lhs);
			auto RhsKeyIt = m_LocationMap.find(Rhs);
			return LhsKeyIt->second.GetOffset() < RhsKeyIt->second.GetOffset();
		});

		uint64_t					 NewTotalSize = 0;
		std::vector<CasDiskLocation> ChunkLocations;
		ChunkLocations.reserve(ChunkHashes.size());
		for (auto Entry : ChunkHashes)
		{
			auto		KeyIt		  = m_LocationMap.find(Entry);
			const auto& ChunkLocation = KeyIt->second;
			ChunkLocations.push_back(ChunkLocation);
			NewTotalSize += ChunkLocation.GetSize();
		}

		if (!CollectSmallObjects)
		{
			ZEN_INFO("garbage collect from '{}' DISABLED, found #{} {} chunks of total #{} {}",
					 m_Config.RootDirectory / m_ContainerBaseName,
					 TotalChunkCount - ChunkCount,
					 NiceBytes(TotalSize - NewTotalSize),
					 TotalChunkCount,
					 NiceBytes(TotalSize));
			return;
		}

		for (auto ChunkHash : DeletedChunks)
		{
			auto		KeyIt			= m_LocationMap.find(ChunkHash);
			const auto& ChunkLocation	= KeyIt->second;
			uint64_t	NextChunkOffset = ChunkLocation.GetOffset() + ChunkLocation.GetSize();
			m_CasLog.Append({.Key = ChunkHash, .Location = ChunkLocation, .Flags = CasDiskIndexEntry::kTombstone});
			m_LocationMap.erase(ChunkHash);
			if (m_CurrentInsertOffset == NextChunkOffset)
			{
				m_CurrentInsertOffset = ChunkLocation.GetOffset();
			}
			m_TotalSize.fetch_sub(static_cast<uint64_t>(ChunkLocation.GetSize()));
		}

		// We can break here if we only want to remove items without compacting of space

		std::vector<IoHash> MovedChunks;

		uint64_t WriteOffset{};
		uint64_t ChunkIndex{};
		while (ChunkIndex < ChunkHashes.size())
		{
			IoHash		ChunkHash	  = ChunkHashes[ChunkIndex];
			const auto& ChunkLocation = ChunkLocations[ChunkIndex];

			uint64_t NextChunkOffset = AlignPositon(ChunkLocation.GetOffset() + ChunkLocation.GetSize(), m_PayloadAlignment);

			uint64_t FreeChunkSize = ChunkLocation.GetOffset() - WriteOffset;

			// TODO: We could keep some wiggle room here, only try to find the last keep block if there is a reasonable amount of space free
			while (FreeChunkSize >= m_PayloadAlignment)
			{
				// We should move as many keep chunk at the end as we can possibly fit
				uint64_t LastKeepChunkIndex = ChunkHashes.size() - 1;
				if (LastKeepChunkIndex == ChunkIndex)
				{
					break;
				}

				IoHash		LastChunkHash	  = ChunkHashes[LastKeepChunkIndex];
				const auto& LastChunkLocation = ChunkLocations[LastKeepChunkIndex];
				if (LastChunkLocation.GetSize() > FreeChunkSize)
				{
					break;
				}

				// Move the last chunk to our write location
				std::vector<uint8_t> Chunk;
				Chunk.resize(LastChunkLocation.GetSize());
				m_SmallObjectFile.Read(Chunk.data(), Chunk.size(), LastChunkLocation.GetOffset());
				CasDiskLocation NewChunkLocation(WriteOffset, Chunk.size());
				m_SmallObjectFile.Write(Chunk.data(), Chunk.size(), NewChunkLocation.GetOffset());

				CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = NewChunkLocation};
				m_CasLog.Append(IndexEntry);
				m_LocationMap[LastChunkHash] = NewChunkLocation;
				ChunkHashes.pop_back();

				WriteOffset	  = AlignPositon(WriteOffset + Chunk.size(), m_PayloadAlignment);
				FreeChunkSize = ChunkLocation.GetOffset() - WriteOffset;
				MovedChunks.push_back(LastChunkHash);

				uint64_t LastChunkNextChunkOffset = AlignPositon(LastChunkLocation.GetOffset() + Chunk.size(), m_PayloadAlignment);
				if (m_CurrentInsertOffset == LastChunkNextChunkOffset)
				{
					m_CurrentInsertOffset = LastChunkLocation.GetOffset();
				}
			}

			// TODO: We could keep some wiggle room here, don't move chunk if we only move it a very small amount
			if (FreeChunkSize > m_PayloadAlignment)
			{
				std::vector<uint8_t> Chunk;
				Chunk.resize(ChunkLocation.GetSize());
				m_SmallObjectFile.Read(Chunk.data(), Chunk.size(), ChunkLocation.GetOffset());
				CasDiskLocation NewChunkLocation(WriteOffset, Chunk.size());
				m_SmallObjectFile.Write(Chunk.data(), Chunk.size(), NewChunkLocation.GetOffset());

				CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = NewChunkLocation};
				m_CasLog.Append(IndexEntry);
				m_LocationMap[ChunkHash] = NewChunkLocation;

				MovedChunks.push_back(ChunkHash);
				WriteOffset = AlignPositon(NewChunkLocation.GetOffset() + Chunk.size(), m_PayloadAlignment);
			}
			else
			{
				WriteOffset = NextChunkOffset;
			}

			// Update insert location if this is the last chunk in the file
			if (m_CurrentInsertOffset == NextChunkOffset)
			{
				m_CurrentInsertOffset = WriteOffset;
			}

			// We can break here if we want to do incremental GC

			ChunkIndex++;
		}

		if (ChunkCount == 0)
		{
			m_CurrentInsertOffset = 0;
		}

		GcCtx.DeletedCas(DeletedChunks);

		uint64_t CurrentSize = m_SmallObjectFile.FileSize();
		ZEN_INFO("garbage collection complete '{}', space {} to {}, moved {} and delete {} chunks",
				 m_Config.RootDirectory / m_ContainerBaseName,
				 NiceBytes(CurrentSize),
				 NiceBytes(m_CurrentInsertOffset),
				 MovedChunks.size(),
				 DeletedChunks.size());
		// TODO: Should we truncate the file or just keep the size of the file and reuse the space?
	}

	MakeIndexSnapshot();
}

void
CasContainerStrategy::MakeIndexSnapshot()
{
	ZEN_INFO("writing index snapshot for '{}'", m_Config.RootDirectory / m_ContainerBaseName);

	namespace fs = std::filesystem;

	fs::path SlogPath	   = m_Config.RootDirectory / (m_ContainerBaseName + ".ulog");
	fs::path SidxPath	   = m_Config.RootDirectory / (m_ContainerBaseName + ".uidx");
	fs::path STmplogPath  = m_Config.RootDirectory / (m_ContainerBaseName + ".tmp.ulog");
	fs::path STmpSidxPath = m_Config.RootDirectory / (m_ContainerBaseName + ".tmp.uidx");
	fs::path SRecoveredlogPath = m_Config.RootDirectory / (m_ContainerBaseName + ".recover.ulog");

	// Move cas and index away, we keep them if something goes wrong, any new chunks will be added to the new log
	{
		RwLock::ExclusiveLockScope _(m_LocationMapLock);
		m_CasLog.Close();

		if (fs::exists(STmplogPath))
		{
			fs::remove(STmplogPath);
		}
		if (fs::exists(STmpSidxPath))
		{
			fs::remove(STmpSidxPath);
		}

		fs::rename(SlogPath, STmplogPath);
		if (fs::exists(SidxPath))
		{
			fs::rename(SidxPath, STmpSidxPath);
		}

		// Open an new log
		m_CasLog.Open(SlogPath, true);
	}

	try
	{
		// Write the current state of the location map to a new index state
		std::vector<CasDiskIndexEntry> Entries;

		{
			RwLock::SharedLockScope		   _l(m_LocationMapLock);
			Entries.resize(m_LocationMap.size());

			uint64_t EntryIndex = 0;
			for (auto& Entry : m_LocationMap)
			{
				CasDiskIndexEntry& IndexEntry = Entries[EntryIndex++];
				IndexEntry.Key				  = Entry.first;
				IndexEntry.Location			  = Entry.second;
			}
		}

		BasicFile SmallObjectIndex;
		SmallObjectIndex.Open(SidxPath, true);
		SmallObjectIndex.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), 0);
		SmallObjectIndex.Close();
	}
	catch (std::exception& Err)
	{
		ZEN_ERROR("snapshot FAILED, reason '{}'", Err.what());

		// Reconstruct the log from old log and any added log entries
		RwLock::ExclusiveLockScope _(m_LocationMapLock);
		if (fs::exists(STmplogPath))
		{
			std::vector<CasDiskIndexEntry> Records;
			Records.reserve(m_LocationMap.size());
			{
				TCasLogFile<CasDiskIndexEntry> OldCasLog;
				OldCasLog.Open(STmplogPath, false);
				OldCasLog.Replay([&](const CasDiskIndexEntry& Record) { Records.push_back(Record); });
			}
			{
				m_CasLog.Replay([&](const CasDiskIndexEntry& Record) { Records.push_back(Record); });
			}

			TCasLogFile<CasDiskIndexEntry> RecoveredCasLog;
			RecoveredCasLog.Open(SRecoveredlogPath, true);
			for (const auto& Record : Records)
			{
				RecoveredCasLog.Append(Record);
			}
			RecoveredCasLog.Close();

			fs::remove(SlogPath);
			fs::rename(SRecoveredlogPath, SlogPath);
			fs::remove(STmplogPath);
		}

		if (fs::exists(SidxPath))
		{
			fs::remove(SidxPath);
		}

		// Restore any previous snapshot
		if (fs::exists(STmpSidxPath))
		{
			fs::remove(SidxPath);
			fs::rename(STmpSidxPath, SidxPath);
		}
	}
	if (fs::exists(STmpSidxPath))
	{
		fs::remove(STmpSidxPath);
	}
}

void
CasContainerStrategy::OpenContainer(bool IsNewStore)
{
	std::filesystem::path SobsPath = m_Config.RootDirectory / (m_ContainerBaseName + ".ucas");
	std::filesystem::path SlogPath = m_Config.RootDirectory / (m_ContainerBaseName + ".ulog");

	m_SmallObjectFile.Open(SobsPath, IsNewStore);
	m_CasLog.Open(SlogPath, IsNewStore);

	// TODO: should validate integrity of container files here

	m_CurrentInsertOffset = 0;
	m_TotalSize			  = 0;

	m_LocationMap.clear();

	std::filesystem::path SidxPath = m_Config.RootDirectory / (m_ContainerBaseName + ".uidx");
	if (std::filesystem::exists(SidxPath))
	{
		BasicFile SmallObjectIndex;
		SmallObjectIndex.Open(SidxPath, false);
		uint64_t					   Size		  = SmallObjectIndex.FileSize();
		uint64_t					   EntryCount = Size / sizeof(CasDiskIndexEntry);
		std::vector<CasDiskIndexEntry> Entries{EntryCount};
		SmallObjectIndex.Read(Entries.data(), Size, 0);
		for (const auto& Entry : Entries)
		{
			m_LocationMap[Entry.Key] = Entry.Location;
		}
		SmallObjectIndex.Close();
	}

	m_CasLog.Replay([&](const CasDiskIndexEntry& Record) {
		if (Record.Flags & CasDiskIndexEntry::kTombstone)
		{
			m_LocationMap.erase(Record.Key);
		}
		else
		{
			m_LocationMap[Record.Key] = Record.Location;
		}
	});

	uint64_t MaxFileOffset = 0;
	for (const auto& Entry : m_LocationMap)
	{
		const auto& Location = Entry.second;
		MaxFileOffset		 = std::max<uint64_t>(MaxFileOffset, Location.GetOffset() + Location.GetSize());
		m_TotalSize.fetch_add(Location.GetSize());
	}

	m_CurrentInsertOffset = AlignPositon(MaxFileOffset, m_PayloadAlignment);
}

//////////////////////////////////////////////////////////////////////////

#if ZEN_WITH_TESTS

TEST_CASE("cas.compact.gc")
{
	ScopedTemporaryDirectory TempDir;

	CasStoreConfiguration CasConfig;
	CasConfig.RootDirectory = TempDir.Path();

	CreateDirectories(CasConfig.RootDirectory);

	const int kIterationCount = 1000;

	std::vector<IoHash> Keys(kIterationCount);

	{
		CasGc				 Gc;
		CasContainerStrategy Cas(CasConfig, Gc);
		Cas.Initialize("test", 16, true);

		for (int i = 0; i < kIterationCount; ++i)
		{
			CbObjectWriter Cbo;
			Cbo << "id" << i;
			CbObject Obj = Cbo.Save();

			IoBuffer	 ObjBuffer = Obj.GetBuffer().AsIoBuffer();
			const IoHash Hash	   = HashBuffer(ObjBuffer);

			Cas.InsertChunk(ObjBuffer, Hash);

			Keys[i] = Hash;
		}

		for (int i = 0; i < kIterationCount; ++i)
		{
			IoBuffer Chunk = Cas.FindChunk(Keys[i]);

			CHECK(!!Chunk);

			CbObject Value = LoadCompactBinaryObject(Chunk);

			CHECK_EQ(Value["id"].AsInt32(), i);
		}
	}

	// Validate that we can still read the inserted data after closing
	// the original cas store

	{
		CasGc				 Gc;
		CasContainerStrategy Cas(CasConfig, Gc);
		Cas.Initialize("test", 16, false);

		for (int i = 0; i < kIterationCount; ++i)
		{
			IoBuffer Chunk = Cas.FindChunk(Keys[i]);

			CHECK(!!Chunk);

			CbObject Value = LoadCompactBinaryObject(Chunk);

			CHECK_EQ(Value["id"].AsInt32(), i);
		}

		GcContext Ctx;
		Cas.CollectGarbage(Ctx);
	}
}

TEST_CASE("cas.compact.totalsize")
{
	std::random_device rd;
	std::mt19937	   g(rd());

	const auto CreateChunk = [&](uint64_t Size) -> IoBuffer {
		const size_t		  Count = static_cast<size_t>(Size / sizeof(uint32_t));
		std::vector<uint32_t> Values;
		Values.resize(Count);
		for (size_t Idx = 0; Idx < Count; ++Idx)
		{
			Values[Idx] = static_cast<uint32_t>(Idx);
		}
		std::shuffle(Values.begin(), Values.end(), g);

		return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size() * sizeof(uint32_t));
	};

	ScopedTemporaryDirectory TempDir;

	CasStoreConfiguration CasConfig;
	CasConfig.RootDirectory = TempDir.Path();

	CreateDirectories(CasConfig.RootDirectory);

	const uint64_t kChunkSize  = 1024;
	const int32_t  kChunkCount = 16;

	{
		CasGc				 Gc;
		CasContainerStrategy Cas(CasConfig, Gc);
		Cas.Initialize("test", 16, true);

		for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
		{
			IoBuffer	 Chunk		  = CreateChunk(kChunkSize);
			const IoHash Hash		  = HashBuffer(Chunk);
			auto		 InsertResult = Cas.InsertChunk(Chunk, Hash);
			ZEN_ASSERT(InsertResult.New);
		}

		const uint64_t TotalSize = Cas.StorageSize().DiskSize;
		CHECK_EQ(kChunkSize * kChunkCount, TotalSize);
	}

	{
		CasGc				 Gc;
		CasContainerStrategy Cas(CasConfig, Gc);
		Cas.Initialize("test", 16, false);

		const uint64_t TotalSize = Cas.StorageSize().DiskSize;
		CHECK_EQ(kChunkSize * kChunkCount, TotalSize);
	}
}

#endif

void
compactcas_forcelink()
{
}

}  // namespace zen