path: root/zenstore/filecas.cpp

// Copyright Epic Games, Inc. All Rights Reserved.

#include "filecas.h"

#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/memory.h>
#include <zencore/scopeguard.h>
#include <zencore/string.h>
#include <zencore/testing.h>
#include <zencore/testutils.h>
#include <zencore/thread.h>
#include <zencore/timer.h>
#include <zencore/uid.h>
#include <zenstore/basicfile.h>
#include <zenstore/gc.h>

#if ZEN_WITH_TESTS
#	include <zencore/compactbinarybuilder.h>
#endif

#include <gsl/gsl-lite.hpp>

#include <barrier>
#include <filesystem>
#include <functional>
#include <unordered_map>

ZEN_THIRD_PARTY_INCLUDES_START
#if ZEN_PLATFORM_WINDOWS
#	include <atlfile.h>
#endif
ZEN_THIRD_PARTY_INCLUDES_END

namespace zen {

FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash)
{
	ShardedPath.Append(RootPath.c_str());

	ExtendableStringBuilder<64> HashString;
	ChunkHash.ToHexString(HashString);

	const char* str = HashString.c_str();

	// Shard into a path with two directory levels containing 12 bits and 8 bits
	// respectively.
	//
	// This results in a maximum of 4096 * 256 directories
	//
	// The numbers have been chosen somewhat arbitrarily but are large to scale
	// to very large chunk repositories without creating too many directories
	// on a single level since NTFS does not deal very well with this.
	//
	// It may or may not make sense to make this a configurable policy, and it
	// would probably be a good idea to measure performance for different
	// policies and chunk counts

	ShardedPath.AppendSeparator();
	ShardedPath.AppendAsciiRange(str, str + 3);

	ShardedPath.AppendSeparator();
	ShardedPath.AppendAsciiRange(str + 3, str + 5);
	Shard2len = ShardedPath.Size();

	ShardedPath.AppendSeparator();
	ShardedPath.AppendAsciiRange(str + 5, str + 40);
}

//////////////////////////////////////////////////////////////////////////

FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config, CasGc& Gc)
: GcStorage(Gc)
, m_Config(Config)
, m_Log(logging::Get("filecas"))
{
}

FileCasStrategy::~FileCasStrategy()
{
}

void
FileCasStrategy::Initialize(bool IsNewStore)
{
	m_IsInitialized = true;

	CreateDirectories(m_Config.RootDirectory);

	m_CasLog.Open(m_Config.RootDirectory / "cas.ulog", IsNewStore ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite);

	Stopwatch  Timer;
	const auto _ = MakeGuard([&] {
		ZEN_INFO("read log {} containing {}", m_Config.RootDirectory / "cas.ulog", NiceBytes(m_TotalSize.load(std::memory_order::relaxed)));
	});

	std::unordered_set<IoHash> FoundEntries;
	FoundEntries.reserve(10000);
	m_CasLog.Replay(
		[&](const FileCasIndexEntry& Entry) {
			if (Entry.IsFlagSet(FileCasIndexEntry::kTombStone))
			{
				if (!FoundEntries.contains(Entry.Key))
				{
					return;
				}
				m_TotalSize.fetch_sub(Entry.Size, std::memory_order_relaxed);
				FoundEntries.erase(Entry.Key);
			}
			else
			{
				if (FoundEntries.contains(Entry.Key))
				{
					return;
				}
				FoundEntries.insert(Entry.Key);
				m_TotalSize.fetch_add(Entry.Size, std::memory_order_relaxed);
			}
		},
		0);
}

CasStore::InsertResult
FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
{
	ZEN_ASSERT(m_IsInitialized);

	// File-based chunks have special case handling whereby we move the file into
	// place in the file store directory, thus avoiding unnecessary copying

	IoBufferFileReference FileRef;
	if (Chunk.IsWholeFile() && Chunk.GetFileReference(/* out */ FileRef))
	{
		ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);

		RwLock::ExclusiveLockScope _(LockForHash(ChunkHash));

#if ZEN_PLATFORM_WINDOWS
		const HANDLE ChunkFileHandle = FileRef.FileHandle;

		auto DeletePayloadFileOnClose = [&] {
			// This will cause the file to be deleted when the last handle to it is closed
			FILE_DISPOSITION_INFO Fdi{};
			Fdi.DeleteFile = TRUE;
			BOOL Success   = SetFileInformationByHandle(ChunkFileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);

			if (!Success)
			{
				ZEN_WARN("Failed to flag temporary payload file '{}' for deletion: '{}'",
						 PathFromHandle(ChunkFileHandle),
						 GetLastErrorAsString());
			}
		};

		// See if file already exists
		//
		// Future improvement: maintain Bloom filter to avoid expensive file system probes?

		{
			CAtlFile PayloadFile;

			if (HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); SUCCEEDED(hRes))
			{
				// If we succeeded in opening the target file then we don't need to do anything else because it already exists
				// and should contain the content we were about to insert

				// We do need to ensure the source file goes away on close, however

				uint64_t FileSize = 0;
				if (HRESULT hSizeRes = PayloadFile.GetSize(FileSize); SUCCEEDED(hSizeRes))
				{
					m_TotalSize.fetch_add(static_cast<int64_t>(FileSize));
				}
				else
				{
					ZEN_WARN("get file size FAILED, file cas '{}'", Name.ShardedPath.ToUtf8());
				}

				DeletePayloadFileOnClose();

				return CasStore::InsertResult{.New = false};
			}
			else
			{
				if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
				{
					// Shard directory does not exist
				}
				else if (hRes == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND))
				{
					// Shard directory exists, but not the file
				}
				else if (hRes == HRESULT_FROM_WIN32(ERROR_SHARING_VIOLATION))
				{
					// Sharing violation, likely because we are trying to open a file
					// which has been renamed on another thread, and the file handle
					// used to rename it is still open. We handle this case below
					// instead of here
				}
				else
				{
					ZEN_INFO("Unexpected error opening file '{}': {}", Name.ShardedPath.ToUtf8(), hRes);
				}
			}
		}

		std::filesystem::path FullPath(Name.ShardedPath.c_str());

		std::filesystem::path FilePath = FullPath.parent_path();
		std::wstring		  FileName = FullPath.native();

		const DWORD		  BufferSize = sizeof(FILE_RENAME_INFO) + gsl::narrow<DWORD>(FileName.size() * sizeof(WCHAR));
		FILE_RENAME_INFO* RenameInfo = reinterpret_cast<FILE_RENAME_INFO*>(Memory::Alloc(BufferSize));
		memset(RenameInfo, 0, BufferSize);

		RenameInfo->ReplaceIfExists = FALSE;
		RenameInfo->FileNameLength	= gsl::narrow<DWORD>(FileName.size());
		memcpy(RenameInfo->FileName, FileName.c_str(), FileName.size() * sizeof(WCHAR));
		RenameInfo->FileName[FileName.size()] = 0;

		auto $ = MakeGuard([&] { Memory::Free(RenameInfo); });

		// Try to move file into place

		BOOL Success = SetFileInformationByHandle(ChunkFileHandle, FileRenameInfo, RenameInfo, BufferSize);

		if (!Success)
		{
			// The rename/move could fail because the target directory does not yet exist. This code attempts
			// to create it

			CAtlFile DirHandle;

			auto InternalCreateDirectoryHandle = [&] {
				return DirHandle.Create(FilePath.c_str(),
										GENERIC_READ | GENERIC_WRITE,
										FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
										OPEN_EXISTING,
										FILE_FLAG_BACKUP_SEMANTICS);
			};

			// It's possible for several threads to enter this logic trying to create the same
			// directory. Only one will create the directory of course, but all threads will
			// make it through okay

			HRESULT hRes = InternalCreateDirectoryHandle();

			if (FAILED(hRes))
			{
				// TODO: we can handle directory creation more intelligently and efficiently than
				// this currently does

				CreateDirectories(FilePath.c_str());

				hRes = InternalCreateDirectoryHandle();
			}

			if (FAILED(hRes))
			{
				ThrowSystemException(hRes, fmt::format("Failed to open shard directory '{}'", FilePath));
			}

			// Retry rename/move

			Success = SetFileInformationByHandle(ChunkFileHandle, FileRenameInfo, RenameInfo, BufferSize);
		}

		if (Success)
		{
			m_TotalSize.fetch_add(Chunk.Size(), std::memory_order::relaxed);
			m_CasLog.Append({.Key = ChunkHash, .Size = Chunk.Size()});

			return CasStore::InsertResult{.New = true};
		}

		const DWORD LastError = GetLastError();

		if ((LastError == ERROR_FILE_EXISTS) || (LastError == ERROR_ALREADY_EXISTS))
		{
			DeletePayloadFileOnClose();

			return CasStore::InsertResult{.New = false};
		}

		ZEN_WARN("rename of CAS payload file failed ('{}'), falling back to regular write for insert of {}",
				 GetSystemErrorAsString(LastError),
				 ChunkHash);

		DeletePayloadFileOnClose();
#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
		std::filesystem::path SourcePath = PathFromHandle(FileRef.FileHandle);
		std::filesystem::path DestPath	 = Name.ShardedPath.c_str();
		int					  Ret		 = link(SourcePath.c_str(), DestPath.c_str());
		if (Ret < 0 && zen::GetLastError() == ENOENT)
		{
			// Destination directory doesn't exist. Create it any try again.
			CreateDirectories(DestPath.parent_path().c_str());
			Ret = link(SourcePath.c_str(), DestPath.c_str());
		}
		int LinkError = zen::GetLastError();

		// Unlink the file. If the path to unlink didn't exist someone else
		// beat us to it and that is hunky-dory.
		if (unlink(SourcePath.c_str()) < 0)
		{
			int UnlinkError = zen::GetLastError();
			if (UnlinkError != ENOENT)
			{
				ZEN_WARN("unlink of CAS payload file failed ('{}')", GetSystemErrorAsString(UnlinkError));
			}
		}

		// It is possible that someone beat us to it in linking the file. In that
		// case a "file exists" error is okay. All others are not.
		if (Ret < 0)
		{
			if (LinkError == EEXIST)
			{
				return CasStore::InsertResult{.New = false};
			}

			ZEN_WARN("link of CAS payload file failed ('{}'), falling back to regular write for insert of {}",
					 GetSystemErrorAsString(LinkError),
					 ChunkHash);
		}
		else
		{
			return CasStore::InsertResult{.New = true};
		}
#endif	// ZEN_PLATFORM_*
	}

	return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash);
}

CasStore::InsertResult
FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize, const IoHash& ChunkHash)
{
	ZEN_ASSERT(m_IsInitialized);

	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);

	// See if file already exists
	//
	// Future improvement: maintain Bloom filter to avoid expensive file system probes?

#if ZEN_PLATFORM_WINDOWS
	CAtlFile PayloadFile;

	HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);

	if (SUCCEEDED(hRes))
	{
		// If we succeeded in opening the file then we don't need to do anything else because it already exists and should contain the
		// content we were about to insert

		m_TotalSize.fetch_add(static_cast<int64_t>(ChunkSize));
		return CasStore::InsertResult{.New = false};
	}

	PayloadFile.Close();
#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
	if (access(Name.ShardedPath.c_str(), F_OK) == 0)
	{
		return CasStore::InsertResult{.New = false};
	}
#endif

	RwLock::ExclusiveLockScope _(LockForHash(ChunkHash));

#if ZEN_PLATFORM_WINDOWS
	// For now, use double-checked locking to see if someone else was first

	hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);

	if (SUCCEEDED(hRes))
	{
		// If we succeeded in opening the file then we don't need to do anything
		// else because someone else managed to create the file before we did. Just return.

		m_TotalSize.fetch_add(static_cast<int64_t>(ChunkSize));
		return {.New = false};
	}

	if ((hRes != HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) && (hRes != HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND)))
	{
		ZEN_WARN("Unexpected error code when opening shard file for read: {:#x}", uint32_t(hRes));
	}

	auto InternalCreateFile = [&] { return PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_WRITE, FILE_SHARE_DELETE, CREATE_ALWAYS); };

	hRes = InternalCreateFile();

	if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
	{
		// Ensure parent directories exist and retry file creation

		std::filesystem::create_directories(std::wstring_view(Name.ShardedPath.c_str(), Name.Shard2len));

		hRes = InternalCreateFile();
	}

	if (FAILED(hRes))
	{
		ThrowSystemException(hRes, fmt::format("Failed to open shard file '{}'", Name.ShardedPath.ToUtf8()));
	}
#else
	// Attempt to exclusively create the file.
	auto InternalCreateFile = [&] {
		int Fd = open(Name.ShardedPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666);
		if (Fd >= 0)
		{
			fchmod(Fd, 0666);
		}
		return Fd;
	};
	int Fd = InternalCreateFile();
	if (Fd < 0)
	{
		switch (zen::GetLastError())
		{
			case EEXIST:
				// Another thread has beat us to it so we're golden.
				return {.New = false};

			case ENOENT:
				if (zen::CreateDirectories(std::string_view(Name.ShardedPath.c_str(), Name.Shard2len)))
				{
					Fd = InternalCreateFile();
					if (Fd >= 0)
					{
						break;
					}
				}
				ThrowLastError(fmt::format("Failed creating shard directory '{}'", Name.ShardedPath));

			default:
				ThrowLastError(fmt::format("Unexpected error occurred opening shard file '{}'", Name.ShardedPath));
		}
	}

	struct FdWrapper
	{
		~FdWrapper() { Close(); }
		void Write(const void* Cursor, size_t Size) { (void)!write(Fd, Cursor, Size); }
		void Close()
		{
			if (Fd >= 0)
			{
				close(Fd);
				Fd = -1;
			}
		}
		int Fd;
	} PayloadFile = {Fd};
#endif	// ZEN_PLATFORM_WINDOWS

	size_t ChunkRemain = ChunkSize;
	auto   ChunkCursor = reinterpret_cast<const uint8_t*>(ChunkData);

	while (ChunkRemain != 0)
	{
		uint32_t ByteCount = uint32_t(std::min<size_t>(4 * 1024 * 1024ull, ChunkRemain));

		PayloadFile.Write(ChunkCursor, ByteCount);

		ChunkCursor += ByteCount;
		ChunkRemain -= ByteCount;
	}

	// We cannot rely on RAII to close the file handle since it would be closed
	// *after* the lock is released due to the initialization order
	PayloadFile.Close();

	m_TotalSize.fetch_add(ChunkSize, std::memory_order::relaxed);
	m_CasLog.Append({.Key = ChunkHash, .Size = ChunkSize});

	return {.New = true};
}

IoBuffer
FileCasStrategy::FindChunk(const IoHash& ChunkHash)
{
	ZEN_ASSERT(m_IsInitialized);

	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);

	RwLock::SharedLockScope _(LockForHash(ChunkHash));

	return IoBufferBuilder::MakeFromFile(Name.ShardedPath.c_str());
}

bool
FileCasStrategy::HaveChunk(const IoHash& ChunkHash)
{
	ZEN_ASSERT(m_IsInitialized);

	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);

	RwLock::SharedLockScope _(LockForHash(ChunkHash));

	std::error_code Ec;
	if (std::filesystem::exists(Name.ShardedPath.c_str(), Ec))
	{
		return true;
	}

	return false;
}

void
FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec)
{
	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);

	uint64_t FileSize = static_cast<uint64_t>(std::filesystem::file_size(Name.ShardedPath.c_str(), Ec));
	if (Ec)
	{
		ZEN_WARN("get file size FAILED, file cas '{}'", Name.ShardedPath.ToUtf8());
		FileSize = 0;
	}

	ZEN_DEBUG("deleting CAS payload file '{}' {}", Name.ShardedPath.ToUtf8(), NiceBytes(FileSize));

	std::filesystem::remove(Name.ShardedPath.c_str(), Ec);

	if (!Ec)
	{
		m_TotalSize.fetch_sub(FileSize);
		m_CasLog.Append({.Key = ChunkHash, .Flags = FileCasIndexEntry::kTombStone, .Size = FileSize});
	}
}

void
FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks)
{
	ZEN_ASSERT(m_IsInitialized);

	// NOTE: it's not a problem now, but in the future if a GC should happen while this
	// is in flight, the result could be wrong since chunks could go away in the meantime.
	//
	// It would be good to have a pinning mechanism to make this less likely but
	// given that chunks could go away at any point after the results are returned to
	// a caller, this is something which needs to be taken into account by anyone consuming
	// this functionality in any case

	InOutChunks.RemoveChunksIf([&](const IoHash& Hash) { return HaveChunk(Hash); });
}

void
FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& PayloadFile)>&& Callback)
{
	ZEN_ASSERT(m_IsInitialized);

	struct Visitor : public FileSystemTraversal::TreeVisitor
	{
		Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {}
		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
		{
			ZEN_UNUSED(FileSize);

			std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory);

			std::filesystem::path::string_type PathString = RelPath.native();

			if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2)))
			{
				if (PathString.at(3) == std::filesystem::path::preferred_separator)
				{
					PathString.erase(3, 1);
				}
				PathString.append(File);

				// TODO: should validate that we're actually dealing with a valid hex string here

#if ZEN_PLATFORM_WINDOWS
				StringBuilder<64> Utf8;
				WideToUtf8(PathString, Utf8);
				IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()});
#else
				IoHash NameHash = IoHash::FromHexString(PathString);
#endif

				BasicFile		PayloadFile;
				std::error_code Ec;
				PayloadFile.Open(Parent / File, BasicFile::Mode::kWrite, Ec);

				if (!Ec)
				{
					Callback(NameHash, PayloadFile);
				}
			}
		}

		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
									[[maybe_unused]] const path_view&			  DirectoryName) override
		{
			return true;
		}

		const std::filesystem::path&									RootDirectory;
		std::function<void(const IoHash& Hash, BasicFile& PayloadFile)> Callback;
	} CasVisitor{m_Config.RootDirectory};

	CasVisitor.Callback = std::move(Callback);

	FileSystemTraversal Traversal;
	Traversal.TraverseFileSystem(m_Config.RootDirectory, CasVisitor);
}

void
FileCasStrategy::Flush()
{
	// Since we don't keep files open after writing there's nothing specific
	// to flush here.
	//
	// Depending on what semantics we want Flush() to provide, it could be
	// argued that this should just flush the volume which we are using to
	// store the CAS files on here, to ensure metadata is flushed along
	// with file data
	//
	// Related: to facilitate more targeted validation during recovery we could
	// maintain a log of when chunks were created
}

void
FileCasStrategy::Scrub(ScrubContext& Ctx)
{
	ZEN_ASSERT(m_IsInitialized);

	std::vector<IoHash>	  BadHashes;
	std::atomic<uint64_t> ChunkCount{0}, ChunkBytes{0};

	IterateChunks([&](const IoHash& Hash, BasicFile& Payload) {
		IoHashStream Hasher;
		Payload.StreamFile([&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
		IoHash ComputedHash = Hasher.GetHash();

		if (ComputedHash != Hash)
		{
			BadHashes.push_back(Hash);
		}

		++ChunkCount;
		ChunkBytes.fetch_add(Payload.FileSize());
	});

	Ctx.ReportScrubbed(ChunkCount, ChunkBytes);

	if (!BadHashes.empty())
	{
		ZEN_ERROR("file CAS scrubbing: {} bad chunks found", BadHashes.size());

		if (Ctx.RunRecovery())
		{
			ZEN_WARN("recovery: deleting backing files for {} bad chunks which were identified as bad", BadHashes.size());

			for (const IoHash& Hash : BadHashes)
			{
				std::error_code Ec;
				DeleteChunk(Hash, Ec);

				if (Ec)
				{
					ZEN_WARN("failed to delete file for chunk {}", Hash);
				}
			}
		}
	}

	Ctx.ReportBadCasChunks(BadHashes);

	ZEN_INFO("file CAS scrubbed: {} chunks ({})", ChunkCount.load(), NiceBytes(ChunkBytes));
}

void
FileCasStrategy::CollectGarbage(GcContext& GcCtx)
{
	ZEN_ASSERT(m_IsInitialized);

	ZEN_INFO("collecting garbage from {}", m_Config.RootDirectory);

	std::vector<IoHash>	  ChunksToDelete;
	std::atomic<uint64_t> ChunksToDeleteBytes{0};
	std::atomic<uint64_t> ChunkCount{0}, ChunkBytes{0};

	std::vector<IoHash> CandidateCas;

	uint64_t DeletedCount = 0;
	uint64_t OldTotalSize = m_TotalSize.load(std::memory_order::relaxed);

	Stopwatch  TotalTimer;
	const auto _ = MakeGuard([&] {
		ZEN_INFO("garbage collect for '{}' DONE after {}, deleted {} out of {} files, removed {} out of {}",
				 m_Config.RootDirectory,
				 NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()),
				 DeletedCount,
				 ChunkCount,
				 NiceBytes(OldTotalSize - m_TotalSize.load(std::memory_order::relaxed)),
				 NiceBytes(OldTotalSize));
	});

	IterateChunks([&](const IoHash& Hash, BasicFile& Payload) {
		bool KeepThis = false;
		CandidateCas.clear();
		CandidateCas.push_back(Hash);
		GcCtx.FilterCas(CandidateCas, [&](const IoHash& Hash) {
			ZEN_UNUSED(Hash);
			KeepThis = true;
		});

		const uint64_t FileSize = Payload.FileSize();

		if (!KeepThis)
		{
			ChunksToDelete.push_back(Hash);
			ChunksToDeleteBytes.fetch_add(FileSize);
		}

		++ChunkCount;
		ChunkBytes.fetch_add(FileSize);
	});

	if (ChunksToDelete.empty())
	{
		ZEN_INFO("gc for '{}' SKIPPED, nothing to delete", m_Config.RootDirectory);
		return;
	}

	ZEN_INFO("deleting file CAS garbage for '{}': {} out of {} chunks ({})",
			 m_Config.RootDirectory,
			 ChunksToDelete.size(),
			 ChunkCount.load(),
			 NiceBytes(ChunksToDeleteBytes));

	if (GcCtx.IsDeletionMode() == false)
	{
		ZEN_INFO("NOTE: not actually deleting anything since deletion is disabled");

		return;
	}

	for (const IoHash& Hash : ChunksToDelete)
	{
		ZEN_TRACE("deleting chunk {}", Hash);

		std::error_code Ec;
		DeleteChunk(Hash, Ec);

		if (Ec)
		{
			ZEN_WARN("gc for '{}' failed to delete file for chunk {}: '{}'", m_Config.RootDirectory, Hash, Ec.message());
			continue;
		}
		DeletedCount++;
	}

	GcCtx.DeletedCas(ChunksToDelete);
}

//////////////////////////////////////////////////////////////////////////

#if ZEN_WITH_TESTS

TEST_CASE("cas.file.move")
{
	// specifying an absolute path here can be helpful when using procmon to dig into things
	ScopedTemporaryDirectory TempDir;  // {"d:\\filecas_testdir"};

	CasGc Gc;

	CasStoreConfiguration CasConfig;
	CasConfig.RootDirectory = TempDir.Path() / "cas";

	FileCasStrategy FileCas(CasConfig, Gc);
	FileCas.Initialize(/* IsNewStore */ true);

	{
		std::filesystem::path Payload1Path{TempDir.Path() / "payload_1"};

		IoBuffer ZeroBytes{1024 * 1024};
		IoHash	 ZeroHash = IoHash::HashBuffer(ZeroBytes);

		BasicFile PayloadFile;
		PayloadFile.Open(Payload1Path, BasicFile::Mode::kTruncate);
		PayloadFile.Write(ZeroBytes, 0);
		PayloadFile.Close();

		IoBuffer Payload1 = IoBufferBuilder::MakeFromTemporaryFile(Payload1Path);

		CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, ZeroHash);
		CHECK_EQ(Result.New, true);
	}

#	if 0
	SUBCASE("stresstest")
	{
		std::vector<IoHash> PayloadHashes;

		const int kWorkers = 64;
		const int kItemCount = 128;

		for (int w = 0; w < kWorkers; ++w)
		{
			for (int i = 0; i < kItemCount; ++i)
			{
				IoBuffer Payload{1024};
				*reinterpret_cast<int*>(Payload.MutableData()) = i;
				PayloadHashes.push_back(IoHash::HashBuffer(Payload));

				std::filesystem::path PayloadPath{TempDir.Path() / fmt::format("payload_{}_{}", w, i)};
				WriteFile(PayloadPath, Payload);
			}
		}

		std::barrier Sync{kWorkers};

		auto PopulateAll = [&](int w) {
			std::vector<IoBuffer> Buffers;

			for (int i = 0; i < kItemCount; ++i)
			{
				std::filesystem::path PayloadPath{TempDir.Path() / fmt::format("payload_{}_{}", w, i)};
				IoBuffer			  Payload = IoBufferBuilder::MakeFromTemporaryFile(PayloadPath);
				Buffers.push_back(Payload);
				Sync.arrive_and_wait();
				CasStore::InsertResult Result = FileCas.InsertChunk(Payload, PayloadHashes[i]);
			}
		};

		std::vector<std::jthread> Threads;

		for (int i = 0; i < kWorkers; ++i)
		{
			Threads.push_back(std::jthread(PopulateAll, i));
		}

		for (std::jthread& Thread : Threads)
		{
			Thread.join();
		}
	}
#	endif
}

TEST_CASE("cas.file.gc")
{
	// specifying an absolute path here can be helpful when using procmon to dig into things
	ScopedTemporaryDirectory TempDir;  // {"d:\\filecas_testdir"};

	CasStoreConfiguration CasConfig;
	CasConfig.RootDirectory = TempDir.Path() / "cas";

	CasGc			Gc;
	FileCasStrategy FileCas(CasConfig, Gc);
	FileCas.Initialize(/* IsNewStore */ true);

	const int			kIterationCount = 1000;
	std::vector<IoHash> Keys{kIterationCount};

	auto InsertChunks = [&] {
		for (int i = 0; i < kIterationCount; ++i)
		{
			CbObjectWriter Cbo;
			Cbo << "id" << i;
			CbObject Obj = Cbo.Save();

			IoBuffer ObjBuffer = Obj.GetBuffer().AsIoBuffer();
			IoHash	 Hash	   = HashBuffer(ObjBuffer);

			FileCas.InsertChunk(ObjBuffer, Hash);

			Keys[i] = Hash;
		}
	};

	// Drop everything

	{
		InsertChunks();

		GcContext Ctx;
		FileCas.CollectGarbage(Ctx);

		for (const IoHash& Key : Keys)
		{
			IoBuffer Chunk = FileCas.FindChunk(Key);

			CHECK(!Chunk);
		}
	}

	// Keep roughly half of the chunks

	{
		InsertChunks();

		GcContext Ctx;

		for (const IoHash& Key : Keys)
		{
			if (Key.Hash[0] & 1)
			{
				Ctx.ContributeCas(std::vector<IoHash>{Key});
			}
		}

		FileCas.CollectGarbage(Ctx);

		for (const IoHash& Key : Keys)
		{
			if (Key.Hash[0] & 1)
			{
				CHECK(FileCas.FindChunk(Key));
			}
			else
			{
				CHECK(!FileCas.FindChunk(Key));
			}
		}
	}
}

#endif

void
filecas_forcelink()
{
}

}  // namespace zen