moved source directories into `/src` (#264)

* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees

1 files changed, 1304 insertions, 0 deletions

diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
new file mode 100644
index 000000000..a17773024
--- /dev/null
+++ b/src/zencore/filesystem.cpp
@@ -0,0 +1,1304 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/filesystem.h>
+
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/iobuffer.h>
+#include <zencore/logging.h>
+#include <zencore/stream.h>
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <atlbase.h>
+#	include <atlfile.h>
+#	include <winioctl.h>
+#	include <winnt.h>
+#endif
+
+#if ZEN_PLATFORM_LINUX
+#	include <dirent.h>
+#	include <fcntl.h>
+#	include <sys/resource.h>
+#	include <sys/stat.h>
+#	include <unistd.h>
+#endif
+
+#if ZEN_PLATFORM_MAC
+#	include <dirent.h>
+#	include <fcntl.h>
+#	include <libproc.h>
+#	include <sys/resource.h>
+#	include <sys/stat.h>
+#	include <sys/syslimits.h>
+#	include <unistd.h>
+#endif
+
+#include <filesystem>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+using namespace std::literals;
+
+#if ZEN_PLATFORM_WINDOWS
+
+static bool
+DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
+{
+	CHandle hDir(CreateFileW(Path,
+							 GENERIC_WRITE,
+							 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+							 nullptr,
+							 OPEN_EXISTING,
+							 FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT,
+							 nullptr));
+
+	if (hDir != INVALID_HANDLE_VALUE)
+	{
+		REPARSE_GUID_DATA_BUFFER Rgdb = {};
+		Rgdb.ReparseTag				  = dwReparseTag;
+
+		DWORD	   dwBytes;
+		const BOOL bOK =
+			DeviceIoControl(hDir, FSCTL_DELETE_REPARSE_POINT, &Rgdb, REPARSE_GUID_DATA_BUFFER_HEADER_SIZE, nullptr, 0, &dwBytes, nullptr);
+
+		return bOK == TRUE;
+	}
+
+	return false;
+}
+
+bool
+CreateDirectories(const wchar_t* Dir)
+{
+	// This may be suboptimal, in that it appears to try and create directories
+	// from the root on up instead of from some directory which is known to
+	// be present
+	//
+	// We should implement a smarter version at some point since this can be
+	// pretty expensive in aggregate
+
+	return std::filesystem::create_directories(Dir);
+}
+
+// Erase all files and directories in a given directory, leaving an empty directory
+// behind
+
+static bool
+WipeDirectory(const wchar_t* DirPath)
+{
+	ExtendableWideStringBuilder<128> Pattern;
+	Pattern.Append(DirPath);
+	Pattern.Append(L"\\*");
+
+	WIN32_FIND_DATAW FindData;
+	HANDLE			 hFind = FindFirstFileW(Pattern.c_str(), &FindData);
+
+	if (hFind != nullptr)
+	{
+		do
+		{
+			bool IsRegular = true;
+
+			if (FindData.cFileName[0] == L'.')
+			{
+				if (FindData.cFileName[1] == L'.')
+				{
+					if (FindData.cFileName[2] == L'\0')
+					{
+						IsRegular = false;
+					}
+				}
+				else if (FindData.cFileName[1] == L'\0')
+				{
+					IsRegular = false;
+				}
+			}
+
+			if (IsRegular)
+			{
+				ExtendableWideStringBuilder<128> Path;
+				Path.Append(DirPath);
+				Path.Append(L'\\');
+				Path.Append(FindData.cFileName);
+
+				// if (fd.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN)
+				//	deleteReparsePoint(path.c_str(), fd.dwReserved0);
+
+				if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+				{
+					if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN)
+					{
+						DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
+					}
+
+					if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS)
+					{
+						DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
+					}
+
+					bool Success = DeleteDirectories(Path.c_str());
+
+					if (!Success)
+					{
+						if (FindData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+						{
+							DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
+						}
+					}
+				}
+				else
+				{
+					DeleteFileW(Path.c_str());
+				}
+			}
+		} while (FindNextFileW(hFind, &FindData) == TRUE);
+
+		FindClose(hFind);
+	}
+
+	return true;
+}
+
+bool
+DeleteDirectories(const wchar_t* DirPath)
+{
+	return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
+}
+
+bool
+CleanDirectory(const wchar_t* DirPath)
+{
+	if (std::filesystem::exists(DirPath))
+	{
+		return WipeDirectory(DirPath);
+	}
+
+	return CreateDirectories(DirPath);
+}
+
+#endif	// ZEN_PLATFORM_WINDOWS
+
+bool
+CreateDirectories(const std::filesystem::path& Dir)
+{
+	while (!std::filesystem::is_directory(Dir))
+	{
+		if (Dir.has_parent_path())
+		{
+			CreateDirectories(Dir.parent_path());
+		}
+		std::error_code ErrorCode;
+		std::filesystem::create_directory(Dir, ErrorCode);
+		if (ErrorCode)
+		{
+			throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string()));
+		}
+		return true;
+	}
+	return false;
+}
+
+bool
+DeleteDirectories(const std::filesystem::path& Dir)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return DeleteDirectories(Dir.c_str());
+#else
+	std::error_code ErrorCode;
+	return std::filesystem::remove_all(Dir, ErrorCode);
+#endif
+}
+
+bool
+CleanDirectory(const std::filesystem::path& Dir)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return CleanDirectory(Dir.c_str());
+#else
+	if (std::filesystem::exists(Dir))
+	{
+		bool Success = true;
+
+		std::error_code ErrorCode;
+		for (const auto& Item : std::filesystem::directory_iterator(Dir))
+		{
+			Success &= std::filesystem::remove_all(Item, ErrorCode);
+		}
+
+		return Success;
+	}
+
+	return CreateDirectories(Dir);
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+bool
+SupportsBlockRefCounting(std::filesystem::path Path)
+{
+#if ZEN_PLATFORM_WINDOWS
+	ATL::CHandle Handle(CreateFileW(Path.c_str(),
+									GENERIC_READ,
+									FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+									nullptr,
+									OPEN_EXISTING,
+									FILE_FLAG_BACKUP_SEMANTICS,
+									nullptr));
+
+	if (Handle == INVALID_HANDLE_VALUE)
+	{
+		Handle.Detach();
+		return false;
+	}
+
+	ULONG FileSystemFlags = 0;
+	if (!GetVolumeInformationByHandleW(Handle, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0))
+	{
+		return false;
+	}
+
+	if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING))
+	{
+		return false;
+	}
+
+	return true;
+#else
+	ZEN_UNUSED(Path);
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
+{
+#if ZEN_PLATFORM_WINDOWS
+	ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	if (FromFile == INVALID_HANDLE_VALUE)
+	{
+		FromFile.Detach();
+		return false;
+	}
+
+	ULONG FileSystemFlags;
+	if (!GetVolumeInformationByHandleW(FromFile, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0))
+	{
+		return false;
+	}
+	if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING))
+	{
+		SetLastError(ERROR_NOT_CAPABLE);
+		return false;
+	}
+
+	FILE_END_OF_FILE_INFO FileSize;
+	if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile))
+	{
+		return false;
+	}
+
+	FILE_BASIC_INFO BasicInfo;
+	if (!GetFileInformationByHandleEx(FromFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo))
+	{
+		return false;
+	}
+
+	DWORD								   dwBytesReturned = 0;
+	FSCTL_GET_INTEGRITY_INFORMATION_BUFFER GetIntegrityInfoBuffer;
+	if (!DeviceIoControl(FromFile,
+						 FSCTL_GET_INTEGRITY_INFORMATION,
+						 nullptr,
+						 0,
+						 &GetIntegrityInfoBuffer,
+						 sizeof GetIntegrityInfoBuffer,
+						 &dwBytesReturned,
+						 nullptr))
+	{
+		return false;
+	}
+
+	SetFileAttributesW(ToPath.c_str(), FILE_ATTRIBUTE_NORMAL);
+
+	ATL::CHandle TargetFile(CreateFileW(ToPath.c_str(),
+										GENERIC_READ | GENERIC_WRITE | DELETE,
+										/* no sharing */ FILE_SHARE_READ,
+										nullptr,
+										OPEN_ALWAYS,
+										0,
+										/* hTemplateFile */ FromFile));
+
+	if (TargetFile == INVALID_HANDLE_VALUE)
+	{
+		TargetFile.Detach();
+		return false;
+	}
+
+	// Delete target file when handle is closed (we only reset this if the copy succeeds)
+	FILE_DISPOSITION_INFO FileDisposition = {TRUE};
+	if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
+	{
+		TargetFile.Close();
+		DeleteFileW(ToPath.c_str());
+		return false;
+	}
+
+	// Make file sparse so we don't end up allocating space when we change the file size
+	if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dwBytesReturned, nullptr))
+	{
+		return false;
+	}
+
+	// Copy integrity checking information
+	FSCTL_SET_INTEGRITY_INFORMATION_BUFFER IntegritySet = {GetIntegrityInfoBuffer.ChecksumAlgorithm,
+														   GetIntegrityInfoBuffer.Reserved,
+														   GetIntegrityInfoBuffer.Flags};
+	if (!DeviceIoControl(TargetFile, FSCTL_SET_INTEGRITY_INFORMATION, &IntegritySet, sizeof IntegritySet, nullptr, 0, nullptr, nullptr))
+	{
+		return false;
+	}
+
+	// Resize file - note that the file is sparse at this point so no additional data will be written
+	if (!SetFileInformationByHandle(TargetFile, FileEndOfFileInfo, &FileSize, sizeof FileSize))
+	{
+		return false;
+	}
+
+	constexpr auto RoundToClusterSize = [](LONG64 FileSize, ULONG ClusterSize) -> LONG64 {
+		return (FileSize + ClusterSize - 1) / ClusterSize * ClusterSize;
+	};
+	static_assert(RoundToClusterSize(5678, 4 * 1024) == 8 * 1024);
+
+	// Loop for cloning file contents. This is necessary as the API has a 32-bit size
+	// limit for some reason
+
+	const LONG64 SplitThreshold = (1LL << 32) - GetIntegrityInfoBuffer.ClusterSizeInBytes;
+
+	DUPLICATE_EXTENTS_DATA DuplicateExtentsData{.FileHandle = FromFile};
+
+	for (LONG64 CurrentByteOffset = 0,
+				RemainingBytes	  = RoundToClusterSize(FileSize.EndOfFile.QuadPart, GetIntegrityInfoBuffer.ClusterSizeInBytes);
+		 RemainingBytes > 0;
+		 CurrentByteOffset += SplitThreshold, RemainingBytes -= SplitThreshold)
+	{
+		DuplicateExtentsData.SourceFileOffset.QuadPart = CurrentByteOffset;
+		DuplicateExtentsData.TargetFileOffset.QuadPart = CurrentByteOffset;
+		DuplicateExtentsData.ByteCount.QuadPart		   = std::min(SplitThreshold, RemainingBytes);
+
+		if (!DeviceIoControl(TargetFile,
+							 FSCTL_DUPLICATE_EXTENTS_TO_FILE,
+							 &DuplicateExtentsData,
+							 sizeof DuplicateExtentsData,
+							 nullptr,
+							 0,
+							 &dwBytesReturned,
+							 nullptr))
+		{
+			return false;
+		}
+	}
+
+	// Make the file not sparse again now that we have populated the contents
+	if (!(BasicInfo.FileAttributes & FILE_ATTRIBUTE_SPARSE_FILE))
+	{
+		FILE_SET_SPARSE_BUFFER SetSparse = {FALSE};
+
+		if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, &SetSparse, sizeof SetSparse, nullptr, 0, &dwBytesReturned, nullptr))
+		{
+			return false;
+		}
+	}
+
+	// Update timestamps (but don't lie about the creation time)
+	BasicInfo.CreationTime.QuadPart = 0;
+	if (!SetFileInformationByHandle(TargetFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo))
+	{
+		return false;
+	}
+
+	if (!FlushFileBuffers(TargetFile))
+	{
+		return false;
+	}
+
+	// Finally now everything is done - make sure the file is not deleted on close!
+
+	FileDisposition = {FALSE};
+
+	const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition));
+
+	return AllOk;
+#elif ZEN_PLATFORM_LINUX
+#	if 0
+	struct ScopedFd
+	{
+		~ScopedFd() { close(Fd); }
+		int Fd;
+	};
+
+	// The 'from' file
+	int FromFd = open(FromPath.c_str(), O_RDONLY|O_CLOEXEC);
+	if (FromFd < 0)
+	{
+		return false;
+	}
+	ScopedFd $From = { FromFd };
+
+	// The 'to' file
+	int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0666);
+	if (ToFd < 0)
+	{
+		return false;
+	}
+	fchmod(ToFd, 0666);
+	ScopedFd $To = { FromFd };
+
+	ioctl(ToFd, FICLONE, FromFd);
+
+	return false;
+#	endif	// 0
+	ZEN_UNUSED(FromPath, ToPath);
+	ZEN_ERROR("CloneFile() is not implemented on this platform");
+	return false;
+#elif ZEN_PLATFORM_MAC
+	/* clonefile() syscall if APFS */
+	ZEN_UNUSED(FromPath, ToPath);
+	ZEN_ERROR("CloneFile() is not implemented on this platform");
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
+{
+	bool Success = false;
+
+	if (Options.EnableClone)
+	{
+		Success = CloneFile(FromPath.native(), ToPath.native());
+
+		if (Success)
+		{
+			return true;
+		}
+	}
+
+	if (Options.MustClone)
+	{
+		return false;
+	}
+
+#if ZEN_PLATFORM_WINDOWS
+	BOOL CancelFlag = FALSE;
+	Success			= !!::CopyFileExW(FromPath.c_str(),
+							  ToPath.c_str(),
+							  /* lpProgressRoutine */ nullptr,
+							  /* lpData */ nullptr,
+							  &CancelFlag,
+							  /* dwCopyFlags */ 0);
+#else
+	struct ScopedFd
+	{
+		~ScopedFd() { close(Fd); }
+		int Fd;
+	};
+
+	// From file
+	int FromFd = open(FromPath.c_str(), O_RDONLY | O_CLOEXEC);
+	if (FromFd < 0)
+	{
+		ThrowLastError(fmt::format("failed to open file {}", FromPath));
+	}
+	ScopedFd $From = {FromFd};
+
+	// To file
+	int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666);
+	if (ToFd < 0)
+	{
+		ThrowLastError(fmt::format("failed to create file {}", ToPath));
+	}
+	fchmod(ToFd, 0666);
+	ScopedFd $To = {ToFd};
+
+	// Copy impl
+	static const size_t BufferSize = 64 << 10;
+	void*				Buffer	   = malloc(BufferSize);
+	while (true)
+	{
+		int BytesRead = read(FromFd, Buffer, BufferSize);
+		if (BytesRead <= 0)
+		{
+			Success = (BytesRead == 0);
+			break;
+		}
+
+		if (write(ToFd, Buffer, BytesRead) != BufferSize)
+		{
+			Success = false;
+			break;
+		}
+	}
+	free(Buffer);
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	if (!Success)
+	{
+		ThrowLastError("file copy failed"sv);
+	}
+
+	return true;
+}
+
+void
+WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount)
+{
+#if ZEN_PLATFORM_WINDOWS
+	CAtlFile Outfile;
+	HRESULT	 hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
+	if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
+	{
+		CreateDirectories(Path.parent_path());
+
+		hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
+	}
+
+	if (FAILED(hRes))
+	{
+		ThrowSystemException(hRes, fmt::format("File open failed for '{}'", Path).c_str());
+	}
+
+#else
+	int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC;
+	int Fd		  = open(Path.c_str(), OpenFlags, 0666);
+	if (Fd < 0)
+	{
+		zen::CreateDirectories(Path.parent_path());
+		Fd = open(Path.c_str(), OpenFlags, 0666);
+	}
+
+	if (Fd < 0)
+	{
+		ThrowLastError(fmt::format("File open failed for '{}'", Path));
+	}
+
+	fchmod(Fd, 0666);
+#endif
+
+	// TODO: this should be block-enlightened
+
+	for (size_t i = 0; i < BufferCount; ++i)
+	{
+		uint64_t	WriteSize = Data[i]->Size();
+		const void* DataPtr	  = Data[i]->Data();
+
+		while (WriteSize)
+		{
+			const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024);
+
+#if ZEN_PLATFORM_WINDOWS
+			hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize));
+			if (FAILED(hRes))
+			{
+				ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str());
+			}
+#else
+			if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize))
+			{
+				ThrowLastError(fmt::format("File write failed for '{}'", Path));
+			}
+#endif	// ZEN_PLATFORM_WINDOWS
+
+			WriteSize -= ChunkSize;
+			DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize;
+		}
+	}
+
+#if !ZEN_PLATFORM_WINDOWS
+	close(Fd);
+#endif
+}
+
+void
+WriteFile(std::filesystem::path Path, IoBuffer Data)
+{
+	const IoBuffer* const DataPtr = &Data;
+
+	WriteFile(Path, &DataPtr, 1);
+}
+
+IoBuffer
+FileContents::Flatten()
+{
+	if (Data.size() == 1)
+	{
+		return Data[0];
+	}
+	else if (Data.empty())
+	{
+		return {};
+	}
+	else
+	{
+		ZEN_NOT_IMPLEMENTED();
+	}
+}
+
+FileContents
+ReadStdIn()
+{
+	BinaryWriter Writer;
+
+	do
+	{
+		uint8_t ReadBuffer[1024];
+
+		size_t BytesRead = fread(ReadBuffer, 1, sizeof ReadBuffer, stdin);
+		Writer.Write(ReadBuffer, BytesRead);
+	} while (!feof(stdin));
+
+	FileContents Contents;
+	Contents.Data.emplace_back(IoBuffer(IoBuffer::Clone, Writer.GetData(), Writer.GetSize()));
+
+	return Contents;
+}
+
+FileContents
+ReadFile(std::filesystem::path Path)
+{
+	uint64_t FileSizeBytes;
+	void*	 Handle;
+
+#if ZEN_PLATFORM_WINDOWS
+	ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	if (FromFile == INVALID_HANDLE_VALUE)
+	{
+		FromFile.Detach();
+		return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
+	}
+
+	FILE_END_OF_FILE_INFO FileSize;
+	if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile))
+	{
+		return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
+	}
+
+	FileSizeBytes = FileSize.EndOfFile.QuadPart;
+	Handle		  = FromFile.Detach();
+#else
+	int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC);
+	if (Fd < 0)
+	{
+		FileContents Ret;
+		Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category());
+		return Ret;
+	}
+
+	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+	struct stat Stat;
+	fstat(Fd, &Stat);
+
+	FileSizeBytes = Stat.st_size;
+	Handle		  = (void*)uintptr_t(Fd);
+#endif
+
+	FileContents Contents;
+	Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes));
+	return Contents;
+}
+
+bool
+ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
+{
+#if ZEN_PLATFORM_WINDOWS
+	ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	if (FromFile == INVALID_HANDLE_VALUE)
+	{
+		FromFile.Detach();
+		return false;
+	}
+
+	std::vector<uint8_t> ReadBuffer(ChunkSize);
+
+	for (;;)
+	{
+		DWORD dwBytesRead = 0;
+		BOOL  Success	  = ::ReadFile(FromFile, ReadBuffer.data(), (DWORD)ReadBuffer.size(), &dwBytesRead, nullptr);
+
+		if (!Success)
+		{
+			throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed");
+		}
+
+		if (dwBytesRead == 0)
+			break;
+
+		ProcessFunc(ReadBuffer.data(), dwBytesRead);
+	}
+#else
+	int Fd		  = open(Path.c_str(), O_RDONLY | O_CLOEXEC);
+	if (Fd < 0)
+	{
+		return false;
+	}
+
+	bool Success = true;
+
+	void* Buffer = malloc(ChunkSize);
+	while (true)
+	{
+		int BytesRead = read(Fd, Buffer, ChunkSize);
+		if (BytesRead < 0)
+		{
+			Success = false;
+			break;
+		}
+
+		if (BytesRead == 0)
+		{
+			break;
+		}
+
+		ProcessFunc(Buffer, BytesRead);
+	}
+
+	free(Buffer);
+	close(Fd);
+
+	if (!Success)
+	{
+		ThrowLastError("file scan failed");
+	}
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	return true;
+}
+
+void
+PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out)
+{
+#if ZEN_PLATFORM_WINDOWS
+	WideToUtf8(Path.native().c_str(), Out);
+#else
+	Out << Path.c_str();
+#endif
+}
+
+std::string
+PathToUtf8(const std::filesystem::path& Path)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return WideToUtf8(Path.native().c_str());
+#else
+	return Path.string();
+#endif
+}
+
+DiskSpace
+DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error)
+{
+	using namespace std::filesystem;
+
+	space_info SpaceInfo = space(Directory, Error);
+	if (Error)
+	{
+		return {};
+	}
+
+	return {
+		.Free  = uint64_t(SpaceInfo.available),
+		.Total = uint64_t(SpaceInfo.capacity),
+	};
+}
+
+void
+FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor)
+{
+#if ZEN_PLATFORM_WINDOWS
+	uint64_t FileInfoBuffer[8 * 1024];
+
+	FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo;
+	bool					  Continue = true;
+
+	CAtlFile RootDirHandle;
+	HRESULT	 hRes =
+		RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS);
+
+	if (FAILED(hRes))
+	{
+		ThrowSystemException(hRes, "Failed to open handle to volume root");
+	}
+
+	while (Continue)
+	{
+		BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer);
+		FibClass	 = FileIdBothDirectoryInfo;	 // Set up for next iteration
+
+		uint64_t EntryOffset = 0;
+
+		if (!Success)
+		{
+			DWORD LastError = GetLastError();
+
+			if (LastError == ERROR_NO_MORE_FILES)
+			{
+				break;
+			}
+
+			throw std::system_error(std::error_code(LastError, std::system_category()), "file system traversal error");
+		}
+
+		for (;;)
+		{
+			const FILE_ID_BOTH_DIR_INFO* DirInfo =
+				reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset);
+
+			std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t));
+
+			if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+			{
+				if (FileName == L"."sv || FileName == L".."sv)
+				{
+					// Not very interesting
+				}
+				else
+				{
+					const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName);
+
+					if (ShouldDescend)
+					{
+						// Note that this recursion combined with the buffer could
+						// blow the stack, we should consider a different strategy
+
+						std::filesystem::path FullPath = RootDir / FileName;
+
+						TraverseFileSystem(FullPath, Visitor);
+					}
+				}
+			}
+			else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE)
+			{
+				ZEN_WARN("encountered device node during file system traversal: '{}' found in '{}'", WideToUtf8(FileName), RootDir);
+			}
+			else
+			{
+				Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
+			}
+
+			const uint64_t NextOffset = DirInfo->NextEntryOffset;
+
+			if (NextOffset == 0)
+			{
+				break;
+			}
+
+			EntryOffset += NextOffset;
+		}
+	}
+#else
+	/* Could also implement this using Linux's getdents() syscall */
+
+	DIR* Dir = opendir(RootDir.c_str());
+	if (Dir == nullptr)
+	{
+		ThrowLastError(fmt::format("Failed to open directory for traversal: {}", RootDir.c_str()));
+	}
+
+	for (struct dirent* Entry; (Entry = readdir(Dir));)
+	{
+		const char* FileName = Entry->d_name;
+
+		struct stat			  Stat;
+		std::filesystem::path FullPath = RootDir / FileName;
+		stat(FullPath.c_str(), &Stat);
+
+		if (S_ISDIR(Stat.st_mode))
+		{
+			if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0)
+			{
+				/* nop */
+			}
+			else if (Visitor.VisitDirectory(RootDir, FileName))
+			{
+				TraverseFileSystem(FullPath, Visitor);
+			}
+		}
+		else if (S_ISREG(Stat.st_mode))
+		{
+			Visitor.VisitFile(RootDir, FileName, Stat.st_size);
+		}
+		else
+		{
+			ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}",
+					 Stat.st_mode,
+					 FileName,
+					 RootDir.c_str());
+		}
+	}
+
+	closedir(Dir);
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+std::filesystem::path
+PathFromHandle(void* NativeHandle)
+{
+#if ZEN_PLATFORM_WINDOWS
+	if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE)
+	{
+		return std::filesystem::path();
+	}
+
+	auto GetFinalPathNameByHandleWRetry = [](HANDLE hFile, LPWSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags) -> DWORD {
+		while (true)
+		{
+			DWORD Res = GetFinalPathNameByHandleW(hFile, lpszFilePath, cchFilePath, dwFlags);
+			if (Res == 0)
+			{
+				DWORD LastError = zen::GetLastError();
+				// Under heavy concurrent loads we might get access denied on a file handle while trying to get path name.
+				// Retry if that is the case.
+				if (LastError != ERROR_ACCESS_DENIED)
+				{
+					ThrowSystemError(LastError, fmt::format("failed to get path from file handle {}", hFile));
+				}
+				// Retry
+				continue;
+			}
+			ZEN_ASSERT(Res != 1);  // We don't accept empty path names
+			return Res;
+		}
+	};
+
+	static const DWORD PathDataSize = 512;
+	wchar_t			   PathData[PathDataSize];
+	DWORD			   RequiredLengthIncludingNul = GetFinalPathNameByHandleWRetry(NativeHandle, PathData, PathDataSize, FILE_NAME_OPENED);
+	if (RequiredLengthIncludingNul == 0)
+	{
+		ThrowLastError(fmt::format("failed to get path from file handle {}", NativeHandle));
+	}
+
+	if (RequiredLengthIncludingNul < PathDataSize)
+	{
+		std::wstring FullPath(PathData, gsl::narrow<size_t>(RequiredLengthIncludingNul));
+		return FullPath;
+	}
+
+	std::wstring FullPath;
+	FullPath.resize(RequiredLengthIncludingNul - 1);
+
+	const DWORD FinalLength = GetFinalPathNameByHandleWRetry(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED);
+	ZEN_UNUSED(FinalLength);
+	return FullPath;
+
+#elif ZEN_PLATFORM_LINUX
+	char Link[PATH_MAX];
+	char Path[64];
+
+	sprintf(Path, "/proc/self/fd/%d", int(uintptr_t(NativeHandle)));
+	ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1);
+	if (BytesRead <= 0)
+	{
+		return std::filesystem::path();
+	}
+
+	Link[BytesRead] = '\0';
+	return Link;
+#elif ZEN_PLATFORM_MAC
+	int	 Fd = int(uintptr_t(NativeHandle));
+	char Path[MAXPATHLEN];
+	if (fcntl(Fd, F_GETPATH, Path) < 0)
+	{
+		return std::filesystem::path();
+	}
+
+	return Path;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+std::filesystem::path
+GetRunningExecutablePath()
+{
+#if ZEN_PLATFORM_WINDOWS
+	TCHAR ExePath[MAX_PATH];
+	DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath));
+
+	return {std::wstring_view(ExePath, PathLength)};
+#elif ZEN_PLATFORM_LINUX
+	char	Link[256];
+	ssize_t BytesRead = readlink("/proc/self/exe", Link, sizeof(Link) - 1);
+	if (BytesRead < 0)
+		return {};
+
+	Link[BytesRead] = '\0';
+	return Link;
+#elif ZEN_PLATFORM_MAC
+	char Buffer[PROC_PIDPATHINFO_MAXSIZE];
+
+	int SelfPid = GetCurrentProcessId();
+	if (proc_pidpath(SelfPid, Buffer, sizeof(Buffer)) <= 0)
+		return {};
+
+	return Buffer;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+void
+MaximizeOpenFileCount()
+{
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	struct rlimit Limit;
+	int			  Error = getrlimit(RLIMIT_NOFILE, &Limit);
+	if (Error)
+	{
+		ZEN_WARN("failed getting rlimit RLIMIT_NOFILE, reason '{}'", zen::MakeErrorCode(Error).message());
+	}
+	else
+	{
+		struct rlimit NewLimit = Limit;
+		NewLimit.rlim_cur	   = NewLimit.rlim_max;
+		ZEN_INFO("changing RLIMIT_NOFILE from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}",
+				 Limit.rlim_cur,
+				 Limit.rlim_max,
+				 NewLimit.rlim_cur,
+				 NewLimit.rlim_max);
+
+		Error = setrlimit(RLIMIT_NOFILE, &NewLimit);
+		if (Error != 0)
+		{
+			ZEN_WARN("failed to set RLIMIT_NOFILE limits from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}, reason '{}'",
+					 Limit.rlim_cur,
+					 Limit.rlim_max,
+					 NewLimit.rlim_cur,
+					 NewLimit.rlim_max,
+					 zen::MakeErrorCode(Error).message());
+		}
+	}
+#endif
+}
+
+void
+GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
+{
+	FileSystemTraversal Traversal;
+	struct Visitor : public FileSystemTraversal::TreeVisitor
+	{
+		Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
+
+		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
+							   [[maybe_unused]] const path_view&			 File,
+							   [[maybe_unused]] uint64_t					 FileSize) override
+		{
+			if (Flags & DirectoryContent::IncludeFilesFlag)
+			{
+				Content.Files.push_back(Parent / File);
+			}
+		}
+
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		{
+			if (Flags & DirectoryContent::IncludeDirsFlag)
+			{
+				Content.Directories.push_back(Parent / DirectoryName);
+			}
+			return (Flags & DirectoryContent::RecursiveFlag) != 0;
+		}
+
+		const uint8_t	  Flags;
+		DirectoryContent& Content;
+	} Visit(Flags, OutContent);
+
+	Traversal.TraverseFileSystem(RootDir, Visit);
+}
+
+std::string
+GetEnvVariable(std::string_view VariableName)
+{
+	ZEN_ASSERT(!VariableName.empty());
+#if ZEN_PLATFORM_WINDOWS
+
+	CHAR  EnvVariableBuffer[1023 + 1];
+	DWORD RESULT = GetEnvironmentVariableA(std::string(VariableName).c_str(), EnvVariableBuffer, sizeof(EnvVariableBuffer));
+	if (RESULT > 0 && RESULT < sizeof(EnvVariableBuffer))
+	{
+		return std::string(EnvVariableBuffer);
+	}
+#endif
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	char* EnvVariable = getenv(std::string(VariableName).c_str());
+	if (EnvVariable)
+	{
+		return std::string(EnvVariable);
+	}
+#endif
+	return "";
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+#if ZEN_WITH_TESTS
+
+void
+filesystem_forcelink()
+{
+}
+
+TEST_CASE("filesystem")
+{
+	using namespace std::filesystem;
+
+	// GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in
+	path	   BinPath	   = GetRunningExecutablePath();
+	const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver";
+	CHECK(ExpectedExe);
+	CHECK(is_regular_file(BinPath));
+
+	// PathFromHandle
+	void* Handle;
+#	if ZEN_PLATFORM_WINDOWS
+	Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
+	CHECK(Handle != INVALID_HANDLE_VALUE);
+#	else
+	int Fd = open(BinPath.c_str(), O_RDONLY | O_CLOEXEC);
+	CHECK(Fd >= 0);
+	Handle = (void*)uintptr_t(Fd);
+#	endif
+
+	auto FromHandle = PathFromHandle((void*)uintptr_t(Handle));
+	CHECK(equivalent(FromHandle, BinPath));
+
+#	if ZEN_PLATFORM_WINDOWS
+	CloseHandle(Handle);
+#	else
+	close(int(uintptr_t(Handle)));
+#	endif
+
+	// Traversal
+	struct : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
+		{
+			bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
+		}
+
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+
+		bool				  bFoundExpected = false;
+		std::filesystem::path Expected;
+	} Visitor;
+	Visitor.Expected = BinPath;
+
+	FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor);
+	CHECK(Visitor.bFoundExpected);
+
+	// Scan/read file
+	FileContents		 BinRead = ReadFile(BinPath);
+	std::vector<uint8_t> BinScan;
+	ScanFile(BinPath, 16 << 10, [&](const void* Data, size_t Size) {
+		const auto* Ptr = (uint8_t*)Data;
+		BinScan.insert(BinScan.end(), Ptr, Ptr + Size);
+	});
+	CHECK_EQ(BinRead.Data.size(), 1);
+	CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize());
+}
+
+TEST_CASE("WriteFile")
+{
+	std::filesystem::path TempFile = GetRunningExecutablePath().parent_path();
+	TempFile /= "write_file_test";
+
+	uint64_t Magics[] = {
+		0x0'a9e'a9e'a9e'a9e'a9e,
+		0x0'493'493'493'493'493,
+	};
+
+	struct
+	{
+		const void* Data;
+		size_t		Size;
+	} MagicTests[] = {
+		{
+			Magics,
+			sizeof(Magics),
+		},
+		{
+			Magics + 1,
+			sizeof(Magics[0]),
+		},
+	};
+	for (auto& MagicTest : MagicTests)
+	{
+		WriteFile(TempFile, IoBuffer(IoBuffer::Wrap, MagicTest.Data, MagicTest.Size));
+
+		FileContents MagicsReadback = ReadFile(TempFile);
+		CHECK_EQ(MagicsReadback.Data.size(), 1);
+		CHECK_EQ(MagicsReadback.Data[0].GetSize(), MagicTest.Size);
+		CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0);
+	}
+
+	std::filesystem::remove(TempFile);
+}
+
+TEST_CASE("DiskSpaceInfo")
+{
+	std::filesystem::path BinPath = GetRunningExecutablePath();
+
+	DiskSpace Space = {};
+
+	std::error_code Error;
+	Space = DiskSpaceInfo(BinPath, Error);
+	CHECK(!Error);
+
+	bool Okay = DiskSpaceInfo(BinPath, Space);
+	CHECK(Okay);
+
+	CHECK(int64_t(Space.Total) > 0);
+	CHECK(int64_t(Space.Free) > 0);	 // Hopefully there's at least one byte free
+}
+
+TEST_CASE("PathBuilder")
+{
+#	if ZEN_PLATFORM_WINDOWS
+	const char* foo_bar = "/foo\\bar";
+#	else
+	const char* foo_bar = "/foo/bar";
+#	endif
+
+	ExtendablePathBuilder<32> Path;
+	for (const char* Prefix : {"/foo", "/foo/"})
+	{
+		Path.Reset();
+		Path.Append(Prefix);
+		Path /= "bar";
+		CHECK(Path.ToPath() == foo_bar);
+	}
+
+	using fspath = std::filesystem::path;
+
+	Path.Reset();
+	Path.Append(fspath("/foo/"));
+	Path /= (fspath("bar"));
+	CHECK(Path.ToPath() == foo_bar);
+
+#	if ZEN_PLATFORM_WINDOWS
+	Path.Reset();
+	Path.Append(fspath(L"/\u0119oo/"));
+	Path /= L"bar";
+	printf("%ls\n", Path.ToPath().c_str());
+	CHECK(Path.ToView() == L"/\u0119oo/bar");
+	CHECK(Path.ToPath() == L"\\\u0119oo\\bar");
+#	endif
+}
+
+#endif
+
+}  // namespace zen