moved source directories into `/src` (#264)

* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees

1 files changed, 575 insertions, 0 deletions

diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
new file mode 100644
index 000000000..1e6043d7e
--- /dev/null
+++ b/src/zenutil/basicfile.cpp
@@ -0,0 +1,575 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenutil/basicfile.h"
+
+#include <zencore/compactbinary.h>
+#include <zencore/except.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#else
+#	include <fcntl.h>
+#	include <sys/file.h>
+#	include <sys/stat.h>
+#	include <unistd.h>
+#endif
+
+#include <fmt/format.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+BasicFile::~BasicFile()
+{
+	Close();
+}
+
+void
+BasicFile::Open(const std::filesystem::path& FileName, Mode Mode)
+{
+	std::error_code Ec;
+	Open(FileName, Mode, Ec);
+
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to open file '{}'", FileName));
+	}
+}
+
+void
+BasicFile::Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec)
+{
+	Ec.clear();
+
+#if ZEN_PLATFORM_WINDOWS
+	DWORD dwCreationDisposition = 0;
+	DWORD dwDesiredAccess		= 0;
+	switch (Mode)
+	{
+		case Mode::kRead:
+			dwCreationDisposition |= OPEN_EXISTING;
+			dwDesiredAccess |= GENERIC_READ;
+			break;
+		case Mode::kWrite:
+			dwCreationDisposition |= OPEN_ALWAYS;
+			dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE);
+			break;
+		case Mode::kDelete:
+			dwCreationDisposition |= OPEN_ALWAYS;
+			dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE);
+			break;
+		case Mode::kTruncate:
+			dwCreationDisposition |= CREATE_ALWAYS;
+			dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE);
+			break;
+		case Mode::kTruncateDelete:
+			dwCreationDisposition |= CREATE_ALWAYS;
+			dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE);
+			break;
+	}
+
+	const DWORD dwShareMode			 = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
+	const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+	HANDLE		hTemplateFile		 = nullptr;
+
+	HANDLE FileHandle = CreateFile(FileName.c_str(),
+								   dwDesiredAccess,
+								   dwShareMode,
+								   /* lpSecurityAttributes */ nullptr,
+								   dwCreationDisposition,
+								   dwFlagsAndAttributes,
+								   hTemplateFile);
+
+	if (FileHandle == INVALID_HANDLE_VALUE)
+	{
+		Ec = MakeErrorCodeFromLastError();
+
+		return;
+	}
+#else
+	int OpenFlags = O_CLOEXEC;
+	switch (Mode)
+	{
+		case Mode::kRead:
+			OpenFlags |= O_RDONLY;
+			break;
+		case Mode::kWrite:
+		case Mode::kDelete:
+			OpenFlags |= (O_RDWR | O_CREAT);
+			break;
+		case Mode::kTruncate:
+		case Mode::kTruncateDelete:
+			OpenFlags |= (O_RDWR | O_CREAT | O_TRUNC);
+			break;
+	}
+
+	int Fd = open(FileName.c_str(), OpenFlags, 0666);
+	if (Fd < 0)
+	{
+		Ec = MakeErrorCodeFromLastError();
+		return;
+	}
+	if (Mode != Mode::kRead)
+	{
+		fchmod(Fd, 0666);
+	}
+
+	void* FileHandle = (void*)(uintptr_t(Fd));
+#endif
+
+	m_FileHandle = FileHandle;
+}
+
+void
+BasicFile::Close()
+{
+	if (m_FileHandle)
+	{
+#if ZEN_PLATFORM_WINDOWS
+		::CloseHandle(m_FileHandle);
+#else
+		int Fd = int(uintptr_t(m_FileHandle));
+		close(Fd);
+#endif
+		m_FileHandle = nullptr;
+	}
+}
+
+void
+BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
+{
+	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+
+	while (BytesToRead)
+	{
+		const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize);
+
+#if ZEN_PLATFORM_WINDOWS
+		OVERLAPPED Ovl{};
+
+		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
+		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+		DWORD dwNumberOfBytesRead = 0;
+		BOOL  Success			  = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
+
+		ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead);
+#else
+		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+		int	 Fd		   = int(uintptr_t(m_FileHandle));
+		int	 BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
+		bool Success   = (BytesRead > 0);
+#endif
+
+		if (!Success)
+		{
+			ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle)));
+		}
+
+		BytesToRead -= NumberOfBytesToRead;
+		FileOffset += NumberOfBytesToRead;
+		Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+	}
+}
+
+IoBuffer
+BasicFile::ReadAll()
+{
+	IoBuffer Buffer(FileSize());
+	Read(Buffer.MutableData(), Buffer.Size(), 0);
+	return Buffer;
+}
+
+void
+BasicFile::StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+	StreamByteRange(0, FileSize(), std::move(ChunkFun));
+}
+
+void
+BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+	const uint64_t ChunkSize = 128 * 1024;
+	IoBuffer	   ReadBuffer{ChunkSize};
+	void*		   BufferPtr = ReadBuffer.MutableData();
+
+	uint64_t RemainBytes   = Size;
+	uint64_t CurrentOffset = FileOffset;
+
+	while (RemainBytes)
+	{
+		const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes);
+
+		Read(BufferPtr, ThisChunkBytes, CurrentOffset);
+
+		ChunkFun(BufferPtr, ThisChunkBytes);
+
+		CurrentOffset += ThisChunkBytes;
+		RemainBytes -= ThisChunkBytes;
+	}
+}
+
+void
+BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
+{
+	Write(Data.GetData(), Data.GetSize(), FileOffset, Ec);
+}
+
+void
+BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
+{
+	Ec.clear();
+
+	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+
+	while (Size)
+	{
+		const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize);
+
+#if ZEN_PLATFORM_WINDOWS
+		OVERLAPPED Ovl{};
+
+		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
+		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+		DWORD dwNumberOfBytesWritten = 0;
+
+		BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
+#else
+		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+		int	 Fd			  = int(uintptr_t(m_FileHandle));
+		int	 BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset);
+		bool Success	  = (BytesWritten > 0);
+#endif
+
+		if (!Success)
+		{
+			Ec = MakeErrorCodeFromLastError();
+
+			return;
+		}
+
+		Size -= NumberOfBytesToWrite;
+		FileOffset += NumberOfBytesToWrite;
+		Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesToWrite;
+	}
+}
+
+void
+BasicFile::Write(MemoryView Data, uint64_t FileOffset)
+{
+	Write(Data.GetData(), Data.GetSize(), FileOffset);
+}
+
+void
+BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
+{
+	std::error_code Ec;
+	Write(Data, Size, Offset, Ec);
+
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle)));
+	}
+}
+
+void
+BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec)
+{
+	Write(Data.Data(), Data.Size(), 0, Ec);
+}
+
+void
+BasicFile::Flush()
+{
+#if ZEN_PLATFORM_WINDOWS
+	FlushFileBuffers(m_FileHandle);
+#else
+	int Fd = int(uintptr_t(m_FileHandle));
+	fsync(Fd);
+#endif
+}
+
+uint64_t
+BasicFile::FileSize()
+{
+#if ZEN_PLATFORM_WINDOWS
+	ULARGE_INTEGER liFileSize;
+	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
+	if (liFileSize.LowPart == INVALID_FILE_SIZE)
+	{
+		int Error = zen::GetLastError();
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle)));
+		}
+	}
+	return uint64_t(liFileSize.QuadPart);
+#else
+	int Fd = int(uintptr_t(m_FileHandle));
+	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+	struct stat Stat;
+	fstat(Fd, &Stat);
+	return uint64_t(Stat.st_size);
+#endif
+}
+
+void
+BasicFile::SetFileSize(uint64_t FileSize)
+{
+#if ZEN_PLATFORM_WINDOWS
+	LARGE_INTEGER liFileSize;
+	liFileSize.QuadPart = FileSize;
+	BOOL OK				= ::SetFilePointerEx(m_FileHandle, liFileSize, 0, FILE_BEGIN);
+	if (OK == FALSE)
+	{
+		int Error = zen::GetLastError();
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+		}
+	}
+	OK = ::SetEndOfFile(m_FileHandle);
+	if (OK == FALSE)
+	{
+		int Error = zen::GetLastError();
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+		}
+	}
+#elif ZEN_PLATFORM_MAC
+	int Fd = int(intptr_t(m_FileHandle));
+	if (ftruncate(Fd, (off_t)FileSize) < 0)
+	{
+		int Error = zen::GetLastError();
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+		}
+	}
+#else
+	int Fd = int(intptr_t(m_FileHandle));
+	if (ftruncate64(Fd, (off64_t)FileSize) < 0)
+	{
+		int Error = zen::GetLastError();
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+		}
+	}
+	if (FileSize > 0)
+	{
+		int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize);
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+		}
+	}
+#endif
+}
+
+void*
+BasicFile::Detach()
+{
+	void* FileHandle = m_FileHandle;
+	m_FileHandle	 = 0;
+	return FileHandle;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+TemporaryFile::~TemporaryFile()
+{
+	Close();
+}
+
+void
+TemporaryFile::Close()
+{
+	if (m_FileHandle)
+	{
+#if ZEN_PLATFORM_WINDOWS
+		// Mark file for deletion when final handle is closed
+
+		FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE};
+
+		SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
+#else
+		std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle);
+		unlink(FilePath.c_str());
+#endif
+
+		BasicFile::Close();
+	}
+}
+
+void
+TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec)
+{
+	StringBuilder<64> TempName;
+	Oid::NewOid().ToString(TempName);
+
+	m_TempPath = TempDirName / TempName.c_str();
+
+	Open(m_TempPath, BasicFile::Mode::kTruncateDelete, Ec);
+}
+
+void
+TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec)
+{
+	// We intentionally call the base class Close() since otherwise we'll end up
+	// deleting the temporary file
+	BasicFile::Close();
+
+	std::filesystem::rename(m_TempPath, FinalFileName, Ec);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+LockFile::LockFile()
+{
+}
+
+LockFile::~LockFile()
+{
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	int Fd = int(intptr_t(m_FileHandle));
+	flock(Fd, LOCK_UN | LOCK_NB);
+#endif
+}
+
+void
+LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+	Ec.clear();
+
+	const DWORD dwCreationDisposition = CREATE_ALWAYS;
+	DWORD		dwDesiredAccess		  = GENERIC_READ | GENERIC_WRITE | DELETE;
+	const DWORD dwShareMode			  = FILE_SHARE_READ;
+	const DWORD dwFlagsAndAttributes  = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE;
+	HANDLE		hTemplateFile		  = nullptr;
+
+	HANDLE FileHandle = CreateFile(FileName.c_str(),
+								   dwDesiredAccess,
+								   dwShareMode,
+								   /* lpSecurityAttributes */ nullptr,
+								   dwCreationDisposition,
+								   dwFlagsAndAttributes,
+								   hTemplateFile);
+
+	if (FileHandle == INVALID_HANDLE_VALUE)
+	{
+		Ec = zen::MakeErrorCodeFromLastError();
+
+		return;
+	}
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	int Fd = open(FileName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666);
+	if (Fd < 0)
+	{
+		Ec = zen::MakeErrorCodeFromLastError();
+		return;
+	}
+	fchmod(Fd, 0666);
+
+	int LockRet = flock(Fd, LOCK_EX | LOCK_NB);
+	if (LockRet < 0)
+	{
+		Ec = zen::MakeErrorCodeFromLastError();
+		close(Fd);
+		return;
+	}
+
+	void* FileHandle = (void*)uintptr_t(Fd);
+#endif
+
+	m_FileHandle = FileHandle;
+
+	BasicFile::Write(Payload.GetBuffer(), 0, Ec);
+}
+
+void
+LockFile::Update(CbObject Payload, std::error_code& Ec)
+{
+	BasicFile::Write(Payload.GetBuffer(), 0, Ec);
+}
+
+/*
+		___________              __
+		\__    ___/___   _______/  |_  ______
+		  |    |_/ __ \ /  ___/\   __\/  ___/
+		  |    |\  ___/ \___ \  |  |  \___ \
+		  |____| \___  >____  > |__| /____  >
+					 \/     \/            \/
+*/
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("BasicFile")
+{
+	ScopedCurrentDirectoryChange _;
+
+	BasicFile File1;
+	CHECK_THROWS(File1.Open("zonk", BasicFile::Mode::kRead));
+	CHECK_NOTHROW(File1.Open("zonk", BasicFile::Mode::kTruncate));
+	CHECK_NOTHROW(File1.Write("abcd", 4, 0));
+	CHECK(File1.FileSize() == 4);
+	{
+		IoBuffer Data = File1.ReadAll();
+		CHECK(Data.Size() == 4);
+		CHECK_EQ(memcmp(Data.Data(), "abcd", 4), 0);
+	}
+	CHECK_NOTHROW(File1.Write("efgh", 4, 2));
+	CHECK(File1.FileSize() == 6);
+	{
+		IoBuffer Data = File1.ReadAll();
+		CHECK(Data.Size() == 6);
+		CHECK_EQ(memcmp(Data.Data(), "abefgh", 6), 0);
+	}
+}
+
+TEST_CASE("TemporaryFile")
+{
+	ScopedCurrentDirectoryChange _;
+
+	SUBCASE("DeleteOnClose")
+	{
+		TemporaryFile	TmpFile;
+		std::error_code Ec;
+		TmpFile.CreateTemporary(std::filesystem::current_path(), Ec);
+		CHECK(!Ec);
+		CHECK(std::filesystem::exists(TmpFile.GetPath()));
+		TmpFile.Close();
+		CHECK(std::filesystem::exists(TmpFile.GetPath()) == false);
+	}
+
+	SUBCASE("MoveIntoPlace")
+	{
+		TemporaryFile	TmpFile;
+		std::error_code Ec;
+		TmpFile.CreateTemporary(std::filesystem::current_path(), Ec);
+		CHECK(!Ec);
+		std::filesystem::path TempPath	= TmpFile.GetPath();
+		std::filesystem::path FinalPath = std::filesystem::current_path() / "final";
+		CHECK(std::filesystem::exists(TempPath));
+		TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec);
+		CHECK(!Ec);
+		CHECK(std::filesystem::exists(TempPath) == false);
+		CHECK(std::filesystem::exists(FinalPath));
+	}
+}
+
+void
+basicfile_forcelink()
+{
+}
+
+#endif
+
+}  // namespace zen