Merge branch 'main' of https://github.com/EpicGames/zen

1 files changed, 262 insertions, 25 deletions

diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index f6e410ee2..d1b8b7aeb 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -7,14 +7,25 @@
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
-#include <zencore/windows.h>
+#include <zencore/testing.h>
 
-#include <atlbase.h>
-#include <atlfile.h>
-#include <winioctl.h>
-#include <winnt.h>
-#include <filesystem>
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <atlbase.h>
+#	include <atlfile.h>
+#	include <winioctl.h>
+#	include <winnt.h>
+#else
+#	include <dirent.h>
+#	include <fcntl.h>
+#	include <sys/stat.h>
+#	include <unistd.h>
+#endif
 
+#include <filesystem>
 #include <gsl/gsl-lite.hpp>
 
 namespace zen {
@@ -22,6 +33,7 @@ namespace zen {
 using namespace std::literals;
 
 #if ZEN_PLATFORM_WINDOWS
+
 static bool
 DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
 {
@@ -48,6 +60,12 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
 	return false;
 }
 
+bool
+CreateDirectories(const wchar_t* Dir)
+{
+	return std::filesystem::create_directories(Dir);
+}
+
 // Erase all files and directories in a given directory, leaving an empty directory
 // behind
 
@@ -126,12 +144,11 @@ WipeDirectory(const wchar_t* DirPath)
 
 	return true;
 }
-#endif
 
 bool
-CreateDirectories(const std::filesystem::path& Dir)
+DeleteDirectories(const wchar_t* DirPath)
 {
-	return std::filesystem::create_directories(Dir);
+	return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
 }
 
 bool
@@ -145,16 +162,46 @@ CleanDirectory(const wchar_t* DirPath)
 	return CreateDirectories(DirPath);
 }
 
+#endif	// ZEN_PLATFORM_WINDOWS
+
+bool
+CreateDirectories(const std::filesystem::path& Dir)
+{
+	return std::filesystem::create_directories(Dir);
+}
+
 bool
 DeleteDirectories(const std::filesystem::path& Dir)
 {
-	return WipeDirectory(Dir.c_str()) && RemoveDirectoryW(Dir.c_str()) == TRUE;
+#if ZEN_PLATFORM_WINDOWS
+	return DeleteDirectories(Dir.c_str());
+#else
+	std::error_code ErrorCode;
+	return std::filesystem::remove_all(Dir, ErrorCode);
+#endif
 }
 
 bool
 CleanDirectory(const std::filesystem::path& Dir)
 {
+#if ZEN_PLATFORM_WINDOWS
 	return CleanDirectory(Dir.c_str());
+#else
+	if (std::filesystem::exists(Dir))
+	{
+		bool Success = true;
+
+		std::error_code ErrorCode;
+		for (const auto& Item : std::filesystem::directory_iterator(Dir))
+		{
+			Success &= std::filesystem::remove_all(Item, ErrorCode);
+		}
+
+		return Success;
+	}
+
+	return CreateDirectories(Dir);
+#endif
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -191,12 +238,13 @@ SupportsBlockRefCounting(std::filesystem::path Path)
 	return true;
 #else
 	return false;
-#endif
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 bool
 CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 {
+#if ZEN_PLATFORM_WINDOWS
 	ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
@@ -351,11 +399,16 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 	const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition));
 
 	return AllOk;
+#else
+	ZEN_ERROR("CloneFile() is not implemented on this platform");
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 bool
 CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
 {
+#if ZEN_PLATFORM_WINDOWS
 	bool Success = false;
 
 	if (Options.EnableClone)
@@ -387,6 +440,10 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 	}
 
 	return Success;
+#else
+	ZEN_ERROR("CopyFile() is not implemented on this platform");
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 void
@@ -394,6 +451,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
 {
 	using namespace fmt::literals;
 
+#if ZEN_PLATFORM_WINDOWS
 	CAtlFile Outfile;
 	HRESULT	 hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
 	if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
@@ -408,6 +466,20 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
 		ThrowSystemException(hRes, "File open failed for '{}'"_format(Path).c_str());
 	}
 
+#else
+	int Fd = open(Path.c_str(), O_WRONLY);
+	if (Fd < 0)
+	{
+		zen::CreateDirectories(Path.parent_path());
+		Fd = open(Path.c_str(), O_WRONLY);
+	}
+
+	if (Fd < 0)
+	{
+		ThrowLastError("File open failed for '{}'"_format(Path));
+	}
+#endif
+
 	// TODO: this should be block-enlightened
 
 	for (size_t i = 0; i < BufferCount; ++i)
@@ -419,17 +491,27 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
 		{
 			const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024);
 
+#if ZEN_PLATFORM_WINDOWS
 			hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize));
-
 			if (FAILED(hRes))
 			{
 				ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str());
 			}
+#else
+			if (write(Fd, DataPtr, WriteSize) != WriteSize)
+			{
+				ThrowLastError("File write failed for '{}'"_format(Path));
+			}
+#endif	// ZEN_PLATFORM_WINDOWS
 
 			WriteSize -= ChunkSize;
 			DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize;
 		}
 	}
+
+#if !ZEN_PLATFORM_WINDOWS
+	close(Fd);
+#endif
 }
 
 void
@@ -443,6 +525,10 @@ WriteFile(std::filesystem::path Path, IoBuffer Data)
 FileContents
 ReadFile(std::filesystem::path Path)
 {
+	uint64_t FileSizeBytes;
+	void*	 Handle;
+
+#if ZEN_PLATFORM_WINDOWS
 	ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
@@ -456,18 +542,32 @@ ReadFile(std::filesystem::path Path)
 		return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
 	}
 
-	const uint64_t FileSizeBytes = FileSize.EndOfFile.QuadPart;
+	FileSizeBytes = FileSize.EndOfFile.QuadPart;
+	Handle		  = FromFile.Detach();
+#else
+	int Fd = open(Path.c_str(), O_RDONLY);
+	if (Fd < 0)
+	{
+		return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())};
+	}
 
-	FileContents Contents;
+	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+	struct stat Stat;
+	fstat(Fd, &Stat);
 
-	Contents.Data.emplace_back(IoBuffer(IoBuffer::File, FromFile.Detach(), 0, FileSizeBytes));
+	FileSizeBytes = Stat.st_size;
+	Handle		  = (void*)uintptr_t(Fd);
+#endif
 
+	FileContents Contents;
+	Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes));
 	return Contents;
 }
 
 bool
 ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
 {
+#if ZEN_PLATFORM_WINDOWS
 	ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
@@ -494,30 +594,34 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi
 	}
 
 	return true;
+#else
+	ZEN_ERROR("ScanFile() is not implemented on this platform");
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 std::string
 ToUtf8(const std::filesystem::path& Path)
 {
+#if ZEN_PLATFORM_WINDOWS
 	return WideToUtf8(Path.native().c_str());
+#else
+	return Path.string();
+#endif
 }
 
 void
 FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor)
 {
+#if ZEN_PLATFORM_WINDOWS
 	uint64_t FileInfoBuffer[8 * 1024];
 
 	FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo;
 	bool					  Continue = true;
 
-	std::wstring RootDirPath = RootDir.native();
-
 	CAtlFile RootDirHandle;
-	HRESULT	 hRes = RootDirHandle.Create(RootDirPath.c_str(),
-										 GENERIC_READ,
-										 FILE_SHARE_READ | FILE_SHARE_WRITE,
-										 OPEN_EXISTING,
-										 FILE_FLAG_BACKUP_SEMANTICS);
+	HRESULT	 hRes =
+		RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS);
 
 	if (FAILED(hRes))
 	{
@@ -573,12 +677,12 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 			}
 			else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE)
 			{
-				ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir);
+				ZEN_WARN("encountered device node during file system traversal: {} found in {}",
+						 WideToUtf8(FileName),
+						 WideToUtf8(RootDir.c_str()));
 			}
 			else
 			{
-				std::filesystem::path FullPath = RootDir / FileName;
-
 				Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
 			}
 
@@ -592,11 +696,57 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 			EntryOffset += NextOffset;
 		}
 	}
+#else
+	using namespace fmt::literals;
+
+	/* Could also implement this using Linux's getdents() syscall */
+
+	DIR* Dir = opendir(RootDir.c_str());
+	if (Dir == nullptr)
+	{
+		ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str()));
+	}
+
+	for (struct dirent* Entry; Entry = readdir(Dir);)
+	{
+		const char* FileName = Entry->d_name;
+
+		struct stat			  Stat;
+		std::filesystem::path FullPath = RootDir / FileName;
+		stat(FullPath.c_str(), &Stat);
+
+		if (S_ISDIR(Stat.st_mode))
+		{
+			if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0)
+			{
+				/* nop */
+			}
+			else if (Visitor.VisitDirectory(RootDir, FileName))
+			{
+				TraverseFileSystem(FullPath, Visitor);
+			}
+		}
+		else if (S_ISREG(Stat.st_mode))
+		{
+			Visitor.VisitFile(RootDir, FileName, Stat.st_size);
+		}
+		else
+		{
+			ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}",
+					 Stat.st_mode,
+					 FileName,
+					 RootDir.c_str());
+		}
+	}
+
+	closedir(Dir);
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 std::filesystem::path
 PathFromHandle(void* NativeHandle)
 {
+#if ZEN_PLATFORM_WINDOWS
 	if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE)
 	{
 		return std::filesystem::path();
@@ -612,15 +762,102 @@ PathFromHandle(void* NativeHandle)
 	ZEN_UNUSED(FinalLength);
 
 	return FullPath;
+#elif ZEN_PLATFORM_LINUX
+	char Buffer[256];
+	sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle)));
+	ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1);
+	if (BytesRead <= 0)
+		return std::filesystem::path();
+
+	Buffer[BytesRead] = '\0';
+	return Buffer;
+#else
+#	error Unimplemented platform
+#endif	// ZEN_PLATFORM_WINDOWS
 }
 
 std::filesystem::path
 GetRunningExecutablePath()
 {
+#if ZEN_PLATFORM_WINDOWS
 	TCHAR ExePath[MAX_PATH];
 	DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath));
 
 	return {std::wstring_view(ExePath, PathLength)};
+#elif ZEN_PLATFORM_LINUX
+	char Buffer[256];
+	sprintf(Buffer, "/proc/%d/exe", getpid());
+	ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1);
+	if (BytesRead < 0)
+		return {};
+
+	Buffer[BytesRead] = '\0';
+	return Buffer;
+#else
+#	error Unimplemented platform
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+#if ZEN_WITH_TESTS
+
+void
+filesystem_forcelink()
+{
 }
 
+TEST_CASE("filesystem")
+{
+	using namespace std::filesystem;
+
+	// GetExePath
+	path BinPath = GetRunningExecutablePath();
+	CHECK(BinPath.stem() == "zencore-test");
+	CHECK(is_regular_file(BinPath));
+
+	// PathFromHandle
+	void* Handle;
+#	if ZEN_PLATFORM_WINDOWS
+	Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
+	CHECK(Handle != INVALID_HANDLE_VALUE);
+#	else
+	int Fd = open(BinPath.c_str(), O_RDONLY);
+	CHECK(Fd >= 0);
+	Handle = (void*)uintptr_t(Fd);
+#	endif
+
+	auto FromHandle = PathFromHandle((void*)uintptr_t(Handle));
+	CHECK(equivalent(FromHandle, BinPath));
+
+#	if ZEN_PLATFORM_WINDOWS
+	CloseHandle(Handle);
+#	else
+	close(int(uintptr_t(Handle)));
+#	endif
+
+	// Traversal
+	struct : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
+		{
+			bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
+		}
+
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+
+		bool				  bFoundExpected = false;
+		std::filesystem::path Expected;
+	} Visitor;
+	Visitor.Expected = BinPath;
+
+	FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor);
+	CHECK(Visitor.bFoundExpected);
+}
+
+#endif
+
 }  // namespace zen