aboutsummaryrefslogtreecommitdiff
path: root/zencore/filesystem.cpp
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-05-02 10:01:47 +0200
committerGitHub <[email protected]>2023-05-02 10:01:47 +0200
commit075d17f8ada47e990fe94606c3d21df409223465 (patch)
treee50549b766a2f3c354798a54ff73404217b4c9af /zencore/filesystem.cpp
parentfix: bundle shouldn't append content zip to zen (diff)
downloadzen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz
zen-075d17f8ada47e990fe94606c3d21df409223465.zip
moved source directories into `/src` (#264)
* moved source directories into `/src` * updated bundle.lua for new `src` path * moved some docs, icon * removed old test trees
Diffstat (limited to 'zencore/filesystem.cpp')
-rw-r--r--zencore/filesystem.cpp1304
1 files changed, 0 insertions, 1304 deletions
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
deleted file mode 100644
index a17773024..000000000
--- a/zencore/filesystem.cpp
+++ /dev/null
@@ -1,1304 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include <zencore/filesystem.h>
-
-#include <zencore/except.h>
-#include <zencore/fmtutils.h>
-#include <zencore/iobuffer.h>
-#include <zencore/logging.h>
-#include <zencore/stream.h>
-#include <zencore/string.h>
-#include <zencore/testing.h>
-
-#if ZEN_PLATFORM_WINDOWS
-# include <zencore/windows.h>
-#endif
-
-#if ZEN_PLATFORM_WINDOWS
-# include <atlbase.h>
-# include <atlfile.h>
-# include <winioctl.h>
-# include <winnt.h>
-#endif
-
-#if ZEN_PLATFORM_LINUX
-# include <dirent.h>
-# include <fcntl.h>
-# include <sys/resource.h>
-# include <sys/stat.h>
-# include <unistd.h>
-#endif
-
-#if ZEN_PLATFORM_MAC
-# include <dirent.h>
-# include <fcntl.h>
-# include <libproc.h>
-# include <sys/resource.h>
-# include <sys/stat.h>
-# include <sys/syslimits.h>
-# include <unistd.h>
-#endif
-
-#include <filesystem>
-#include <gsl/gsl-lite.hpp>
-
-namespace zen {
-
-using namespace std::literals;
-
-#if ZEN_PLATFORM_WINDOWS
-
-static bool
-DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
-{
- CHandle hDir(CreateFileW(Path,
- GENERIC_WRITE,
- FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
- nullptr,
- OPEN_EXISTING,
- FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT,
- nullptr));
-
- if (hDir != INVALID_HANDLE_VALUE)
- {
- REPARSE_GUID_DATA_BUFFER Rgdb = {};
- Rgdb.ReparseTag = dwReparseTag;
-
- DWORD dwBytes;
- const BOOL bOK =
- DeviceIoControl(hDir, FSCTL_DELETE_REPARSE_POINT, &Rgdb, REPARSE_GUID_DATA_BUFFER_HEADER_SIZE, nullptr, 0, &dwBytes, nullptr);
-
- return bOK == TRUE;
- }
-
- return false;
-}
-
-bool
-CreateDirectories(const wchar_t* Dir)
-{
- // This may be suboptimal, in that it appears to try and create directories
- // from the root on up instead of from some directory which is known to
- // be present
- //
- // We should implement a smarter version at some point since this can be
- // pretty expensive in aggregate
-
- return std::filesystem::create_directories(Dir);
-}
-
-// Erase all files and directories in a given directory, leaving an empty directory
-// behind
-
-static bool
-WipeDirectory(const wchar_t* DirPath)
-{
- ExtendableWideStringBuilder<128> Pattern;
- Pattern.Append(DirPath);
- Pattern.Append(L"\\*");
-
- WIN32_FIND_DATAW FindData;
- HANDLE hFind = FindFirstFileW(Pattern.c_str(), &FindData);
-
- if (hFind != nullptr)
- {
- do
- {
- bool IsRegular = true;
-
- if (FindData.cFileName[0] == L'.')
- {
- if (FindData.cFileName[1] == L'.')
- {
- if (FindData.cFileName[2] == L'\0')
- {
- IsRegular = false;
- }
- }
- else if (FindData.cFileName[1] == L'\0')
- {
- IsRegular = false;
- }
- }
-
- if (IsRegular)
- {
- ExtendableWideStringBuilder<128> Path;
- Path.Append(DirPath);
- Path.Append(L'\\');
- Path.Append(FindData.cFileName);
-
- // if (fd.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN)
- // deleteReparsePoint(path.c_str(), fd.dwReserved0);
-
- if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
- {
- if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN)
- {
- DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
- }
-
- if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS)
- {
- DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
- }
-
- bool Success = DeleteDirectories(Path.c_str());
-
- if (!Success)
- {
- if (FindData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
- {
- DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
- }
- }
- }
- else
- {
- DeleteFileW(Path.c_str());
- }
- }
- } while (FindNextFileW(hFind, &FindData) == TRUE);
-
- FindClose(hFind);
- }
-
- return true;
-}
-
-bool
-DeleteDirectories(const wchar_t* DirPath)
-{
- return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
-}
-
-bool
-CleanDirectory(const wchar_t* DirPath)
-{
- if (std::filesystem::exists(DirPath))
- {
- return WipeDirectory(DirPath);
- }
-
- return CreateDirectories(DirPath);
-}
-
-#endif // ZEN_PLATFORM_WINDOWS
-
-bool
-CreateDirectories(const std::filesystem::path& Dir)
-{
- while (!std::filesystem::is_directory(Dir))
- {
- if (Dir.has_parent_path())
- {
- CreateDirectories(Dir.parent_path());
- }
- std::error_code ErrorCode;
- std::filesystem::create_directory(Dir, ErrorCode);
- if (ErrorCode)
- {
- throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string()));
- }
- return true;
- }
- return false;
-}
-
-bool
-DeleteDirectories(const std::filesystem::path& Dir)
-{
-#if ZEN_PLATFORM_WINDOWS
- return DeleteDirectories(Dir.c_str());
-#else
- std::error_code ErrorCode;
- return std::filesystem::remove_all(Dir, ErrorCode);
-#endif
-}
-
-bool
-CleanDirectory(const std::filesystem::path& Dir)
-{
-#if ZEN_PLATFORM_WINDOWS
- return CleanDirectory(Dir.c_str());
-#else
- if (std::filesystem::exists(Dir))
- {
- bool Success = true;
-
- std::error_code ErrorCode;
- for (const auto& Item : std::filesystem::directory_iterator(Dir))
- {
- Success &= std::filesystem::remove_all(Item, ErrorCode);
- }
-
- return Success;
- }
-
- return CreateDirectories(Dir);
-#endif
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-bool
-SupportsBlockRefCounting(std::filesystem::path Path)
-{
-#if ZEN_PLATFORM_WINDOWS
- ATL::CHandle Handle(CreateFileW(Path.c_str(),
- GENERIC_READ,
- FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
- nullptr,
- OPEN_EXISTING,
- FILE_FLAG_BACKUP_SEMANTICS,
- nullptr));
-
- if (Handle == INVALID_HANDLE_VALUE)
- {
- Handle.Detach();
- return false;
- }
-
- ULONG FileSystemFlags = 0;
- if (!GetVolumeInformationByHandleW(Handle, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0))
- {
- return false;
- }
-
- if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING))
- {
- return false;
- }
-
- return true;
-#else
- ZEN_UNUSED(Path);
- return false;
-#endif // ZEN_PLATFORM_WINDOWS
-}
-
-bool
-CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
-{
-#if ZEN_PLATFORM_WINDOWS
- ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
- if (FromFile == INVALID_HANDLE_VALUE)
- {
- FromFile.Detach();
- return false;
- }
-
- ULONG FileSystemFlags;
- if (!GetVolumeInformationByHandleW(FromFile, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0))
- {
- return false;
- }
- if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING))
- {
- SetLastError(ERROR_NOT_CAPABLE);
- return false;
- }
-
- FILE_END_OF_FILE_INFO FileSize;
- if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile))
- {
- return false;
- }
-
- FILE_BASIC_INFO BasicInfo;
- if (!GetFileInformationByHandleEx(FromFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo))
- {
- return false;
- }
-
- DWORD dwBytesReturned = 0;
- FSCTL_GET_INTEGRITY_INFORMATION_BUFFER GetIntegrityInfoBuffer;
- if (!DeviceIoControl(FromFile,
- FSCTL_GET_INTEGRITY_INFORMATION,
- nullptr,
- 0,
- &GetIntegrityInfoBuffer,
- sizeof GetIntegrityInfoBuffer,
- &dwBytesReturned,
- nullptr))
- {
- return false;
- }
-
- SetFileAttributesW(ToPath.c_str(), FILE_ATTRIBUTE_NORMAL);
-
- ATL::CHandle TargetFile(CreateFileW(ToPath.c_str(),
- GENERIC_READ | GENERIC_WRITE | DELETE,
- /* no sharing */ FILE_SHARE_READ,
- nullptr,
- OPEN_ALWAYS,
- 0,
- /* hTemplateFile */ FromFile));
-
- if (TargetFile == INVALID_HANDLE_VALUE)
- {
- TargetFile.Detach();
- return false;
- }
-
- // Delete target file when handle is closed (we only reset this if the copy succeeds)
- FILE_DISPOSITION_INFO FileDisposition = {TRUE};
- if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
- {
- TargetFile.Close();
- DeleteFileW(ToPath.c_str());
- return false;
- }
-
- // Make file sparse so we don't end up allocating space when we change the file size
- if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dwBytesReturned, nullptr))
- {
- return false;
- }
-
- // Copy integrity checking information
- FSCTL_SET_INTEGRITY_INFORMATION_BUFFER IntegritySet = {GetIntegrityInfoBuffer.ChecksumAlgorithm,
- GetIntegrityInfoBuffer.Reserved,
- GetIntegrityInfoBuffer.Flags};
- if (!DeviceIoControl(TargetFile, FSCTL_SET_INTEGRITY_INFORMATION, &IntegritySet, sizeof IntegritySet, nullptr, 0, nullptr, nullptr))
- {
- return false;
- }
-
- // Resize file - note that the file is sparse at this point so no additional data will be written
- if (!SetFileInformationByHandle(TargetFile, FileEndOfFileInfo, &FileSize, sizeof FileSize))
- {
- return false;
- }
-
- constexpr auto RoundToClusterSize = [](LONG64 FileSize, ULONG ClusterSize) -> LONG64 {
- return (FileSize + ClusterSize - 1) / ClusterSize * ClusterSize;
- };
- static_assert(RoundToClusterSize(5678, 4 * 1024) == 8 * 1024);
-
- // Loop for cloning file contents. This is necessary as the API has a 32-bit size
- // limit for some reason
-
- const LONG64 SplitThreshold = (1LL << 32) - GetIntegrityInfoBuffer.ClusterSizeInBytes;
-
- DUPLICATE_EXTENTS_DATA DuplicateExtentsData{.FileHandle = FromFile};
-
- for (LONG64 CurrentByteOffset = 0,
- RemainingBytes = RoundToClusterSize(FileSize.EndOfFile.QuadPart, GetIntegrityInfoBuffer.ClusterSizeInBytes);
- RemainingBytes > 0;
- CurrentByteOffset += SplitThreshold, RemainingBytes -= SplitThreshold)
- {
- DuplicateExtentsData.SourceFileOffset.QuadPart = CurrentByteOffset;
- DuplicateExtentsData.TargetFileOffset.QuadPart = CurrentByteOffset;
- DuplicateExtentsData.ByteCount.QuadPart = std::min(SplitThreshold, RemainingBytes);
-
- if (!DeviceIoControl(TargetFile,
- FSCTL_DUPLICATE_EXTENTS_TO_FILE,
- &DuplicateExtentsData,
- sizeof DuplicateExtentsData,
- nullptr,
- 0,
- &dwBytesReturned,
- nullptr))
- {
- return false;
- }
- }
-
- // Make the file not sparse again now that we have populated the contents
- if (!(BasicInfo.FileAttributes & FILE_ATTRIBUTE_SPARSE_FILE))
- {
- FILE_SET_SPARSE_BUFFER SetSparse = {FALSE};
-
- if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, &SetSparse, sizeof SetSparse, nullptr, 0, &dwBytesReturned, nullptr))
- {
- return false;
- }
- }
-
- // Update timestamps (but don't lie about the creation time)
- BasicInfo.CreationTime.QuadPart = 0;
- if (!SetFileInformationByHandle(TargetFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo))
- {
- return false;
- }
-
- if (!FlushFileBuffers(TargetFile))
- {
- return false;
- }
-
- // Finally now everything is done - make sure the file is not deleted on close!
-
- FileDisposition = {FALSE};
-
- const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition));
-
- return AllOk;
-#elif ZEN_PLATFORM_LINUX
-# if 0
- struct ScopedFd
- {
- ~ScopedFd() { close(Fd); }
- int Fd;
- };
-
- // The 'from' file
- int FromFd = open(FromPath.c_str(), O_RDONLY|O_CLOEXEC);
- if (FromFd < 0)
- {
- return false;
- }
- ScopedFd $From = { FromFd };
-
- // The 'to' file
- int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0666);
- if (ToFd < 0)
- {
- return false;
- }
- fchmod(ToFd, 0666);
- ScopedFd $To = { FromFd };
-
- ioctl(ToFd, FICLONE, FromFd);
-
- return false;
-# endif // 0
- ZEN_UNUSED(FromPath, ToPath);
- ZEN_ERROR("CloneFile() is not implemented on this platform");
- return false;
-#elif ZEN_PLATFORM_MAC
- /* clonefile() syscall if APFS */
- ZEN_UNUSED(FromPath, ToPath);
- ZEN_ERROR("CloneFile() is not implemented on this platform");
- return false;
-#endif // ZEN_PLATFORM_WINDOWS
-}
-
-bool
-CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
-{
- bool Success = false;
-
- if (Options.EnableClone)
- {
- Success = CloneFile(FromPath.native(), ToPath.native());
-
- if (Success)
- {
- return true;
- }
- }
-
- if (Options.MustClone)
- {
- return false;
- }
-
-#if ZEN_PLATFORM_WINDOWS
- BOOL CancelFlag = FALSE;
- Success = !!::CopyFileExW(FromPath.c_str(),
- ToPath.c_str(),
- /* lpProgressRoutine */ nullptr,
- /* lpData */ nullptr,
- &CancelFlag,
- /* dwCopyFlags */ 0);
-#else
- struct ScopedFd
- {
- ~ScopedFd() { close(Fd); }
- int Fd;
- };
-
- // From file
- int FromFd = open(FromPath.c_str(), O_RDONLY | O_CLOEXEC);
- if (FromFd < 0)
- {
- ThrowLastError(fmt::format("failed to open file {}", FromPath));
- }
- ScopedFd $From = {FromFd};
-
- // To file
- int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666);
- if (ToFd < 0)
- {
- ThrowLastError(fmt::format("failed to create file {}", ToPath));
- }
- fchmod(ToFd, 0666);
- ScopedFd $To = {ToFd};
-
- // Copy impl
- static const size_t BufferSize = 64 << 10;
- void* Buffer = malloc(BufferSize);
- while (true)
- {
- int BytesRead = read(FromFd, Buffer, BufferSize);
- if (BytesRead <= 0)
- {
- Success = (BytesRead == 0);
- break;
- }
-
- if (write(ToFd, Buffer, BytesRead) != BufferSize)
- {
- Success = false;
- break;
- }
- }
- free(Buffer);
-#endif // ZEN_PLATFORM_WINDOWS
-
- if (!Success)
- {
- ThrowLastError("file copy failed"sv);
- }
-
- return true;
-}
-
-void
-WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount)
-{
-#if ZEN_PLATFORM_WINDOWS
- CAtlFile Outfile;
- HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
- if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
- {
- CreateDirectories(Path.parent_path());
-
- hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
- }
-
- if (FAILED(hRes))
- {
- ThrowSystemException(hRes, fmt::format("File open failed for '{}'", Path).c_str());
- }
-
-#else
- int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC;
- int Fd = open(Path.c_str(), OpenFlags, 0666);
- if (Fd < 0)
- {
- zen::CreateDirectories(Path.parent_path());
- Fd = open(Path.c_str(), OpenFlags, 0666);
- }
-
- if (Fd < 0)
- {
- ThrowLastError(fmt::format("File open failed for '{}'", Path));
- }
-
- fchmod(Fd, 0666);
-#endif
-
- // TODO: this should be block-enlightened
-
- for (size_t i = 0; i < BufferCount; ++i)
- {
- uint64_t WriteSize = Data[i]->Size();
- const void* DataPtr = Data[i]->Data();
-
- while (WriteSize)
- {
- const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024);
-
-#if ZEN_PLATFORM_WINDOWS
- hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize));
- if (FAILED(hRes))
- {
- ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str());
- }
-#else
- if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize))
- {
- ThrowLastError(fmt::format("File write failed for '{}'", Path));
- }
-#endif // ZEN_PLATFORM_WINDOWS
-
- WriteSize -= ChunkSize;
- DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize;
- }
- }
-
-#if !ZEN_PLATFORM_WINDOWS
- close(Fd);
-#endif
-}
-
-void
-WriteFile(std::filesystem::path Path, IoBuffer Data)
-{
- const IoBuffer* const DataPtr = &Data;
-
- WriteFile(Path, &DataPtr, 1);
-}
-
-IoBuffer
-FileContents::Flatten()
-{
- if (Data.size() == 1)
- {
- return Data[0];
- }
- else if (Data.empty())
- {
- return {};
- }
- else
- {
- ZEN_NOT_IMPLEMENTED();
- }
-}
-
-FileContents
-ReadStdIn()
-{
- BinaryWriter Writer;
-
- do
- {
- uint8_t ReadBuffer[1024];
-
- size_t BytesRead = fread(ReadBuffer, 1, sizeof ReadBuffer, stdin);
- Writer.Write(ReadBuffer, BytesRead);
- } while (!feof(stdin));
-
- FileContents Contents;
- Contents.Data.emplace_back(IoBuffer(IoBuffer::Clone, Writer.GetData(), Writer.GetSize()));
-
- return Contents;
-}
-
-FileContents
-ReadFile(std::filesystem::path Path)
-{
- uint64_t FileSizeBytes;
- void* Handle;
-
-#if ZEN_PLATFORM_WINDOWS
- ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
- if (FromFile == INVALID_HANDLE_VALUE)
- {
- FromFile.Detach();
- return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
- }
-
- FILE_END_OF_FILE_INFO FileSize;
- if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile))
- {
- return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
- }
-
- FileSizeBytes = FileSize.EndOfFile.QuadPart;
- Handle = FromFile.Detach();
-#else
- int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC);
- if (Fd < 0)
- {
- FileContents Ret;
- Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category());
- return Ret;
- }
-
- static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
- struct stat Stat;
- fstat(Fd, &Stat);
-
- FileSizeBytes = Stat.st_size;
- Handle = (void*)uintptr_t(Fd);
-#endif
-
- FileContents Contents;
- Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes));
- return Contents;
-}
-
-bool
-ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
-{
-#if ZEN_PLATFORM_WINDOWS
- ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
- if (FromFile == INVALID_HANDLE_VALUE)
- {
- FromFile.Detach();
- return false;
- }
-
- std::vector<uint8_t> ReadBuffer(ChunkSize);
-
- for (;;)
- {
- DWORD dwBytesRead = 0;
- BOOL Success = ::ReadFile(FromFile, ReadBuffer.data(), (DWORD)ReadBuffer.size(), &dwBytesRead, nullptr);
-
- if (!Success)
- {
- throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed");
- }
-
- if (dwBytesRead == 0)
- break;
-
- ProcessFunc(ReadBuffer.data(), dwBytesRead);
- }
-#else
- int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC);
- if (Fd < 0)
- {
- return false;
- }
-
- bool Success = true;
-
- void* Buffer = malloc(ChunkSize);
- while (true)
- {
- int BytesRead = read(Fd, Buffer, ChunkSize);
- if (BytesRead < 0)
- {
- Success = false;
- break;
- }
-
- if (BytesRead == 0)
- {
- break;
- }
-
- ProcessFunc(Buffer, BytesRead);
- }
-
- free(Buffer);
- close(Fd);
-
- if (!Success)
- {
- ThrowLastError("file scan failed");
- }
-#endif // ZEN_PLATFORM_WINDOWS
-
- return true;
-}
-
-void
-PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out)
-{
-#if ZEN_PLATFORM_WINDOWS
- WideToUtf8(Path.native().c_str(), Out);
-#else
- Out << Path.c_str();
-#endif
-}
-
-std::string
-PathToUtf8(const std::filesystem::path& Path)
-{
-#if ZEN_PLATFORM_WINDOWS
- return WideToUtf8(Path.native().c_str());
-#else
- return Path.string();
-#endif
-}
-
-DiskSpace
-DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error)
-{
- using namespace std::filesystem;
-
- space_info SpaceInfo = space(Directory, Error);
- if (Error)
- {
- return {};
- }
-
- return {
- .Free = uint64_t(SpaceInfo.available),
- .Total = uint64_t(SpaceInfo.capacity),
- };
-}
-
-void
-FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor)
-{
-#if ZEN_PLATFORM_WINDOWS
- uint64_t FileInfoBuffer[8 * 1024];
-
- FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo;
- bool Continue = true;
-
- CAtlFile RootDirHandle;
- HRESULT hRes =
- RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS);
-
- if (FAILED(hRes))
- {
- ThrowSystemException(hRes, "Failed to open handle to volume root");
- }
-
- while (Continue)
- {
- BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer);
- FibClass = FileIdBothDirectoryInfo; // Set up for next iteration
-
- uint64_t EntryOffset = 0;
-
- if (!Success)
- {
- DWORD LastError = GetLastError();
-
- if (LastError == ERROR_NO_MORE_FILES)
- {
- break;
- }
-
- throw std::system_error(std::error_code(LastError, std::system_category()), "file system traversal error");
- }
-
- for (;;)
- {
- const FILE_ID_BOTH_DIR_INFO* DirInfo =
- reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset);
-
- std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t));
-
- if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY)
- {
- if (FileName == L"."sv || FileName == L".."sv)
- {
- // Not very interesting
- }
- else
- {
- const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName);
-
- if (ShouldDescend)
- {
- // Note that this recursion combined with the buffer could
- // blow the stack, we should consider a different strategy
-
- std::filesystem::path FullPath = RootDir / FileName;
-
- TraverseFileSystem(FullPath, Visitor);
- }
- }
- }
- else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE)
- {
- ZEN_WARN("encountered device node during file system traversal: '{}' found in '{}'", WideToUtf8(FileName), RootDir);
- }
- else
- {
- Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
- }
-
- const uint64_t NextOffset = DirInfo->NextEntryOffset;
-
- if (NextOffset == 0)
- {
- break;
- }
-
- EntryOffset += NextOffset;
- }
- }
-#else
- /* Could also implement this using Linux's getdents() syscall */
-
- DIR* Dir = opendir(RootDir.c_str());
- if (Dir == nullptr)
- {
- ThrowLastError(fmt::format("Failed to open directory for traversal: {}", RootDir.c_str()));
- }
-
- for (struct dirent* Entry; (Entry = readdir(Dir));)
- {
- const char* FileName = Entry->d_name;
-
- struct stat Stat;
- std::filesystem::path FullPath = RootDir / FileName;
- stat(FullPath.c_str(), &Stat);
-
- if (S_ISDIR(Stat.st_mode))
- {
- if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0)
- {
- /* nop */
- }
- else if (Visitor.VisitDirectory(RootDir, FileName))
- {
- TraverseFileSystem(FullPath, Visitor);
- }
- }
- else if (S_ISREG(Stat.st_mode))
- {
- Visitor.VisitFile(RootDir, FileName, Stat.st_size);
- }
- else
- {
- ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}",
- Stat.st_mode,
- FileName,
- RootDir.c_str());
- }
- }
-
- closedir(Dir);
-#endif // ZEN_PLATFORM_WINDOWS
-}
-
-std::filesystem::path
-PathFromHandle(void* NativeHandle)
-{
-#if ZEN_PLATFORM_WINDOWS
- if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE)
- {
- return std::filesystem::path();
- }
-
- auto GetFinalPathNameByHandleWRetry = [](HANDLE hFile, LPWSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags) -> DWORD {
- while (true)
- {
- DWORD Res = GetFinalPathNameByHandleW(hFile, lpszFilePath, cchFilePath, dwFlags);
- if (Res == 0)
- {
- DWORD LastError = zen::GetLastError();
- // Under heavy concurrent loads we might get access denied on a file handle while trying to get path name.
- // Retry if that is the case.
- if (LastError != ERROR_ACCESS_DENIED)
- {
- ThrowSystemError(LastError, fmt::format("failed to get path from file handle {}", hFile));
- }
- // Retry
- continue;
- }
- ZEN_ASSERT(Res != 1); // We don't accept empty path names
- return Res;
- }
- };
-
- static const DWORD PathDataSize = 512;
- wchar_t PathData[PathDataSize];
- DWORD RequiredLengthIncludingNul = GetFinalPathNameByHandleWRetry(NativeHandle, PathData, PathDataSize, FILE_NAME_OPENED);
- if (RequiredLengthIncludingNul == 0)
- {
- ThrowLastError(fmt::format("failed to get path from file handle {}", NativeHandle));
- }
-
- if (RequiredLengthIncludingNul < PathDataSize)
- {
- std::wstring FullPath(PathData, gsl::narrow<size_t>(RequiredLengthIncludingNul));
- return FullPath;
- }
-
- std::wstring FullPath;
- FullPath.resize(RequiredLengthIncludingNul - 1);
-
- const DWORD FinalLength = GetFinalPathNameByHandleWRetry(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED);
- ZEN_UNUSED(FinalLength);
- return FullPath;
-
-#elif ZEN_PLATFORM_LINUX
- char Link[PATH_MAX];
- char Path[64];
-
- sprintf(Path, "/proc/self/fd/%d", int(uintptr_t(NativeHandle)));
- ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1);
- if (BytesRead <= 0)
- {
- return std::filesystem::path();
- }
-
- Link[BytesRead] = '\0';
- return Link;
-#elif ZEN_PLATFORM_MAC
- int Fd = int(uintptr_t(NativeHandle));
- char Path[MAXPATHLEN];
- if (fcntl(Fd, F_GETPATH, Path) < 0)
- {
- return std::filesystem::path();
- }
-
- return Path;
-#endif // ZEN_PLATFORM_WINDOWS
-}
-
-std::filesystem::path
-GetRunningExecutablePath()
-{
-#if ZEN_PLATFORM_WINDOWS
- TCHAR ExePath[MAX_PATH];
- DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath));
-
- return {std::wstring_view(ExePath, PathLength)};
-#elif ZEN_PLATFORM_LINUX
- char Link[256];
- ssize_t BytesRead = readlink("/proc/self/exe", Link, sizeof(Link) - 1);
- if (BytesRead < 0)
- return {};
-
- Link[BytesRead] = '\0';
- return Link;
-#elif ZEN_PLATFORM_MAC
- char Buffer[PROC_PIDPATHINFO_MAXSIZE];
-
- int SelfPid = GetCurrentProcessId();
- if (proc_pidpath(SelfPid, Buffer, sizeof(Buffer)) <= 0)
- return {};
-
- return Buffer;
-#endif // ZEN_PLATFORM_WINDOWS
-}
-
-void
-MaximizeOpenFileCount()
-{
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
- struct rlimit Limit;
- int Error = getrlimit(RLIMIT_NOFILE, &Limit);
- if (Error)
- {
- ZEN_WARN("failed getting rlimit RLIMIT_NOFILE, reason '{}'", zen::MakeErrorCode(Error).message());
- }
- else
- {
- struct rlimit NewLimit = Limit;
- NewLimit.rlim_cur = NewLimit.rlim_max;
- ZEN_INFO("changing RLIMIT_NOFILE from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}",
- Limit.rlim_cur,
- Limit.rlim_max,
- NewLimit.rlim_cur,
- NewLimit.rlim_max);
-
- Error = setrlimit(RLIMIT_NOFILE, &NewLimit);
- if (Error != 0)
- {
- ZEN_WARN("failed to set RLIMIT_NOFILE limits from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}, reason '{}'",
- Limit.rlim_cur,
- Limit.rlim_max,
- NewLimit.rlim_cur,
- NewLimit.rlim_max,
- zen::MakeErrorCode(Error).message());
- }
- }
-#endif
-}
-
-void
-GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
-{
- FileSystemTraversal Traversal;
- struct Visitor : public FileSystemTraversal::TreeVisitor
- {
- Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
-
- virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
- [[maybe_unused]] const path_view& File,
- [[maybe_unused]] uint64_t FileSize) override
- {
- if (Flags & DirectoryContent::IncludeFilesFlag)
- {
- Content.Files.push_back(Parent / File);
- }
- }
-
- virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
- {
- if (Flags & DirectoryContent::IncludeDirsFlag)
- {
- Content.Directories.push_back(Parent / DirectoryName);
- }
- return (Flags & DirectoryContent::RecursiveFlag) != 0;
- }
-
- const uint8_t Flags;
- DirectoryContent& Content;
- } Visit(Flags, OutContent);
-
- Traversal.TraverseFileSystem(RootDir, Visit);
-}
-
-std::string
-GetEnvVariable(std::string_view VariableName)
-{
- ZEN_ASSERT(!VariableName.empty());
-#if ZEN_PLATFORM_WINDOWS
-
- CHAR EnvVariableBuffer[1023 + 1];
- DWORD RESULT = GetEnvironmentVariableA(std::string(VariableName).c_str(), EnvVariableBuffer, sizeof(EnvVariableBuffer));
- if (RESULT > 0 && RESULT < sizeof(EnvVariableBuffer))
- {
- return std::string(EnvVariableBuffer);
- }
-#endif
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
- char* EnvVariable = getenv(std::string(VariableName).c_str());
- if (EnvVariable)
- {
- return std::string(EnvVariable);
- }
-#endif
- return "";
-}
-
-//////////////////////////////////////////////////////////////////////////
-//
-// Testing related code follows...
-//
-
-#if ZEN_WITH_TESTS
-
-void
-filesystem_forcelink()
-{
-}
-
-TEST_CASE("filesystem")
-{
- using namespace std::filesystem;
-
- // GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in
- path BinPath = GetRunningExecutablePath();
- const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver";
- CHECK(ExpectedExe);
- CHECK(is_regular_file(BinPath));
-
- // PathFromHandle
- void* Handle;
-# if ZEN_PLATFORM_WINDOWS
- Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
- CHECK(Handle != INVALID_HANDLE_VALUE);
-# else
- int Fd = open(BinPath.c_str(), O_RDONLY | O_CLOEXEC);
- CHECK(Fd >= 0);
- Handle = (void*)uintptr_t(Fd);
-# endif
-
- auto FromHandle = PathFromHandle((void*)uintptr_t(Handle));
- CHECK(equivalent(FromHandle, BinPath));
-
-# if ZEN_PLATFORM_WINDOWS
- CloseHandle(Handle);
-# else
- close(int(uintptr_t(Handle)));
-# endif
-
- // Traversal
- struct : public FileSystemTraversal::TreeVisitor
- {
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
- {
- bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
- }
-
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
-
- bool bFoundExpected = false;
- std::filesystem::path Expected;
- } Visitor;
- Visitor.Expected = BinPath;
-
- FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor);
- CHECK(Visitor.bFoundExpected);
-
- // Scan/read file
- FileContents BinRead = ReadFile(BinPath);
- std::vector<uint8_t> BinScan;
- ScanFile(BinPath, 16 << 10, [&](const void* Data, size_t Size) {
- const auto* Ptr = (uint8_t*)Data;
- BinScan.insert(BinScan.end(), Ptr, Ptr + Size);
- });
- CHECK_EQ(BinRead.Data.size(), 1);
- CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize());
-}
-
-TEST_CASE("WriteFile")
-{
- std::filesystem::path TempFile = GetRunningExecutablePath().parent_path();
- TempFile /= "write_file_test";
-
- uint64_t Magics[] = {
- 0x0'a9e'a9e'a9e'a9e'a9e,
- 0x0'493'493'493'493'493,
- };
-
- struct
- {
- const void* Data;
- size_t Size;
- } MagicTests[] = {
- {
- Magics,
- sizeof(Magics),
- },
- {
- Magics + 1,
- sizeof(Magics[0]),
- },
- };
- for (auto& MagicTest : MagicTests)
- {
- WriteFile(TempFile, IoBuffer(IoBuffer::Wrap, MagicTest.Data, MagicTest.Size));
-
- FileContents MagicsReadback = ReadFile(TempFile);
- CHECK_EQ(MagicsReadback.Data.size(), 1);
- CHECK_EQ(MagicsReadback.Data[0].GetSize(), MagicTest.Size);
- CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0);
- }
-
- std::filesystem::remove(TempFile);
-}
-
-TEST_CASE("DiskSpaceInfo")
-{
- std::filesystem::path BinPath = GetRunningExecutablePath();
-
- DiskSpace Space = {};
-
- std::error_code Error;
- Space = DiskSpaceInfo(BinPath, Error);
- CHECK(!Error);
-
- bool Okay = DiskSpaceInfo(BinPath, Space);
- CHECK(Okay);
-
- CHECK(int64_t(Space.Total) > 0);
- CHECK(int64_t(Space.Free) > 0); // Hopefully there's at least one byte free
-}
-
-TEST_CASE("PathBuilder")
-{
-# if ZEN_PLATFORM_WINDOWS
- const char* foo_bar = "/foo\\bar";
-# else
- const char* foo_bar = "/foo/bar";
-# endif
-
- ExtendablePathBuilder<32> Path;
- for (const char* Prefix : {"/foo", "/foo/"})
- {
- Path.Reset();
- Path.Append(Prefix);
- Path /= "bar";
- CHECK(Path.ToPath() == foo_bar);
- }
-
- using fspath = std::filesystem::path;
-
- Path.Reset();
- Path.Append(fspath("/foo/"));
- Path /= (fspath("bar"));
- CHECK(Path.ToPath() == foo_bar);
-
-# if ZEN_PLATFORM_WINDOWS
- Path.Reset();
- Path.Append(fspath(L"/\u0119oo/"));
- Path /= L"bar";
- printf("%ls\n", Path.ToPath().c_str());
- CHECK(Path.ToView() == L"/\u0119oo/bar");
- CHECK(Path.ToPath() == L"\\\u0119oo\\bar");
-# endif
-}
-
-#endif
-
-} // namespace zen