diff options
| author | Stefan Boberg <[email protected]> | 2023-05-02 10:01:47 +0200 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-05-02 10:01:47 +0200 |
| commit | 075d17f8ada47e990fe94606c3d21df409223465 (patch) | |
| tree | e50549b766a2f3c354798a54ff73404217b4c9af /zencore/filesystem.cpp | |
| parent | fix: bundle shouldn't append content zip to zen (diff) | |
| download | zen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz zen-075d17f8ada47e990fe94606c3d21df409223465.zip | |
moved source directories into `/src` (#264)
* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees
Diffstat (limited to 'zencore/filesystem.cpp')
| -rw-r--r-- | zencore/filesystem.cpp | 1304 |
1 files changed, 0 insertions, 1304 deletions
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp deleted file mode 100644 index a17773024..000000000 --- a/zencore/filesystem.cpp +++ /dev/null @@ -1,1304 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/filesystem.h> - -#include <zencore/except.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/logging.h> -#include <zencore/stream.h> -#include <zencore/string.h> -#include <zencore/testing.h> - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#endif - -#if ZEN_PLATFORM_WINDOWS -# include <atlbase.h> -# include <atlfile.h> -# include <winioctl.h> -# include <winnt.h> -#endif - -#if ZEN_PLATFORM_LINUX -# include <dirent.h> -# include <fcntl.h> -# include <sys/resource.h> -# include <sys/stat.h> -# include <unistd.h> -#endif - -#if ZEN_PLATFORM_MAC -# include <dirent.h> -# include <fcntl.h> -# include <libproc.h> -# include <sys/resource.h> -# include <sys/stat.h> -# include <sys/syslimits.h> -# include <unistd.h> -#endif - -#include <filesystem> -#include <gsl/gsl-lite.hpp> - -namespace zen { - -using namespace std::literals; - -#if ZEN_PLATFORM_WINDOWS - -static bool -DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) -{ - CHandle hDir(CreateFileW(Path, - GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, - nullptr)); - - if (hDir != INVALID_HANDLE_VALUE) - { - REPARSE_GUID_DATA_BUFFER Rgdb = {}; - Rgdb.ReparseTag = dwReparseTag; - - DWORD dwBytes; - const BOOL bOK = - DeviceIoControl(hDir, FSCTL_DELETE_REPARSE_POINT, &Rgdb, REPARSE_GUID_DATA_BUFFER_HEADER_SIZE, nullptr, 0, &dwBytes, nullptr); - - return bOK == TRUE; - } - - return false; -} - -bool -CreateDirectories(const wchar_t* Dir) -{ - // This may be suboptimal, in that it appears to try and create directories - // from the root on up instead of from some directory which is known to - // be present - // - // We should implement a smarter version at some point since this can be - // pretty expensive in aggregate - - return std::filesystem::create_directories(Dir); -} - -// Erase all files and directories in a given directory, leaving an empty directory -// behind - -static bool -WipeDirectory(const wchar_t* DirPath) -{ - ExtendableWideStringBuilder<128> Pattern; - Pattern.Append(DirPath); - Pattern.Append(L"\\*"); - - WIN32_FIND_DATAW FindData; - HANDLE hFind = FindFirstFileW(Pattern.c_str(), &FindData); - - if (hFind != nullptr) - { - do - { - bool IsRegular = true; - - if (FindData.cFileName[0] == L'.') - { - if (FindData.cFileName[1] == L'.') - { - if (FindData.cFileName[2] == L'\0') - { - IsRegular = false; - } - } - else if (FindData.cFileName[1] == L'\0') - { - IsRegular = false; - } - } - - if (IsRegular) - { - ExtendableWideStringBuilder<128> Path; - Path.Append(DirPath); - Path.Append(L'\\'); - Path.Append(FindData.cFileName); - - // if (fd.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN) - // deleteReparsePoint(path.c_str(), fd.dwReserved0); - - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - { - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN) - { - DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); - } - - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) - { - DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); - } - - bool Success = DeleteDirectories(Path.c_str()); - - if (!Success) - { - if (FindData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) - { - DeleteReparsePoint(Path.c_str(), FindData.dwReserved0); - } - } - } - else - { - DeleteFileW(Path.c_str()); - } - } - } while (FindNextFileW(hFind, &FindData) == TRUE); - - FindClose(hFind); - } - - return true; -} - -bool -DeleteDirectories(const wchar_t* DirPath) -{ - return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE; -} - -bool -CleanDirectory(const wchar_t* DirPath) -{ - if (std::filesystem::exists(DirPath)) - { - return WipeDirectory(DirPath); - } - - return CreateDirectories(DirPath); -} - -#endif // ZEN_PLATFORM_WINDOWS - -bool -CreateDirectories(const std::filesystem::path& Dir) -{ - while (!std::filesystem::is_directory(Dir)) - { - if (Dir.has_parent_path()) - { - CreateDirectories(Dir.parent_path()); - } - std::error_code ErrorCode; - std::filesystem::create_directory(Dir, ErrorCode); - if (ErrorCode) - { - throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string())); - } - return true; - } - return false; -} - -bool -DeleteDirectories(const std::filesystem::path& Dir) -{ -#if ZEN_PLATFORM_WINDOWS - return DeleteDirectories(Dir.c_str()); -#else - std::error_code ErrorCode; - return std::filesystem::remove_all(Dir, ErrorCode); -#endif -} - -bool -CleanDirectory(const std::filesystem::path& Dir) -{ -#if ZEN_PLATFORM_WINDOWS - return CleanDirectory(Dir.c_str()); -#else - if (std::filesystem::exists(Dir)) - { - bool Success = true; - - std::error_code ErrorCode; - for (const auto& Item : std::filesystem::directory_iterator(Dir)) - { - Success &= std::filesystem::remove_all(Item, ErrorCode); - } - - return Success; - } - - return CreateDirectories(Dir); -#endif -} - -////////////////////////////////////////////////////////////////////////// - -bool -SupportsBlockRefCounting(std::filesystem::path Path) -{ -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle Handle(CreateFileW(Path.c_str(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, - nullptr)); - - if (Handle == INVALID_HANDLE_VALUE) - { - Handle.Detach(); - return false; - } - - ULONG FileSystemFlags = 0; - if (!GetVolumeInformationByHandleW(Handle, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0)) - { - return false; - } - - if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING)) - { - return false; - } - - return true; -#else - ZEN_UNUSED(Path); - return false; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) -{ -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); - if (FromFile == INVALID_HANDLE_VALUE) - { - FromFile.Detach(); - return false; - } - - ULONG FileSystemFlags; - if (!GetVolumeInformationByHandleW(FromFile, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0)) - { - return false; - } - if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING)) - { - SetLastError(ERROR_NOT_CAPABLE); - return false; - } - - FILE_END_OF_FILE_INFO FileSize; - if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile)) - { - return false; - } - - FILE_BASIC_INFO BasicInfo; - if (!GetFileInformationByHandleEx(FromFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo)) - { - return false; - } - - DWORD dwBytesReturned = 0; - FSCTL_GET_INTEGRITY_INFORMATION_BUFFER GetIntegrityInfoBuffer; - if (!DeviceIoControl(FromFile, - FSCTL_GET_INTEGRITY_INFORMATION, - nullptr, - 0, - &GetIntegrityInfoBuffer, - sizeof GetIntegrityInfoBuffer, - &dwBytesReturned, - nullptr)) - { - return false; - } - - SetFileAttributesW(ToPath.c_str(), FILE_ATTRIBUTE_NORMAL); - - ATL::CHandle TargetFile(CreateFileW(ToPath.c_str(), - GENERIC_READ | GENERIC_WRITE | DELETE, - /* no sharing */ FILE_SHARE_READ, - nullptr, - OPEN_ALWAYS, - 0, - /* hTemplateFile */ FromFile)); - - if (TargetFile == INVALID_HANDLE_VALUE) - { - TargetFile.Detach(); - return false; - } - - // Delete target file when handle is closed (we only reset this if the copy succeeds) - FILE_DISPOSITION_INFO FileDisposition = {TRUE}; - if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)) - { - TargetFile.Close(); - DeleteFileW(ToPath.c_str()); - return false; - } - - // Make file sparse so we don't end up allocating space when we change the file size - if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dwBytesReturned, nullptr)) - { - return false; - } - - // Copy integrity checking information - FSCTL_SET_INTEGRITY_INFORMATION_BUFFER IntegritySet = {GetIntegrityInfoBuffer.ChecksumAlgorithm, - GetIntegrityInfoBuffer.Reserved, - GetIntegrityInfoBuffer.Flags}; - if (!DeviceIoControl(TargetFile, FSCTL_SET_INTEGRITY_INFORMATION, &IntegritySet, sizeof IntegritySet, nullptr, 0, nullptr, nullptr)) - { - return false; - } - - // Resize file - note that the file is sparse at this point so no additional data will be written - if (!SetFileInformationByHandle(TargetFile, FileEndOfFileInfo, &FileSize, sizeof FileSize)) - { - return false; - } - - constexpr auto RoundToClusterSize = [](LONG64 FileSize, ULONG ClusterSize) -> LONG64 { - return (FileSize + ClusterSize - 1) / ClusterSize * ClusterSize; - }; - static_assert(RoundToClusterSize(5678, 4 * 1024) == 8 * 1024); - - // Loop for cloning file contents. This is necessary as the API has a 32-bit size - // limit for some reason - - const LONG64 SplitThreshold = (1LL << 32) - GetIntegrityInfoBuffer.ClusterSizeInBytes; - - DUPLICATE_EXTENTS_DATA DuplicateExtentsData{.FileHandle = FromFile}; - - for (LONG64 CurrentByteOffset = 0, - RemainingBytes = RoundToClusterSize(FileSize.EndOfFile.QuadPart, GetIntegrityInfoBuffer.ClusterSizeInBytes); - RemainingBytes > 0; - CurrentByteOffset += SplitThreshold, RemainingBytes -= SplitThreshold) - { - DuplicateExtentsData.SourceFileOffset.QuadPart = CurrentByteOffset; - DuplicateExtentsData.TargetFileOffset.QuadPart = CurrentByteOffset; - DuplicateExtentsData.ByteCount.QuadPart = std::min(SplitThreshold, RemainingBytes); - - if (!DeviceIoControl(TargetFile, - FSCTL_DUPLICATE_EXTENTS_TO_FILE, - &DuplicateExtentsData, - sizeof DuplicateExtentsData, - nullptr, - 0, - &dwBytesReturned, - nullptr)) - { - return false; - } - } - - // Make the file not sparse again now that we have populated the contents - if (!(BasicInfo.FileAttributes & FILE_ATTRIBUTE_SPARSE_FILE)) - { - FILE_SET_SPARSE_BUFFER SetSparse = {FALSE}; - - if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, &SetSparse, sizeof SetSparse, nullptr, 0, &dwBytesReturned, nullptr)) - { - return false; - } - } - - // Update timestamps (but don't lie about the creation time) - BasicInfo.CreationTime.QuadPart = 0; - if (!SetFileInformationByHandle(TargetFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo)) - { - return false; - } - - if (!FlushFileBuffers(TargetFile)) - { - return false; - } - - // Finally now everything is done - make sure the file is not deleted on close! - - FileDisposition = {FALSE}; - - const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); - - return AllOk; -#elif ZEN_PLATFORM_LINUX -# if 0 - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - - // The 'from' file - int FromFd = open(FromPath.c_str(), O_RDONLY|O_CLOEXEC); - if (FromFd < 0) - { - return false; - } - ScopedFd $From = { FromFd }; - - // The 'to' file - int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0666); - if (ToFd < 0) - { - return false; - } - fchmod(ToFd, 0666); - ScopedFd $To = { FromFd }; - - ioctl(ToFd, FICLONE, FromFd); - - return false; -# endif // 0 - ZEN_UNUSED(FromPath, ToPath); - ZEN_ERROR("CloneFile() is not implemented on this platform"); - return false; -#elif ZEN_PLATFORM_MAC - /* clonefile() syscall if APFS */ - ZEN_UNUSED(FromPath, ToPath); - ZEN_ERROR("CloneFile() is not implemented on this platform"); - return false; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) -{ - bool Success = false; - - if (Options.EnableClone) - { - Success = CloneFile(FromPath.native(), ToPath.native()); - - if (Success) - { - return true; - } - } - - if (Options.MustClone) - { - return false; - } - -#if ZEN_PLATFORM_WINDOWS - BOOL CancelFlag = FALSE; - Success = !!::CopyFileExW(FromPath.c_str(), - ToPath.c_str(), - /* lpProgressRoutine */ nullptr, - /* lpData */ nullptr, - &CancelFlag, - /* dwCopyFlags */ 0); -#else - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - - // From file - int FromFd = open(FromPath.c_str(), O_RDONLY | O_CLOEXEC); - if (FromFd < 0) - { - ThrowLastError(fmt::format("failed to open file {}", FromPath)); - } - ScopedFd $From = {FromFd}; - - // To file - int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666); - if (ToFd < 0) - { - ThrowLastError(fmt::format("failed to create file {}", ToPath)); - } - fchmod(ToFd, 0666); - ScopedFd $To = {ToFd}; - - // Copy impl - static const size_t BufferSize = 64 << 10; - void* Buffer = malloc(BufferSize); - while (true) - { - int BytesRead = read(FromFd, Buffer, BufferSize); - if (BytesRead <= 0) - { - Success = (BytesRead == 0); - break; - } - - if (write(ToFd, Buffer, BytesRead) != BufferSize) - { - Success = false; - break; - } - } - free(Buffer); -#endif // ZEN_PLATFORM_WINDOWS - - if (!Success) - { - ThrowLastError("file copy failed"sv); - } - - return true; -} - -void -WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount) -{ -#if ZEN_PLATFORM_WINDOWS - CAtlFile Outfile; - HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); - if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND)) - { - CreateDirectories(Path.parent_path()); - - hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); - } - - if (FAILED(hRes)) - { - ThrowSystemException(hRes, fmt::format("File open failed for '{}'", Path).c_str()); - } - -#else - int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC; - int Fd = open(Path.c_str(), OpenFlags, 0666); - if (Fd < 0) - { - zen::CreateDirectories(Path.parent_path()); - Fd = open(Path.c_str(), OpenFlags, 0666); - } - - if (Fd < 0) - { - ThrowLastError(fmt::format("File open failed for '{}'", Path)); - } - - fchmod(Fd, 0666); -#endif - - // TODO: this should be block-enlightened - - for (size_t i = 0; i < BufferCount; ++i) - { - uint64_t WriteSize = Data[i]->Size(); - const void* DataPtr = Data[i]->Data(); - - while (WriteSize) - { - const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); - -#if ZEN_PLATFORM_WINDOWS - hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize)); - if (FAILED(hRes)) - { - ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); - } -#else - if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) - { - ThrowLastError(fmt::format("File write failed for '{}'", Path)); - } -#endif // ZEN_PLATFORM_WINDOWS - - WriteSize -= ChunkSize; - DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize; - } - } - -#if !ZEN_PLATFORM_WINDOWS - close(Fd); -#endif -} - -void -WriteFile(std::filesystem::path Path, IoBuffer Data) -{ - const IoBuffer* const DataPtr = &Data; - - WriteFile(Path, &DataPtr, 1); -} - -IoBuffer -FileContents::Flatten() -{ - if (Data.size() == 1) - { - return Data[0]; - } - else if (Data.empty()) - { - return {}; - } - else - { - ZEN_NOT_IMPLEMENTED(); - } -} - -FileContents -ReadStdIn() -{ - BinaryWriter Writer; - - do - { - uint8_t ReadBuffer[1024]; - - size_t BytesRead = fread(ReadBuffer, 1, sizeof ReadBuffer, stdin); - Writer.Write(ReadBuffer, BytesRead); - } while (!feof(stdin)); - - FileContents Contents; - Contents.Data.emplace_back(IoBuffer(IoBuffer::Clone, Writer.GetData(), Writer.GetSize())); - - return Contents; -} - -FileContents -ReadFile(std::filesystem::path Path) -{ - uint64_t FileSizeBytes; - void* Handle; - -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); - if (FromFile == INVALID_HANDLE_VALUE) - { - FromFile.Detach(); - return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; - } - - FILE_END_OF_FILE_INFO FileSize; - if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile)) - { - return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; - } - - FileSizeBytes = FileSize.EndOfFile.QuadPart; - Handle = FromFile.Detach(); -#else - int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) - { - FileContents Ret; - Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category()); - return Ret; - } - - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - fstat(Fd, &Stat); - - FileSizeBytes = Stat.st_size; - Handle = (void*)uintptr_t(Fd); -#endif - - FileContents Contents; - Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes)); - return Contents; -} - -bool -ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc) -{ -#if ZEN_PLATFORM_WINDOWS - ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); - if (FromFile == INVALID_HANDLE_VALUE) - { - FromFile.Detach(); - return false; - } - - std::vector<uint8_t> ReadBuffer(ChunkSize); - - for (;;) - { - DWORD dwBytesRead = 0; - BOOL Success = ::ReadFile(FromFile, ReadBuffer.data(), (DWORD)ReadBuffer.size(), &dwBytesRead, nullptr); - - if (!Success) - { - throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed"); - } - - if (dwBytesRead == 0) - break; - - ProcessFunc(ReadBuffer.data(), dwBytesRead); - } -#else - int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) - { - return false; - } - - bool Success = true; - - void* Buffer = malloc(ChunkSize); - while (true) - { - int BytesRead = read(Fd, Buffer, ChunkSize); - if (BytesRead < 0) - { - Success = false; - break; - } - - if (BytesRead == 0) - { - break; - } - - ProcessFunc(Buffer, BytesRead); - } - - free(Buffer); - close(Fd); - - if (!Success) - { - ThrowLastError("file scan failed"); - } -#endif // ZEN_PLATFORM_WINDOWS - - return true; -} - -void -PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out) -{ -#if ZEN_PLATFORM_WINDOWS - WideToUtf8(Path.native().c_str(), Out); -#else - Out << Path.c_str(); -#endif -} - -std::string -PathToUtf8(const std::filesystem::path& Path) -{ -#if ZEN_PLATFORM_WINDOWS - return WideToUtf8(Path.native().c_str()); -#else - return Path.string(); -#endif -} - -DiskSpace -DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error) -{ - using namespace std::filesystem; - - space_info SpaceInfo = space(Directory, Error); - if (Error) - { - return {}; - } - - return { - .Free = uint64_t(SpaceInfo.available), - .Total = uint64_t(SpaceInfo.capacity), - }; -} - -void -FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) -{ -#if ZEN_PLATFORM_WINDOWS - uint64_t FileInfoBuffer[8 * 1024]; - - FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; - bool Continue = true; - - CAtlFile RootDirHandle; - HRESULT hRes = - RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS); - - if (FAILED(hRes)) - { - ThrowSystemException(hRes, "Failed to open handle to volume root"); - } - - while (Continue) - { - BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); - FibClass = FileIdBothDirectoryInfo; // Set up for next iteration - - uint64_t EntryOffset = 0; - - if (!Success) - { - DWORD LastError = GetLastError(); - - if (LastError == ERROR_NO_MORE_FILES) - { - break; - } - - throw std::system_error(std::error_code(LastError, std::system_category()), "file system traversal error"); - } - - for (;;) - { - const FILE_ID_BOTH_DIR_INFO* DirInfo = - reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset); - - std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); - - if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) - { - if (FileName == L"."sv || FileName == L".."sv) - { - // Not very interesting - } - else - { - const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName); - - if (ShouldDescend) - { - // Note that this recursion combined with the buffer could - // blow the stack, we should consider a different strategy - - std::filesystem::path FullPath = RootDir / FileName; - - TraverseFileSystem(FullPath, Visitor); - } - } - } - else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) - { - ZEN_WARN("encountered device node during file system traversal: '{}' found in '{}'", WideToUtf8(FileName), RootDir); - } - else - { - Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart); - } - - const uint64_t NextOffset = DirInfo->NextEntryOffset; - - if (NextOffset == 0) - { - break; - } - - EntryOffset += NextOffset; - } - } -#else - /* Could also implement this using Linux's getdents() syscall */ - - DIR* Dir = opendir(RootDir.c_str()); - if (Dir == nullptr) - { - ThrowLastError(fmt::format("Failed to open directory for traversal: {}", RootDir.c_str())); - } - - for (struct dirent* Entry; (Entry = readdir(Dir));) - { - const char* FileName = Entry->d_name; - - struct stat Stat; - std::filesystem::path FullPath = RootDir / FileName; - stat(FullPath.c_str(), &Stat); - - if (S_ISDIR(Stat.st_mode)) - { - if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0) - { - /* nop */ - } - else if (Visitor.VisitDirectory(RootDir, FileName)) - { - TraverseFileSystem(FullPath, Visitor); - } - } - else if (S_ISREG(Stat.st_mode)) - { - Visitor.VisitFile(RootDir, FileName, Stat.st_size); - } - else - { - ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}", - Stat.st_mode, - FileName, - RootDir.c_str()); - } - } - - closedir(Dir); -#endif // ZEN_PLATFORM_WINDOWS -} - -std::filesystem::path -PathFromHandle(void* NativeHandle) -{ -#if ZEN_PLATFORM_WINDOWS - if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE) - { - return std::filesystem::path(); - } - - auto GetFinalPathNameByHandleWRetry = [](HANDLE hFile, LPWSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags) -> DWORD { - while (true) - { - DWORD Res = GetFinalPathNameByHandleW(hFile, lpszFilePath, cchFilePath, dwFlags); - if (Res == 0) - { - DWORD LastError = zen::GetLastError(); - // Under heavy concurrent loads we might get access denied on a file handle while trying to get path name. - // Retry if that is the case. - if (LastError != ERROR_ACCESS_DENIED) - { - ThrowSystemError(LastError, fmt::format("failed to get path from file handle {}", hFile)); - } - // Retry - continue; - } - ZEN_ASSERT(Res != 1); // We don't accept empty path names - return Res; - } - }; - - static const DWORD PathDataSize = 512; - wchar_t PathData[PathDataSize]; - DWORD RequiredLengthIncludingNul = GetFinalPathNameByHandleWRetry(NativeHandle, PathData, PathDataSize, FILE_NAME_OPENED); - if (RequiredLengthIncludingNul == 0) - { - ThrowLastError(fmt::format("failed to get path from file handle {}", NativeHandle)); - } - - if (RequiredLengthIncludingNul < PathDataSize) - { - std::wstring FullPath(PathData, gsl::narrow<size_t>(RequiredLengthIncludingNul)); - return FullPath; - } - - std::wstring FullPath; - FullPath.resize(RequiredLengthIncludingNul - 1); - - const DWORD FinalLength = GetFinalPathNameByHandleWRetry(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED); - ZEN_UNUSED(FinalLength); - return FullPath; - -#elif ZEN_PLATFORM_LINUX - char Link[PATH_MAX]; - char Path[64]; - - sprintf(Path, "/proc/self/fd/%d", int(uintptr_t(NativeHandle))); - ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); - if (BytesRead <= 0) - { - return std::filesystem::path(); - } - - Link[BytesRead] = '\0'; - return Link; -#elif ZEN_PLATFORM_MAC - int Fd = int(uintptr_t(NativeHandle)); - char Path[MAXPATHLEN]; - if (fcntl(Fd, F_GETPATH, Path) < 0) - { - return std::filesystem::path(); - } - - return Path; -#endif // ZEN_PLATFORM_WINDOWS -} - -std::filesystem::path -GetRunningExecutablePath() -{ -#if ZEN_PLATFORM_WINDOWS - TCHAR ExePath[MAX_PATH]; - DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath)); - - return {std::wstring_view(ExePath, PathLength)}; -#elif ZEN_PLATFORM_LINUX - char Link[256]; - ssize_t BytesRead = readlink("/proc/self/exe", Link, sizeof(Link) - 1); - if (BytesRead < 0) - return {}; - - Link[BytesRead] = '\0'; - return Link; -#elif ZEN_PLATFORM_MAC - char Buffer[PROC_PIDPATHINFO_MAXSIZE]; - - int SelfPid = GetCurrentProcessId(); - if (proc_pidpath(SelfPid, Buffer, sizeof(Buffer)) <= 0) - return {}; - - return Buffer; -#endif // ZEN_PLATFORM_WINDOWS -} - -void -MaximizeOpenFileCount() -{ -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - struct rlimit Limit; - int Error = getrlimit(RLIMIT_NOFILE, &Limit); - if (Error) - { - ZEN_WARN("failed getting rlimit RLIMIT_NOFILE, reason '{}'", zen::MakeErrorCode(Error).message()); - } - else - { - struct rlimit NewLimit = Limit; - NewLimit.rlim_cur = NewLimit.rlim_max; - ZEN_INFO("changing RLIMIT_NOFILE from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}", - Limit.rlim_cur, - Limit.rlim_max, - NewLimit.rlim_cur, - NewLimit.rlim_max); - - Error = setrlimit(RLIMIT_NOFILE, &NewLimit); - if (Error != 0) - { - ZEN_WARN("failed to set RLIMIT_NOFILE limits from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}, reason '{}'", - Limit.rlim_cur, - Limit.rlim_max, - NewLimit.rlim_cur, - NewLimit.rlim_max, - zen::MakeErrorCode(Error).message()); - } - } -#endif -} - -void -GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent) -{ - FileSystemTraversal Traversal; - struct Visitor : public FileSystemTraversal::TreeVisitor - { - Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {} - - virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent, - [[maybe_unused]] const path_view& File, - [[maybe_unused]] uint64_t FileSize) override - { - if (Flags & DirectoryContent::IncludeFilesFlag) - { - Content.Files.push_back(Parent / File); - } - } - - virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override - { - if (Flags & DirectoryContent::IncludeDirsFlag) - { - Content.Directories.push_back(Parent / DirectoryName); - } - return (Flags & DirectoryContent::RecursiveFlag) != 0; - } - - const uint8_t Flags; - DirectoryContent& Content; - } Visit(Flags, OutContent); - - Traversal.TraverseFileSystem(RootDir, Visit); -} - -std::string -GetEnvVariable(std::string_view VariableName) -{ - ZEN_ASSERT(!VariableName.empty()); -#if ZEN_PLATFORM_WINDOWS - - CHAR EnvVariableBuffer[1023 + 1]; - DWORD RESULT = GetEnvironmentVariableA(std::string(VariableName).c_str(), EnvVariableBuffer, sizeof(EnvVariableBuffer)); - if (RESULT > 0 && RESULT < sizeof(EnvVariableBuffer)) - { - return std::string(EnvVariableBuffer); - } -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - char* EnvVariable = getenv(std::string(VariableName).c_str()); - if (EnvVariable) - { - return std::string(EnvVariable); - } -#endif - return ""; -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -void -filesystem_forcelink() -{ -} - -TEST_CASE("filesystem") -{ - using namespace std::filesystem; - - // GetExePath -- this is not a great test as it's so dependent on where the this code gets linked in - path BinPath = GetRunningExecutablePath(); - const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver"; - CHECK(ExpectedExe); - CHECK(is_regular_file(BinPath)); - - // PathFromHandle - void* Handle; -# if ZEN_PLATFORM_WINDOWS - Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); - CHECK(Handle != INVALID_HANDLE_VALUE); -# else - int Fd = open(BinPath.c_str(), O_RDONLY | O_CLOEXEC); - CHECK(Fd >= 0); - Handle = (void*)uintptr_t(Fd); -# endif - - auto FromHandle = PathFromHandle((void*)uintptr_t(Handle)); - CHECK(equivalent(FromHandle, BinPath)); - -# if ZEN_PLATFORM_WINDOWS - CloseHandle(Handle); -# else - close(int(uintptr_t(Handle))); -# endif - - // Traversal - struct : public FileSystemTraversal::TreeVisitor - { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override - { - bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); - } - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } - - bool bFoundExpected = false; - std::filesystem::path Expected; - } Visitor; - Visitor.Expected = BinPath; - - FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor); - CHECK(Visitor.bFoundExpected); - - // Scan/read file - FileContents BinRead = ReadFile(BinPath); - std::vector<uint8_t> BinScan; - ScanFile(BinPath, 16 << 10, [&](const void* Data, size_t Size) { - const auto* Ptr = (uint8_t*)Data; - BinScan.insert(BinScan.end(), Ptr, Ptr + Size); - }); - CHECK_EQ(BinRead.Data.size(), 1); - CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); -} - -TEST_CASE("WriteFile") -{ - std::filesystem::path TempFile = GetRunningExecutablePath().parent_path(); - TempFile /= "write_file_test"; - - uint64_t Magics[] = { - 0x0'a9e'a9e'a9e'a9e'a9e, - 0x0'493'493'493'493'493, - }; - - struct - { - const void* Data; - size_t Size; - } MagicTests[] = { - { - Magics, - sizeof(Magics), - }, - { - Magics + 1, - sizeof(Magics[0]), - }, - }; - for (auto& MagicTest : MagicTests) - { - WriteFile(TempFile, IoBuffer(IoBuffer::Wrap, MagicTest.Data, MagicTest.Size)); - - FileContents MagicsReadback = ReadFile(TempFile); - CHECK_EQ(MagicsReadback.Data.size(), 1); - CHECK_EQ(MagicsReadback.Data[0].GetSize(), MagicTest.Size); - CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0); - } - - std::filesystem::remove(TempFile); -} - -TEST_CASE("DiskSpaceInfo") -{ - std::filesystem::path BinPath = GetRunningExecutablePath(); - - DiskSpace Space = {}; - - std::error_code Error; - Space = DiskSpaceInfo(BinPath, Error); - CHECK(!Error); - - bool Okay = DiskSpaceInfo(BinPath, Space); - CHECK(Okay); - - CHECK(int64_t(Space.Total) > 0); - CHECK(int64_t(Space.Free) > 0); // Hopefully there's at least one byte free -} - -TEST_CASE("PathBuilder") -{ -# if ZEN_PLATFORM_WINDOWS - const char* foo_bar = "/foo\\bar"; -# else - const char* foo_bar = "/foo/bar"; -# endif - - ExtendablePathBuilder<32> Path; - for (const char* Prefix : {"/foo", "/foo/"}) - { - Path.Reset(); - Path.Append(Prefix); - Path /= "bar"; - CHECK(Path.ToPath() == foo_bar); - } - - using fspath = std::filesystem::path; - - Path.Reset(); - Path.Append(fspath("/foo/")); - Path /= (fspath("bar")); - CHECK(Path.ToPath() == foo_bar); - -# if ZEN_PLATFORM_WINDOWS - Path.Reset(); - Path.Append(fspath(L"/\u0119oo/")); - Path /= L"bar"; - printf("%ls\n", Path.ToPath().c_str()); - CHECK(Path.ToView() == L"/\u0119oo/bar"); - CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); -# endif -} - -#endif - -} // namespace zen |