From 4c95b15f90bf99104c8b528ef0ac2da05c86c612 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 17 Nov 2022 13:13:46 +0100 Subject: move BasicFile to zenutil to remove zenstore dependency from zen command (#190) --- zenstore/basicfile.cpp | 575 ------------------------------------------------- 1 file changed, 575 deletions(-) delete mode 100644 zenstore/basicfile.cpp (limited to 'zenstore/basicfile.cpp') diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp deleted file mode 100644 index e5a2adc41..000000000 --- a/zenstore/basicfile.cpp +++ /dev/null @@ -1,575 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zenstore/basicfile.h" - -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -#else -# include -# include -# include -# include -#endif - -#include -#include - -namespace zen { - -BasicFile::~BasicFile() -{ - Close(); -} - -void -BasicFile::Open(const std::filesystem::path& FileName, Mode Mode) -{ - std::error_code Ec; - Open(FileName, Mode, Ec); - - if (Ec) - { - throw std::system_error(Ec, fmt::format("failed to open file '{}'", FileName)); - } -} - -void -BasicFile::Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec) -{ - Ec.clear(); - -#if ZEN_PLATFORM_WINDOWS - DWORD dwCreationDisposition = 0; - DWORD dwDesiredAccess = 0; - switch (Mode) - { - case Mode::kRead: - dwCreationDisposition |= OPEN_EXISTING; - dwDesiredAccess |= GENERIC_READ; - break; - case Mode::kWrite: - dwCreationDisposition |= OPEN_ALWAYS; - dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE); - break; - case Mode::kDelete: - dwCreationDisposition |= OPEN_ALWAYS; - dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE); - break; - case Mode::kTruncate: - dwCreationDisposition |= CREATE_ALWAYS; - dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE); - break; - case Mode::kTruncateDelete: - dwCreationDisposition |= CREATE_ALWAYS; - dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE); - break; - } - - const DWORD dwShareMode = FILE_SHARE_READ; - const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; - HANDLE hTemplateFile = nullptr; - - HANDLE FileHandle = CreateFile(FileName.c_str(), - dwDesiredAccess, - dwShareMode, - /* lpSecurityAttributes */ nullptr, - dwCreationDisposition, - dwFlagsAndAttributes, - hTemplateFile); - - if (FileHandle == INVALID_HANDLE_VALUE) - { - Ec = MakeErrorCodeFromLastError(); - - return; - } -#else - int OpenFlags = O_CLOEXEC; - switch (Mode) - { - case Mode::kRead: - OpenFlags |= O_RDONLY; - break; - case Mode::kWrite: - case Mode::kDelete: - OpenFlags |= (O_RDWR | O_CREAT); - break; - case Mode::kTruncate: - case Mode::kTruncateDelete: - OpenFlags |= (O_RDWR | O_CREAT | O_TRUNC); - break; - } - - int Fd = open(FileName.c_str(), OpenFlags, 0666); - if (Fd < 0) - { - Ec = MakeErrorCodeFromLastError(); - return; - } - if (Mode != Mode::kRead) - { - fchmod(Fd, 0666); - } - - void* FileHandle = (void*)(uintptr_t(Fd)); -#endif - - m_FileHandle = FileHandle; -} - -void -BasicFile::Close() -{ - if (m_FileHandle) - { -#if ZEN_PLATFORM_WINDOWS - ::CloseHandle(m_FileHandle); -#else - int Fd = int(uintptr_t(m_FileHandle)); - close(Fd); -#endif - m_FileHandle = nullptr; - } -} - -void -BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) -{ - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - - while (BytesToRead) - { - const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesRead = 0; - BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); - - ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset); - bool Success = (BytesRead > 0); -#endif - - if (!Success) - { - ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle))); - } - - BytesToRead -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast(Data) + NumberOfBytesToRead; - } -} - -IoBuffer -BasicFile::ReadAll() -{ - IoBuffer Buffer(FileSize()); - Read(Buffer.MutableData(), Buffer.Size(), 0); - return Buffer; -} - -void -BasicFile::StreamFile(std::function&& ChunkFun) -{ - StreamByteRange(0, FileSize(), std::move(ChunkFun)); -} - -void -BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun) -{ - const uint64_t ChunkSize = 128 * 1024; - IoBuffer ReadBuffer{ChunkSize}; - void* BufferPtr = ReadBuffer.MutableData(); - - uint64_t RemainBytes = Size; - uint64_t CurrentOffset = FileOffset; - - while (RemainBytes) - { - const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes); - - Read(BufferPtr, ThisChunkBytes, CurrentOffset); - - ChunkFun(BufferPtr, ThisChunkBytes); - - CurrentOffset += ThisChunkBytes; - RemainBytes -= ThisChunkBytes; - } -} - -void -BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) -{ - Write(Data.GetData(), Data.GetSize(), FileOffset, Ec); -} - -void -BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) -{ - Ec.clear(); - - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - - while (Size) - { - const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesWritten = 0; - - BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); - bool Success = (BytesWritten > 0); -#endif - - if (!Success) - { - Ec = MakeErrorCodeFromLastError(); - - return; - } - - Size -= NumberOfBytesToWrite; - FileOffset += NumberOfBytesToWrite; - Data = reinterpret_cast(Data) + NumberOfBytesToWrite; - } -} - -void -BasicFile::Write(MemoryView Data, uint64_t FileOffset) -{ - Write(Data.GetData(), Data.GetSize(), FileOffset); -} - -void -BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) -{ - std::error_code Ec; - Write(Data, Size, Offset, Ec); - - if (Ec) - { - throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle))); - } -} - -void -BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) -{ - Write(Data.Data(), Data.Size(), 0, Ec); -} - -void -BasicFile::Flush() -{ -#if ZEN_PLATFORM_WINDOWS - FlushFileBuffers(m_FileHandle); -#else - int Fd = int(uintptr_t(m_FileHandle)); - fsync(Fd); -#endif -} - -uint64_t -BasicFile::FileSize() -{ -#if ZEN_PLATFORM_WINDOWS - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart == INVALID_FILE_SIZE) - { - int Error = zen::GetLastError(); - if (Error) - { - ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle))); - } - } - return uint64_t(liFileSize.QuadPart); -#else - int Fd = int(uintptr_t(m_FileHandle)); - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - fstat(Fd, &Stat); - return uint64_t(Stat.st_size); -#endif -} - -void -BasicFile::SetFileSize(uint64_t FileSize) -{ -#if ZEN_PLATFORM_WINDOWS - LARGE_INTEGER liFileSize; - liFileSize.QuadPart = FileSize; - BOOL OK = ::SetFilePointerEx(m_FileHandle, liFileSize, 0, FILE_BEGIN); - if (OK == FALSE) - { - int Error = zen::GetLastError(); - if (Error) - { - ThrowSystemError(Error, fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); - } - } - OK = ::SetEndOfFile(m_FileHandle); - if (OK == FALSE) - { - int Error = zen::GetLastError(); - if (Error) - { - ThrowSystemError(Error, fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); - } - } -#elif ZEN_PLATFORM_MAC - int Fd = int(intptr_t(m_FileHandle)); - if (ftruncate(Fd, (off_t)FileSize) < 0) - { - int Error = zen::GetLastError(); - if (Error) - { - ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); - } - } -#else - int Fd = int(intptr_t(m_FileHandle)); - if (ftruncate64(Fd, (off64_t)FileSize) < 0) - { - int Error = zen::GetLastError(); - if (Error) - { - ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); - } - } - if (FileSize > 0) - { - int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize); - if (Error) - { - ThrowSystemError(Error, fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle))); - } - } -#endif -} - -void* -BasicFile::Detach() -{ - void* FileHandle = m_FileHandle; - m_FileHandle = 0; - return FileHandle; -} - -////////////////////////////////////////////////////////////////////////// - -TemporaryFile::~TemporaryFile() -{ - Close(); -} - -void -TemporaryFile::Close() -{ - if (m_FileHandle) - { -#if ZEN_PLATFORM_WINDOWS - // Mark file for deletion when final handle is closed - - FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; - - SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); -#else - std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); - unlink(FilePath.c_str()); -#endif - - BasicFile::Close(); - } -} - -void -TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec) -{ - StringBuilder<64> TempName; - Oid::NewOid().ToString(TempName); - - m_TempPath = TempDirName / TempName.c_str(); - - Open(m_TempPath, BasicFile::Mode::kTruncateDelete, Ec); -} - -void -TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec) -{ - // We intentionally call the base class Close() since otherwise we'll end up - // deleting the temporary file - BasicFile::Close(); - - std::filesystem::rename(m_TempPath, FinalFileName, Ec); -} - -////////////////////////////////////////////////////////////////////////// - -LockFile::LockFile() -{ -} - -LockFile::~LockFile() -{ -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Fd = int(intptr_t(m_FileHandle)); - flock(Fd, LOCK_UN | LOCK_NB); -#endif -} - -void -LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec) -{ -#if ZEN_PLATFORM_WINDOWS - Ec.clear(); - - const DWORD dwCreationDisposition = CREATE_ALWAYS; - DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE | DELETE; - const DWORD dwShareMode = FILE_SHARE_READ; - const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE; - HANDLE hTemplateFile = nullptr; - - HANDLE FileHandle = CreateFile(FileName.c_str(), - dwDesiredAccess, - dwShareMode, - /* lpSecurityAttributes */ nullptr, - dwCreationDisposition, - dwFlagsAndAttributes, - hTemplateFile); - - if (FileHandle == INVALID_HANDLE_VALUE) - { - Ec = zen::MakeErrorCodeFromLastError(); - - return; - } -#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Fd = open(FileName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); - if (Fd < 0) - { - Ec = zen::MakeErrorCodeFromLastError(); - return; - } - fchmod(Fd, 0666); - - int LockRet = flock(Fd, LOCK_EX | LOCK_NB); - if (LockRet < 0) - { - Ec = zen::MakeErrorCodeFromLastError(); - close(Fd); - return; - } - - void* FileHandle = (void*)uintptr_t(Fd); -#endif - - m_FileHandle = FileHandle; - - BasicFile::Write(Payload.GetBuffer(), 0, Ec); -} - -void -LockFile::Update(CbObject Payload, std::error_code& Ec) -{ - BasicFile::Write(Payload.GetBuffer(), 0, Ec); -} - -/* - ___________ __ - \__ ___/___ _______/ |_ ______ - | |_/ __ \ / ___/\ __\/ ___/ - | |\ ___/ \___ \ | | \___ \ - |____| \___ >____ > |__| /____ > - \/ \/ \/ -*/ - -#if ZEN_WITH_TESTS - -TEST_CASE("BasicFile") -{ - ScopedCurrentDirectoryChange _; - - BasicFile File1; - CHECK_THROWS(File1.Open("zonk", BasicFile::Mode::kRead)); - CHECK_NOTHROW(File1.Open("zonk", BasicFile::Mode::kTruncate)); - CHECK_NOTHROW(File1.Write("abcd", 4, 0)); - CHECK(File1.FileSize() == 4); - { - IoBuffer Data = File1.ReadAll(); - CHECK(Data.Size() == 4); - CHECK_EQ(memcmp(Data.Data(), "abcd", 4), 0); - } - CHECK_NOTHROW(File1.Write("efgh", 4, 2)); - CHECK(File1.FileSize() == 6); - { - IoBuffer Data = File1.ReadAll(); - CHECK(Data.Size() == 6); - CHECK_EQ(memcmp(Data.Data(), "abefgh", 6), 0); - } -} - -TEST_CASE("TemporaryFile") -{ - ScopedCurrentDirectoryChange _; - - SUBCASE("DeleteOnClose") - { - TemporaryFile TmpFile; - std::error_code Ec; - TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); - CHECK(!Ec); - CHECK(std::filesystem::exists(TmpFile.GetPath())); - TmpFile.Close(); - CHECK(std::filesystem::exists(TmpFile.GetPath()) == false); - } - - SUBCASE("MoveIntoPlace") - { - TemporaryFile TmpFile; - std::error_code Ec; - TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); - CHECK(!Ec); - std::filesystem::path TempPath = TmpFile.GetPath(); - std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; - CHECK(std::filesystem::exists(TempPath)); - TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); - CHECK(!Ec); - CHECK(std::filesystem::exists(TempPath) == false); - CHECK(std::filesystem::exists(FinalPath)); - } -} - -void -basicfile_forcelink() -{ -} - -#endif - -} // namespace zen -- cgit v1.2.3