// Copyright Epic Games, Inc. All Rights Reserved. #include "zenstore/basicfile.h" #include #include #include #include #include #include #if ZEN_PLATFORM_WINDOWS # include #else # include # include # include # include #endif #include #include namespace zen { BasicFile::~BasicFile() { Close(); } void BasicFile::Open(const std::filesystem::path& FileName, Mode Mode) { std::error_code Ec; Open(FileName, Mode, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("failed to open file '{}'", FileName)); } } void BasicFile::Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec) { Ec.clear(); #if ZEN_PLATFORM_WINDOWS DWORD dwCreationDisposition = 0; DWORD dwDesiredAccess = 0; switch (Mode) { case Mode::kRead: dwCreationDisposition |= OPEN_EXISTING; dwDesiredAccess |= GENERIC_READ; break; case Mode::kWrite: dwCreationDisposition |= OPEN_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE); break; case Mode::kDelete: dwCreationDisposition |= OPEN_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE); break; case Mode::kTruncate: dwCreationDisposition |= CREATE_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE); break; case Mode::kTruncateDelete: dwCreationDisposition |= CREATE_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE); break; } const DWORD dwShareMode = FILE_SHARE_READ; const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; HANDLE hTemplateFile = nullptr; HANDLE FileHandle = CreateFile(FileName.c_str(), dwDesiredAccess, dwShareMode, /* lpSecurityAttributes */ nullptr, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile); if (FileHandle == INVALID_HANDLE_VALUE) { Ec = MakeErrorCodeFromLastError(); return; } #else int OpenFlags = O_CLOEXEC; switch (Mode) { case Mode::kRead: OpenFlags |= O_RDONLY; break; case Mode::kWrite: case Mode::kDelete: OpenFlags |= (O_RDWR | O_CREAT); break; case Mode::kTruncate: case Mode::kTruncateDelete: OpenFlags |= (O_RDWR | O_CREAT | O_TRUNC); break; } int Fd = open(FileName.c_str(), OpenFlags, 0666); if (Fd < 0) { Ec = MakeErrorCodeFromLastError(); return; } if (Mode != Mode::kRead) { fchmod(Fd, 0666); } void* FileHandle = (void*)(uintptr_t(Fd)); #endif m_FileHandle = FileHandle; } void BasicFile::Close() { if (m_FileHandle) { #if ZEN_PLATFORM_WINDOWS ::CloseHandle(m_FileHandle); #else int Fd = int(uintptr_t(m_FileHandle)); close(Fd); #endif m_FileHandle = nullptr; } } void BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; while (BytesToRead) { const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(FileOffset >> 32); DWORD dwNumberOfBytesRead = 0; BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); #else static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); int Fd = int(uintptr_t(m_FileHandle)); int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset); bool Success = (BytesRead > 0); #endif if (!Success) { ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle))); } BytesToRead -= NumberOfBytesToRead; FileOffset += NumberOfBytesToRead; Data = reinterpret_cast(Data) + NumberOfBytesToRead; } } IoBuffer BasicFile::ReadAll() { IoBuffer Buffer(FileSize()); Read(Buffer.MutableData(), Buffer.Size(), 0); return Buffer; } void BasicFile::StreamFile(std::function&& ChunkFun) { StreamByteRange(0, FileSize(), std::move(ChunkFun)); } void BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun) { const uint64_t ChunkSize = 128 * 1024; IoBuffer ReadBuffer{ChunkSize}; void* BufferPtr = ReadBuffer.MutableData(); uint64_t RemainBytes = Size; uint64_t CurrentOffset = FileOffset; while (RemainBytes) { const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes); Read(BufferPtr, ThisChunkBytes, CurrentOffset); ChunkFun(BufferPtr, ThisChunkBytes); CurrentOffset += ThisChunkBytes; RemainBytes -= ThisChunkBytes; } } void BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) { Write(Data.GetData(), Data.GetSize(), FileOffset, Ec); } void BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) { Ec.clear(); const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; while (Size) { const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(FileOffset >> 32); DWORD dwNumberOfBytesWritten = 0; BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); #else static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); int Fd = int(uintptr_t(m_FileHandle)); int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); bool Success = (BytesWritten > 0); #endif if (!Success) { Ec = MakeErrorCodeFromLastError(); return; } Size -= NumberOfBytesToWrite; FileOffset += NumberOfBytesToWrite; Data = reinterpret_cast(Data) + NumberOfBytesToWrite; } } void BasicFile::Write(MemoryView Data, uint64_t FileOffset) { Write(Data.GetData(), Data.GetSize(), FileOffset); } void BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) { std::error_code Ec; Write(Data, Size, Offset, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle))); } } void BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) { Write(Data.Data(), Data.Size(), 0, Ec); } void BasicFile::Flush() { #if ZEN_PLATFORM_WINDOWS FlushFileBuffers(m_FileHandle); #else int Fd = int(uintptr_t(m_FileHandle)); fsync(Fd); #endif } uint64_t BasicFile::FileSize() { #if ZEN_PLATFORM_WINDOWS ULARGE_INTEGER liFileSize; liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); if (liFileSize.LowPart == INVALID_FILE_SIZE) { int Error = zen::GetLastError(); if (Error) { ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle))); } } return uint64_t(liFileSize.QuadPart); #else int Fd = int(uintptr_t(m_FileHandle)); static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); struct stat Stat; fstat(Fd, &Stat); return uint64_t(Stat.st_size); #endif } void BasicFile::SetFileSize(uint64_t FileSize) { #if ZEN_PLATFORM_WINDOWS LARGE_INTEGER liFileSize; liFileSize.QuadPart = FileSize; BOOL OK = ::SetFilePointerEx(m_FileHandle, liFileSize, 0, FILE_BEGIN); if (OK == FALSE) { int Error = zen::GetLastError(); if (Error) { ThrowSystemError(Error, fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); } } OK = ::SetEndOfFile(m_FileHandle); if (OK == FALSE) { int Error = zen::GetLastError(); if (Error) { ThrowSystemError(Error, fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); } } #elif ZEN_PLATFORM_MAC int Fd = int(intptr_t(m_FileHandle)); if (ftruncate(Fd, (off_t)FileSize) < 0) { int Error = zen::GetLastError(); if (Error) { ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); } } #else int Fd = int(intptr_t(m_FileHandle)); if (ftruncate64(Fd, (off64_t)FileSize) < 0) { int Error = zen::GetLastError(); if (Error) { ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle))); } } if (FileSize > 0) { int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize); if (Error) { ThrowSystemError(Error, fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle))); } } #endif } void* BasicFile::Detach() { void* FileHandle = m_FileHandle; m_FileHandle = 0; return FileHandle; } ////////////////////////////////////////////////////////////////////////// TemporaryFile::~TemporaryFile() { Close(); } void TemporaryFile::Close() { if (m_FileHandle) { #if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); #else std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); unlink(FilePath.c_str()); #endif BasicFile::Close(); } } void TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec) { StringBuilder<64> TempName; Oid::NewOid().ToString(TempName); m_TempPath = TempDirName / TempName.c_str(); Open(m_TempPath, BasicFile::Mode::kTruncateDelete, Ec); } void TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec) { // We intentionally call the base class Close() since otherwise we'll end up // deleting the temporary file BasicFile::Close(); std::filesystem::rename(m_TempPath, FinalFileName, Ec); } ////////////////////////////////////////////////////////////////////////// LockFile::LockFile() { } LockFile::~LockFile() { #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC int Fd = int(intptr_t(m_FileHandle)); flock(Fd, LOCK_UN | LOCK_NB); #endif } void LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS Ec.clear(); const DWORD dwCreationDisposition = CREATE_ALWAYS; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE | DELETE; const DWORD dwShareMode = FILE_SHARE_READ; const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE; HANDLE hTemplateFile = nullptr; HANDLE FileHandle = CreateFile(FileName.c_str(), dwDesiredAccess, dwShareMode, /* lpSecurityAttributes */ nullptr, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile); if (FileHandle == INVALID_HANDLE_VALUE) { Ec = zen::MakeErrorCodeFromLastError(); return; } #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC int Fd = open(FileName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (Fd < 0) { Ec = zen::MakeErrorCodeFromLastError(); return; } fchmod(Fd, 0666); int LockRet = flock(Fd, LOCK_EX | LOCK_NB); if (LockRet < 0) { Ec = zen::MakeErrorCodeFromLastError(); close(Fd); return; } void* FileHandle = (void*)uintptr_t(Fd); #endif m_FileHandle = FileHandle; BasicFile::Write(Payload.GetBuffer(), 0, Ec); } void LockFile::Update(CbObject Payload, std::error_code& Ec) { BasicFile::Write(Payload.GetBuffer(), 0, Ec); } /* ___________ __ \__ ___/___ _______/ |_ ______ | |_/ __ \ / ___/\ __\/ ___/ | |\ ___/ \___ \ | | \___ \ |____| \___ >____ > |__| /____ > \/ \/ \/ */ #if ZEN_WITH_TESTS TEST_CASE("BasicFile") { ScopedCurrentDirectoryChange _; BasicFile File1; CHECK_THROWS(File1.Open("zonk", BasicFile::Mode::kRead)); CHECK_NOTHROW(File1.Open("zonk", BasicFile::Mode::kTruncate)); CHECK_NOTHROW(File1.Write("abcd", 4, 0)); CHECK(File1.FileSize() == 4); { IoBuffer Data = File1.ReadAll(); CHECK(Data.Size() == 4); CHECK_EQ(memcmp(Data.Data(), "abcd", 4), 0); } CHECK_NOTHROW(File1.Write("efgh", 4, 2)); CHECK(File1.FileSize() == 6); { IoBuffer Data = File1.ReadAll(); CHECK(Data.Size() == 6); CHECK_EQ(memcmp(Data.Data(), "abefgh", 6), 0); } } TEST_CASE("TemporaryFile") { ScopedCurrentDirectoryChange _; SUBCASE("DeleteOnClose") { TemporaryFile TmpFile; std::error_code Ec; TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); CHECK(std::filesystem::exists(TmpFile.GetPath())); TmpFile.Close(); CHECK(std::filesystem::exists(TmpFile.GetPath()) == false); } SUBCASE("MoveIntoPlace") { TemporaryFile TmpFile; std::error_code Ec; TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; CHECK(std::filesystem::exists(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); CHECK(std::filesystem::exists(TempPath) == false); CHECK(std::filesystem::exists(FinalPath)); } } void basicfile_forcelink() { } #endif } // namespace zen