// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include #include #if ZEN_PLATFORM_WINDOWS # include #else # include # include # include # include #endif #include #include namespace zen { BasicFile::~BasicFile() { Close(); } BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode) { Open(FileName, Mode); } BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec) { Open(FileName, Mode, Ec); } BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback) { Open(FileName, Mode, std::move(RetryCallback)); } void BasicFile::Open(const std::filesystem::path& FileName, Mode Mode) { std::error_code Ec; Open(FileName, Mode, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("failed to open file '{}', mode: {:x}", FileName, uint32_t(Mode))); } } void BasicFile::Open(const std::filesystem::path& FileName, Mode InMode, std::error_code& Ec) { Ec.clear(); Mode Mode = InMode & Mode::kModeMask; #if ZEN_PLATFORM_WINDOWS DWORD dwCreationDisposition = 0; DWORD dwDesiredAccess = 0; switch (Mode) { case Mode::kRead: dwCreationDisposition |= OPEN_EXISTING; dwDesiredAccess |= GENERIC_READ; break; case Mode::kWrite: dwCreationDisposition |= OPEN_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE); break; case Mode::kDelete: dwCreationDisposition |= OPEN_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE); break; case Mode::kTruncate: dwCreationDisposition |= CREATE_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE); break; case Mode::kTruncateDelete: dwCreationDisposition |= CREATE_ALWAYS; dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE); break; } const DWORD dwShareMode = FILE_SHARE_READ | (EnumHasAllFlags(InMode, Mode::kPreventWrite) ? 0 : FILE_SHARE_WRITE) | (EnumHasAllFlags(InMode, Mode::kPreventDelete) ? 0 : FILE_SHARE_DELETE); const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; const HANDLE hTemplateFile = nullptr; const HANDLE FileHandle = CreateFile(FileName.c_str(), dwDesiredAccess, dwShareMode, /* lpSecurityAttributes */ nullptr, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile); if (FileHandle == INVALID_HANDLE_VALUE) { Ec = MakeErrorCodeFromLastError(); return; } #else int OpenFlags = O_CLOEXEC; switch (Mode) { case Mode::kRead: OpenFlags |= O_RDONLY; break; case Mode::kWrite: case Mode::kDelete: OpenFlags |= (O_RDWR | O_CREAT); break; case Mode::kTruncate: case Mode::kTruncateDelete: OpenFlags |= (O_RDWR | O_CREAT | O_TRUNC); break; } int Fd = open(FileName.c_str(), OpenFlags, 0666); if (Fd < 0) { Ec = MakeErrorCodeFromLastError(); return; } if (Mode != Mode::kRead) { fchmod(Fd, 0666); } void* FileHandle = (void*)(uintptr_t(Fd)); #endif m_FileHandle = FileHandle; } void BasicFile::Open(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback) { std::error_code Ec; Open(FileName, Mode, Ec); while (Ec && RetryCallback(Ec)) { Ec.clear(); Open(FileName, Mode, Ec); } if (Ec) { throw std::system_error(Ec, fmt::format("failed to open file '{}', mode: {:x}", FileName, uint32_t(Mode))); } } void BasicFile::Close() { if (m_FileHandle) { #if ZEN_PLATFORM_WINDOWS ::CloseHandle(m_FileHandle); #else int Fd = int(uintptr_t(m_FileHandle)); close(Fd); #endif m_FileHandle = nullptr; } } IoBuffer BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount) { return IoBufferBuilder::MakeFromFileHandle(m_FileHandle, FileOffset, ByteCount); } void BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; while (BytesToRead) { const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); int32_t Error = 0; size_t BytesRead = 0; #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(FileOffset >> 32); DWORD dwNumberOfBytesRead = 0; BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); if (Success) { BytesRead = size_t(dwNumberOfBytesRead); } else { Error = zen::GetLastError(); } #else static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); int Fd = int(uintptr_t(m_FileHandle)); ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset); if (ReadResult != -1) { BytesRead = size_t(ReadResult); } else { Error = zen::GetLastError(); } #endif if (Error || (BytesRead != NumberOfBytesToRead)) { std::error_code DummyEc; throw std::system_error(std::error_code(Error, std::system_category()), fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", FileOffset, NumberOfBytesToRead, PathFromHandle(m_FileHandle, DummyEc), FileSizeFromHandle(m_FileHandle))); } BytesToRead -= NumberOfBytesToRead; FileOffset += NumberOfBytesToRead; Data = reinterpret_cast(Data) + NumberOfBytesToRead; } } IoBuffer BasicFile::ReadAll() { if (const uint64_t Size = FileSize()) { IoBuffer Buffer(Size); Read(Buffer.MutableData(), Size, 0); return Buffer; } else { return {}; } } void BasicFile::StreamFile(std::function&& ChunkFun) { StreamByteRange(0, FileSize(), std::move(ChunkFun)); } void BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun) { const uint64_t ChunkSize = 128 * 1024; IoBuffer ReadBuffer{ChunkSize}; void* BufferPtr = ReadBuffer.MutableData(); uint64_t RemainBytes = Size; uint64_t CurrentOffset = FileOffset; while (RemainBytes) { const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes); Read(BufferPtr, ThisChunkBytes, CurrentOffset); ChunkFun(BufferPtr, ThisChunkBytes); CurrentOffset += ThisChunkBytes; RemainBytes -= ThisChunkBytes; } } uint64_t BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset) { std::error_code Ec; uint64_t WrittenBytes = Write(Data, FileOffset, Ec); if (Ec) { std::error_code Dummy; throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy))); } return WrittenBytes; } uint64_t BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec) { uint64_t WrittenBytes = 0; for (const SharedBuffer& Buffer : Data.GetSegments()) { MemoryView BlockView = Buffer.GetView(); Write(BlockView, FileOffset + WrittenBytes, Ec); if (Ec) { return WrittenBytes; } WrittenBytes += BlockView.GetSize(); } return WrittenBytes; } void BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) { Write(Data.GetData(), Data.GetSize(), FileOffset, Ec); } void BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) { ZEN_ASSERT(m_FileHandle != nullptr); Ec.clear(); const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; while (Size) { const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(FileOffset >> 32); DWORD dwNumberOfBytesWritten = 0; BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); #else static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); int Fd = int(uintptr_t(m_FileHandle)); int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); bool Success = (BytesWritten > 0); #endif if (!Success) { Ec = MakeErrorCodeFromLastError(); return; } Size -= NumberOfBytesToWrite; FileOffset += NumberOfBytesToWrite; Data = reinterpret_cast(Data) + NumberOfBytesToWrite; } } void BasicFile::Write(MemoryView Data, uint64_t FileOffset) { Write(Data.GetData(), Data.GetSize(), FileOffset); } void BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) { std::error_code Ec; Write(Data, Size, Offset, Ec); if (Ec) { std::error_code Dummy; throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy))); } } void BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) { Write(Data.Data(), Data.Size(), 0, Ec); } void BasicFile::Flush() { if (m_FileHandle == nullptr) { return; } #if ZEN_PLATFORM_WINDOWS FlushFileBuffers(m_FileHandle); #else int Fd = int(uintptr_t(m_FileHandle)); fsync(Fd); #endif } uint64_t BasicFile::FileSize() const { #if ZEN_PLATFORM_WINDOWS ULARGE_INTEGER liFileSize; liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); if (liFileSize.LowPart == INVALID_FILE_SIZE) { int Error = zen::GetLastError(); if (Error) { std::error_code Dummy; ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); } } return uint64_t(liFileSize.QuadPart); #else int Fd = int(uintptr_t(m_FileHandle)); static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); struct stat Stat; if (fstat(Fd, &Stat) == -1) { std::error_code Dummy; ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); } return uint64_t(Stat.st_size); #endif } uint64_t BasicFile::FileSize(std::error_code& Ec) const { #if ZEN_PLATFORM_WINDOWS ULARGE_INTEGER liFileSize; liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); if (liFileSize.LowPart == INVALID_FILE_SIZE) { int Error = zen::GetLastError(); if (Error) { Ec = MakeErrorCode(Error); return 0; } } return uint64_t(liFileSize.QuadPart); #else int Fd = int(uintptr_t(m_FileHandle)); static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); struct stat Stat; if (fstat(Fd, &Stat) == -1) { Ec = MakeErrorCodeFromLastError(); return 0; } return uint64_t(Stat.st_size); #endif } void BasicFile::SetFileSize(uint64_t FileSize) { #if ZEN_PLATFORM_WINDOWS LARGE_INTEGER liFileSize; liFileSize.QuadPart = FileSize; BOOL OK = ::SetFilePointerEx(m_FileHandle, liFileSize, 0, FILE_BEGIN); if (OK == FALSE) { int Error = zen::GetLastError(); if (Error) { std::error_code Dummy; ThrowSystemError(Error, fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy))); } } OK = ::SetEndOfFile(m_FileHandle); if (OK == FALSE) { int Error = zen::GetLastError(); if (Error) { std::error_code Dummy; ThrowSystemError(Error, fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy))); } } #elif ZEN_PLATFORM_MAC int Fd = int(intptr_t(m_FileHandle)); if (ftruncate(Fd, (off_t)FileSize) < 0) { int Error = zen::GetLastError(); if (Error) { std::error_code Dummy; ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy))); } } #else int Fd = int(intptr_t(m_FileHandle)); if (ftruncate64(Fd, (off64_t)FileSize) < 0) { int Error = zen::GetLastError(); if (Error) { std::error_code Dummy; ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy))); } } if (FileSize > 0) { int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize); if (Error) { std::error_code Dummy; ThrowSystemError(Error, fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy))); } } #endif } void BasicFile::Attach(void* Handle) { ZEN_ASSERT(Handle != nullptr); ZEN_ASSERT(m_FileHandle == nullptr); m_FileHandle = Handle; } void* BasicFile::Detach() { void* FileHandle = m_FileHandle; m_FileHandle = 0; return FileHandle; } ////////////////////////////////////////////////////////////////////////// TemporaryFile::~TemporaryFile() { Close(); } void TemporaryFile::Close() { if (m_FileHandle) { #if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); #else std::error_code Ec; std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle, Ec); if (!Ec) { unlink(FilePath.c_str()); } #endif BasicFile::Close(); } } void TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec) { StringBuilder<64> TempName; Oid::NewOid().ToString(TempName); m_TempPath = TempDirName / TempName.c_str(); Open(m_TempPath, BasicFile::Mode::kTruncateDelete, Ec); } void TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec) { // We intentionally call the base class Close() since otherwise we'll end up // deleting the temporary file BasicFile::Close(); RenameFile(m_TempPath, FinalFileName, Ec); if (Ec) { // Try to re-open the temp file so we clean up after us when TemporaryFile is destructed std::error_code DummyEc; Open(m_TempPath, BasicFile::Mode::kWrite, DummyEc); } } ////////////////////////////////////////////////////////////////////////// void TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data) { std::error_code Ec; SafeWriteFile(Path, Data, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("Failed to safely write file '{}'", Path)); } } void TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc) { TemporaryFile TempFile; if (TempFile.CreateTemporary(Path.parent_path(), OutEc); !OutEc) { if (TempFile.Write(Data, 0, OutEc); !OutEc) { TempFile.MoveTemporaryIntoPlace(Path, OutEc); } } } ////////////////////////////////////////////////////////////////////////// LockFile::LockFile() { } LockFile::~LockFile() { #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC int Fd = int(intptr_t(m_FileHandle)); flock(Fd, LOCK_UN | LOCK_NB); #endif } void LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS Ec.clear(); const DWORD dwCreationDisposition = CREATE_ALWAYS; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE | DELETE; const DWORD dwShareMode = FILE_SHARE_READ; const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE; HANDLE hTemplateFile = nullptr; HANDLE FileHandle = CreateFile(FileName.c_str(), dwDesiredAccess, dwShareMode, /* lpSecurityAttributes */ nullptr, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile); if (FileHandle == INVALID_HANDLE_VALUE) { Ec = zen::MakeErrorCodeFromLastError(); return; } #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC int Fd = open(FileName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (Fd < 0) { Ec = zen::MakeErrorCodeFromLastError(); return; } fchmod(Fd, 0666); int LockRet = flock(Fd, LOCK_EX | LOCK_NB); if (LockRet < 0) { Ec = zen::MakeErrorCodeFromLastError(); close(Fd); return; } void* FileHandle = (void*)uintptr_t(Fd); #endif m_FileHandle = FileHandle; BasicFile::Write(Payload.GetBuffer(), 0, Ec); } void LockFile::Update(CbObject Payload, std::error_code& Ec) { BasicFile::Write(Payload.GetBuffer(), 0, Ec); } ////////////////////////////////////////////////////////////////////////// BasicFileBuffer::BasicFileBuffer(BasicFile& Base, uint64_t BufferSize) : m_Base(Base) , m_Buffer(nullptr) , m_BufferSize(BufferSize) , m_Size(Base.FileSize()) , m_BufferStart(0) , m_BufferEnd(0) { m_Buffer = (uint8_t*)Memory::Alloc(m_BufferSize); } BasicFileBuffer::~BasicFileBuffer() { Memory::Free(m_Buffer); } void BasicFileBuffer::Read(void* Data, uint64_t Size, uint64_t FileOffset) { if (m_Buffer == nullptr || (Size > m_BufferSize) || (FileOffset + Size > m_Size)) { m_Base.Read(Data, Size, FileOffset); return; } uint8_t* WritePtr = ((uint8_t*)Data); uint64_t Begin = FileOffset; uint64_t End = FileOffset + Size; if (FileOffset <= m_BufferStart) { if (End > m_BufferStart) { uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart; memcpy(WritePtr + End - Count - FileOffset, m_Buffer, Count); End -= Count; if (Begin == End) { return; } } } else if (FileOffset < m_BufferEnd) { uint64_t Count = Min(m_BufferEnd, End) - FileOffset; memcpy(WritePtr + Begin - FileOffset, m_Buffer + Begin - m_BufferStart, Count); Begin += Count; if (Begin == End) { return; } } m_BufferStart = Begin; m_BufferEnd = Min(Begin + m_BufferSize, m_Size); m_Base.Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart); uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart; memcpy(WritePtr + Begin - FileOffset, m_Buffer, Count); ZEN_ASSERT(Begin + Count == End); } MemoryView BasicFileBuffer::MakeView(uint64_t Size, uint64_t FileOffset) { if (FileOffset < m_BufferStart || (FileOffset + Size) > m_BufferEnd) { if (m_Buffer == nullptr || (Size > m_BufferSize) || (FileOffset + Size > m_Size)) { return {}; } m_BufferStart = FileOffset; m_BufferEnd = Min(m_BufferStart + m_BufferSize, m_Size); m_Base.Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart); } return MemoryView(m_Buffer + (FileOffset - m_BufferStart), Size); } ////////////////////////////////////////////////////////////////////////// BasicFileWriter::BasicFileWriter(BasicFile& Base, uint64_t BufferSize) : m_Base(Base) , m_Buffer(nullptr) , m_BufferSize(BufferSize) , m_BufferStart(0) , m_BufferEnd(0) { m_Buffer = (uint8_t*)Memory::Alloc(m_BufferSize); } BasicFileWriter::~BasicFileWriter() { Flush(); Memory::Free(m_Buffer); } void BasicFileWriter::AddPadding(uint64_t Padding) { while (Padding) { const uint64_t BufferOffset = m_BufferEnd - m_BufferStart; const uint64_t RemainingBufferCapacity = m_BufferSize - BufferOffset; const uint64_t BlockPadBytes = Min(RemainingBufferCapacity, Padding); memset(m_Buffer + BufferOffset, 0, BlockPadBytes); m_BufferEnd += BlockPadBytes; Padding -= BlockPadBytes; if ((BufferOffset + BlockPadBytes) == m_BufferSize) { Flush(); } } } uint64_t BasicFileWriter::AlignTo(uint64_t Alignment) { uint64_t AlignedPos = RoundUp(m_BufferEnd, Alignment); uint64_t Padding = AlignedPos - m_BufferEnd; AddPadding(Padding); return AlignedPos; } void BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset) { if (m_Buffer == nullptr || (Size >= m_BufferSize)) { if (FileOffset == m_BufferEnd) { Flush(); m_BufferStart = m_BufferEnd = FileOffset + Size; } m_Base.Write(Data, Size, FileOffset); return; } // Note that this only supports buffering of sequential writes! if (FileOffset != m_BufferEnd) { Flush(); m_BufferStart = m_BufferEnd = FileOffset; } const uint8_t* DataPtr = (const uint8_t*)Data; while (Size) { const uint64_t RemainingBufferCapacity = m_BufferStart + m_BufferSize - m_BufferEnd; const uint64_t BlockWriteBytes = Min(RemainingBufferCapacity, Size); const uint64_t BufferWriteOffset = FileOffset - m_BufferStart; ZEN_ASSERT_SLOW(BufferWriteOffset < m_BufferSize); ZEN_ASSERT_SLOW((BufferWriteOffset + BlockWriteBytes) <= m_BufferSize); memcpy(m_Buffer + BufferWriteOffset, DataPtr, BlockWriteBytes); Size -= BlockWriteBytes; m_BufferEnd += BlockWriteBytes; FileOffset += BlockWriteBytes; DataPtr += BlockWriteBytes; if ((m_BufferEnd - m_BufferStart) == m_BufferSize) { Flush(); } } } void BasicFileWriter::Write(const CompositeBuffer& Data, uint64_t FileOffset) { for (const SharedBuffer& Segment : Data.GetSegments()) { const uint64_t SegmentSize = Segment.GetSize(); Write(Segment.GetData(), SegmentSize, FileOffset); FileOffset += SegmentSize; } } void BasicFileWriter::Flush() { const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart; if (BufferedBytes == 0) return; const uint64_t WriteOffset = m_BufferStart; m_BufferStart = m_BufferEnd; m_Base.Write(m_Buffer, BufferedBytes, WriteOffset); } IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) { TemporaryFile Temp; std::error_code Ec; Temp.CreateTemporary(Path.parent_path(), Ec); if (Ec) { throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path)); } uint64_t BufferSize = Buffer.GetSize(); { uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; // BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); for (const SharedBuffer& Segment : Buffer.GetSegments()) { size_t SegmentSize = Segment.GetSize(); IoBufferFileReference FileRef; if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef)) { ScanFile(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, [&Temp, &Offset](const void* Data, size_t Size) { Temp.Write(Data, Size, Offset); Offset += Size; }); } else { Temp.Write(Segment.GetData(), SegmentSize, Offset); Offset += SegmentSize; } } } Temp.MoveTemporaryIntoPlace(Path, Ec); if (Ec) { Ec.clear(); BasicFile OpenTemp(Path, BasicFile::Mode::kDelete, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); } if (OpenTemp.FileSize() != BufferSize) { throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file size already exists", Path)); } IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true); IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); if (ExistingHash != ExpectedHash) { throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file hash already exists", Path)); } Buffer = CompositeBuffer{}; TmpBuffer.SetDeleteOnClose(true); return TmpBuffer; } Buffer = CompositeBuffer{}; BasicFile OpenTemp(Path, BasicFile::Mode::kDelete); IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true); TmpBuffer.SetDeleteOnClose(true); return TmpBuffer; } ////////////////////////////////////////////////////////////////////////// /* ___________ __ \__ ___/___ _______/ |_ ______ | |_/ __ \ / ___/\ __\/ ___/ | |\ ___/ \___ \ | | \___ \ |____| \___ >____ > |__| /____ > \/ \/ \/ */ #if ZEN_WITH_TESTS TEST_CASE("BasicFile") { ScopedCurrentDirectoryChange _; BasicFile File1; CHECK_THROWS(File1.Open("zonk", BasicFile::Mode::kRead)); CHECK_NOTHROW(File1.Open("zonk", BasicFile::Mode::kTruncate)); CHECK_NOTHROW(File1.Write("abcd", 4, 0)); CHECK(File1.FileSize() == 4); { IoBuffer Data = File1.ReadAll(); CHECK(Data.Size() == 4); CHECK_EQ(memcmp(Data.Data(), "abcd", 4), 0); } CHECK_NOTHROW(File1.Write("efgh", 4, 2)); CHECK(File1.FileSize() == 6); { IoBuffer Data = File1.ReadAll(); CHECK(Data.Size() == 6); CHECK_EQ(memcmp(Data.Data(), "abefgh", 6), 0); } } TEST_CASE("TemporaryFile") { ScopedCurrentDirectoryChange _; SUBCASE("DeleteOnClose") { std::filesystem::path Path; { TemporaryFile TmpFile; std::error_code Ec; TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); Path = TmpFile.GetPath(); CHECK(IsFile(Path)); } CHECK(IsFile(Path) == false); } SUBCASE("MoveIntoPlace") { TemporaryFile TmpFile; std::error_code Ec; TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; CHECK(IsFile(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); CHECK(IsFile(TempPath) == false); CHECK(IsFile(FinalPath)); } } TEST_CASE("BasicFileBuffer") { ScopedCurrentDirectoryChange _; { BasicFile File1; const std::string_view Data = "0123456789abcdef"; File1.Open("buffered", BasicFile::Mode::kTruncate); for (uint32_t I = 0; I < 16; ++I) { File1.Write(Data.data(), Data.size(), I * Data.size()); } } SUBCASE("EvenBuffer") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); // Non-primed { char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 1 * 16); std::string_view Verify(Buffer, 16); CHECK(Verify == "0123456789abcdef"); } // Primed { char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 1 * 16); std::string_view Verify(Buffer, 16); CHECK(Verify == "0123456789abcdef"); } } SUBCASE("UnevenBuffer") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); // Non-primed { char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 7); std::string_view Verify(Buffer, 16); CHECK(Verify == "789abcdef0123456"); } // Primed { char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 7); std::string_view Verify(Buffer, 16); CHECK(Verify == "789abcdef0123456"); } } SUBCASE("BiggerThanBuffer") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[17] = {0}; File1Buffer.Read(Buffer, 17, 0 * 16); std::string_view Verify(Buffer, 17); CHECK(Verify == "0123456789abcdef0"); } SUBCASE("InsideBuffer") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 0 * 16); File1Buffer.Read(Buffer, 8, 2); std::string_view Verify(Buffer, 8); CHECK(Verify == "23456789"); } SUBCASE("BeginningOfBuffer") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 8); File1Buffer.Read(Buffer, 8, 8); std::string_view Verify(Buffer, 8); CHECK(Verify == "89abcdef"); } SUBCASE("EndOfBuffer") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 0 * 16); File1Buffer.Read(Buffer, 8, 8); std::string_view Verify(Buffer, 8); CHECK(Verify == "89abcdef"); } SUBCASE("OverEnd") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 0 * 16); File1Buffer.Read(Buffer, 16, 8); std::string_view Verify(Buffer, 16); CHECK(Verify == "89abcdef01234567"); } SUBCASE("OverBegin") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 1 * 16); File1Buffer.Read(Buffer, 16, 8); std::string_view Verify(Buffer, 16); CHECK(Verify == "89abcdef01234567"); } SUBCASE("EndOfFile") { BasicFile File1; File1.Open("buffered", BasicFile::Mode::kRead); BasicFileBuffer File1Buffer(File1, 16); char Buffer[16] = {0}; File1Buffer.Read(Buffer, 16, 0 * 16); File1Buffer.Read(Buffer, 8, 256 - 8); std::string_view Verify(Buffer, 8); CHECK(Verify == "89abcdef"); } } void basicfile_forcelink() { } #endif } // namespace zen