// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include #include #include #include #include #include #if ZEN_USE_MIMALLOC ZEN_THIRD_PARTY_INCLUDES_START # include ZEN_THIRD_PARTY_INCLUDES_END #endif #if ZEN_PLATFORM_WINDOWS # include #else # include # include # include # include #endif #include namespace zen { ////////////////////////////////////////////////////////////////////////// void IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) const { #if ZEN_PLATFORM_WINDOWS if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000)) { m_Flags.fetch_or(kLowLevelAlloc, std::memory_order_relaxed); void* Ptr = VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE); if (!Ptr) { ThrowLastError(fmt::format("VirtualAlloc failed for {:#x} bytes aligned to {:#x}", InSize, Alignment)); } m_DataPtr = Ptr; return; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_USE_MIMALLOC void* Ptr = mi_aligned_alloc(Alignment, RoundUp(InSize, Alignment)); m_Flags.fetch_or(kIoBufferAlloc, std::memory_order_relaxed); #else void* Ptr = Memory::Alloc(InSize, Alignment); #endif if (!Ptr) { ThrowOutOfMemory(fmt::format("failed allocating {:#x} bytes aligned to {:#x}", InSize, Alignment)); } m_DataPtr = Ptr; } void IoBufferCore::FreeBuffer() { if (!m_DataPtr) { return; } const uint32_t LocalFlags = m_Flags.load(std::memory_order_relaxed); #if ZEN_PLATFORM_WINDOWS if (LocalFlags & kLowLevelAlloc) { VirtualFree(const_cast(m_DataPtr), 0, MEM_DECOMMIT); return; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_USE_MIMALLOC if (LocalFlags & kIoBufferAlloc) { return mi_free(const_cast(m_DataPtr)); } #endif ZEN_UNUSED(LocalFlags); return Memory::Free(const_cast(m_DataPtr)); } ////////////////////////////////////////////////////////////////////////// static_assert(sizeof(IoBufferCore) == 32); IoBufferCore::IoBufferCore(size_t InSize) { ZEN_ASSERT(InSize); AllocateBuffer(InSize, sizeof(void*)); m_DataBytes = InSize; SetIsOwnedByThis(true); } IoBufferCore::IoBufferCore(size_t InSize, size_t Alignment) { ZEN_ASSERT(InSize); AllocateBuffer(InSize, Alignment); m_DataBytes = InSize; SetIsOwnedByThis(true); } IoBufferCore::~IoBufferCore() { if (IsOwnedByThis() && m_DataPtr) { FreeBuffer(); m_DataPtr = nullptr; } } void IoBufferCore::DeleteThis() const { // We do this just to avoid paying for the cost of a vtable if (const IoBufferExtendedCore* _ = ExtendedCore()) { delete _; } else { delete this; } } void IoBufferCore::Materialize() const { if (const IoBufferExtendedCore* _ = ExtendedCore()) { _->Materialize(); } } void IoBufferCore::MakeOwned(bool Immutable) { if (!IsOwned()) { ZEN_TRACE_CPU("IoBufferCore::MakeOwned"); const void* OldDataPtr = m_DataPtr; AllocateBuffer(m_DataBytes, sizeof(void*)); memcpy(const_cast(m_DataPtr), OldDataPtr, m_DataBytes); SetIsOwnedByThis(true); } SetIsImmutable(Immutable); } void* IoBufferCore::MutableDataPointer() const { EnsureDataValid(); ZEN_ASSERT(!IsImmutable()); return const_cast(m_DataPtr); } ////////////////////////////////////////////////////////////////////////// IoBufferExtendedCore::IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership) : IoBufferCore(nullptr, Size) , m_FileHandle(FileHandle) , m_FileOffset(Offset) { uint32_t NewFlags = kIsOwnedByThis | kIsExtended; if (TransferHandleOwnership) { NewFlags |= kOwnsFile; } m_Flags.fetch_or(NewFlags, std::memory_order_relaxed); } IoBufferExtendedCore::IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size) : IoBufferCore(Outer, nullptr, Size) , m_FileHandle(Outer->m_FileHandle) , m_FileOffset(Outer->m_FileOffset + Offset) { m_Flags.fetch_or(kIsExtended, std::memory_order_relaxed); } IoBufferExtendedCore::~IoBufferExtendedCore() { if (m_MappedPointer) { #if ZEN_PLATFORM_WINDOWS UnmapViewOfFile(m_MappedPointer); #else uint64_t MapSize = ~uint64_t(uintptr_t(m_MmapHandle)); munmap(m_MappedPointer, MapSize); #endif m_DataPtr = nullptr; // prevent any buffer deallocation attempts } const uint32_t LocalFlags = m_Flags.load(std::memory_order_relaxed); #if ZEN_PLATFORM_WINDOWS if (LocalFlags & kOwnsMmap) { CloseHandle(m_MmapHandle); } #endif if (LocalFlags & kOwnsFile) { if (LocalFlags & kDeleteOnClose) { #if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); #else std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); unlink(FilePath.c_str()); #endif } #if ZEN_PLATFORM_WINDOWS BOOL Success = CloseHandle(m_FileHandle); #else int Fd = int(uintptr_t(m_FileHandle)); bool Success = (close(Fd) == 0); #endif if (!Success) { ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString()); } } } static constexpr size_t MappingLockCount = 128; static_assert(IsPow2(MappingLockCount), "MappingLockCount must be power of two"); static RwLock g_MappingLocks[MappingLockCount]; static uint64_t HashPtr64(uint64_t x) { x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); x = x ^ (x >> 31); return x; } static RwLock& MappingLockForInstance(const IoBufferExtendedCore* instance) { intptr_t base = (intptr_t)instance; uint32_t lock_index = uint32_t(HashPtr64(uint64_t(base)) & (MappingLockCount - 1u)); return g_MappingLocks[lock_index]; } void IoBufferExtendedCore::Materialize() const { // The synchronization scheme here is very primitive, if we end up with // a lot of contention we can make it more fine-grained if (m_Flags.load(std::memory_order_acquire) & kIsMaterialized) return; ZEN_TRACE_CPU("IoBufferExtendedCore::Materialize"); RwLock::ExclusiveLockScope _(MappingLockForInstance(this)); // Someone could have gotten here first // We can use memory_order_relaxed on this load because the mutex has already provided the fence if (m_Flags.load(std::memory_order_relaxed) & kIsMaterialized) return; uint32_t NewFlags = kIsMaterialized; if (m_DataBytes == 0) { // Fake a "valid" pointer, nobody should read this as size is zero m_DataPtr = reinterpret_cast(&m_MmapHandle); m_Flags.fetch_or(NewFlags, std::memory_order_release); return; } const size_t DisableMMapSizeLimit = 0x1000ull; if (m_DataBytes < DisableMMapSizeLimit) { ZEN_TRACE_CPU("IoBufferExtendedCore::Materialize::Read"); AllocateBuffer(m_DataBytes, sizeof(void*)); NewFlags |= kIsOwnedByThis; int32_t Error = 0; size_t BytesRead = 0; #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(m_FileOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(m_FileOffset >> 32); DWORD dwNumberOfBytesRead = 0; BOOL Success = ::ReadFile(m_FileHandle, (void*)m_DataPtr, DWORD(m_DataBytes), &dwNumberOfBytesRead, &Ovl) == TRUE; if (Success) { BytesRead = size_t(dwNumberOfBytesRead); } else { Error = zen::GetLastError(); } #else static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); int Fd = int(uintptr_t(m_FileHandle)); ssize_t ReadResult = pread(Fd, (void*)m_DataPtr, m_DataBytes, m_FileOffset); if (ReadResult != -1) { BytesRead = size_t(ReadResult); } else { Error = zen::GetLastError(); } #endif // ZEN_PLATFORM_WINDOWS if (Error || (BytesRead != m_DataBytes)) { std::error_code DummyEc; ZEN_WARN("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}", m_FileOffset, m_DataBytes, zen::PathFromHandle(m_FileHandle, DummyEc), zen::FileSizeFromHandle(m_FileHandle), GetSystemErrorAsString(Error)); throw std::system_error(std::error_code(Error, std::system_category()), fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", m_FileOffset, m_DataBytes, PathFromHandle(m_FileHandle, DummyEc), FileSizeFromHandle(m_FileHandle))); } m_Flags.fetch_or(NewFlags, std::memory_order_release); return; } ZEN_TRACE_CPU("IoBufferExtendedCore::Materialize::MMap"); void* NewMmapHandle; const uint64_t MapOffset = m_FileOffset & ~0xffffull; const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement; ZEN_ASSERT(MapSize > 0); #if ZEN_PLATFORM_WINDOWS NewMmapHandle = CreateFileMapping(m_FileHandle, /* lpFileMappingAttributes */ nullptr, /* flProtect */ PAGE_READONLY, /* dwMaximumSizeLow */ 0, /* dwMaximumSizeHigh */ 0, /* lpName */ nullptr); if (NewMmapHandle == nullptr) { int32_t Error = zen::GetLastError(); std::error_code DummyEc; ZEN_WARN("CreateFileMapping failed on file '{}', {}", zen::PathFromHandle(m_FileHandle, DummyEc), GetSystemErrorAsString(Error)); throw std::system_error(std::error_code(Error, std::system_category()), fmt::format("CreateFileMapping failed on file '{}'", zen::PathFromHandle(m_FileHandle, DummyEc))); } NewFlags |= kOwnsMmap; void* MappedBase = MapViewOfFile(NewMmapHandle, /* dwDesiredAccess */ FILE_MAP_READ, /* FileOffsetHigh */ uint32_t(MapOffset >> 32), /* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu), /* dwNumberOfBytesToMap */ MapSize); #else NewMmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0) NewFlags |= kOwnsMmap; void* MappedBase = mmap( /* addr */ nullptr, /* length */ MapSize, /* prot */ PROT_READ, /* flags */ MAP_SHARED | MAP_NORESERVE, /* fd */ int(uintptr_t(m_FileHandle)), /* offset */ MapOffset); #endif // ZEN_PLATFORM_WINDOWS if (MappedBase == nullptr) { int32_t Error = zen::GetLastError(); #if ZEN_PLATFORM_WINDOWS CloseHandle(NewMmapHandle); #endif // ZEN_PLATFORM_WINDOWS std::error_code DummyEc; ZEN_WARN("MapViewOfFile/mmap failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}", MapOffset, MapSize, zen::PathFromHandle(m_FileHandle, DummyEc), zen::FileSizeFromHandle(m_FileHandle), GetSystemErrorAsString(Error)); throw std::system_error(std::error_code(Error, std::system_category()), fmt::format("MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", MapOffset, MapSize, zen::PathFromHandle(m_FileHandle, DummyEc), zen::FileSizeFromHandle(m_FileHandle))); } m_MappedPointer = MappedBase; m_DataPtr = reinterpret_cast(MappedBase) + MappedOffsetDisplacement; m_MmapHandle = NewMmapHandle; m_Flags.fetch_or(NewFlags, std::memory_order_release); } bool IoBufferExtendedCore::GetFileReference(IoBufferFileReference& OutRef) const { if (m_FileHandle == nullptr) { return false; } OutRef.FileHandle = m_FileHandle; OutRef.FileChunkOffset = m_FileOffset; OutRef.FileChunkSize = m_DataBytes; return true; } void IoBufferExtendedCore::SetDeleteOnClose(bool DeleteOnClose) { if (DeleteOnClose && (m_Flags & kOwnsFile)) { m_Flags.fetch_or(kDeleteOnClose, std::memory_order_release); } else { m_Flags.fetch_and(~static_cast(kDeleteOnClose), std::memory_order_release); } } ////////////////////////////////////////////////////////////////////////// RefPtr IoBuffer::NullBufferCore(new IoBufferCore); IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize)) { m_Core->SetIsImmutable(false); } IoBuffer::IoBuffer(size_t InSize, uint64_t InAlignment) : m_Core(new IoBufferCore(InSize, InAlignment)) { m_Core->SetIsImmutable(false); } IoBuffer::IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t Size) { if (Size == ~(0ull)) { Size = std::clamp(Size, 0, OuterBuffer.Size() - Offset); } ZEN_ASSERT(Offset <= OuterBuffer.Size()); ZEN_ASSERT((Offset + Size) <= OuterBuffer.Size()); if (IoBufferExtendedCore* Extended = OuterBuffer.m_Core->ExtendedCore()) { m_Core = new IoBufferExtendedCore(Extended, Offset, Size); } else { m_Core = new IoBufferCore(OuterBuffer.m_Core, reinterpret_cast(OuterBuffer.Data()) + Offset, Size); } } IoBuffer::IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize, bool IsWholeFile) : m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ true)) { m_Core->SetIsWholeFile(IsWholeFile); } IoBuffer::IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize) : m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ false)) { } bool IoBuffer::GetFileReference(IoBufferFileReference& OutRef) const { if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore()) { if (ExtCore->GetFileReference(OutRef)) { return true; } } // Not a file reference OutRef.FileHandle = 0; OutRef.FileChunkOffset = ~0ull; OutRef.FileChunkSize = 0; return false; } void IoBuffer::SetDeleteOnClose(bool DeleteOnClose) { if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore()) { ExtCore->SetDeleteOnClose(DeleteOnClose); } } ////////////////////////////////////////////////////////////////////////// IoBuffer IoBufferBuilder::ReadFromFileMaybe(const IoBuffer& InBuffer) { IoBufferFileReference FileRef; if (InBuffer.GetFileReference(/* out */ FileRef)) { IoBuffer OutBuffer(FileRef.FileChunkSize); #if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; const uint64_t NumberOfBytesToRead = FileRef.FileChunkSize; const uint64_t& FileOffset = FileRef.FileChunkOffset; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(FileOffset >> 32); DWORD dwNumberOfBytesRead = 0; BOOL Success = ::ReadFile(FileRef.FileHandle, OutBuffer.MutableData(), DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); #else int Fd = int(intptr_t(FileRef.FileHandle)); int Result = pread(Fd, OutBuffer.MutableData(), size_t(FileRef.FileChunkSize), off_t(FileRef.FileChunkOffset)); bool Success = (Result >= 0); uint32_t dwNumberOfBytesRead = uint32_t(Result); #endif if (!Success) { ThrowLastError(fmt::format("file read failed in IoBufferBuilder::ReadFromFileMaybe (handle: {}, offset: {}, length: {})", intptr_t(FileRef.FileHandle), FileRef.FileChunkOffset, FileRef.FileChunkSize)); } ZEN_ASSERT(dwNumberOfBytesRead == FileRef.FileChunkSize); OutBuffer.SetContentType(InBuffer.GetContentType()); return OutBuffer; } else { return InBuffer; } } IoBuffer IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFileHandle"); return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); } IoBuffer IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFile"); uint64_t FileSize; #if ZEN_PLATFORM_WINDOWS windows::FileHandle DataFile; DWORD ShareOptions = FILE_SHARE_DELETE | FILE_SHARE_WRITE | FILE_SHARE_DELETE | FILE_SHARE_READ; HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, ShareOptions, OPEN_EXISTING); if (FAILED(hRes)) { return {}; } DataFile.GetSize((ULONGLONG&)FileSize); #else int Flags = O_RDONLY | O_CLOEXEC; int Fd = open(FileName.c_str(), Flags); if (Fd < 0) { return {}; } static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); struct stat Stat; fstat(Fd, &Stat); FileSize = Stat.st_size; #endif // ZEN_PLATFORM_WINDOWS // TODO: should validate that offset is in range if (Size == ~0ull) { Size = FileSize - Offset; } else { // Clamp size if ((Offset + Size) > FileSize) { Size = FileSize - Offset; } } if (Size) { #if ZEN_PLATFORM_WINDOWS void* Fd = DataFile.Detach(); #endif return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); } #if !ZEN_PLATFORM_WINDOWS close(Fd); #endif // For an empty file, we may as well just return an empty memory IoBuffer return IoBuffer(IoBuffer::Wrap, "", 0); } IoBuffer IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromTemporaryFile"); uint64_t FileSize; void* Handle; #if ZEN_PLATFORM_WINDOWS windows::FileHandle DataFile; // We need to open with DELETE since this is used for the case // when a file has been written to a staging directory, and is going // to be moved in place HRESULT hRes = DataFile.Create(FileName.native().c_str(), GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING); if (FAILED(hRes)) { return {}; } DataFile.GetSize((ULONGLONG&)FileSize); Handle = DataFile.Detach(); #else int Fd = open(FileName.native().c_str(), O_RDONLY); if (Fd < 0) { return {}; } static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); struct stat Stat; fstat(Fd, &Stat); FileSize = Stat.st_size; Handle = (void*)uintptr_t(Fd); #endif // ZEN_PLATFORM_WINDOWS return IoBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); } IoHash HashBuffer(IoBuffer& Buffer) { size_t BufferSize = Buffer.Size(); static const size_t BufferingSize = 512 * 1024; if (BufferSize >= (BufferingSize + BufferingSize / 2)) { IoBufferFileReference _; if (Buffer.GetFileReference(/* out */ _)) { size_t Offset = 0; IoHashStream HashStream; while (Offset < BufferSize) { size_t ChunkSize = Min(BufferSize - Offset, BufferingSize); IoBuffer SubRange(Buffer, Offset, ChunkSize); HashStream.Append(SubRange.GetData(), SubRange.GetSize()); Offset += ChunkSize; } return HashStream.GetHash(); } } return IoHash::HashBuffer(Buffer.Data(), BufferSize); } ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS void iobuffer_forcelink() { } TEST_CASE("IoBuffer") { zen::IoBuffer buffer1; zen::IoBuffer buffer2(16384); zen::IoBuffer buffer3(buffer2, 0, buffer2.Size()); } TEST_CASE("IoBuffer.mmap") { zen::IoBuffer Buffer1{65536}; uint8_t* Mutate = Buffer1.MutableData(); memcpy(Mutate, "abc123", 6); zen::WriteFile("test_file.data", Buffer1); SUBCASE("in-range") { zen::IoBuffer FileBuffer = IoBufferBuilder::MakeFromFile("test_file.data", 0, 65536); const void* Data = FileBuffer.GetData(); CHECK(Data != nullptr); CHECK_EQ(memcmp(Data, "abc123", 6), 0); } // Linux/MacOS offers different semantics when calling mmap with out-of-range so // for now let's ignore whether that makes sense or not # if ZEN_PLATFORM_WINDOWS SUBCASE("out-of-range") { zen::IoBuffer FileBuffer = IoBufferBuilder::MakeFromFile("test_file.data", 131072, 65536); const void* Data = nullptr; CHECK_THROWS(Data = FileBuffer.GetData()); CHECK(Data == nullptr); } # endif } #endif } // namespace zen