aboutsummaryrefslogtreecommitdiff
path: root/src/zencore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zencore')
-rw-r--r--src/zencore/basicfile.cpp1033
-rw-r--r--src/zencore/filesystem.cpp164
-rw-r--r--src/zencore/include/zencore/basicfile.h185
-rw-r--r--src/zencore/include/zencore/filesystem.h58
-rw-r--r--src/zencore/iobuffer.cpp2
-rw-r--r--src/zencore/memory/llm.cpp4
-rw-r--r--src/zencore/memtrack/memorytrace.cpp4
-rw-r--r--src/zencore/process.cpp2
-rw-r--r--src/zencore/zencore.cpp2
9 files changed, 1421 insertions, 33 deletions
diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
new file mode 100644
index 000000000..c2a21ae90
--- /dev/null
+++ b/src/zencore/basicfile.cpp
@@ -0,0 +1,1033 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/basicfile.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/except.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/memory/memory.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
+
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#else
+# include <fcntl.h>
+# include <sys/file.h>
+# include <sys/stat.h>
+# include <unistd.h>
+#endif
+
+#include <fmt/format.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+BasicFile::~BasicFile()
+{
+ Close();
+}
+
+void
+BasicFile::Open(const std::filesystem::path& FileName, Mode Mode)
+{
+ std::error_code Ec;
+ Open(FileName, Mode, Ec);
+
+ if (Ec)
+ {
+ throw std::system_error(Ec, fmt::format("failed to open file '{}', mode: {:x}", FileName, uint32_t(Mode)));
+ }
+}
+
+void
+BasicFile::Open(const std::filesystem::path& FileName, Mode InMode, std::error_code& Ec)
+{
+ Ec.clear();
+
+ Mode Mode = InMode & Mode::kModeMask;
+
+#if ZEN_PLATFORM_WINDOWS
+ DWORD dwCreationDisposition = 0;
+ DWORD dwDesiredAccess = 0;
+ switch (Mode)
+ {
+ case Mode::kRead:
+ dwCreationDisposition |= OPEN_EXISTING;
+ dwDesiredAccess |= GENERIC_READ;
+ break;
+ case Mode::kWrite:
+ dwCreationDisposition |= OPEN_ALWAYS;
+ dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE);
+ break;
+ case Mode::kDelete:
+ dwCreationDisposition |= OPEN_ALWAYS;
+ dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE);
+ break;
+ case Mode::kTruncate:
+ dwCreationDisposition |= CREATE_ALWAYS;
+ dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE);
+ break;
+ case Mode::kTruncateDelete:
+ dwCreationDisposition |= CREATE_ALWAYS;
+ dwDesiredAccess |= (GENERIC_READ | GENERIC_WRITE | DELETE);
+ break;
+ }
+
+ const DWORD dwShareMode = FILE_SHARE_READ | (EnumHasAllFlags(InMode, Mode::kPreventWrite) ? 0 : FILE_SHARE_WRITE) |
+ (EnumHasAllFlags(InMode, Mode::kPreventDelete) ? 0 : FILE_SHARE_DELETE);
+ const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+ const HANDLE hTemplateFile = nullptr;
+ const HANDLE FileHandle = CreateFile(FileName.c_str(),
+ dwDesiredAccess,
+ dwShareMode,
+ /* lpSecurityAttributes */ nullptr,
+ dwCreationDisposition,
+ dwFlagsAndAttributes,
+ hTemplateFile);
+
+ if (FileHandle == INVALID_HANDLE_VALUE)
+ {
+ Ec = MakeErrorCodeFromLastError();
+
+ return;
+ }
+#else
+ int OpenFlags = O_CLOEXEC;
+ switch (Mode)
+ {
+ case Mode::kRead:
+ OpenFlags |= O_RDONLY;
+ break;
+ case Mode::kWrite:
+ case Mode::kDelete:
+ OpenFlags |= (O_RDWR | O_CREAT);
+ break;
+ case Mode::kTruncate:
+ case Mode::kTruncateDelete:
+ OpenFlags |= (O_RDWR | O_CREAT | O_TRUNC);
+ break;
+ }
+
+ int Fd = open(FileName.c_str(), OpenFlags, 0666);
+ if (Fd < 0)
+ {
+ Ec = MakeErrorCodeFromLastError();
+ return;
+ }
+ if (Mode != Mode::kRead)
+ {
+ fchmod(Fd, 0666);
+ }
+
+ void* FileHandle = (void*)(uintptr_t(Fd));
+#endif
+
+ m_FileHandle = FileHandle;
+}
+
+void
+BasicFile::Open(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback)
+{
+ std::error_code Ec;
+ Open(FileName, Mode, Ec);
+ while (Ec && RetryCallback(Ec))
+ {
+ Ec.clear();
+ Open(FileName, Mode, Ec);
+ }
+ if (Ec)
+ {
+ throw std::system_error(Ec, fmt::format("failed to open file '{}', mode: {:x}", FileName, uint32_t(Mode)));
+ }
+}
+
+void
+BasicFile::Close()
+{
+ if (m_FileHandle)
+ {
+#if ZEN_PLATFORM_WINDOWS
+ ::CloseHandle(m_FileHandle);
+#else
+ int Fd = int(uintptr_t(m_FileHandle));
+ close(Fd);
+#endif
+ m_FileHandle = nullptr;
+ }
+}
+
+IoBuffer
+BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount)
+{
+ return IoBufferBuilder::MakeFromFileHandle(m_FileHandle, FileOffset, ByteCount);
+}
+
+void
+BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
+{
+ const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+
+ while (BytesToRead)
+ {
+ const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize);
+ int32_t Error = 0;
+ size_t BytesRead = 0;
+
+#if ZEN_PLATFORM_WINDOWS
+ OVERLAPPED Ovl{};
+
+ Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu);
+ Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+ DWORD dwNumberOfBytesRead = 0;
+ BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
+ if (Success)
+ {
+ BytesRead = size_t(dwNumberOfBytesRead);
+ }
+ else
+ {
+ Error = zen::GetLastError();
+ }
+#else
+ static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+ int Fd = int(uintptr_t(m_FileHandle));
+ ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
+ if (ReadResult != -1)
+ {
+ BytesRead = size_t(ReadResult);
+ }
+ else
+ {
+ Error = zen::GetLastError();
+ }
+#endif
+
+ if (Error || (BytesRead != NumberOfBytesToRead))
+ {
+ std::error_code DummyEc;
+ throw std::system_error(std::error_code(Error, std::system_category()),
+ fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
+ FileOffset,
+ NumberOfBytesToRead,
+ PathFromHandle(m_FileHandle, DummyEc),
+ FileSizeFromHandle(m_FileHandle)));
+ }
+
+ BytesToRead -= NumberOfBytesToRead;
+ FileOffset += NumberOfBytesToRead;
+ Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+ }
+}
+
+IoBuffer
+BasicFile::ReadAll()
+{
+ if (const uint64_t Size = FileSize())
+ {
+ IoBuffer Buffer(Size);
+ Read(Buffer.MutableData(), Size, 0);
+ return Buffer;
+ }
+ else
+ {
+ return {};
+ }
+}
+
+void
+BasicFile::StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+ StreamByteRange(0, FileSize(), std::move(ChunkFun));
+}
+
+void
+BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+ const uint64_t ChunkSize = 128 * 1024;
+ IoBuffer ReadBuffer{ChunkSize};
+ void* BufferPtr = ReadBuffer.MutableData();
+
+ uint64_t RemainBytes = Size;
+ uint64_t CurrentOffset = FileOffset;
+
+ while (RemainBytes)
+ {
+ const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes);
+
+ Read(BufferPtr, ThisChunkBytes, CurrentOffset);
+
+ ChunkFun(BufferPtr, ThisChunkBytes);
+
+ CurrentOffset += ThisChunkBytes;
+ RemainBytes -= ThisChunkBytes;
+ }
+}
+
+uint64_t
+BasicFile::Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec)
+{
+ uint64_t WrittenBytes = 0;
+ for (const SharedBuffer& Buffer : Data.GetSegments())
+ {
+ MemoryView BlockView = Buffer.GetView();
+ Write(BlockView, FileOffset + WrittenBytes, Ec);
+
+ if (Ec)
+ {
+ return WrittenBytes;
+ }
+
+ WrittenBytes += BlockView.GetSize();
+ }
+
+ return WrittenBytes;
+}
+
+void
+BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
+{
+ Write(Data.GetData(), Data.GetSize(), FileOffset, Ec);
+}
+
+void
+BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
+{
+ Ec.clear();
+
+ const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+
+ while (Size)
+ {
+ const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize);
+
+#if ZEN_PLATFORM_WINDOWS
+ OVERLAPPED Ovl{};
+
+ Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu);
+ Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+ DWORD dwNumberOfBytesWritten = 0;
+
+ BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
+#else
+ static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+ int Fd = int(uintptr_t(m_FileHandle));
+ int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset);
+ bool Success = (BytesWritten > 0);
+#endif
+
+ if (!Success)
+ {
+ Ec = MakeErrorCodeFromLastError();
+
+ return;
+ }
+
+ Size -= NumberOfBytesToWrite;
+ FileOffset += NumberOfBytesToWrite;
+ Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesToWrite;
+ }
+}
+
+void
+BasicFile::Write(MemoryView Data, uint64_t FileOffset)
+{
+ Write(Data.GetData(), Data.GetSize(), FileOffset);
+}
+
+void
+BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
+{
+ std::error_code Ec;
+ Write(Data, Size, Offset, Ec);
+
+ if (Ec)
+ {
+ std::error_code Dummy;
+ throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
+ }
+}
+
+void
+BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec)
+{
+ Write(Data.Data(), Data.Size(), 0, Ec);
+}
+
+void
+BasicFile::Flush()
+{
+ if (m_FileHandle == nullptr)
+ {
+ return;
+ }
+#if ZEN_PLATFORM_WINDOWS
+ FlushFileBuffers(m_FileHandle);
+#else
+ int Fd = int(uintptr_t(m_FileHandle));
+ fsync(Fd);
+#endif
+}
+
+uint64_t
+BasicFile::FileSize() const
+{
+#if ZEN_PLATFORM_WINDOWS
+ ULARGE_INTEGER liFileSize;
+ liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
+ if (liFileSize.LowPart == INVALID_FILE_SIZE)
+ {
+ int Error = zen::GetLastError();
+ if (Error)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
+ }
+ }
+ return uint64_t(liFileSize.QuadPart);
+#else
+ int Fd = int(uintptr_t(m_FileHandle));
+ static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+ struct stat Stat;
+ if (fstat(Fd, &Stat) == -1)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
+ }
+ return uint64_t(Stat.st_size);
+#endif
+}
+
+uint64_t
+BasicFile::FileSize(std::error_code& Ec) const
+{
+#if ZEN_PLATFORM_WINDOWS
+ ULARGE_INTEGER liFileSize;
+ liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
+ if (liFileSize.LowPart == INVALID_FILE_SIZE)
+ {
+ int Error = zen::GetLastError();
+ if (Error)
+ {
+ Ec = MakeErrorCode(Error);
+ return 0;
+ }
+ }
+ return uint64_t(liFileSize.QuadPart);
+#else
+ int Fd = int(uintptr_t(m_FileHandle));
+ static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+ struct stat Stat;
+ if (fstat(Fd, &Stat) == -1)
+ {
+ Ec = MakeErrorCodeFromLastError();
+ return 0;
+ }
+ return uint64_t(Stat.st_size);
+#endif
+}
+
+void
+BasicFile::SetFileSize(uint64_t FileSize)
+{
+#if ZEN_PLATFORM_WINDOWS
+ LARGE_INTEGER liFileSize;
+ liFileSize.QuadPart = FileSize;
+ BOOL OK = ::SetFilePointerEx(m_FileHandle, liFileSize, 0, FILE_BEGIN);
+ if (OK == FALSE)
+ {
+ int Error = zen::GetLastError();
+ if (Error)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(Error,
+ fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
+ }
+ }
+ OK = ::SetEndOfFile(m_FileHandle);
+ if (OK == FALSE)
+ {
+ int Error = zen::GetLastError();
+ if (Error)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(Error,
+ fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
+ }
+ }
+#elif ZEN_PLATFORM_MAC
+ int Fd = int(intptr_t(m_FileHandle));
+ if (ftruncate(Fd, (off_t)FileSize) < 0)
+ {
+ int Error = zen::GetLastError();
+ if (Error)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(Error,
+ fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
+ }
+ }
+#else
+ int Fd = int(intptr_t(m_FileHandle));
+ if (ftruncate64(Fd, (off64_t)FileSize) < 0)
+ {
+ int Error = zen::GetLastError();
+ if (Error)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(Error,
+ fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
+ }
+ }
+ if (FileSize > 0)
+ {
+ int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize);
+ if (Error)
+ {
+ std::error_code Dummy;
+ ThrowSystemError(Error,
+ fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
+ }
+ }
+#endif
+}
+
+void
+BasicFile::Attach(void* Handle)
+{
+ ZEN_ASSERT(Handle != nullptr);
+ ZEN_ASSERT(m_FileHandle == nullptr);
+ m_FileHandle = Handle;
+}
+
+void*
+BasicFile::Detach()
+{
+ void* FileHandle = m_FileHandle;
+ m_FileHandle = 0;
+ return FileHandle;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+TemporaryFile::~TemporaryFile()
+{
+ Close();
+}
+
+void
+TemporaryFile::Close()
+{
+ if (m_FileHandle)
+ {
+#if ZEN_PLATFORM_WINDOWS
+ // Mark file for deletion when final handle is closed
+
+ FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE};
+
+ SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
+#else
+ std::error_code Ec;
+ std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle, Ec);
+ if (!Ec)
+ {
+ unlink(FilePath.c_str());
+ }
+#endif
+
+ BasicFile::Close();
+ }
+}
+
+void
+TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec)
+{
+ StringBuilder<64> TempName;
+ Oid::NewOid().ToString(TempName);
+
+ m_TempPath = TempDirName / TempName.c_str();
+
+ Open(m_TempPath, BasicFile::Mode::kTruncateDelete, Ec);
+}
+
+void
+TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec)
+{
+ // We intentionally call the base class Close() since otherwise we'll end up
+ // deleting the temporary file
+ BasicFile::Close();
+
+ std::filesystem::rename(m_TempPath, FinalFileName, Ec);
+
+ if (Ec)
+ {
+ // Try to re-open the temp file so we clean up after us when TemporaryFile is destructed
+ std::error_code DummyEc;
+ Open(m_TempPath, BasicFile::Mode::kWrite, DummyEc);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data)
+{
+ TemporaryFile TempFile;
+ std::error_code Ec;
+ SafeWriteFile(Path, Data, Ec);
+ if (Ec)
+ {
+ throw std::system_error(Ec, fmt::format("Failed to safely write file '{}'", Path));
+ }
+}
+
+void
+TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc)
+{
+ TemporaryFile TempFile;
+ if (TempFile.CreateTemporary(Path.parent_path(), OutEc); !OutEc)
+ {
+ if (TempFile.Write(Data, 0, OutEc); !OutEc)
+ {
+ TempFile.MoveTemporaryIntoPlace(Path, OutEc);
+ }
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+LockFile::LockFile()
+{
+}
+
+LockFile::~LockFile()
+{
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ int Fd = int(intptr_t(m_FileHandle));
+ flock(Fd, LOCK_UN | LOCK_NB);
+#endif
+}
+
+void
+LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec)
+{
+#if ZEN_PLATFORM_WINDOWS
+ Ec.clear();
+
+ const DWORD dwCreationDisposition = CREATE_ALWAYS;
+ DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE | DELETE;
+ const DWORD dwShareMode = FILE_SHARE_READ;
+ const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE;
+ HANDLE hTemplateFile = nullptr;
+
+ HANDLE FileHandle = CreateFile(FileName.c_str(),
+ dwDesiredAccess,
+ dwShareMode,
+ /* lpSecurityAttributes */ nullptr,
+ dwCreationDisposition,
+ dwFlagsAndAttributes,
+ hTemplateFile);
+
+ if (FileHandle == INVALID_HANDLE_VALUE)
+ {
+ Ec = zen::MakeErrorCodeFromLastError();
+
+ return;
+ }
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ int Fd = open(FileName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666);
+ if (Fd < 0)
+ {
+ Ec = zen::MakeErrorCodeFromLastError();
+ return;
+ }
+ fchmod(Fd, 0666);
+
+ int LockRet = flock(Fd, LOCK_EX | LOCK_NB);
+ if (LockRet < 0)
+ {
+ Ec = zen::MakeErrorCodeFromLastError();
+ close(Fd);
+ return;
+ }
+
+ void* FileHandle = (void*)uintptr_t(Fd);
+#endif
+
+ m_FileHandle = FileHandle;
+
+ BasicFile::Write(Payload.GetBuffer(), 0, Ec);
+}
+
+void
+LockFile::Update(CbObject Payload, std::error_code& Ec)
+{
+ BasicFile::Write(Payload.GetBuffer(), 0, Ec);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+BasicFileBuffer::BasicFileBuffer(BasicFile& Base, uint64_t BufferSize)
+: m_Base(Base)
+, m_Buffer(nullptr)
+, m_BufferSize(BufferSize)
+, m_Size(Base.FileSize())
+, m_BufferStart(0)
+, m_BufferEnd(0)
+{
+ m_Buffer = (uint8_t*)Memory::Alloc(m_BufferSize);
+}
+
+BasicFileBuffer::~BasicFileBuffer()
+{
+ Memory::Free(m_Buffer);
+}
+
+void
+BasicFileBuffer::Read(void* Data, uint64_t Size, uint64_t FileOffset)
+{
+ if (m_Buffer == nullptr || (Size > m_BufferSize) || (FileOffset + Size > m_Size))
+ {
+ m_Base.Read(Data, Size, FileOffset);
+ return;
+ }
+ uint8_t* WritePtr = ((uint8_t*)Data);
+ uint64_t Begin = FileOffset;
+ uint64_t End = FileOffset + Size;
+ if (FileOffset <= m_BufferStart)
+ {
+ if (End > m_BufferStart)
+ {
+ uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart;
+ memcpy(WritePtr + End - Count - FileOffset, m_Buffer, Count);
+ End -= Count;
+ if (Begin == End)
+ {
+ return;
+ }
+ }
+ }
+ else if (FileOffset < m_BufferEnd)
+ {
+ uint64_t Count = Min(m_BufferEnd, End) - FileOffset;
+ memcpy(WritePtr + Begin - FileOffset, m_Buffer + Begin - m_BufferStart, Count);
+ Begin += Count;
+ if (Begin == End)
+ {
+ return;
+ }
+ }
+ m_BufferStart = Begin;
+ m_BufferEnd = Min(Begin + m_BufferSize, m_Size);
+ m_Base.Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart);
+ uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart;
+ memcpy(WritePtr + Begin - FileOffset, m_Buffer, Count);
+ ZEN_ASSERT(Begin + Count == End);
+}
+
+MemoryView
+BasicFileBuffer::MakeView(uint64_t Size, uint64_t FileOffset)
+{
+ if (FileOffset < m_BufferStart || (FileOffset + Size) > m_BufferEnd)
+ {
+ if (m_Buffer == nullptr || (Size > m_BufferSize) || (FileOffset + Size > m_Size))
+ {
+ return {};
+ }
+ m_BufferStart = FileOffset;
+ m_BufferEnd = Min(m_BufferStart + m_BufferSize, m_Size);
+ m_Base.Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart);
+ }
+ return MemoryView(m_Buffer + (FileOffset - m_BufferStart), Size);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+BasicFileWriter::BasicFileWriter(BasicFile& Base, uint64_t BufferSize)
+: m_Base(Base)
+, m_Buffer(nullptr)
+, m_BufferSize(BufferSize)
+, m_BufferStart(0)
+, m_BufferEnd(0)
+{
+ m_Buffer = (uint8_t*)Memory::Alloc(m_BufferSize);
+}
+
+BasicFileWriter::~BasicFileWriter()
+{
+ Flush();
+ Memory::Free(m_Buffer);
+}
+
+void
+BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
+{
+ if (m_Buffer == nullptr || (Size >= m_BufferSize))
+ {
+ m_Base.Write(Data, Size, FileOffset);
+ return;
+ }
+
+ // Note that this only supports buffering of sequential writes!
+
+ if (FileOffset != m_BufferEnd)
+ {
+ Flush();
+ m_BufferStart = m_BufferEnd = FileOffset;
+ }
+
+ const uint8_t* DataPtr = (const uint8_t*)Data;
+ while (Size)
+ {
+ const uint64_t RemainingBufferCapacity = m_BufferStart + m_BufferSize - m_BufferEnd;
+ const uint64_t BlockWriteBytes = Min(RemainingBufferCapacity, Size);
+ const uint64_t BufferWriteOffset = FileOffset - m_BufferStart;
+
+ ZEN_ASSERT_SLOW(BufferWriteOffset < m_BufferSize);
+ ZEN_ASSERT_SLOW((BufferWriteOffset + BlockWriteBytes) <= m_BufferSize);
+
+ memcpy(m_Buffer + BufferWriteOffset, DataPtr, BlockWriteBytes);
+
+ Size -= BlockWriteBytes;
+ m_BufferEnd += BlockWriteBytes;
+ FileOffset += BlockWriteBytes;
+ DataPtr += BlockWriteBytes;
+
+ if ((m_BufferEnd - m_BufferStart) == m_BufferSize)
+ {
+ Flush();
+ }
+ }
+}
+
+void
+BasicFileWriter::Flush()
+{
+ const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart;
+
+ if (BufferedBytes == 0)
+ return;
+
+ const uint64_t WriteOffset = m_BufferStart;
+ m_BufferStart = m_BufferEnd;
+
+ m_Base.Write(m_Buffer, BufferedBytes, WriteOffset);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+/*
+ ___________ __
+ \__ ___/___ _______/ |_ ______
+ | |_/ __ \ / ___/\ __\/ ___/
+ | |\ ___/ \___ \ | | \___ \
+ |____| \___ >____ > |__| /____ >
+ \/ \/ \/
+*/
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("BasicFile")
+{
+ ScopedCurrentDirectoryChange _;
+
+ BasicFile File1;
+ CHECK_THROWS(File1.Open("zonk", BasicFile::Mode::kRead));
+ CHECK_NOTHROW(File1.Open("zonk", BasicFile::Mode::kTruncate));
+ CHECK_NOTHROW(File1.Write("abcd", 4, 0));
+ CHECK(File1.FileSize() == 4);
+ {
+ IoBuffer Data = File1.ReadAll();
+ CHECK(Data.Size() == 4);
+ CHECK_EQ(memcmp(Data.Data(), "abcd", 4), 0);
+ }
+ CHECK_NOTHROW(File1.Write("efgh", 4, 2));
+ CHECK(File1.FileSize() == 6);
+ {
+ IoBuffer Data = File1.ReadAll();
+ CHECK(Data.Size() == 6);
+ CHECK_EQ(memcmp(Data.Data(), "abefgh", 6), 0);
+ }
+}
+
+TEST_CASE("TemporaryFile")
+{
+ ScopedCurrentDirectoryChange _;
+
+ SUBCASE("DeleteOnClose")
+ {
+ std::filesystem::path Path;
+ {
+ TemporaryFile TmpFile;
+ std::error_code Ec;
+ TmpFile.CreateTemporary(std::filesystem::current_path(), Ec);
+ CHECK(!Ec);
+ Path = TmpFile.GetPath();
+ CHECK(std::filesystem::exists(Path));
+ }
+ CHECK(std::filesystem::exists(Path) == false);
+ }
+
+ SUBCASE("MoveIntoPlace")
+ {
+ TemporaryFile TmpFile;
+ std::error_code Ec;
+ TmpFile.CreateTemporary(std::filesystem::current_path(), Ec);
+ CHECK(!Ec);
+ std::filesystem::path TempPath = TmpFile.GetPath();
+ std::filesystem::path FinalPath = std::filesystem::current_path() / "final";
+ CHECK(std::filesystem::exists(TempPath));
+ TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec);
+ CHECK(!Ec);
+ CHECK(std::filesystem::exists(TempPath) == false);
+ CHECK(std::filesystem::exists(FinalPath));
+ }
+}
+
+TEST_CASE("BasicFileBuffer")
+{
+ ScopedCurrentDirectoryChange _;
+ {
+ BasicFile File1;
+ const std::string_view Data = "0123456789abcdef";
+ File1.Open("buffered", BasicFile::Mode::kTruncate);
+ for (uint32_t I = 0; I < 16; ++I)
+ {
+ File1.Write(Data.data(), Data.size(), I * Data.size());
+ }
+ }
+ SUBCASE("EvenBuffer")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ // Non-primed
+ {
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 1 * 16);
+ std::string_view Verify(Buffer, 16);
+ CHECK(Verify == "0123456789abcdef");
+ }
+ // Primed
+ {
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 1 * 16);
+ std::string_view Verify(Buffer, 16);
+ CHECK(Verify == "0123456789abcdef");
+ }
+ }
+ SUBCASE("UnevenBuffer")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ // Non-primed
+ {
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 7);
+ std::string_view Verify(Buffer, 16);
+ CHECK(Verify == "789abcdef0123456");
+ }
+ // Primed
+ {
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 7);
+ std::string_view Verify(Buffer, 16);
+ CHECK(Verify == "789abcdef0123456");
+ }
+ }
+ SUBCASE("BiggerThanBuffer")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[17] = {0};
+ File1Buffer.Read(Buffer, 17, 0 * 16);
+ std::string_view Verify(Buffer, 17);
+ CHECK(Verify == "0123456789abcdef0");
+ }
+ SUBCASE("InsideBuffer")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 0 * 16);
+
+ File1Buffer.Read(Buffer, 8, 2);
+ std::string_view Verify(Buffer, 8);
+ CHECK(Verify == "23456789");
+ }
+ SUBCASE("BeginningOfBuffer")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 8);
+
+ File1Buffer.Read(Buffer, 8, 8);
+ std::string_view Verify(Buffer, 8);
+ CHECK(Verify == "89abcdef");
+ }
+ SUBCASE("EndOfBuffer")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 0 * 16);
+
+ File1Buffer.Read(Buffer, 8, 8);
+ std::string_view Verify(Buffer, 8);
+ CHECK(Verify == "89abcdef");
+ }
+ SUBCASE("OverEnd")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 0 * 16);
+
+ File1Buffer.Read(Buffer, 16, 8);
+ std::string_view Verify(Buffer, 16);
+ CHECK(Verify == "89abcdef01234567");
+ }
+ SUBCASE("OverBegin")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 1 * 16);
+
+ File1Buffer.Read(Buffer, 16, 8);
+ std::string_view Verify(Buffer, 16);
+ CHECK(Verify == "89abcdef01234567");
+ }
+ SUBCASE("EndOfFile")
+ {
+ BasicFile File1;
+ File1.Open("buffered", BasicFile::Mode::kRead);
+ BasicFileBuffer File1Buffer(File1, 16);
+ char Buffer[16] = {0};
+ File1Buffer.Read(Buffer, 16, 0 * 16);
+
+ File1Buffer.Read(Buffer, 8, 256 - 8);
+ std::string_view Verify(Buffer, 8);
+ CHECK(Verify == "89abcdef");
+ }
+}
+
+void
+basicfile_forcelink()
+{
+}
+
+#endif
+
+} // namespace zen
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 52f2c4adc..b8c35212f 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -9,9 +9,11 @@
#include <zencore/logging.h>
#include <zencore/memory/memory.h>
#include <zencore/process.h>
+#include <zencore/scopeguard.h>
#include <zencore/stream.h>
#include <zencore/string.h>
#include <zencore/testing.h>
+#include <zencore/workthreadpool.h>
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
@@ -681,7 +683,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
{
}
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
{
std::error_code Ec;
const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
@@ -727,7 +729,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
}
}
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
std::filesystem::path BasePath;
std::filesystem::path TargetPath;
@@ -1215,7 +1217,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
}
else
{
- const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName);
+ const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
if (ShouldDescend)
{
@@ -1234,7 +1236,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
}
else
{
- Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
+ Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
}
const uint64_t NextOffset = DirInfo->NextEntryOffset;
@@ -1276,14 +1278,14 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
{
/* nop */
}
- else if (Visitor.VisitDirectory(RootDir, FileName))
+ else if (Visitor.VisitDirectory(RootDir, FileName, gsl::narrow<uint32_t>(Stat.st_mode)))
{
TraverseFileSystem(FullPath, Visitor);
}
}
else if (S_ISREG(Stat.st_mode))
{
- Visitor.VisitFile(RootDir, FileName, Stat.st_size);
+ Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode));
}
else
{
@@ -1527,39 +1529,163 @@ MaximizeOpenFileCount()
}
void
-GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
+GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent)
{
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeDirs));
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)
+ ? true
+ : (!EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes)));
+
FileSystemTraversal Traversal;
struct Visitor : public FileSystemTraversal::TreeVisitor
{
- Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
+ Visitor(DirectoryContentFlags Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
- virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
- [[maybe_unused]] const path_view& File,
- [[maybe_unused]] uint64_t FileSize) override
+ virtual void VisitFile(const std::filesystem::path& Parent,
+ const path_view& File,
+ uint64_t FileSize,
+ uint32_t NativeModeOrAttributes) override
{
- if (Flags & DirectoryContent::IncludeFilesFlag)
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
{
Content.Files.push_back(Parent / File);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes))
+ {
+ Content.FileSizes.push_back(FileSize);
+ }
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.FileAttributes.push_back(NativeModeOrAttributes);
+ }
}
}
- virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+ virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
+ const path_view& DirectoryName,
+ uint32_t NativeModeOrAttributes) override
{
- if (Flags & DirectoryContent::IncludeDirsFlag)
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeDirs))
{
Content.Directories.push_back(Parent / DirectoryName);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.DirectoryAttributes.push_back(NativeModeOrAttributes);
+ }
}
- return (Flags & DirectoryContent::RecursiveFlag) != 0;
+ return EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive);
}
- const uint8_t Flags;
- DirectoryContent& Content;
+ const DirectoryContentFlags Flags;
+ DirectoryContent& Content;
} Visit(Flags, OutContent);
Traversal.TraverseFileSystem(RootDir, Visit);
}
+void
+GetDirectoryContent(const std::filesystem::path& RootDir,
+ DirectoryContentFlags Flags,
+ GetDirectoryContentVisitor& Visitor,
+ WorkerThreadPool& WorkerPool,
+ Latch& PendingWorkCount)
+{
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeDirs));
+ ZEN_ASSERT(EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)
+ ? true
+ : (!EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes)));
+
+ struct MultithreadedVisitor : public FileSystemTraversal::TreeVisitor
+ {
+ MultithreadedVisitor(WorkerThreadPool& InWorkerPool,
+ Latch& InOutPendingWorkCount,
+ const std::filesystem::path& InRelativeRoot,
+ DirectoryContentFlags InFlags,
+ GetDirectoryContentVisitor* InVisitor)
+ : WorkerPool(InWorkerPool)
+ , PendingWorkCount(InOutPendingWorkCount)
+ , RelativeRoot(InRelativeRoot)
+ , Flags(InFlags)
+ , Visitor(InVisitor)
+ {
+ }
+
+ virtual void VisitFile(const std::filesystem::path&,
+ const path_view& File,
+ uint64_t FileSize,
+ uint32_t NativeModeOrAttributes) override
+ {
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
+ {
+ Content.FileNames.push_back(File);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFileSizes))
+ {
+ Content.FileSizes.push_back(FileSize);
+ }
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.FileAttributes.push_back(NativeModeOrAttributes);
+ }
+ }
+ }
+
+ virtual bool VisitDirectory(const std::filesystem::path& Parent,
+ const path_view& DirectoryName,
+ uint32_t NativeModeOrAttributes) override
+ {
+ std::filesystem::path Path = Parent / DirectoryName;
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeDirs))
+ {
+ Content.DirectoryNames.push_back(DirectoryName);
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeAttributes))
+ {
+ Content.DirectoryAttributes.push_back(NativeModeOrAttributes);
+ }
+ }
+ if (EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive))
+ {
+ PendingWorkCount.AddCount(1);
+ try
+ {
+ WorkerPool.ScheduleWork([WorkerPool = &WorkerPool,
+ PendingWorkCount = &PendingWorkCount,
+ Visitor = Visitor,
+ Flags = Flags,
+ Path = std::move(Path),
+ RelativeRoot = RelativeRoot / DirectoryName]() {
+ ZEN_ASSERT(Visitor);
+ auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); });
+ {
+ MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor);
+ FileSystemTraversal Traversal;
+ Traversal.TraverseFileSystem(Path, SubVisitor);
+ Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content));
+ }
+ });
+ }
+ catch (const std::exception Ex)
+ {
+ ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what());
+ PendingWorkCount.CountDown();
+ }
+ }
+ return false;
+ }
+
+ WorkerThreadPool& WorkerPool;
+ Latch& PendingWorkCount;
+
+ const std::filesystem::path RelativeRoot;
+ const DirectoryContentFlags Flags;
+
+ GetDirectoryContentVisitor::DirectoryContent Content;
+ GetDirectoryContentVisitor* Visitor;
+ } WrapperVisitor(WorkerPool, PendingWorkCount, {}, Flags, &Visitor);
+
+ FileSystemTraversal Traversal;
+ Traversal.TraverseFileSystem(RootDir, WrapperVisitor);
+ Visitor.AsyncVisitDirectory(WrapperVisitor.RelativeRoot, std::move(WrapperVisitor.Content));
+}
+
std::string
GetEnvVariable(std::string_view VariableName)
{
@@ -1802,12 +1928,12 @@ TEST_CASE("filesystem")
// Traversal
struct : public FileSystemTraversal::TreeVisitor
{
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override
{
bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
}
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
bool bFoundExpected = false;
std::filesystem::path Expected;
diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
new file mode 100644
index 000000000..03c5605df
--- /dev/null
+++ b/src/zencore/include/zencore/basicfile.h
@@ -0,0 +1,185 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compositebuffer.h>
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+
+#include <filesystem>
+#include <functional>
+
+namespace zen {
+
+class CbObject;
+
+/**
+ * Probably the most basic file abstraction in the universe
+ *
+ * One thing of note is that there is no notion of a "current file position"
+ * in this API -- all reads and writes are done from explicit offsets in
+ * the file. This avoids concurrency issues which can occur otherwise.
+ *
+ */
+
+class BasicFile
+{
+public:
+ BasicFile() = default;
+ ~BasicFile();
+
+ BasicFile(const BasicFile&) = delete;
+ BasicFile& operator=(const BasicFile&) = delete;
+
+ enum class Mode : uint32_t
+ {
+ kRead = 0, // Opens a existing file for read only
+ kWrite = 1, // Opens (or creates) a file for read and write
+ kTruncate = 2, // Opens (or creates) a file for read and write and sets the size to zero
+ kDelete = 3, // Opens (or creates) a file for read and write allowing .DeleteFile file disposition to be set
+ kTruncateDelete =
+ 4, // Opens (or creates) a file for read and write and sets the size to zero allowing .DeleteFile file disposition to be set
+ kModeMask = 0x0007,
+ kPreventDelete = 0x1000'0000, // Do not open with delete sharing mode (prevent other processes from deleting file while open)
+ kPreventWrite = 0x2000'0000, // Do not open with write sharing mode (prevent other processes from writing to file while open)
+ };
+
+ void Open(const std::filesystem::path& FileName, Mode Mode);
+ void Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec);
+ void Open(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback);
+ void Close();
+ void Read(void* Data, uint64_t Size, uint64_t FileOffset);
+ IoBuffer ReadRange(uint64_t FileOffset, uint64_t ByteCount);
+ void StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+ void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+ void Write(MemoryView Data, uint64_t FileOffset);
+ void Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec);
+ uint64_t Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec);
+ void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+ void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
+ void Flush();
+ [[nodiscard]] uint64_t FileSize() const;
+ [[nodiscard]] uint64_t FileSize(std::error_code& Ec) const;
+ void SetFileSize(uint64_t FileSize);
+ IoBuffer ReadAll();
+ void WriteAll(IoBuffer Data, std::error_code& Ec);
+ void Attach(void* Handle);
+ void* Detach();
+
+ inline void* Handle() { return m_FileHandle; }
+ bool IsOpen() const { return m_FileHandle != nullptr; }
+
+protected:
+ void* m_FileHandle = nullptr; // This is either null or valid
+private:
+};
+
+ENUM_CLASS_FLAGS(BasicFile::Mode);
+
+/**
+ * Simple abstraction for a temporary file
+ *
+ * Works like a regular BasicFile but implements a simple mechanism to allow creating
+ * a temporary file for writing in a directory which may later be moved atomically
+ * into the intended location after it has been fully written to.
+ *
+ */
+
+class TemporaryFile : public BasicFile
+{
+public:
+ TemporaryFile() = default;
+ ~TemporaryFile();
+
+ TemporaryFile(const TemporaryFile&) = delete;
+ TemporaryFile& operator=(const TemporaryFile&) = delete;
+
+ void CreateTemporary(std::filesystem::path TempDirName, std::error_code& Ec);
+ void MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::error_code& Ec);
+ const std::filesystem::path& GetPath() const { return m_TempPath; }
+
+ static void SafeWriteFile(const std::filesystem::path& Path, MemoryView Data);
+ static void SafeWriteFile(const std::filesystem::path& Path, MemoryView Data, std::error_code& OutEc);
+
+private:
+ void Close();
+ std::filesystem::path m_TempPath;
+
+ using BasicFile::Open;
+};
+
+/** Lock file abstraction
+
+ */
+
+class LockFile : protected BasicFile
+{
+public:
+ LockFile();
+ ~LockFile();
+
+ void Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec);
+ void Update(CbObject Payload, std::error_code& Ec);
+
+private:
+};
+
+/** Adds a layer of buffered reading to a BasicFile
+
+This class is not intended for concurrent access, it is not thread safe.
+
+*/
+
+class BasicFileBuffer
+{
+public:
+ BasicFileBuffer(BasicFile& Base, uint64_t BufferSize);
+ ~BasicFileBuffer();
+
+ void Read(void* Data, uint64_t Size, uint64_t FileOffset);
+ MemoryView MakeView(uint64_t Size, uint64_t FileOffset);
+
+ template<typename T>
+ const T* MakeView(uint64_t FileOffset)
+ {
+ MemoryView View = MakeView(sizeof(T), FileOffset);
+ return reinterpret_cast<const T*>(View.GetData());
+ }
+
+private:
+ BasicFile& m_Base;
+ uint8_t* m_Buffer;
+ const uint64_t m_BufferSize;
+ uint64_t m_Size;
+ uint64_t m_BufferStart;
+ uint64_t m_BufferEnd;
+};
+
+/** Adds a layer of buffered writing to a BasicFile
+
+This class is not intended for concurrent access, it is not thread safe.
+
+*/
+
+class BasicFileWriter
+{
+public:
+ BasicFileWriter(BasicFile& Base, uint64_t BufferSize);
+ ~BasicFileWriter();
+
+ void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+ void Flush();
+
+private:
+ BasicFile& m_Base;
+ uint8_t* m_Buffer;
+ const uint64_t m_BufferSize;
+ uint64_t m_BufferStart;
+ uint64_t m_BufferEnd;
+};
+
+ZENCORE_API void basicfile_forcelink();
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index dba4981f0..ca8682cd7 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -4,6 +4,7 @@
#include "zencore.h"
+#include <zencore/enumflags.h>
#include <zencore/iobuffer.h>
#include <zencore/string.h>
@@ -12,8 +13,10 @@
namespace zen {
-class IoBuffer;
class CompositeBuffer;
+class IoBuffer;
+class Latch;
+class WorkerThreadPool;
/** Delete directory (after deleting any contents)
*/
@@ -195,28 +198,65 @@ class FileSystemTraversal
public:
struct TreeVisitor
{
- using path_view = std::basic_string_view<std::filesystem::path::value_type>;
- using path_string = std::filesystem::path::string_type;
+ using path_view = std::basic_string_view<std::filesystem::path::value_type>;
- virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0;
+ virtual void VisitFile(const std::filesystem::path& Parent,
+ const path_view& File,
+ uint64_t FileSize,
+ uint32_t NativeModeOrAttributes) = 0;
// This should return true if we should recurse into the directory
- virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0;
+ virtual bool VisitDirectory(const std::filesystem::path& Parent,
+ const path_view& DirectoryName,
+ uint32_t NativeModeOrAttributes) = 0;
};
void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
};
+enum class DirectoryContentFlags : uint8_t
+{
+ None = 0,
+ IncludeDirs = 1u << 0,
+ IncludeFiles = 1u << 1,
+ Recursive = 1u << 2,
+ IncludeFileSizes = 1u << 3,
+ IncludeAttributes = 1u << 4,
+ IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+};
+
+ENUM_CLASS_FLAGS(DirectoryContentFlags)
+
struct DirectoryContent
{
- static const uint8_t IncludeDirsFlag = 1u << 0;
- static const uint8_t IncludeFilesFlag = 1u << 1;
- static const uint8_t RecursiveFlag = 1u << 2;
std::vector<std::filesystem::path> Files;
+ std::vector<uint64_t> FileSizes;
+ std::vector<uint32_t> FileAttributes;
std::vector<std::filesystem::path> Directories;
+ std::vector<uint32_t> DirectoryAttributes;
+};
+
+void GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent);
+
+struct GetDirectoryContentVisitor
+{
+public:
+ struct DirectoryContent
+ {
+ std::vector<std::filesystem::path> FileNames;
+ std::vector<uint64_t> FileSizes;
+ std::vector<uint32_t> FileAttributes;
+ std::vector<std::filesystem::path> DirectoryNames;
+ std::vector<uint32_t> DirectoryAttributes;
+ };
+ virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0;
};
-void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent);
+void GetDirectoryContent(const std::filesystem::path& RootDir,
+ DirectoryContentFlags Flags,
+ GetDirectoryContentVisitor& Visitor,
+ WorkerThreadPool& WorkerPool,
+ Latch& PendingWorkCount);
std::string GetEnvVariable(std::string_view VariableName);
diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 3d9d6706a..3b5c89c3e 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -289,7 +289,7 @@ IoBufferExtendedCore::Materialize() const
return;
}
- const size_t DisableMMapSizeLimit = 0x1000ull;
+ const size_t DisableMMapSizeLimit = 0x2000ull;
if (m_DataBytes < DisableMMapSizeLimit)
{
diff --git a/src/zencore/memory/llm.cpp b/src/zencore/memory/llm.cpp
index fe4853d49..61fa29a66 100644
--- a/src/zencore/memory/llm.cpp
+++ b/src/zencore/memory/llm.cpp
@@ -15,6 +15,7 @@ static const int32_t TagNamesBaseIndex = 256;
static const int32_t TrackedTagNameCount = 256;
static const char* TagNames[TrackedTagNameCount];
static uint32_t TagNameHashes[TrackedTagNameCount];
+static int32_t ParentTags[TrackedTagNameCount];
static RwLock TableLock;
@@ -46,7 +47,7 @@ FLLMTag::AssignAndAnnounceNewTag(const char* TagName)
for (int TagIndex = 0; TagIndex <= CurrentMaxTagIndex; ++TagIndex)
{
- if (TagNameHashes[TagIndex] == TagNameHash)
+ if (TagNameHashes[TagIndex] == TagNameHash && ParentTags[TagIndex] == m_ParentTag)
{
m_Tag = TagIndex + TagNamesBaseIndex;
// could verify the string matches here to catch hash collisions
@@ -64,6 +65,7 @@ FLLMTag::AssignAndAnnounceNewTag(const char* TagName)
{
TagNameHashes[TagIndex] = TagNameHash;
TagNames[TagIndex] = TagName;
+ ParentTags[TagIndex] = m_ParentTag;
}
else
{
diff --git a/src/zencore/memtrack/memorytrace.cpp b/src/zencore/memtrack/memorytrace.cpp
index 7089c356a..e4ae8148e 100644
--- a/src/zencore/memtrack/memorytrace.cpp
+++ b/src/zencore/memtrack/memorytrace.cpp
@@ -41,8 +41,8 @@ void MemoryTrace_EnableTracePump();
////////////////////////////////////////////////////////////////////////////////
namespace {
-// Controls how often time markers are emitted (default: every 4095 allocations).
-constexpr uint32_t MarkerSamplePeriod = (4 << 10) - 1;
+// Controls how often time markers are emitted (must be POW2-1 as this is used as a mask)
+constexpr uint32_t MarkerSamplePeriod = 128 - 1;
// Number of shifted bits to SizeLower
constexpr uint32_t SizeShift = 3;
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index b1da034d2..c51e8f69d 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -1012,7 +1012,7 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
#if ZEN_PLATFORM_LINUX
std::vector<uint32_t> RunningPids;
DirectoryContent ProcList;
- GetDirectoryContent("/proc", DirectoryContent::IncludeDirsFlag, ProcList);
+ GetDirectoryContent("/proc", DirectoryContentFlags::IncludeDirs, ProcList);
for (const std::filesystem::path& EntryPath : ProcList.Directories)
{
std::string EntryName = EntryPath.stem();
diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp
index db821bff8..82d28c0e3 100644
--- a/src/zencore/zencore.cpp
+++ b/src/zencore/zencore.cpp
@@ -7,6 +7,7 @@
#endif
#include <zencore/assertfmt.h>
+#include <zencore/basicfile.h>
#include <zencore/blake3.h>
#include <zencore/callstack.h>
#include <zencore/compactbinary.h>
@@ -241,6 +242,7 @@ ApplicationExitCode()
void
zencore_forcelinktests()
{
+ zen::basicfile_forcelink();
zen::blake3_forcelink();
zen::callstack_forcelink();
zen::compositebuffer_forcelink();
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 16:58:16 +0000