From 800305f7f2afe01e9ca038a71a68e9318c52ee77 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 09:05:07 +0100 Subject: move WriteToTempFile to basicfile.h (#283) add helper constructors to BasicFile --- src/zencore/basicfile.cpp | 64 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index c2a21ae90..b3bffd34d 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -28,6 +28,20 @@ BasicFile::~BasicFile() { Close(); } +BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode) +{ + Open(FileName, Mode); +} + +BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec) +{ + Open(FileName, Mode, Ec); +} + +BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback) +{ + Open(FileName, Mode, std::move(RetryCallback)); +} void BasicFile::Open(const std::filesystem::path& FileName, Mode Mode) @@ -267,7 +281,7 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& RetryCallback) +{ + if (std::filesystem::is_regular_file(Path)) + { + IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path)); + if (ExistingTempFile && ExistingTempFile.GetSize() == Buffer.GetSize()) + { + ExistingTempFile.SetDeleteOnClose(true); + return ExistingTempFile; + } + } + BasicFile BlockFile; + BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, std::move(RetryCallback)); + uint64_t Offset = 0; + { + static const uint64_t BufferingSize = 256u * 1024u; + BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); + for (const SharedBuffer& Segment : Buffer.GetSegments()) + { + size_t SegmentSize = Segment.GetSize(); + + IoBufferFileReference FileRef; + if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef)) + { + ScanFile(FileRef.FileHandle, + FileRef.FileChunkOffset, + FileRef.FileChunkSize, + BufferingSize, + [&BufferedOutput, &Offset](const void* Data, size_t Size) { + BufferedOutput.Write(Data, Size, Offset); + Offset += Size; + }); + } + else + { + BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset); + Offset += SegmentSize; + } + } + } + void* FileHandle = BlockFile.Detach(); + IoBuffer BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); + BlockBuffer.SetDeleteOnClose(true); + return BlockBuffer; +} + ////////////////////////////////////////////////////////////////////////// /* -- cgit v1.2.3 From 5bc5b0dd59c0f02afe553e5074dfe57951b19044 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 25 Feb 2025 15:48:43 +0100 Subject: improvements and infrastructure for upcoming builds api command line (#284) * add modification tick to filesystem traversal * add ShowDetails option to ProgressBar * log callstack if we terminate process * handle chunking if MaxSize > 1MB * BasicFile write helpers and WriteToTempFile simplifications * bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data revert of earlier optimization * faster compress/decompress for large disk-based files * enable progress feedback in IoHash::HashBuffer * add payload validation in HttpClient::Get * fix range requests (range is including end byte) * remove BuildPartId for blob/block related operations in builds api --- src/zencore/basicfile.cpp | 79 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 19 deletions(-) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index b3bffd34d..6e879ca0d 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -280,6 +280,20 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& RetryCallback) +WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path) { - if (std::filesystem::is_regular_file(Path)) + TemporaryFile Temp; + std::error_code Ec; + Temp.CreateTemporary(Path.parent_path(), Ec); + if (Ec) { - IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path)); - if (ExistingTempFile && ExistingTempFile.GetSize() == Buffer.GetSize()) - { - ExistingTempFile.SetDeleteOnClose(true); - return ExistingTempFile; - } + throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path)); } - BasicFile BlockFile; - BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, std::move(RetryCallback)); - uint64_t Offset = 0; + { + uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; - BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); + // BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); for (const SharedBuffer& Segment : Buffer.GetSegments()) { size_t SegmentSize = Segment.GetSize(); @@ -859,22 +883,39 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path, std FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, - [&BufferedOutput, &Offset](const void* Data, size_t Size) { - BufferedOutput.Write(Data, Size, Offset); + [&Temp, &Offset](const void* Data, size_t Size) { + Temp.Write(Data, Size, Offset); Offset += Size; }); } else { - BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset); + Temp.Write(Segment.GetData(), SegmentSize, Offset); Offset += SegmentSize; } } } - void* FileHandle = BlockFile.Detach(); - IoBuffer BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); - BlockBuffer.SetDeleteOnClose(true); - return BlockBuffer; + + Temp.MoveTemporaryIntoPlace(Path, Ec); + if (Ec) + { + IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); + if (TmpBuffer) + { + IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); + const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); + if (ExistingHash == ExpectedHash) + { + TmpBuffer.SetDeleteOnClose(true); + return TmpBuffer; + } + } + throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); + } + + IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); + TmpBuffer.SetDeleteOnClose(true); + return TmpBuffer; } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 7de3d4218ee5969af6147f9ab20bda538a136d9a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 10 Mar 2025 18:33:24 +0100 Subject: pick up existing cache (#299) - Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run - Improvement: Faster abort during stream compression - Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write - Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails --- src/zencore/basicfile.cpp | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 6e879ca0d..95876cff4 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -858,7 +858,7 @@ BasicFileWriter::Flush() } IoBuffer -WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path) +WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) { TemporaryFile Temp; std::error_code Ec; @@ -868,6 +868,7 @@ WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path)); } + uint64_t BufferSize = Buffer.GetSize(); { uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; @@ -899,21 +900,31 @@ WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path Temp.MoveTemporaryIntoPlace(Path, Ec); if (Ec) { - IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); - if (TmpBuffer) + Ec.clear(); + BasicFile OpenTemp(Path, BasicFile::Mode::kDelete, Ec); + if (Ec) { - IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); - const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); - if (ExistingHash == ExpectedHash) - { - TmpBuffer.SetDeleteOnClose(true); - return TmpBuffer; - } + throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); } - throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); - } + if (OpenTemp.FileSize() != BufferSize) + { + throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file size already exists", Path)); + } + IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true); - IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); + IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); + const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); + if (ExistingHash != ExpectedHash) + { + throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file hash already exists", Path)); + } + Buffer = CompositeBuffer{}; + TmpBuffer.SetDeleteOnClose(true); + return TmpBuffer; + } + Buffer = CompositeBuffer{}; + BasicFile OpenTemp(Path, BasicFile::Mode::kDelete); + IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true); TmpBuffer.SetDeleteOnClose(true); return TmpBuffer; } -- cgit v1.2.3 From 0295cd45276c9b99f72dd0f35f3b5d5ffe7cb0df Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 09:48:52 +0100 Subject: collapse local writes (#310) * collapse read/writes during local data copy --- src/zencore/basicfile.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 95876cff4..a181bbd66 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -796,6 +796,12 @@ BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset) { if (m_Buffer == nullptr || (Size >= m_BufferSize)) { + if (FileOffset == m_BufferEnd) + { + Flush(); + m_BufferStart = m_BufferEnd = FileOffset + Size; + } + m_Base.Write(Data, Size, FileOffset); return; } -- cgit v1.2.3 From fd2efb5af872a357dbc0f729f4101a330dcb4fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 10:24:39 +0200 Subject: long filename support (#330) - Bugfix: Long file paths now works correctly on Windows --- src/zencore/basicfile.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index a181bbd66..ea526399c 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -590,7 +590,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std:: // deleting the temporary file BasicFile::Close(); - std::filesystem::rename(m_TempPath, FinalFileName, Ec); + RenameFile(m_TempPath, FinalFileName, Ec); if (Ec) { @@ -984,9 +984,9 @@ TEST_CASE("TemporaryFile") TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); Path = TmpFile.GetPath(); - CHECK(std::filesystem::exists(Path)); + CHECK(IsFile(Path)); } - CHECK(std::filesystem::exists(Path) == false); + CHECK(IsFile(Path) == false); } SUBCASE("MoveIntoPlace") @@ -997,11 +997,11 @@ TEST_CASE("TemporaryFile") CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; - CHECK(std::filesystem::exists(TempPath)); + CHECK(IsFile(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); - CHECK(std::filesystem::exists(TempPath) == false); - CHECK(std::filesystem::exists(FinalPath)); + CHECK(IsFile(TempPath) == false); + CHECK(IsFile(FinalPath)); } } -- cgit v1.2.3 From 68938614c95635045a394ff0a52786b82f01ffc4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 7 May 2025 10:23:42 +0200 Subject: optimize block store CompactBlocks (#384) - Improvement: Optimize block compact reducing memcpy operations - Improvement: Handle padding of block store blocks when compacting to avoid excessive flusing of write buffer - Improvement: Handle padding when writing oplog index snapshot to avoid unnecessary flushing of write buffer --- src/zencore/basicfile.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index ea526399c..12ee26155 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -791,6 +791,35 @@ BasicFileWriter::~BasicFileWriter() Memory::Free(m_Buffer); } +void +BasicFileWriter::AddPadding(uint64_t Padding) +{ + while (Padding) + { + const uint64_t BufferOffset = m_BufferEnd - m_BufferStart; + const uint64_t RemainingBufferCapacity = m_BufferSize - BufferOffset; + const uint64_t BlockPadBytes = Min(RemainingBufferCapacity, Padding); + + memset(m_Buffer + BufferOffset, 0, BlockPadBytes); + m_BufferEnd += BlockPadBytes; + Padding -= BlockPadBytes; + + if ((BufferOffset + BlockPadBytes) == m_BufferSize) + { + Flush(); + } + } +} + +uint64_t +BasicFileWriter::AlignTo(uint64_t Alignment) +{ + uint64_t AlignedPos = RoundUp(m_BufferEnd, Alignment); + uint64_t Padding = AlignedPos - m_BufferEnd; + AddPadding(Padding); + return AlignedPos; +} + void BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset) { -- cgit v1.2.3 From 33d443f5361d007f4971bf0d98585b81ee691437 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Jun 2025 19:14:52 +0200 Subject: http client streaming upload (#413) - Improvement: Add streaming upload from HttpClient to reduce I/O caused by excessive MMap usage --- src/zencore/basicfile.cpp | 149 ++++++---------------------------------------- 1 file changed, 18 insertions(+), 131 deletions(-) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 12ee26155..993f2b616 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -181,58 +181,18 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount) void BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - - while (BytesToRead) + const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; + std::error_code Ec; + ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec); + if (Ec) { - const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); - int32_t Error = 0; - size_t BytesRead = 0; - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesRead = 0; - BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); - if (Success) - { - BytesRead = size_t(dwNumberOfBytesRead); - } - else - { - Error = zen::GetLastError(); - } -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset); - if (ReadResult != -1) - { - BytesRead = size_t(ReadResult); - } - else - { - Error = zen::GetLastError(); - } -#endif - - if (Error || (BytesRead != NumberOfBytesToRead)) - { - std::error_code DummyEc; - throw std::system_error(std::error_code(Error, std::system_category()), - fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", - FileOffset, - NumberOfBytesToRead, - PathFromHandle(m_FileHandle, DummyEc), - FileSizeFromHandle(m_FileHandle))); - } - - BytesToRead -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast(Data) + NumberOfBytesToRead; + std::error_code DummyEc; + throw std::system_error(Ec, + fmt::format("BasicFile::Read: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", + FileOffset, + BytesToRead, + PathFromHandle(m_FileHandle, DummyEc), + FileSizeFromHandle(m_FileHandle))); } } @@ -323,43 +283,9 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) void BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) { - ZEN_ASSERT(m_FileHandle != nullptr); - - Ec.clear(); - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - while (Size) - { - const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesWritten = 0; - - BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); - bool Success = (BytesWritten > 0); -#endif - - if (!Success) - { - Ec = MakeErrorCodeFromLastError(); - - return; - } - - Size -= NumberOfBytesToWrite; - FileOffset += NumberOfBytesToWrite; - Data = reinterpret_cast(Data) + NumberOfBytesToWrite; - } + WriteFile(m_FileHandle, Data, Size, FileOffset, MaxChunkSize, Ec); } void @@ -405,59 +331,20 @@ BasicFile::Flush() uint64_t BasicFile::FileSize() const { -#if ZEN_PLATFORM_WINDOWS - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart == INVALID_FILE_SIZE) - { - int Error = zen::GetLastError(); - if (Error) - { - std::error_code Dummy; - ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); - } - } - return uint64_t(liFileSize.QuadPart); -#else - int Fd = int(uintptr_t(m_FileHandle)); - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - if (fstat(Fd, &Stat) == -1) + std::error_code Ec; + uint64_t FileSize = FileSizeFromHandle(m_FileHandle, Ec); + if (Ec) { std::error_code Dummy; - ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); + ThrowSystemError(Ec.value(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); } - return uint64_t(Stat.st_size); -#endif + return FileSize; } uint64_t BasicFile::FileSize(std::error_code& Ec) const { -#if ZEN_PLATFORM_WINDOWS - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart == INVALID_FILE_SIZE) - { - int Error = zen::GetLastError(); - if (Error) - { - Ec = MakeErrorCode(Error); - return 0; - } - } - return uint64_t(liFileSize.QuadPart); -#else - int Fd = int(uintptr_t(m_FileHandle)); - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - if (fstat(Fd, &Stat) == -1) - { - Ec = MakeErrorCodeFromLastError(); - return 0; - } - return uint64_t(Stat.st_size); -#endif + return FileSizeFromHandle(m_FileHandle, Ec); } void -- cgit v1.2.3 From a0b10b046095d57ffbdb46c83084601a832f4562 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Jun 2025 16:21:01 +0200 Subject: fixed size chunking for encrypted files (#410) - Improvement: Use fixed size block chunking for know encrypted/compressed file types - Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed - Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80% --- src/zencore/basicfile.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/zencore/basicfile.cpp') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 993f2b616..6989da67e 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -283,7 +283,7 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) void BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) { - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; + const uint64_t MaxChunkSize = 2u * 1024 * 1024; WriteFile(m_FileHandle, Data, Size, FileOffset, MaxChunkSize, Ec); } @@ -794,7 +794,7 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) { uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; - // BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); + BasicFileWriter BufferedOutput(Temp, Min(BufferingSize, BufferSize)); for (const SharedBuffer& Segment : Buffer.GetSegments()) { size_t SegmentSize = Segment.GetSize(); @@ -806,14 +806,14 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, - [&Temp, &Offset](const void* Data, size_t Size) { - Temp.Write(Data, Size, Offset); + [&BufferedOutput, &Offset](const void* Data, size_t Size) { + BufferedOutput.Write(Data, Size, Offset); Offset += Size; }); } else { - Temp.Write(Segment.GetData(), SegmentSize, Offset); + BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset); Offset += SegmentSize; } } -- cgit v1.2.3