stream oplog attachments from jupiter (#384)

* stream large downloads from jupiter to temporary file
* rework DeleteOnClose - top level marks file for delete and if lower level parts wants to keep it it clears that flag
* changelog
* log number of attachments to download
* add delay on jupiter request failure when retrying
* make sure we upload all attachments even if Needs are empty when ForceUpload is true
release TempAttachment as soon as it is used
* sort attachments so we get predictable blocks for the same oplog

1 files changed, 16 insertions, 13 deletions

diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 22bc61395..efec06f7f 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -217,7 +217,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
 
 	if (LocalFlags & kOwnsFile)
 	{
-		if (m_DeleteOnClose)
+		if (LocalFlags & kDeleteOnClose)
 		{
 #if ZEN_PLATFORM_WINDOWS
 			// Mark file for deletion when final handle is closed
@@ -438,9 +438,16 @@ IoBufferExtendedCore::GetFileReference(IoBufferFileReference& OutRef) const
 }
 
 void
-IoBufferExtendedCore::MarkAsDeleteOnClose()
+IoBufferExtendedCore::SetDeleteOnClose(bool DeleteOnClose)
 {
-	m_DeleteOnClose = true;
+	if (DeleteOnClose && (m_Flags & kOwnsFile))
+	{
+		m_Flags.fetch_or(kDeleteOnClose, std::memory_order_release);
+	}
+	else
+	{
+		m_Flags.fetch_and(~static_cast<uint32_t>(kDeleteOnClose), std::memory_order_release);
+	}
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -475,9 +482,10 @@ IoBuffer::IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t Size)
 	}
 }
 
-IoBuffer::IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize)
+IoBuffer::IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize, bool IsWholeFile)
 : m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ true))
 {
+	m_Core->SetIsWholeFile(IsWholeFile);
 }
 
 IoBuffer::IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize)
@@ -506,11 +514,11 @@ IoBuffer::GetFileReference(IoBufferFileReference& OutRef) const
 }
 
 void
-IoBuffer::MarkAsDeleteOnClose()
+IoBuffer::SetDeleteOnClose(bool DeleteOnClose)
 {
 	if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore())
 	{
-		ExtCore->MarkAsDeleteOnClose();
+		ExtCore->SetDeleteOnClose(DeleteOnClose);
 	}
 }
 
@@ -615,9 +623,7 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of
 #if ZEN_PLATFORM_WINDOWS
 		void* Fd = DataFile.Detach();
 #endif
-		IoBuffer Iob(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size);
-		Iob.m_Core->SetIsWholeFile(Offset == 0 && Size == FileSize);
-		return Iob;
+		return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize);
 	}
 
 #if !ZEN_PLATFORM_WINDOWS
@@ -666,10 +672,7 @@ IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName)
 	Handle = (void*)uintptr_t(Fd);
 #endif	// ZEN_PLATFORM_WINDOWS
 
-	IoBuffer Iob(IoBuffer::File, Handle, 0, FileSize);
-	Iob.m_Core->SetIsWholeFile(true);
-
-	return Iob;
+	return IoBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true);
 }
 
 IoHash