import oplog improvements (#54)

* report down/up transfer speed during progress
* add disk buffering in http client
* offload block decoding and chunk writing form network worker pool threads
add block hash verification for blocks recevied at oplog import
* separate download-latch from write-latch to get more accurate download speed
* check headers when downloading with http client to go directly to file writing for large payloads
* we must clear write callback even if we only provide it as an argument to the Download() call
* make timeout optional in AddSponsorProcess
* check return codes when creating windows threadpool

1 files changed, 1 insertions, 32 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index e452c658e..3a7922aaf 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3210,37 +3210,6 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 	return ConvertResult(ContainerResult);
 }
 
-std::pair<HttpResponseCode, std::string>
-ProjectStore::WriteBlock(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload)
-{
-	ZEN_TRACE_CPU("Store::WriteBlock");
-
-	Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
-	if (!Project)
-	{
-		return {HttpResponseCode::NotFound, fmt::format("Write block request for unknown project '{}'", ProjectId)};
-	}
-	Project->TouchProject();
-
-	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId);
-	if (!Oplog)
-	{
-		return {HttpResponseCode::NotFound, fmt::format("Write block request for unknown oplog '{}/{}'", ProjectId, OplogId)};
-	}
-	Project->TouchOplog(OplogId);
-
-	if (!IterateBlock(std::move(Payload), [this](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) {
-			IoBuffer Compressed = Chunk.GetCompressed().Flatten().AsIoBuffer();
-			m_CidStore.AddChunk(Compressed, AttachmentRawHash);
-			ZEN_DEBUG("Saved attachment {} from block, size {}", AttachmentRawHash, Compressed.GetSize());
-		}))
-	{
-		return {HttpResponseCode::BadRequest, "Invalid chunk in block"};
-	}
-
-	return {HttpResponseCode::OK, {}};
-}
-
 bool
 ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 				  const std::string_view ProjectId,
@@ -4736,7 +4705,7 @@ TEST_CASE("project.store.block")
 	}
 	CompressedBuffer Block		 = GenerateBlock(std::move(Chunks));
 	IoBuffer		 BlockBuffer = Block.GetCompressed().Flatten().AsIoBuffer();
-	CHECK(IterateBlock(std::move(BlockBuffer), [](CompressedBuffer&&, const IoHash&) {}));
+	CHECK(IterateBlock(Block.DecodeRawHash(), std::move(BlockBuffer), [](CompressedBuffer&&, const IoHash&) {}));
 }
 
 #endif