From aa0b0d3cbfc6c4561591df856396703f7177292e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 20 Apr 2024 13:22:05 +0200 Subject: import oplog improvements (#54) * report down/up transfer speed during progress * add disk buffering in http client * offload block decoding and chunk writing form network worker pool threads add block hash verification for blocks recevied at oplog import * separate download-latch from write-latch to get more accurate download speed * check headers when downloading with http client to go directly to file writing for large payloads * we must clear write callback even if we only provide it as an argument to the Download() call * make timeout optional in AddSponsorProcess * check return codes when creating windows threadpool --- src/zenserver/projectstore/projectstore.cpp | 33 +---------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) (limited to 'src/zenserver/projectstore/projectstore.cpp') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index e452c658e..3a7922aaf 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -3210,37 +3210,6 @@ ProjectStore::ReadOplog(const std::string_view ProjectId, return ConvertResult(ContainerResult); } -std::pair -ProjectStore::WriteBlock(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload) -{ - ZEN_TRACE_CPU("Store::WriteBlock"); - - Ref Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Write block request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId); - if (!Oplog) - { - return {HttpResponseCode::NotFound, fmt::format("Write block request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - if (!IterateBlock(std::move(Payload), [this](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { - IoBuffer Compressed = Chunk.GetCompressed().Flatten().AsIoBuffer(); - m_CidStore.AddChunk(Compressed, AttachmentRawHash); - ZEN_DEBUG("Saved attachment {} from block, size {}", AttachmentRawHash, Compressed.GetSize()); - })) - { - return {HttpResponseCode::BadRequest, "Invalid chunk in block"}; - } - - return {HttpResponseCode::OK, {}}; -} - bool ProjectStore::Rpc(HttpServerRequest& HttpReq, const std::string_view ProjectId, @@ -4736,7 +4705,7 @@ TEST_CASE("project.store.block") } CompressedBuffer Block = GenerateBlock(std::move(Chunks)); IoBuffer BlockBuffer = Block.GetCompressed().Flatten().AsIoBuffer(); - CHECK(IterateBlock(std::move(BlockBuffer), [](CompressedBuffer&&, const IoHash&) {})); + CHECK(IterateBlock(Block.DecodeRawHash(), std::move(BlockBuffer), [](CompressedBuffer&&, const IoHash&) {})); } #endif -- cgit v1.2.3