From 2b0bc6def92b846effbed29a34f284a3ec2ac01a Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Fri, 1 Dec 2023 14:49:10 -0700 Subject: Add endpoint for all chunk infos Add endpoint for querying all chunk infos in an oplog. --- src/zenserver/projectstore/httpprojectstore.cpp | 40 +++++++++++++++++++++++++ src/zenserver/projectstore/httpprojectstore.h | 1 + src/zenserver/projectstore/projectstore.cpp | 12 +++----- 3 files changed, 45 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 261485834..3994a9a46 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -275,6 +275,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, [this](HttpRouterRequest& Req) { HandleFilesRequest(Req); }, HttpVerb::kGet); + m_Router.RegisterRoute( + "{project}/oplog/{log}/chunks", + [this](HttpRouterRequest& Req) { HandleChunksRequest(Req); }, + HttpVerb::kGet); + m_Router.RegisterRoute( "{project}/oplog/{log}/{chunk}/info", [this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); }, @@ -642,6 +647,41 @@ HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } +void +HttpProjectService::HandleChunksRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Chunks"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + CbObject ResponsePayload; + std::pair Result = m_ProjectStore->GetProjectChunks(ProjectId, OplogId, ResponsePayload); + if (Result.first == HttpResponseCode::OK) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } + else + { + if (Result.first == HttpResponseCode::BadRequest) + { + m_ProjectStats.BadRequestCount++; + } + ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", + ToString(HttpReq.RequestVerb()), + HttpReq.QueryString(), + static_cast(Result.first), + Result.second); + } + if (Result.second.empty()) + { + return HttpReq.WriteResponse(Result.first); + } + return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); +} + void HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) { diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 9998ae83e..9845164d7 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -64,6 +64,7 @@ private: void HandleProjectListRequest(HttpRouterRequest& Req); void HandleChunkBatchRequest(HttpRouterRequest& Req); void HandleFilesRequest(HttpRouterRequest& Req); + void HandleChunksRequest(HttpRouterRequest& Req); void HandleChunkInfoRequest(HttpRouterRequest& Req); void HandleChunkByIdRequest(HttpRouterRequest& Req); void HandleChunkByCidRequest(HttpRouterRequest& Req); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index c6097dea2..b19913747 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -2242,14 +2242,10 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri Response.BeginArray("chunks"sv); for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo) { - Response << Info.ChunkId; - } - Response.EndArray(); - - Response.BeginArray("sizes"sv); - for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo) - { - Response << Info.ChunkSize; + Response.BeginObject(); + Response << "id"sv << Info.ChunkId; + Response << "size"sv << Info.ChunkSize; + Response.EndObject(); } Response.EndArray(); -- cgit v1.2.3 From 722b4f350621dfe81b053f8904de17e029a2e662 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:33:47 -0700 Subject: Get hash when retrieving chunks Also changes the returned fields for each chunk from size->rawsize. Backwards compatibility is not a concern as this was unused in past zenserver releases. --- src/zenserver/projectstore/projectstore.cpp | 38 +++++++++++++++++++++++------ src/zenserver/projectstore/projectstore.h | 1 + 2 files changed, 32 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index b19913747..cf827b22e 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -751,6 +751,22 @@ ProjectStore::Oplog::GetAllChunksInfo() return InfoArray; } +void +ProjectStore::Oplog::IterateChunkMap( + std::function&& Fn) +{ + RwLock::SharedLockScope _(m_OplogLock); + if (!m_Storage) + { + return; + } + + for (const auto& Kv : m_ChunkMap) + { + Fn(Kv.first, Kv.second); + } +} + void ProjectStore::Oplog::IterateFileMap( std::function&& Fn) @@ -2235,17 +2251,25 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri } Project->TouchOplog(OplogId); - std::vector ChunkInfo = FoundLog->GetAllChunksInfo(); + std::vector> ChunkInfos; + FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { + ChunkInfos.push_back({Id, Hash}); + }); - CbObjectWriter Response; + CbObjectWriter Response; Response.BeginArray("chunks"sv); - for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo) + + for (const auto& ChunkInfo : ChunkInfos) { - Response.BeginObject(); - Response << "id"sv << Info.ChunkId; - Response << "size"sv << Info.ChunkSize; - Response.EndObject(); + if (IoBuffer Chunk = FoundLog->FindChunk(ChunkInfo.first)) + { + Response.BeginObject(); + Response << "id"sv << ChunkInfo.first; + Response << "rawhash"sv << ChunkInfo.second; + Response << "rawsize"sv << Chunk.GetSize(); + Response.EndObject(); + } } Response.EndArray(); diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 555f8bdf2..813af9b37 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -93,6 +93,7 @@ public: }; std::vector GetAllChunksInfo(); + void IterateChunkMap(std::function&& Fn); void IterateFileMap(std::function&& Fn); void IterateOplog(std::function&& Fn); void IterateOplogWithKey(std::function&& Fn); -- cgit v1.2.3 From 1e3fbd753aaeaeabeb4ef383f64b32042c492337 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:37:58 -0700 Subject: Ran precommit --- src/zenserver/projectstore/projectstore.cpp | 8 ++------ src/zenserver/projectstore/projectstore.h | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index cf827b22e..8bc49c55f 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -752,8 +752,7 @@ ProjectStore::Oplog::GetAllChunksInfo() } void -ProjectStore::Oplog::IterateChunkMap( - std::function&& Fn) +ProjectStore::Oplog::IterateChunkMap(std::function&& Fn) { RwLock::SharedLockScope _(m_OplogLock); if (!m_Storage) @@ -2252,10 +2251,7 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri Project->TouchOplog(OplogId); std::vector> ChunkInfos; - FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { - ChunkInfos.push_back({Id, Hash}); - }); - + FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { ChunkInfos.push_back({Id, Hash}); }); CbObjectWriter Response; Response.BeginArray("chunks"sv); diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 813af9b37..27cb092a2 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -93,7 +93,7 @@ public: }; std::vector GetAllChunksInfo(); - void IterateChunkMap(std::function&& Fn); + void IterateChunkMap(std::function&& Fn); void IterateFileMap(std::function&& Fn); void IterateOplog(std::function&& Fn); void IterateOplogWithKey(std::function&& Fn); -- cgit v1.2.3 From ed465b8d83c5b1a3f996fd8f4f037427574918f2 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:46:00 -0700 Subject: Change naming to ChunkInfos instead of Chunks --- src/zenserver/projectstore/httpprojectstore.cpp | 10 +++++----- src/zenserver/projectstore/httpprojectstore.h | 2 +- src/zenserver/projectstore/projectstore.cpp | 6 +++--- src/zenserver/projectstore/projectstore.h | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 3994a9a46..0ba49cf8a 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -276,8 +276,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpVerb::kGet); m_Router.RegisterRoute( - "{project}/oplog/{log}/chunks", - [this](HttpRouterRequest& Req) { HandleChunksRequest(Req); }, + "{project}/oplog/{log}/chunkinfos", + [this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( @@ -648,9 +648,9 @@ HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req) } void -HttpProjectService::HandleChunksRequest(HttpRouterRequest& Req) +HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req) { - ZEN_TRACE_CPU("ProjectService::Chunks"); + ZEN_TRACE_CPU("ProjectService::ChunkInfos"); HttpServerRequest& HttpReq = Req.ServerRequest(); @@ -658,7 +658,7 @@ HttpProjectService::HandleChunksRequest(HttpRouterRequest& Req) const auto& OplogId = Req.GetCapture(2); CbObject ResponsePayload; - std::pair Result = m_ProjectStore->GetProjectChunks(ProjectId, OplogId, ResponsePayload); + std::pair Result = m_ProjectStore->GetProjectChunkInfos(ProjectId, OplogId, ResponsePayload); if (Result.first == HttpResponseCode::OK) { return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 9845164d7..9990ee264 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -64,7 +64,7 @@ private: void HandleProjectListRequest(HttpRouterRequest& Req); void HandleChunkBatchRequest(HttpRouterRequest& Req); void HandleFilesRequest(HttpRouterRequest& Req); - void HandleChunksRequest(HttpRouterRequest& Req); + void HandleChunkInfosRequest(HttpRouterRequest& Req); void HandleChunkInfoRequest(HttpRouterRequest& Req); void HandleChunkByIdRequest(HttpRouterRequest& Req); void HandleChunkByCidRequest(HttpRouterRequest& Req); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 8bc49c55f..9ba8e3a19 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -2230,9 +2230,9 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin } std::pair -ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload) +ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload) { - ZEN_TRACE_CPU("ProjectStore::GetProjectChunks"); + ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos"); using namespace std::literals; @@ -2254,7 +2254,7 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { ChunkInfos.push_back({Id, Hash}); }); CbObjectWriter Response; - Response.BeginArray("chunks"sv); + Response.BeginArray("chunkinfos"sv); for (const auto& ChunkInfo : ChunkInfos) { diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 27cb092a2..57cda8ae7 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -307,9 +307,9 @@ public: const std::string_view OplogId, bool FilterClient, CbObject& OutPayload); - std::pair GetProjectChunks(const std::string_view ProjectId, - const std::string_view OplogId, - CbObject& OutPayload); + std::pair GetProjectChunkInfos(const std::string_view ProjectId, + const std::string_view OplogId, + CbObject& OutPayload); std::pair GetChunkInfo(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view ChunkId, -- cgit v1.2.3