aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore/projectstore.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-09-30 22:03:31 +0200
committerGitHub Enterprise <[email protected]>2025-09-30 22:03:31 +0200
commit04ce16176630799ebf7035d70348d4b7c9a31bb4 (patch)
treed1332fc2faa87a81462ca573bbb041e1562851ab /src/zenserver/projectstore/projectstore.cpp
parentfix bounds check when finalizing build state (#533) (diff)
downloadzen-04ce16176630799ebf7035d70348d4b7c9a31bb4.tar.xz
zen-04ce16176630799ebf7035d70348d4b7c9a31bb4.zip
projectstore refactor (#531)
* convert ProjectStore::GetProjectFiles to not use http return codes * convert ProjectStore::GetProjectChunkInfos to not use http return codes * convert ProjectStore::GetChunkInfo to not use http return codes * convert ProjectStore::GetChunkRange to not use http return codes * convert ProjectStore::GetChunk to not use http return codes * convert ProjectStore::PutChunk to not use http return codes * convert ProjectStore::WriteOplog to not use http return codes * convert ProjectStore::ReadOplog to not use http return codes
Diffstat (limited to 'src/zenserver/projectstore/projectstore.cpp')
-rw-r--r--src/zenserver/projectstore/projectstore.cpp760
1 files changed, 287 insertions, 473 deletions
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 284a32baa..cf30f32e0 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -4640,7 +4640,7 @@ ProjectStore::GetProjectsList()
CbWriter Response;
Response.BeginArray();
- IterateProjects([&Response](ProjectStore::Project& Prj) {
+ IterateProjects([&Response](Project& Prj) {
Response.BeginObject();
Response << "Id"sv << Prj.Identifier;
Response << "RootDir"sv << Prj.RootDir.string();
@@ -4653,31 +4653,13 @@ ProjectStore::GetProjectsList()
return Response.Save().AsArray();
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetProjectFiles(const std::string_view ProjectId,
- const std::string_view OplogId,
- const std::unordered_set<std::string>& WantedFieldNames,
- CbObject& OutPayload)
+CbObject
+ProjectStore::GetProjectFiles(LoggerRef InLog, Project& Project, Oplog& Oplog, const std::unordered_set<std::string>& WantedFieldNames)
{
- ZEN_MEMSCOPE(GetProjectstoreTag());
- ZEN_TRACE_CPU("Store::GetProjectFiles");
+ auto Log = [&InLog]() { return InLog; };
using namespace std::literals;
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Project files request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
-
- Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
- if (!FoundLog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
-
const bool WantsAllFields = WantedFieldNames.empty();
const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id");
@@ -4693,7 +4675,7 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
std::vector<uint64_t> RawSizes;
size_t Count = 0;
- FoundLog->IterateFileMap([&](const Oid& Id, const std::string_view& ServerPath, const std::string_view& ClientPath) {
+ Oplog.IterateFileMap([&](const Oid& Id, const std::string_view& ServerPath, const std::string_view& ClientPath) {
if (WantsIdField || WantsRawSizeField || WantsSizeField)
{
Ids.push_back(Id);
@@ -4719,8 +4701,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
RawSizes.resize(Ids.size(), (uint64_t)-1);
}
- FoundLog->IterateChunks(
- Project->RootDir,
+ Oplog.IterateChunks(
+ Project.RootDir,
Ids,
false,
[&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) {
@@ -4743,8 +4725,8 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
else
{
ZEN_WARN("oplog '{}/{}': payload for project file info for id {} is not a valid compressed binary.",
- ProjectId,
- OplogId,
+ Project.Identifier,
+ Oplog.OplogId(),
Ids[Index]);
}
}
@@ -4767,14 +4749,17 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
}
else
{
- ZEN_WARN("oplog '{}/{}': failed getting payload for project file info for id {}.", ProjectId, OplogId, Ids[Index]);
+ ZEN_WARN("oplog '{}/{}': failed getting payload for project file info for id {}.",
+ Project.Identifier,
+ Oplog.OplogId(),
+ Ids[Index]);
}
}
catch (const std::exception& Ex)
{
ZEN_WARN("oplog '{}/{}': failed getting project file info for id {}. Reason: '{}'",
- ProjectId,
- OplogId,
+ Project.Identifier,
+ Oplog.OplogId(),
Ids[Index],
Ex.what());
}
@@ -4814,34 +4799,18 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId,
}
Response.EndArray();
- OutPayload = Response.Save();
- return {HttpResponseCode::OK, {}};
+ return Response.Save();
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
- const std::string_view OplogId,
- const std::unordered_set<std::string>& WantedFieldNames,
- CbObject& OutPayload)
+CbObject
+ProjectStore::GetProjectChunkInfos(LoggerRef InLog, Project& Project, Oplog& Oplog, const std::unordered_set<std::string>& WantedFieldNames)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos");
- using namespace std::literals;
+ auto Log = [&InLog]() { return InLog; };
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
-
- Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
- if (!FoundLog)
- {
- return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
+ using namespace std::literals;
const bool WantsAllFields = WantedFieldNames.empty();
@@ -4856,7 +4825,7 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
std::vector<uint64_t> Sizes;
size_t Count = 0;
- size_t EstimatedCount = FoundLog->OplogCount();
+ size_t EstimatedCount = Oplog.OplogCount();
if (WantsIdField)
{
Ids.reserve(EstimatedCount);
@@ -4865,7 +4834,7 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
{
Hashes.reserve(EstimatedCount);
}
- FoundLog->IterateChunkMap([&](const Oid& Id, const IoHash& Hash) {
+ Oplog.IterateChunkMap([&](const Oid& Id, const IoHash& Hash) {
if (WantsIdField)
{
Ids.push_back(Id);
@@ -4889,7 +4858,7 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
}
WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();
- (void)FoundLog->IterateChunks(
+ (void)Oplog.IterateChunks(
Hashes,
false,
[&](size_t Index, const IoBuffer& Payload, uint64_t /*ModTag*/) -> bool {
@@ -4914,8 +4883,8 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
else
{
ZEN_WARN("oplog '{}/{}': payload for project chunk for id {} is not a valid compressed binary.",
- ProjectId,
- OplogId,
+ Project.Identifier,
+ Oplog.OplogId(),
Ids[Index]);
}
}
@@ -4941,14 +4910,17 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
}
else
{
- ZEN_WARN("oplog '{}/{}': failed getting payload for chunk for id {}", ProjectId, OplogId, Ids[Index]);
+ ZEN_WARN("oplog '{}/{}': failed getting payload for chunk for id {}",
+ Project.Identifier,
+ Oplog.OplogId(),
+ Ids[Index]);
}
}
catch (const std::exception& Ex)
{
ZEN_WARN("oplog '{}/{}': failed getting project chunk info for id {}. Reason: '{}'",
- ProjectId,
- OplogId,
+ Project.Identifier,
+ Oplog.OplogId(),
Ids[Index],
Ex.what());
}
@@ -4986,45 +4958,23 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId,
}
Response.EndArray();
- OutPayload = Response.Save();
- return {HttpResponseCode::OK, {}};
+ return Response.Save();
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetChunkInfo(const std::string_view ProjectId,
- const std::string_view OplogId,
- const std::string_view ChunkId,
- CbObject& OutPayload)
+CbObject
+ProjectStore::GetChunkInfo(LoggerRef InLog, Project& Project, Oplog& Oplog, const Oid& ChunkId)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- using namespace std::literals;
+ ZEN_TRACE_CPU("ProjectStore::GetChunkInfo");
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
-
- Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false);
- if (!FoundLog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
-
- if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
- {
- return {HttpResponseCode::BadRequest,
- fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
- }
+ using namespace std::literals;
- const Oid Obj = Oid::FromHexString(ChunkId);
+ auto Log = [&InLog]() { return InLog; };
- IoBuffer Chunk = FoundLog->FindChunk(Project->RootDir, Obj, nullptr);
+ IoBuffer Chunk = Oplog.FindChunk(Project.RootDir, ChunkId, nullptr);
if (!Chunk)
{
- return {HttpResponseCode::NotFound, {}};
+ return {};
}
uint64_t ChunkSize = Chunk.GetSize();
@@ -5035,61 +4985,36 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
bool IsCompressed = CompressedBuffer::ValidateCompressedHeader(Chunk, RawHash, RawSize);
if (!IsCompressed)
{
- return {HttpResponseCode::InternalServerError,
- fmt::format("Chunk info request for malformed chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
+ throw std::runtime_error(
+ fmt::format("Chunk info request for malformed chunk id '{}/{}'/'{}'", Project.Identifier, Oplog.OplogId(), ChunkId));
}
ChunkSize = RawSize;
}
CbObjectWriter Response;
Response << "size"sv << ChunkSize;
- OutPayload = Response.Save();
- return {HttpResponseCode::OK, {}};
+ return Response.Save();
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetChunkRange(const std::string_view ProjectId,
- const std::string_view OplogId,
- const std::string_view ChunkId,
- uint64_t Offset,
- uint64_t Size,
- ZenContentType AcceptType,
- CompositeBuffer& OutChunk,
- ZenContentType& OutContentType,
- uint64_t* OptionalInOutModificationTag)
+static ProjectStore::GetChunkRangeResult
+ExtractRange(IoBuffer&& Chunk, uint64_t Offset, uint64_t Size, ZenContentType AcceptType)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
- {
- return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)};
- }
-
- const Oid Obj = Oid::FromHexString(ChunkId);
- return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType, OptionalInOutModificationTag);
-}
+ ProjectStore::GetChunkRangeResult Result;
-static std::pair<HttpResponseCode, std::string>
-ExtractRange(IoBuffer&& Chunk,
- uint64_t Offset,
- uint64_t Size,
- ZenContentType AcceptType,
- ZenContentType& OutContentType,
- CompositeBuffer& OutChunk,
- IoHash& OutRawHash,
- uint64_t& OutRawSize)
-{
- ZEN_MEMSCOPE(GetProjectstoreTag());
- OutContentType = Chunk.GetContentType();
+ Result.ContentType = Chunk.GetContentType();
- if (OutContentType == ZenContentType::kCompressedBinary)
+ if (Result.ContentType == ZenContentType::kCompressedBinary)
{
IoHash RawHash;
uint64_t RawSize;
CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), RawHash, RawSize);
if (!Compressed)
{
- return {HttpResponseCode::InternalServerError, "malformed compressed buffer"};
+ Result.Error = ProjectStore::GetChunkRangeResult::EError::MalformedContent;
+ Result.ErrorDescription = fmt::format("Malformed payload, not a compressed buffer");
+ return Result;
}
const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == RawSize));
@@ -5098,14 +5023,14 @@ ExtractRange(IoBuffer&& Chunk,
{
if (AcceptType == ZenContentType::kBinary)
{
- OutChunk = Compressed.DecompressToComposite();
- OutContentType = ZenContentType::kBinary;
+ Result.Chunk = Compressed.DecompressToComposite();
+ Result.ContentType = ZenContentType::kBinary;
}
else
{
- OutChunk = Compressed.GetCompressed();
+ Result.Chunk = Compressed.GetCompressed();
}
- OutRawSize = 0;
+ Result.RawSize = 0;
}
else
{
@@ -5123,27 +5048,29 @@ ExtractRange(IoBuffer&& Chunk,
if (Size == 0)
{
- return {HttpResponseCode::NotFound,
- fmt::format("Chunk request for range outside of compressed chunk. Offset: {}, Size: {}, ChunkSize: {}",
- Offset,
- Size,
- RawSize)};
+ Result.Error = ProjectStore::GetChunkRangeResult::EError::OutOfRange;
+ Result.ErrorDescription =
+ fmt::format("Chunk request for range outside of compressed chunk. Offset: {}, Size: {}, ChunkSize: {}",
+ Offset,
+ Size,
+ RawSize);
+ return Result;
}
if (AcceptType == ZenContentType::kBinary)
{
- OutChunk = CompositeBuffer(Compressed.Decompress(Offset, Size));
- OutContentType = ZenContentType::kBinary;
+ Result.Chunk = CompositeBuffer(Compressed.Decompress(Offset, Size));
+ Result.ContentType = ZenContentType::kBinary;
}
else
{
// Value will be a range of compressed blocks that covers the requested range
// The client will have to compensate for any offsets that do not land on an even block size multiple
- OutChunk = Compressed.GetRange(Offset, Size).GetCompressed();
+ Result.Chunk = Compressed.GetRange(Offset, Size).GetCompressed();
}
- OutRawSize = RawSize;
+ Result.RawSize = RawSize;
}
- OutRawHash = RawHash;
+ Result.RawHash = RawHash;
}
else
{
@@ -5152,8 +5079,8 @@ ExtractRange(IoBuffer&& Chunk,
const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == ChunkSize));
if (IsFullRange)
{
- OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk)));
- OutRawSize = 0;
+ Result.Chunk = CompositeBuffer(SharedBuffer(std::move(Chunk)));
+ Result.RawSize = 0;
}
else
{
@@ -5171,159 +5098,126 @@ ExtractRange(IoBuffer&& Chunk,
if (Size == 0)
{
- return {HttpResponseCode::NotFound,
- fmt::format("Chunk request for range outside of chunk. Offset: {}, Size: {}, ChunkSize: {}", Offset, Size, Size)};
+ Result.Error = ProjectStore::GetChunkRangeResult::EError::OutOfRange;
+ Result.ErrorDescription =
+ fmt::format("Chunk request for range outside of compressed chunk. Offset: {}, Size: {}, ChunkSize: {}",
+ Offset,
+ Size,
+ ChunkSize);
+ return Result;
}
- OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size)));
- OutRawSize = ChunkSize;
+ Result.Chunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size)));
+ Result.RawSize = ChunkSize;
}
}
- return {HttpResponseCode::OK, {}};
+ Result.Error = ProjectStore::GetChunkRangeResult::EError::Ok;
+ return Result;
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetChunkRange(const std::string_view ProjectId,
- const std::string_view OplogId,
- Oid ChunkId,
- uint64_t Offset,
- uint64_t Size,
- ZenContentType AcceptType,
- CompositeBuffer& OutChunk,
- ZenContentType& OutContentType,
- uint64_t* OptionalInOutModificationTag)
+ProjectStore::GetChunkRangeResult
+ProjectStore::GetChunkRange(LoggerRef InLog,
+ Project& Project,
+ Oplog& Oplog,
+ const Oid& ChunkId,
+ uint64_t Offset,
+ uint64_t Size,
+ ZenContentType AcceptType,
+ uint64_t* OptionalInOutModificationTag)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
- Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false);
- if (!FoundLog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
+ ZEN_TRACE_CPU("ProjectStore::GetChunkRange");
+
+ auto Log = [&InLog]() { return InLog; };
uint64_t OldTag = OptionalInOutModificationTag == nullptr ? 0 : *OptionalInOutModificationTag;
- IoBuffer Chunk = FoundLog->FindChunk(Project->RootDir, ChunkId, OptionalInOutModificationTag);
+ IoBuffer Chunk = Oplog.FindChunk(Project.RootDir, ChunkId, OptionalInOutModificationTag);
if (!Chunk)
{
- return {HttpResponseCode::NotFound, {}};
+ return GetChunkRangeResult{.Error = GetChunkRangeResult::EError::NotFound,
+ .ErrorDescription = fmt::format("Chunk range request for chunk {}/{}/{} failed, payload not found",
+ Project.Identifier,
+ Oplog.OplogId(),
+ ChunkId)};
}
if (OptionalInOutModificationTag != nullptr && OldTag == *OptionalInOutModificationTag)
{
- return {HttpResponseCode::NotModified, {}};
+ return {.Error = GetChunkRangeResult::EError::NotModified};
}
- IoHash _;
- uint64_t __;
- std::pair<HttpResponseCode, std::string> Result =
- ExtractRange(std::move(Chunk), Offset, Size, AcceptType, OutContentType, OutChunk, /*OutRawHash*/ _, /*OutRawSize*/ __);
- if (Result.first != HttpResponseCode::OK)
- {
- return {Result.first,
- fmt::format("Chunk request for chunk {} in {}/{} failed. Reason: '{}'", ChunkId, ProjectId, OplogId, Result.second)};
- }
- return Result;
+ return ExtractRange(std::move(Chunk), Offset, Size, AcceptType);
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::GetChunk(const std::string_view ProjectId,
- const std::string_view OplogId,
- const std::string_view Cid,
- IoBuffer& OutChunk,
- uint64_t* OptionalInOutModificationTag)
+IoBuffer
+ProjectStore::GetChunk(Project& Project, Oplog& Oplog, const IoHash& ChunkHash)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
+ ZEN_UNUSED(Project, Oplog);
- Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false);
- if (!FoundLog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
+ IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkHash);
- if (Cid.length() != IoHash::StringLength)
+ if (!Chunk)
{
- return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, Cid)};
+ return {};
}
- const IoHash Hash = IoHash::FromHexString(Cid);
- OutChunk = m_CidStore.FindChunkByCid(Hash);
+ Chunk.SetContentType(ZenContentType::kCompressedBinary);
+ return Chunk;
+}
- if (!OutChunk)
+IoBuffer
+ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const Oid& ChunkId)
+{
+ ZEN_MEMSCOPE(GetProjectstoreTag());
+
+ Ref<Project> Project = OpenProject(ProjectId);
+ if (!Project)
{
- return {HttpResponseCode::NotFound, fmt::format("chunk - '{}' MISSING", Cid)};
+ return {};
}
-
- if (OptionalInOutModificationTag != nullptr)
+ Ref<Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact */ false, /*VerifyPathOnDisk*/ false);
+ if (!Oplog)
{
- uint64_t OldTag = *OptionalInOutModificationTag;
- *OptionalInOutModificationTag = GetModificationTagFromRawHash(Hash);
- if (*OptionalInOutModificationTag == OldTag)
- {
- return {HttpResponseCode::NotModified, {}};
- }
+ return {};
}
-
- OutChunk.SetContentType(ZenContentType::kCompressedBinary);
- return {HttpResponseCode::OK, {}};
+ return Oplog->FindChunk(Project->RootDir, ChunkId, /*OptOutModificationTag*/ nullptr);
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::PutChunk(const std::string_view ProjectId,
- const std::string_view OplogId,
- const std::string_view Cid,
- ZenContentType ContentType,
- IoBuffer&& Chunk)
+IoBuffer
+ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const IoHash& Cid)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
- Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false);
- if (!FoundLog)
+ Ref<Project> Project = OpenProject(ProjectId);
+ if (!Project)
{
- return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown oplog '{}/{}'", ProjectId, OplogId)};
+ return {};
}
- Project->TouchOplog(OplogId);
-
- if (Cid.length() != IoHash::StringLength)
+ Ref<Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact */ false, /*VerifyPathOnDisk*/ false);
+ if (!Oplog)
{
- return {HttpResponseCode::BadRequest, fmt::format("Chunk put request for invalid chunk hash '{}'", Cid)};
+ return {};
}
+ return m_CidStore.FindChunkByCid(Cid);
+}
- const IoHash Hash = IoHash::FromHexString(Cid);
-
- if (ContentType != HttpContentType::kCompressedBinary)
- {
- return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid content type for chunk '{}'", Cid)};
- }
+bool
+ProjectStore::PutChunk(Project& Project, Oplog& Oplog, const IoHash& ChunkHash, IoBuffer&& Chunk)
+{
+ ZEN_MEMSCOPE(GetProjectstoreTag());
IoHash RawHash;
uint64_t RawSize;
CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
- if (RawHash != Hash)
+ if (RawHash != ChunkHash)
{
- return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid payload format for chunk '{}'", Cid)};
+ throw std::runtime_error(
+ fmt::format("Chunk request for invalid payload format for chunk {}/{}/{}", Project.Identifier, Oplog.OplogId(), ChunkHash));
}
- FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash});
- CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash);
- return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}};
+ Oplog.CaptureAddedAttachments(std::vector<IoHash>{ChunkHash});
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, ChunkHash);
+ return Result.New;
}
std::pair<HttpResponseCode, std::string>
@@ -5568,28 +5462,20 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
ResponseWriter.AddInteger("ModTag", ChunkRequest.Output.ModTag);
if (!SkipData)
{
- CompositeBuffer ChunkRange;
- ZenContentType ContentType;
- IoHash FullChunkRawHash;
- uint64_t FullChunkSize = 0;
- auto ExtractRangeResult = ExtractRange(std::move(ChunkRequest.Output.ChunkBuffer),
- ChunkRequest.Input.Offset,
- ChunkRequest.Input.Size,
- ZenContentType::kCompressedBinary,
- ContentType,
- ChunkRange,
- FullChunkRawHash,
- FullChunkSize);
- if (ExtractRangeResult.first == HttpResponseCode::OK)
+ auto ExtractRangeResult = ExtractRange(std::move(ChunkRequest.Output.ChunkBuffer),
+ ChunkRequest.Input.Offset,
+ ChunkRequest.Input.Size,
+ ZenContentType::kCompressedBinary);
+ if (ExtractRangeResult.Error == GetChunkRangeResult::EError::Ok)
{
- if (ContentType == ZenContentType::kCompressedBinary)
+ if (ExtractRangeResult.ContentType == ZenContentType::kCompressedBinary)
{
- ZEN_ASSERT(FullChunkRawHash != IoHash::Zero);
+ ZEN_ASSERT(ExtractRangeResult.RawHash != IoHash::Zero);
CompressedBuffer CompressedValue =
- CompressedBuffer::FromCompressedNoValidate(std::move(ChunkRange));
+ CompressedBuffer::FromCompressedNoValidate(std::move(ExtractRangeResult.Chunk));
ZEN_ASSERT(CompressedValue);
- if (FullChunkSize != 0)
+ if (ExtractRangeResult.RawSize != 0)
{
// This really could use some thought so we don't send the same data if we get a request for
// multiple ranges from the same chunk block
@@ -5611,14 +5497,15 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
uint64_t FragmentRawLength = CompressedValue.DecodeRawSize();
IoHashStream FragmentHashStream;
- FragmentHashStream.Append(FullChunkRawHash.Hash, sizeof(FullChunkRawHash.Hash));
+ FragmentHashStream.Append(ExtractRangeResult.RawHash.Hash,
+ sizeof(ExtractRangeResult.RawHash.Hash));
FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset));
FragmentHashStream.Append(&FragmentRawLength, sizeof(FragmentRawLength));
IoHash FragmentHash = FragmentHashStream.GetHash();
ResponseWriter.AddHash("FragmentHash", FragmentHash);
ResponseWriter.AddInteger("FragmentOffset", FragmentRawOffset);
- ResponseWriter.AddInteger("RawSize", FullChunkSize);
+ ResponseWriter.AddInteger("RawSize", ExtractRangeResult.RawSize);
OutResponsePackage.AddAttachment(CbAttachment(CompressedValue, FragmentHash));
}
else
@@ -5631,8 +5518,9 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
}
else
{
- ResponseWriter.AddHash("RawHash"sv, FullChunkRawHash);
- OutResponsePackage.AddAttachment(CbAttachment(std::move(CompressedValue), FullChunkRawHash));
+ ResponseWriter.AddHash("RawHash"sv, ExtractRangeResult.RawHash);
+ OutResponsePackage.AddAttachment(
+ CbAttachment(std::move(CompressedValue), ExtractRangeResult.RawHash));
}
}
else
@@ -5646,16 +5534,17 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
IoHash Hash = HashStream.GetHash();
ResponseWriter.AddHash("Hash"sv, Hash);
- if (FullChunkSize != 0)
+ if (ExtractRangeResult.RawSize != 0)
{
- ResponseWriter.AddInteger("Size", FullChunkSize);
+ ResponseWriter.AddInteger("Size", ExtractRangeResult.RawSize);
}
- OutResponsePackage.AddAttachment(CbAttachment(std::move(ChunkRange), Hash));
+ OutResponsePackage.AddAttachment(CbAttachment(std::move(ExtractRangeResult.Chunk), Hash));
}
}
else
{
- std::string ErrorString = fmt::format("Failed fetchiong chunk range ({})", ExtractRangeResult.second);
+ std::string ErrorString =
+ fmt::format("Failed fetching chunk range ({})", ExtractRangeResult.ErrorDescription);
ResponseWriter.AddString("Error", ErrorString);
ZEN_WARN("oplog '{}/{}': {}", ProjectId, OplogId, ErrorString);
}
@@ -5676,155 +5565,76 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
}
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload, CbObject& OutResponse)
+ProjectStore::WriteOplogResult
+ProjectStore::WriteOplog(Project& Project, Oplog& Oplog, const CbObject& ContainerObject)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::WriteOplog");
+ ZEN_UNUSED(Project);
+
+ CidStore& ChunkStore = m_CidStore;
+ RwLock AttachmentsLock;
+ tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
+
+ auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); };
+ auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
+ RwLock::ExclusiveLockScope _(AttachmentsLock);
+ if (BlockHash != IoHash::Zero)
+ {
+ Attachments.insert(BlockHash);
+ }
+ else
+ {
+ Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
+ }
+ };
+ auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
+ RwLock::ExclusiveLockScope _(AttachmentsLock);
+ Attachments.insert(RawHash);
+ };
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
-
- Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false);
- if (!Oplog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
-
- CbValidateError ValidateResult;
- if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult);
- ValidateResult == CbValidateError::None && ContainerObject)
- {
- CidStore& ChunkStore = m_CidStore;
- RwLock AttachmentsLock;
- tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
-
- auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); };
- auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
- RwLock::ExclusiveLockScope _(AttachmentsLock);
- if (BlockHash != IoHash::Zero)
- {
- Attachments.insert(BlockHash);
- }
- else
- {
- Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
- }
- };
- auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
- RwLock::ExclusiveLockScope _(AttachmentsLock);
- Attachments.insert(RawHash);
- };
-
- auto OnChunkedAttachment = [](const ChunkedInfo&) {};
+ auto OnChunkedAttachment = [](const ChunkedInfo&) {};
- auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); };
- // Make sure we retain any attachments we download before writing the oplog
- Oplog->EnableUpdateCapture();
- auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); });
+ auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog.CaptureAddedAttachments(RawHashes); };
- RemoteProjectStore::Result RemoteResult = SaveOplogContainer(*Oplog,
- ContainerObject,
- OnReferencedAttachments,
- HasAttachment,
- OnNeedBlock,
- OnNeedAttachment,
- OnChunkedAttachment,
- nullptr);
+ // Make sure we retain any attachments we download before writing the oplog
+ Oplog.EnableUpdateCapture();
+ auto _ = MakeGuard([&Oplog]() { Oplog.DisableUpdateCapture(); });
- if (RemoteResult.ErrorCode)
- {
- return ConvertResult(RemoteResult);
- }
-
- CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1);
- Cbo.BeginArray("need");
- {
- for (const IoHash& Hash : Attachments)
- {
- ZEN_DEBUG("Need attachment {}", Hash);
- Cbo << Hash;
- }
- }
- Cbo.EndArray(); // "need"
+ RemoteProjectStore::Result RemoteResult = SaveOplogContainer(Oplog,
+ ContainerObject,
+ OnReferencedAttachments,
+ HasAttachment,
+ OnNeedBlock,
+ OnNeedAttachment,
+ OnChunkedAttachment,
+ nullptr);
- OutResponse = Cbo.Save();
- return {HttpResponseCode::OK, {}};
- }
- else
+ if (RemoteResult.ErrorCode)
{
- return {HttpResponseCode::BadRequest, fmt::format("Invalid payload format ('{}')", ToString(ValidateResult))};
+ return ProjectStore::WriteOplogResult{.ErrorCode = RemoteResult.ErrorCode, .ErrorDescription = RemoteResult.Reason};
}
+
+ return ProjectStore::WriteOplogResult{.Need = std::vector<IoHash>(Attachments.begin(), Attachments.end())};
}
-std::pair<HttpResponseCode, std::string>
-ProjectStore::ReadOplog(const std::string_view ProjectId,
- const std::string_view OplogId,
- const HttpServerRequest::QueryParams& Params,
- CbObject& OutResponse)
+ProjectStore::ReadOplogResult
+ProjectStore::ReadOplog(Project& Project,
+ Oplog& Oplog,
+ size_t MaxBlockSize,
+ size_t MaxChunkEmbedSize,
+ size_t MaxChunksPerBlock,
+ size_t ChunkFileSizeLimit)
{
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::ReadOplog");
- Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
- if (!Project)
- {
- return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown project '{}'", ProjectId)};
- }
- Project->TouchProject();
-
- Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true);
- if (!Oplog)
- {
- return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)};
- }
- Project->TouchOplog(OplogId);
-
- size_t MaxBlockSize = RemoteStoreOptions::DefaultMaxBlockSize;
- if (auto Param = Params.GetValue("maxblocksize"); Param.empty() == false)
- {
- if (auto Value = ParseInt<size_t>(Param))
- {
- MaxBlockSize = Value.value();
- }
- }
- size_t MaxChunkEmbedSize = RemoteStoreOptions::DefaultMaxChunkEmbedSize;
- if (auto Param = Params.GetValue("maxchunkembedsize"); Param.empty() == false)
- {
- if (auto Value = ParseInt<size_t>(Param))
- {
- MaxChunkEmbedSize = Value.value();
- }
- }
- size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock;
- if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false)
- {
- if (auto Value = ParseInt<size_t>(Param))
- {
- MaxChunksPerBlock = Value.value();
- }
- }
-
- size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit;
- if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false)
- {
- if (auto Value = ParseInt<size_t>(Param))
- {
- ChunkFileSizeLimit = Value.value();
- }
- }
-
CidStore& ChunkStore = m_CidStore;
RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer(
ChunkStore,
- *Project.Get(),
- *Oplog,
+ Project,
+ Oplog,
MaxBlockSize,
MaxChunkEmbedSize,
MaxChunksPerBlock,
@@ -5837,8 +5647,12 @@ ProjectStore::ReadOplog(const std::string_view ProjectId,
[](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {},
/* EmbedLooseFiles*/ false);
- OutResponse = std::move(ContainerResult.ContainerObject);
- return ConvertResult(ContainerResult);
+ if (ContainerResult.ErrorCode)
+ {
+ return ProjectStore::ReadOplogResult{.ErrorCode = ContainerResult.ErrorCode, .ErrorDescription = ContainerResult.Reason};
+ }
+
+ return ProjectStore::ReadOplogResult{.ContainerObject = std::move(ContainerResult.ContainerObject)};
}
bool
@@ -9170,80 +8984,84 @@ TEST_CASE("project.store.partial.read")
Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(It.first, It.second));
}
}
+ Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
+ CHECK(Project1);
+ Ref<ProjectStore::Oplog> Oplog1 = Project1->OpenOplog("oplog1"sv, false, true);
+ CHECK(Oplog1);
{
uint64_t ModificationTag = 0;
- IoBuffer Chunk;
- CHECK(ProjectStore
- .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
- .first == HttpResponseCode::OK);
+
+ auto Result = ProjectStore.GetChunkRange(Log(),
+ *Project1,
+ *Oplog1,
+ Attachments[OpIds[1]][0].first,
+ 0,
+ ~0ull,
+ HttpContentType::kCompressedBinary,
+ &ModificationTag);
+
+ CHECK_EQ(ProjectStore::GetChunkRangeResult::EError::Ok, Result.Error);
+
IoHash RawHash;
uint64_t RawSize;
- CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
+ CompressedBuffer Attachment = CompressedBuffer::FromCompressed(Result.Chunk, RawHash, RawSize);
CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize());
- CHECK(ModificationTag != 0);
- CHECK(ProjectStore
- .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
- .first == HttpResponseCode::NotModified);
+ auto Result2 = ProjectStore.GetChunkRange(Log(),
+ *Project1,
+ *Oplog1,
+ Attachments[OpIds[1]][0].first,
+ 0,
+ ~0ull,
+ HttpContentType::kCompressedBinary,
+ &ModificationTag);
+ CHECK_EQ(ProjectStore::GetChunkRangeResult::EError::NotModified, Result2.Error);
}
{
uint64_t FullChunkModificationTag = 0;
{
- CompositeBuffer ChunkResult;
- HttpContentType ContentType;
- CHECK(ProjectStore
- .GetChunkRange("proj1"sv,
- "oplog1"sv,
- OidAsString(Attachments[OpIds[2]][1].first),
- 0,
- ~0ull,
- HttpContentType::kCompressedBinary,
- ChunkResult,
- ContentType,
- &FullChunkModificationTag)
- .first == HttpResponseCode::OK);
- CHECK(ChunkResult);
- CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
+ auto Result = ProjectStore.GetChunkRange(Log(),
+ *Project1,
+ *Oplog1,
+ Attachments[OpIds[2]][1].first,
+ 0,
+ ~0ull,
+ HttpContentType::kCompressedBinary,
+ &FullChunkModificationTag);
+ CHECK_EQ(Result.Error, ProjectStore::GetChunkRangeResult::EError::Ok);
+ CHECK(Result.Chunk);
+ CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(Result.Chunk)).DecodeRawSize() ==
Attachments[OpIds[2]][1].second.DecodeRawSize());
}
{
- CompositeBuffer ChunkResult;
- HttpContentType ContentType;
- CHECK(ProjectStore
- .GetChunkRange("proj1"sv,
- "oplog1"sv,
- OidAsString(Attachments[OpIds[2]][1].first),
- 0,
- ~0ull,
- HttpContentType::kCompressedBinary,
- ChunkResult,
- ContentType,
- &FullChunkModificationTag)
- .first == HttpResponseCode::NotModified);
- }
- }
- {
- CompositeBuffer PartialChunkResult;
- uint64_t PartialChunkModificationTag = 0;
- {
- CompositeBuffer ChunkResult;
- HttpContentType ContentType;
- CHECK(ProjectStore
- .GetChunkRange("proj1"sv,
- "oplog1"sv,
- OidAsString(Attachments[OpIds[2]][1].first),
- 5,
- 1773,
- HttpContentType::kCompressedBinary,
- PartialChunkResult,
- ContentType,
- &PartialChunkModificationTag)
- .first == HttpResponseCode::OK);
- CHECK(PartialChunkResult);
+ auto Result = ProjectStore.GetChunkRange(Log(),
+ *Project1,
+ *Oplog1,
+ Attachments[OpIds[2]][1].first,
+ 0,
+ ~0ull,
+ HttpContentType::kCompressedBinary,
+ &FullChunkModificationTag);
+ CHECK_EQ(Result.Error, ProjectStore::GetChunkRangeResult::EError::NotModified);
+ }
+ }
+ {
+ uint64_t PartialChunkModificationTag = 0;
+ {
+ auto Result = ProjectStore.GetChunkRange(Log(),
+ *Project1,
+ *Oplog1,
+ Attachments[OpIds[2]][1].first,
+ 5,
+ 1773,
+ HttpContentType::kCompressedBinary,
+ &PartialChunkModificationTag);
+ CHECK_EQ(Result.Error, ProjectStore::GetChunkRangeResult::EError::Ok);
+
IoHash PartialRawHash;
uint64_t PartialRawSize;
- CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize);
+ CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(Result.Chunk, PartialRawHash, PartialRawSize);
CHECK(PartialRawSize >= 1773);
uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);
@@ -9255,19 +9073,15 @@ TEST_CASE("project.store.partial.read")
}
{
- CompositeBuffer ChunkResult;
- HttpContentType ContentType;
- CHECK(ProjectStore
- .GetChunkRange("proj1"sv,
- "oplog1"sv,
- OidAsString(Attachments[OpIds[2]][1].first),
- 5,
- 1773,
- HttpContentType::kCompressedBinary,
- PartialChunkResult,
- ContentType,
- &PartialChunkModificationTag)
- .first == HttpResponseCode::NotModified);
+ auto Result = ProjectStore.GetChunkRange(Log(),
+ *Project1,
+ *Oplog1,
+ Attachments[OpIds[2]][1].first,
+ 0,
+ 1773,
+ HttpContentType::kCompressedBinary,
+ &PartialChunkModificationTag);
+ CHECK_EQ(Result.Error, ProjectStore::GetChunkRangeResult::EError::NotModified);
}
}
}