diff options
| author | Martin Ridgers <[email protected]> | 2024-09-17 13:28:53 +0200 |
|---|---|---|
| committer | Martin Ridgers <[email protected]> | 2024-09-19 15:26:13 +0200 |
| commit | b33c39a7a7cda7fdd52582da9d3655f7ca8ef3ad (patch) | |
| tree | a9de8cf8f1f95e3a78fe42690ffd76d4f95bd2fc /src | |
| parent | Mandatory IterateOplog() paging argument (diff) | |
| download | zen-b33c39a7a7cda7fdd52582da9d3655f7ca8ef3ad.tar.xz zen-b33c39a7a7cda7fdd52582da9d3655f7ca8ef3ad.zip | |
clang-format on branch's changed files
Diffstat (limited to 'src')
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.cpp | 3 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 216 | ||||
| -rw-r--r-- | src/zenserver/vfs/vfsimpl.cpp | 10 |
3 files changed, 119 insertions, 110 deletions
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 45a396f02..9e4961889 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -1485,7 +1485,8 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) } else { - FoundLog->IterateOplog([this, &Response, &FilterObject](CbObjectView Op) { Response << FilterObject(Op); }, ProjectStore::Oplog::Paging{}); + FoundLog->IterateOplog([this, &Response, &FilterObject](CbObjectView Op) { Response << FilterObject(Op); }, + ProjectStore::Oplog::Paging{}); } Response.EndArray(); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 75a077e87..a970a0380 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1093,14 +1093,16 @@ ProjectStore::Oplog::GatherReferences(GcContext& GcCtx) std::vector<IoHash> Cids; Cids.reserve(1024); - IterateOplog([&](CbObjectView Op) { - Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); }); - if (Cids.size() >= 1024) - { - GcCtx.AddRetainedCids(Cids); - Cids.clear(); - } - }, Paging{}); + IterateOplog( + [&](CbObjectView Op) { + Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); }); + if (Cids.size() >= 1024) + { + GcCtx.AddRetainedCids(Cids); + Cids.clear(); + } + }, + Paging{}); GcCtx.AddRetainedCids(Cids); } @@ -2063,10 +2065,10 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand }); std::span<OplogEntryAddress> EntrySpan = Entries; - int32_t Size = int32_t(Entries.size()); - int32_t Start = std::clamp(EntryPaging.Start, 0, Size); - int32_t End = std::clamp(EntryPaging.Start + EntryPaging.Count, 0, Size); - EntrySpan = EntrySpan.subspan(Start, End - Start); + int32_t Size = int32_t(Entries.size()); + int32_t Start = std::clamp(EntryPaging.Start, 0, Size); + int32_t End = std::clamp(EntryPaging.Start + EntryPaging.Count, 0, Size); + EntrySpan = EntrySpan.subspan(Start, End - Start); m_Storage->ReplayLogEntries(EntrySpan, [&](CbObjectView Op) { Handler(Op); }); } @@ -2106,21 +2108,23 @@ ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, b std::vector<uint32_t> AttachmentCounts; size_t AttachmentOffset = OutAttachments.size(); - IterateOplogLocked([&](CbObjectView Op) { - using namespace std::literals; - size_t CurrentAttachmentCount = OutAttachments.size(); - Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); }); - if (StoreMetaDataOnDisk) - { - XXH3_128Stream KeyHasher; - Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); }); - XXH3_128 KeyHash128 = KeyHasher.GetHash(); - Oid KeyHash; - memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash); - Keys.push_back(KeyHash); - AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount)); - } - }, Paging{}); + IterateOplogLocked( + [&](CbObjectView Op) { + using namespace std::literals; + size_t CurrentAttachmentCount = OutAttachments.size(); + Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); }); + if (StoreMetaDataOnDisk) + { + XXH3_128Stream KeyHasher; + Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); }); + XXH3_128 KeyHash128 = KeyHasher.GetHash(); + Oid KeyHash; + memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash); + Keys.push_back(KeyHash); + AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount)); + } + }, + Paging{}); if (StoreMetaDataOnDisk) { const IoHash* FirstAttachment = OutAttachments.data() + AttachmentOffset; @@ -4757,110 +4761,112 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, std::vector<CbObject> NewOps; std::unordered_map<Oid, IoHash, Oid::Hasher> NewChunkMappings; - Oplog->IterateOplog([&](CbObjectView Op) { - bool OpRewritten = false; - bool AllOk = true; - - CbWriter Cbo; - Cbo.BeginArray("files"sv); + Oplog->IterateOplog( + [&](CbObjectView Op) { + bool OpRewritten = false; + bool AllOk = true; - for (CbFieldView& Field : Op["files"sv]) - { - bool CopyField = true; + CbWriter Cbo; + Cbo.BeginArray("files"sv); - if (CbObjectView View = Field.AsObjectView()) + for (CbFieldView& Field : Op["files"sv]) { - const IoHash DataHash = View["data"sv].AsHash(); + bool CopyField = true; - if (DataHash == IoHash::Zero) + if (CbObjectView View = Field.AsObjectView()) { - std::string_view ServerPath = View["serverpath"sv].AsString(); - std::filesystem::path FilePath = Project->RootDir / ServerPath; - BasicFile DataFile; - std::error_code Ec; - DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); + const IoHash DataHash = View["data"sv].AsHash(); - if (Ec) + if (DataHash == IoHash::Zero) { - // Error... + std::string_view ServerPath = View["serverpath"sv].AsString(); + std::filesystem::path FilePath = Project->RootDir / ServerPath; + BasicFile DataFile; + std::error_code Ec; + DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); - ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); + if (Ec) + { + // Error... - AllOk = false; - } - else - { - // Read file contents into memory, compress and store in CidStore + ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); - Oid ChunkId = View["id"sv].AsObjectId(); - IoBuffer FileIoBuffer = DataFile.ReadAll(); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); - const IoHash RawHash = Compressed.DecodeRawHash(); - const uint64_t RawSize = Compressed.DecodeRawSize(); - IoBuffer CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer(); - CidStore::InsertResult Result = m_CidStore.AddChunk(CompressedBuffer, RawHash); + AllOk = false; + } + else + { + // Read file contents into memory, compress and store in CidStore - TotalBytes += RawSize; - ++TotalFiles; + Oid ChunkId = View["id"sv].AsObjectId(); + IoBuffer FileIoBuffer = DataFile.ReadAll(); + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); + const IoHash RawHash = Compressed.DecodeRawHash(); + const uint64_t RawSize = Compressed.DecodeRawSize(); + IoBuffer CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer(); + CidStore::InsertResult Result = m_CidStore.AddChunk(CompressedBuffer, RawHash); - if (Result.New) - { - InlinedBytes += RawSize; - ++InlinedFiles; - } + TotalBytes += RawSize; + ++TotalFiles; - // Rewrite file array entry with new data reference - CbObjectWriter Writer; - RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { - if (Field.GetName() == "data"sv) + if (Result.New) { - // omit this field as we will write it explicitly ourselves - return true; + InlinedBytes += RawSize; + ++InlinedFiles; } - return false; - }); - Writer.AddBinaryAttachment("data"sv, RawHash); - CbObject RewrittenOp = Writer.Save(); - Cbo.AddObject(std::move(RewrittenOp)); - CopyField = false; - - NewChunkMappings.insert_or_assign(ChunkId, RawHash); + // Rewrite file array entry with new data reference + CbObjectWriter Writer; + RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { + if (Field.GetName() == "data"sv) + { + // omit this field as we will write it explicitly ourselves + return true; + } + return false; + }); + Writer.AddBinaryAttachment("data"sv, RawHash); + + CbObject RewrittenOp = Writer.Save(); + Cbo.AddObject(std::move(RewrittenOp)); + CopyField = false; + + NewChunkMappings.insert_or_assign(ChunkId, RawHash); + } } } - } - if (CopyField) - { - Cbo.AddField(Field); - } - else - { - OpRewritten = true; + if (CopyField) + { + Cbo.AddField(Field); + } + else + { + OpRewritten = true; + } } - } - if (OpRewritten && AllOk) - { - Cbo.EndArray(); - CbArray FilesArray = Cbo.Save().AsArray(); + if (OpRewritten && AllOk) + { + Cbo.EndArray(); + CbArray FilesArray = Cbo.Save().AsArray(); - CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { - if (Field.GetName() == "files"sv) - { - NewWriter.AddArray("files"sv, FilesArray); + CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { + if (Field.GetName() == "files"sv) + { + NewWriter.AddArray("files"sv, FilesArray); - return true; - } + return true; + } - return false; - }); + return false; + }); - NewOps.push_back(std::move(RewrittenOp)); - } + NewOps.push_back(std::move(RewrittenOp)); + } - OpCount++; - }, Oplog::Paging{}); + OpCount++; + }, + Oplog::Paging{}); // Make sure we have references to our attachments Oplog->AddChunkMappings(NewChunkMappings); diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp index 80de1e1a5..abb7e8f94 100644 --- a/src/zenserver/vfs/vfsimpl.cpp +++ b/src/zenserver/vfs/vfsimpl.cpp @@ -396,10 +396,12 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo } }; - Oplog->IterateOplog([&](CbObjectView Op) { - EmitFilesForDataArray(Op["packagedata"sv].AsArrayView()); - EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView()); - }, ProjectStore::Oplog::Paging{}); + Oplog->IterateOplog( + [&](CbObjectView Op) { + EmitFilesForDataArray(Op["packagedata"sv].AsArrayView()); + EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView()); + }, + ProjectStore::Oplog::Paging{}); DirNode.AddFileNode("stats.json", 42, Oid::Zero); } |