aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorMartin Ridgers <[email protected]>2024-09-17 13:28:53 +0200
committerMartin Ridgers <[email protected]>2024-09-19 15:26:13 +0200
commitb33c39a7a7cda7fdd52582da9d3655f7ca8ef3ad (patch)
treea9de8cf8f1f95e3a78fe42690ffd76d4f95bd2fc /src
parentMandatory IterateOplog() paging argument (diff)
downloadzen-b33c39a7a7cda7fdd52582da9d3655f7ca8ef3ad.tar.xz
zen-b33c39a7a7cda7fdd52582da9d3655f7ca8ef3ad.zip
clang-format on branch's changed files
Diffstat (limited to 'src')
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp3
-rw-r--r--src/zenserver/projectstore/projectstore.cpp216
-rw-r--r--src/zenserver/vfs/vfsimpl.cpp10
3 files changed, 119 insertions, 110 deletions
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 45a396f02..9e4961889 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -1485,7 +1485,8 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req)
}
else
{
- FoundLog->IterateOplog([this, &Response, &FilterObject](CbObjectView Op) { Response << FilterObject(Op); }, ProjectStore::Oplog::Paging{});
+ FoundLog->IterateOplog([this, &Response, &FilterObject](CbObjectView Op) { Response << FilterObject(Op); },
+ ProjectStore::Oplog::Paging{});
}
Response.EndArray();
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 75a077e87..a970a0380 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1093,14 +1093,16 @@ ProjectStore::Oplog::GatherReferences(GcContext& GcCtx)
std::vector<IoHash> Cids;
Cids.reserve(1024);
- IterateOplog([&](CbObjectView Op) {
- Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); });
- if (Cids.size() >= 1024)
- {
- GcCtx.AddRetainedCids(Cids);
- Cids.clear();
- }
- }, Paging{});
+ IterateOplog(
+ [&](CbObjectView Op) {
+ Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); });
+ if (Cids.size() >= 1024)
+ {
+ GcCtx.AddRetainedCids(Cids);
+ Cids.clear();
+ }
+ },
+ Paging{});
GcCtx.AddRetainedCids(Cids);
}
@@ -2063,10 +2065,10 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand
});
std::span<OplogEntryAddress> EntrySpan = Entries;
- int32_t Size = int32_t(Entries.size());
- int32_t Start = std::clamp(EntryPaging.Start, 0, Size);
- int32_t End = std::clamp(EntryPaging.Start + EntryPaging.Count, 0, Size);
- EntrySpan = EntrySpan.subspan(Start, End - Start);
+ int32_t Size = int32_t(Entries.size());
+ int32_t Start = std::clamp(EntryPaging.Start, 0, Size);
+ int32_t End = std::clamp(EntryPaging.Start + EntryPaging.Count, 0, Size);
+ EntrySpan = EntrySpan.subspan(Start, End - Start);
m_Storage->ReplayLogEntries(EntrySpan, [&](CbObjectView Op) { Handler(Op); });
}
@@ -2106,21 +2108,23 @@ ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, b
std::vector<uint32_t> AttachmentCounts;
size_t AttachmentOffset = OutAttachments.size();
- IterateOplogLocked([&](CbObjectView Op) {
- using namespace std::literals;
- size_t CurrentAttachmentCount = OutAttachments.size();
- Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); });
- if (StoreMetaDataOnDisk)
- {
- XXH3_128Stream KeyHasher;
- Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash128 = KeyHasher.GetHash();
- Oid KeyHash;
- memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
- Keys.push_back(KeyHash);
- AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount));
- }
- }, Paging{});
+ IterateOplogLocked(
+ [&](CbObjectView Op) {
+ using namespace std::literals;
+ size_t CurrentAttachmentCount = OutAttachments.size();
+ Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); });
+ if (StoreMetaDataOnDisk)
+ {
+ XXH3_128Stream KeyHasher;
+ Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
+ XXH3_128 KeyHash128 = KeyHasher.GetHash();
+ Oid KeyHash;
+ memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+ Keys.push_back(KeyHash);
+ AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount));
+ }
+ },
+ Paging{});
if (StoreMetaDataOnDisk)
{
const IoHash* FirstAttachment = OutAttachments.data() + AttachmentOffset;
@@ -4757,110 +4761,112 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
std::vector<CbObject> NewOps;
std::unordered_map<Oid, IoHash, Oid::Hasher> NewChunkMappings;
- Oplog->IterateOplog([&](CbObjectView Op) {
- bool OpRewritten = false;
- bool AllOk = true;
-
- CbWriter Cbo;
- Cbo.BeginArray("files"sv);
+ Oplog->IterateOplog(
+ [&](CbObjectView Op) {
+ bool OpRewritten = false;
+ bool AllOk = true;
- for (CbFieldView& Field : Op["files"sv])
- {
- bool CopyField = true;
+ CbWriter Cbo;
+ Cbo.BeginArray("files"sv);
- if (CbObjectView View = Field.AsObjectView())
+ for (CbFieldView& Field : Op["files"sv])
{
- const IoHash DataHash = View["data"sv].AsHash();
+ bool CopyField = true;
- if (DataHash == IoHash::Zero)
+ if (CbObjectView View = Field.AsObjectView())
{
- std::string_view ServerPath = View["serverpath"sv].AsString();
- std::filesystem::path FilePath = Project->RootDir / ServerPath;
- BasicFile DataFile;
- std::error_code Ec;
- DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec);
+ const IoHash DataHash = View["data"sv].AsHash();
- if (Ec)
+ if (DataHash == IoHash::Zero)
{
- // Error...
+ std::string_view ServerPath = View["serverpath"sv].AsString();
+ std::filesystem::path FilePath = Project->RootDir / ServerPath;
+ BasicFile DataFile;
+ std::error_code Ec;
+ DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec);
- ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message());
+ if (Ec)
+ {
+ // Error...
- AllOk = false;
- }
- else
- {
- // Read file contents into memory, compress and store in CidStore
+ ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message());
- Oid ChunkId = View["id"sv].AsObjectId();
- IoBuffer FileIoBuffer = DataFile.ReadAll();
- CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer)));
- const IoHash RawHash = Compressed.DecodeRawHash();
- const uint64_t RawSize = Compressed.DecodeRawSize();
- IoBuffer CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer();
- CidStore::InsertResult Result = m_CidStore.AddChunk(CompressedBuffer, RawHash);
+ AllOk = false;
+ }
+ else
+ {
+ // Read file contents into memory, compress and store in CidStore
- TotalBytes += RawSize;
- ++TotalFiles;
+ Oid ChunkId = View["id"sv].AsObjectId();
+ IoBuffer FileIoBuffer = DataFile.ReadAll();
+ CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer)));
+ const IoHash RawHash = Compressed.DecodeRawHash();
+ const uint64_t RawSize = Compressed.DecodeRawSize();
+ IoBuffer CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer();
+ CidStore::InsertResult Result = m_CidStore.AddChunk(CompressedBuffer, RawHash);
- if (Result.New)
- {
- InlinedBytes += RawSize;
- ++InlinedFiles;
- }
+ TotalBytes += RawSize;
+ ++TotalFiles;
- // Rewrite file array entry with new data reference
- CbObjectWriter Writer;
- RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool {
- if (Field.GetName() == "data"sv)
+ if (Result.New)
{
- // omit this field as we will write it explicitly ourselves
- return true;
+ InlinedBytes += RawSize;
+ ++InlinedFiles;
}
- return false;
- });
- Writer.AddBinaryAttachment("data"sv, RawHash);
- CbObject RewrittenOp = Writer.Save();
- Cbo.AddObject(std::move(RewrittenOp));
- CopyField = false;
-
- NewChunkMappings.insert_or_assign(ChunkId, RawHash);
+ // Rewrite file array entry with new data reference
+ CbObjectWriter Writer;
+ RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool {
+ if (Field.GetName() == "data"sv)
+ {
+ // omit this field as we will write it explicitly ourselves
+ return true;
+ }
+ return false;
+ });
+ Writer.AddBinaryAttachment("data"sv, RawHash);
+
+ CbObject RewrittenOp = Writer.Save();
+ Cbo.AddObject(std::move(RewrittenOp));
+ CopyField = false;
+
+ NewChunkMappings.insert_or_assign(ChunkId, RawHash);
+ }
}
}
- }
- if (CopyField)
- {
- Cbo.AddField(Field);
- }
- else
- {
- OpRewritten = true;
+ if (CopyField)
+ {
+ Cbo.AddField(Field);
+ }
+ else
+ {
+ OpRewritten = true;
+ }
}
- }
- if (OpRewritten && AllOk)
- {
- Cbo.EndArray();
- CbArray FilesArray = Cbo.Save().AsArray();
+ if (OpRewritten && AllOk)
+ {
+ Cbo.EndArray();
+ CbArray FilesArray = Cbo.Save().AsArray();
- CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool {
- if (Field.GetName() == "files"sv)
- {
- NewWriter.AddArray("files"sv, FilesArray);
+ CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool {
+ if (Field.GetName() == "files"sv)
+ {
+ NewWriter.AddArray("files"sv, FilesArray);
- return true;
- }
+ return true;
+ }
- return false;
- });
+ return false;
+ });
- NewOps.push_back(std::move(RewrittenOp));
- }
+ NewOps.push_back(std::move(RewrittenOp));
+ }
- OpCount++;
- }, Oplog::Paging{});
+ OpCount++;
+ },
+ Oplog::Paging{});
// Make sure we have references to our attachments
Oplog->AddChunkMappings(NewChunkMappings);
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
index 80de1e1a5..abb7e8f94 100644
--- a/src/zenserver/vfs/vfsimpl.cpp
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -396,10 +396,12 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo
}
};
- Oplog->IterateOplog([&](CbObjectView Op) {
- EmitFilesForDataArray(Op["packagedata"sv].AsArrayView());
- EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView());
- }, ProjectStore::Oplog::Paging{});
+ Oplog->IterateOplog(
+ [&](CbObjectView Op) {
+ EmitFilesForDataArray(Op["packagedata"sv].AsArrayView());
+ EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView());
+ },
+ ProjectStore::Oplog::Paging{});
DirNode.AddFileNode("stats.json", 42, Oid::Zero);
}