diff options
| author | Dan Engelbrecht <[email protected]> | 2022-12-07 11:21:41 +0100 |
|---|---|---|
| committer | GitHub <[email protected]> | 2022-12-07 02:21:41 -0800 |
| commit | 100c8f966b1c5b2fb190748f0177600562d1c5fe (patch) | |
| tree | fc85e350dea47330149a1d42eb7a6c7ae0a06111 /zenstore/cidstore.cpp | |
| parent | Cache request record/replay (#198) (diff) | |
| download | zen-100c8f966b1c5b2fb190748f0177600562d1c5fe.tar.xz zen-100c8f966b1c5b2fb190748f0177600562d1c5fe.zip | |
optimizations (#200)
* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog
Diffstat (limited to 'zenstore/cidstore.cpp')
| -rw-r--r-- | zenstore/cidstore.cpp | 17 |
1 files changed, 10 insertions, 7 deletions
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index 8b2797ce9..5a5116faf 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -23,14 +23,17 @@ struct CidStore::Impl void Initialize(const CidStoreConfiguration& Config) { m_CasStore.Initialize(Config); } - CidStore::InsertResult AddChunk(const CompressedBuffer& ChunkData, CidStore::InsertMode Mode) + CidStore::InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, CidStore::InsertMode Mode) { - const IoHash DecompressedId = IoHash::FromBLAKE3(ChunkData.GetRawHash()); - IoBuffer Payload = ChunkData.GetCompressed().Flatten().AsIoBuffer(); - +#ifndef NDEBUG + IoHash VerifyRawHash; + uint64_t _; + ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(ChunkData, VerifyRawHash, _) && RawHash == VerifyRawHash); +#endif // NDEBUG + IoBuffer Payload(ChunkData); Payload.SetContentType(ZenContentType::kCompressedBinary); - CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, DecompressedId, static_cast<CasStore::InsertMode>(Mode)); + CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, RawHash, static_cast<CasStore::InsertMode>(Mode)); return {.New = Result.New}; } @@ -78,9 +81,9 @@ CidStore::Initialize(const CidStoreConfiguration& Config) } CidStore::InsertResult -CidStore::AddChunk(const CompressedBuffer& ChunkData, InsertMode Mode) +CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode) { - return m_Impl->AddChunk(ChunkData, Mode); + return m_Impl->AddChunk(ChunkData, RawHash, Mode); } IoBuffer |