// Copyright Epic Games, Inc. All Rights Reserved. #include "zenstore/cidstore.h" #include #include #include #include #include #include #include #include "cas.h" #include namespace zen { struct CidStore::Impl { explicit Impl(CasStore& InCasStore) : m_CasStore(InCasStore) {} CasStore& m_CasStore; void Initialize(const CidStoreConfiguration& Config) { m_CasStore.Initialize(Config); } CidStore::InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, CidStore::InsertMode Mode) { metrics::RequestStats::Scope $(m_AddChunkOps, ChunkData.GetSize()); #if ZEN_BUILD_DEBUG IoHash VerifyRawHash; uint64_t _; ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(ChunkData, VerifyRawHash, _) && RawHash == VerifyRawHash); #endif IoBuffer Payload(ChunkData); Payload.SetContentType(ZenContentType::kCompressedBinary); CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, RawHash, static_cast(Mode)); if (Result.New) { m_WriteCount++; } return {.New = Result.New}; } IoBuffer FindChunkByCid(const IoHash& DecompressedId) { metrics::RequestStats::Scope StatsScope(m_FindChunkOps, 0); IoBuffer Result = m_CasStore.FindChunk(DecompressedId); if (Result) { m_HitCount++; StatsScope.SetBytes(Result.GetSize()); } else { m_MissCount++; } return Result; } bool ContainsChunk(const IoHash& DecompressedId) { // metrics::RequestStats::Scope $(m_ContainChunkOps); return m_CasStore.ContainsChunk(DecompressedId); } void FilterChunks(HashKeySet& InOutChunks) { InOutChunks.RemoveHashesIf([&](const IoHash& Hash) { return ContainsChunk(Hash); }); } void Flush() { m_CasStore.Flush(); } void ScrubStorage(ScrubContext& Ctx) { if (Ctx.ScrubTimestamp() == m_LastScrubTime) { return; } m_LastScrubTime = Ctx.ScrubTimestamp(); m_CasStore.ScrubStorage(Ctx); } CidStoreStats Stats() { return CidStoreStats{ .HitCount = m_HitCount, .MissCount = m_MissCount, .WriteCount = m_WriteCount, .AddChunkOps = m_AddChunkOps.Snapshot(), .FindChunkOps = m_FindChunkOps.Snapshot() // .ContainChunkOps = m_ContainChunkOps.Snapshot() }; } void ReportMetrics(StatsMetrics& Statsd) { const CidStoreStats Now = Stats(); const CidStoreStats& Old = m_LastReportedMetrics; Statsd.Meter("zen.cas_hits", Now.HitCount - Old.HitCount); Statsd.Meter("zen.cas_misses", Now.MissCount - Old.MissCount); Statsd.Meter("zen.cas_writes", Now.WriteCount - Old.WriteCount); m_LastReportedMetrics = Now; } std::atomic_uint64_t m_HitCount{}; std::atomic_uint64_t m_MissCount{}; std::atomic_uint64_t m_WriteCount{}; metrics::RequestStats m_AddChunkOps; metrics::RequestStats m_FindChunkOps; CidStoreStats m_LastReportedMetrics; uint64_t m_LastScrubTime = 0; }; ////////////////////////////////////////////////////////////////////////// CidStore::CidStore(GcManager& Gc) : m_CasStore(CreateCasStore(Gc)), m_Impl(std::make_unique(*m_CasStore)) { } CidStore::~CidStore() { } void CidStore::Initialize(const CidStoreConfiguration& Config) { m_Impl->Initialize(Config); } CidStore::InsertResult CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode) { return m_Impl->AddChunk(ChunkData, RawHash, Mode); } IoBuffer CidStore::FindChunkByCid(const IoHash& DecompressedId) { return m_Impl->FindChunkByCid(DecompressedId); } bool CidStore::ContainsChunk(const IoHash& DecompressedId) { return m_Impl->ContainsChunk(DecompressedId); } void CidStore::FilterChunks(HashKeySet& InOutChunks) { return m_Impl->FilterChunks(InOutChunks); } void CidStore::Flush() { m_Impl->Flush(); } void CidStore::ScrubStorage(ScrubContext& Ctx) { m_Impl->ScrubStorage(Ctx); } CidStoreSize CidStore::TotalSize() const { return m_Impl->m_CasStore.TotalSize(); } CidStoreStats CidStore::Stats() const { return m_Impl->Stats(); } void CidStore::ReportMetrics(StatsMetrics& Statsd) { return m_Impl->ReportMetrics(Statsd); } } // namespace zen