From a0bd0821e7a23f1ff3476151c0702be3691dc582 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Fri, 29 Oct 2021 15:10:28 +0200 Subject: CAS.cpp/h -> cas.cpp/h to keep Zen's file casing consistent --- zenserver/cache/structuredcache.cpp | 2 +- zenserver/compute/apply.cpp | 2 +- zenserver/testing/launch.cpp | 2 +- zenstore/CAS.cpp | 325 ----------------------------------- zenstore/cas.cpp | 325 +++++++++++++++++++++++++++++++++++ zenstore/caslog.cpp | 2 +- zenstore/cidstore.cpp | 2 +- zenstore/compactcas.cpp | 2 +- zenstore/compactcas.h | 2 +- zenstore/filecas.h | 2 +- zenstore/include/zenstore/CAS.h | 129 -------------- zenstore/include/zenstore/cas.h | 129 ++++++++++++++ zenstore/include/zenstore/caslog.h | 2 +- zenstore/include/zenstore/cidstore.h | 2 +- zenstore/zenstore.cpp | 2 +- zenstore/zenstore.vcxproj | 2 +- zenstore/zenstore.vcxproj.filters | 2 +- 17 files changed, 467 insertions(+), 467 deletions(-) delete mode 100644 zenstore/CAS.cpp create mode 100644 zenstore/cas.cpp delete mode 100644 zenstore/include/zenstore/CAS.h create mode 100644 zenstore/include/zenstore/cas.h diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 35cb02cbb..0debea3c6 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include "monitoring/httpstats.h" #include "structuredcache.h" diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index b64a65f14..059cd152d 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #if ZEN_PLATFORM_WINDOWS diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index 1f38257e5..96f01576c 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #if ZEN_PLATFORM_WINDOWS # include diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp deleted file mode 100644 index bc4825419..000000000 --- a/zenstore/CAS.cpp +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include "compactcas.h" -#include "filecas.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -void -CasChunkSet::AddChunkToSet(const IoHash& HashToAdd) -{ - m_ChunkSet.insert(HashToAdd); -} - -void -CasChunkSet::AddChunksToSet(std::span HashesToAdd) -{ - for (const IoHash& Hash : HashesToAdd) - { - m_ChunkSet.insert(Hash); - } -} - -void -CasChunkSet::RemoveChunksIf(std::function&& Predicate) -{ - for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;) - { - if (Predicate(*It)) - { - It = m_ChunkSet.erase(It); - } - else - { - ++It; - } - } -} - -void -CasChunkSet::IterateChunks(std::function&& Callback) -{ - for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd; ++It) - { - Callback(*It); - } -} - -////////////////////////////////////////////////////////////////////////// - -struct GcContext::GcState -{ - CasChunkSet m_CasChunks; - CasChunkSet m_CidChunks; -}; - -GcContext::GcContext() : m_State(std::make_unique()) -{ -} - -GcContext::~GcContext() -{ -} - -void -GcContext::ContributeCids(std::span Cids) -{ - m_State->m_CidChunks.AddChunksToSet(Cids); -} - -void -GcContext::ContributeCas(std::span Cas) -{ - m_State->m_CasChunks.AddChunksToSet(Cas); -} - -////////////////////////////////////////////////////////////////////////// - -void -ScrubContext::ReportBadCasChunks(std::span BadCasChunks) -{ - m_BadCas.AddChunksToSet(BadCasChunks); -} - -void -ScrubContext::ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes) -{ - m_ChunkCount.fetch_add(ChunkCount); - m_ByteCount.fetch_add(ChunkBytes); -} - -/** - * CAS store implementation - * - * Uses a basic strategy of splitting payloads by size, to improve ability to reclaim space - * quickly for unused large chunks and to maintain locality for small chunks which are - * frequently accessed together. - * - */ -class CasImpl : public CasStore -{ -public: - CasImpl(); - virtual ~CasImpl(); - - virtual void Initialize(const CasStoreConfiguration& InConfig) override; - virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) override; - virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; - virtual void FilterChunks(CasChunkSet& InOutChunks) override; - virtual void Flush() override; - virtual void Scrub(ScrubContext& Ctx) override; - -private: - CasContainerStrategy m_TinyStrategy; - CasContainerStrategy m_SmallStrategy; - FileCasStrategy m_LargeStrategy; -}; - -CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) -{ -} - -CasImpl::~CasImpl() -{ -} - -void -CasImpl::Initialize(const CasStoreConfiguration& InConfig) -{ - m_Config = InConfig; - - ZEN_INFO("initializing CAS pool at '{}'", m_Config.RootDirectory); - - // Ensure root directory exists - create if it doesn't exist already - - std::filesystem::create_directories(m_Config.RootDirectory); - - // Open or create manifest - // - // The manifest is not currently fully implemented. The goal is to - // use it for recovery and configuration - - bool IsNewStore = false; - - { - std::filesystem::path ManifestPath = m_Config.RootDirectory; - ManifestPath /= ".ucas_root"; - - std::error_code Ec; - BasicFile Marker; - Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); - - if (Ec) - { - IsNewStore = true; - - ExtendableStringBuilder<128> manifest; - manifest.Append("#CAS_ROOT\n"); - manifest.Append("ID="); - zen::Oid id = zen::Oid::NewOid(); - id.ToString(manifest); - - Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - Marker.Write(manifest.c_str(), uint32_t(manifest.Size()), 0); - } - } - - // Initialize payload storage - - m_TinyStrategy.Initialize("tobs", 16, IsNewStore); - m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); -} - -CasStore::InsertResult -CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) -{ - const uint64_t ChunkSize = Chunk.Size(); - - if (ChunkSize < m_Config.TinyValueThreshold) - { - ZEN_ASSERT(ChunkSize); - - return m_TinyStrategy.InsertChunk(Chunk, ChunkHash); - } - else if (ChunkSize < m_Config.HugeValueThreshold) - { - return m_SmallStrategy.InsertChunk(Chunk, ChunkHash); - } - - return m_LargeStrategy.InsertChunk(Chunk, ChunkHash); -} - -IoBuffer -CasImpl::FindChunk(const IoHash& ChunkHash) -{ - if (IoBuffer Found = m_SmallStrategy.FindChunk(ChunkHash)) - { - return Found; - } - - if (IoBuffer Found = m_TinyStrategy.FindChunk(ChunkHash)) - { - return Found; - } - - if (IoBuffer Found = m_LargeStrategy.FindChunk(ChunkHash)) - { - return Found; - } - - // Not found - return IoBuffer{}; -} - -void -CasImpl::FilterChunks(CasChunkSet& InOutChunks) -{ - m_SmallStrategy.FilterChunks(InOutChunks); - m_TinyStrategy.FilterChunks(InOutChunks); - m_LargeStrategy.FilterChunks(InOutChunks); -} - -void -CasImpl::Flush() -{ - m_SmallStrategy.Flush(); - m_TinyStrategy.Flush(); - m_LargeStrategy.Flush(); -} - -void -CasImpl::Scrub(ScrubContext& Ctx) -{ - if (m_LastScrubTime == Ctx.ScrubTimestamp()) - { - return; - } - - m_LastScrubTime = Ctx.ScrubTimestamp(); - - m_SmallStrategy.Scrub(Ctx); - m_TinyStrategy.Scrub(Ctx); - m_LargeStrategy.Scrub(Ctx); -} - -////////////////////////////////////////////////////////////////////////// - -CasStore* -CreateCasStore() -{ - return new CasImpl(); -} - -////////////////////////////////////////////////////////////////////////// -// -// Testing related code follows... -// - -#if ZEN_WITH_TESTS - -TEST_CASE("CasStore") -{ - ScopedTemporaryDirectory TempDir; - - CasStoreConfiguration config; - config.RootDirectory = TempDir.Path(); - - std::unique_ptr Store{CreateCasStore()}; - Store->Initialize(config); - - ScrubContext Ctx; - Store->Scrub(Ctx); - - IoBuffer Value1{16}; - memcpy(Value1.MutableData(), "1234567890123456", 16); - IoHash Hash1 = IoHash::HashBuffer(Value1.Data(), Value1.Size()); - CasStore::InsertResult Result1 = Store->InsertChunk(Value1, Hash1); - CHECK(Result1.New); - - IoBuffer Value2{16}; - memcpy(Value2.MutableData(), "ABCDEFGHIJKLMNOP", 16); - IoHash Hash2 = IoHash::HashBuffer(Value2.Data(), Value2.Size()); - CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2); - CHECK(Result2.New); - - CasChunkSet ChunkSet; - ChunkSet.AddChunkToSet(Hash1); - ChunkSet.AddChunkToSet(Hash2); - - Store->FilterChunks(ChunkSet); - CHECK(ChunkSet.IsEmpty()); - - IoBuffer Lookup1 = Store->FindChunk(Hash1); - CHECK(Lookup1); - IoBuffer Lookup2 = Store->FindChunk(Hash2); - CHECK(Lookup2); -} - -void -CAS_forcelink() -{ -} - -#endif - -} // namespace zen diff --git a/zenstore/cas.cpp b/zenstore/cas.cpp new file mode 100644 index 000000000..4268e314b --- /dev/null +++ b/zenstore/cas.cpp @@ -0,0 +1,325 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include "compactcas.h" +#include "filecas.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +void +CasChunkSet::AddChunkToSet(const IoHash& HashToAdd) +{ + m_ChunkSet.insert(HashToAdd); +} + +void +CasChunkSet::AddChunksToSet(std::span HashesToAdd) +{ + for (const IoHash& Hash : HashesToAdd) + { + m_ChunkSet.insert(Hash); + } +} + +void +CasChunkSet::RemoveChunksIf(std::function&& Predicate) +{ + for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;) + { + if (Predicate(*It)) + { + It = m_ChunkSet.erase(It); + } + else + { + ++It; + } + } +} + +void +CasChunkSet::IterateChunks(std::function&& Callback) +{ + for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd; ++It) + { + Callback(*It); + } +} + +////////////////////////////////////////////////////////////////////////// + +struct GcContext::GcState +{ + CasChunkSet m_CasChunks; + CasChunkSet m_CidChunks; +}; + +GcContext::GcContext() : m_State(std::make_unique()) +{ +} + +GcContext::~GcContext() +{ +} + +void +GcContext::ContributeCids(std::span Cids) +{ + m_State->m_CidChunks.AddChunksToSet(Cids); +} + +void +GcContext::ContributeCas(std::span Cas) +{ + m_State->m_CasChunks.AddChunksToSet(Cas); +} + +////////////////////////////////////////////////////////////////////////// + +void +ScrubContext::ReportBadCasChunks(std::span BadCasChunks) +{ + m_BadCas.AddChunksToSet(BadCasChunks); +} + +void +ScrubContext::ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes) +{ + m_ChunkCount.fetch_add(ChunkCount); + m_ByteCount.fetch_add(ChunkBytes); +} + +/** + * CAS store implementation + * + * Uses a basic strategy of splitting payloads by size, to improve ability to reclaim space + * quickly for unused large chunks and to maintain locality for small chunks which are + * frequently accessed together. + * + */ +class CasImpl : public CasStore +{ +public: + CasImpl(); + virtual ~CasImpl(); + + virtual void Initialize(const CasStoreConfiguration& InConfig) override; + virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) override; + virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; + virtual void FilterChunks(CasChunkSet& InOutChunks) override; + virtual void Flush() override; + virtual void Scrub(ScrubContext& Ctx) override; + +private: + CasContainerStrategy m_TinyStrategy; + CasContainerStrategy m_SmallStrategy; + FileCasStrategy m_LargeStrategy; +}; + +CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) +{ +} + +CasImpl::~CasImpl() +{ +} + +void +CasImpl::Initialize(const CasStoreConfiguration& InConfig) +{ + m_Config = InConfig; + + ZEN_INFO("initializing CAS pool at '{}'", m_Config.RootDirectory); + + // Ensure root directory exists - create if it doesn't exist already + + std::filesystem::create_directories(m_Config.RootDirectory); + + // Open or create manifest + // + // The manifest is not currently fully implemented. The goal is to + // use it for recovery and configuration + + bool IsNewStore = false; + + { + std::filesystem::path ManifestPath = m_Config.RootDirectory; + ManifestPath /= ".ucas_root"; + + std::error_code Ec; + BasicFile Marker; + Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); + + if (Ec) + { + IsNewStore = true; + + ExtendableStringBuilder<128> manifest; + manifest.Append("#CAS_ROOT\n"); + manifest.Append("ID="); + zen::Oid id = zen::Oid::NewOid(); + id.ToString(manifest); + + Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); + Marker.Write(manifest.c_str(), uint32_t(manifest.Size()), 0); + } + } + + // Initialize payload storage + + m_TinyStrategy.Initialize("tobs", 16, IsNewStore); + m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); +} + +CasStore::InsertResult +CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) +{ + const uint64_t ChunkSize = Chunk.Size(); + + if (ChunkSize < m_Config.TinyValueThreshold) + { + ZEN_ASSERT(ChunkSize); + + return m_TinyStrategy.InsertChunk(Chunk, ChunkHash); + } + else if (ChunkSize < m_Config.HugeValueThreshold) + { + return m_SmallStrategy.InsertChunk(Chunk, ChunkHash); + } + + return m_LargeStrategy.InsertChunk(Chunk, ChunkHash); +} + +IoBuffer +CasImpl::FindChunk(const IoHash& ChunkHash) +{ + if (IoBuffer Found = m_SmallStrategy.FindChunk(ChunkHash)) + { + return Found; + } + + if (IoBuffer Found = m_TinyStrategy.FindChunk(ChunkHash)) + { + return Found; + } + + if (IoBuffer Found = m_LargeStrategy.FindChunk(ChunkHash)) + { + return Found; + } + + // Not found + return IoBuffer{}; +} + +void +CasImpl::FilterChunks(CasChunkSet& InOutChunks) +{ + m_SmallStrategy.FilterChunks(InOutChunks); + m_TinyStrategy.FilterChunks(InOutChunks); + m_LargeStrategy.FilterChunks(InOutChunks); +} + +void +CasImpl::Flush() +{ + m_SmallStrategy.Flush(); + m_TinyStrategy.Flush(); + m_LargeStrategy.Flush(); +} + +void +CasImpl::Scrub(ScrubContext& Ctx) +{ + if (m_LastScrubTime == Ctx.ScrubTimestamp()) + { + return; + } + + m_LastScrubTime = Ctx.ScrubTimestamp(); + + m_SmallStrategy.Scrub(Ctx); + m_TinyStrategy.Scrub(Ctx); + m_LargeStrategy.Scrub(Ctx); +} + +////////////////////////////////////////////////////////////////////////// + +CasStore* +CreateCasStore() +{ + return new CasImpl(); +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +#if ZEN_WITH_TESTS + +TEST_CASE("CasStore") +{ + ScopedTemporaryDirectory TempDir; + + CasStoreConfiguration config; + config.RootDirectory = TempDir.Path(); + + std::unique_ptr Store{CreateCasStore()}; + Store->Initialize(config); + + ScrubContext Ctx; + Store->Scrub(Ctx); + + IoBuffer Value1{16}; + memcpy(Value1.MutableData(), "1234567890123456", 16); + IoHash Hash1 = IoHash::HashBuffer(Value1.Data(), Value1.Size()); + CasStore::InsertResult Result1 = Store->InsertChunk(Value1, Hash1); + CHECK(Result1.New); + + IoBuffer Value2{16}; + memcpy(Value2.MutableData(), "ABCDEFGHIJKLMNOP", 16); + IoHash Hash2 = IoHash::HashBuffer(Value2.Data(), Value2.Size()); + CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2); + CHECK(Result2.New); + + CasChunkSet ChunkSet; + ChunkSet.AddChunkToSet(Hash1); + ChunkSet.AddChunkToSet(Hash2); + + Store->FilterChunks(ChunkSet); + CHECK(ChunkSet.IsEmpty()); + + IoBuffer Lookup1 = Store->FindChunk(Hash1); + CHECK(Lookup1); + IoBuffer Lookup2 = Store->FindChunk(Hash2); + CHECK(Lookup2); +} + +void +CAS_forcelink() +{ +} + +#endif + +} // namespace zen diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp index 449e2fc58..ae2ea973b 100644 --- a/zenstore/caslog.cpp +++ b/zenstore/caslog.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include +#include #include "compactcas.h" diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index 7a5d7bcf4..c67269e0d 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 3631722d4..f80bc85c4 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include +#include #include "compactcas.h" diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index 47e989fe9..a23b80828 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #if ZEN_PLATFORM_WINDOWS diff --git a/zenstore/filecas.h b/zenstore/filecas.h index 04ffffdd2..a86232588 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h deleted file mode 100644 index 7f8b5bfa2..000000000 --- a/zenstore/include/zenstore/CAS.h +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zenstore.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace zen { - -struct CasStoreConfiguration -{ - // Root directory for CAS store - std::filesystem::path RootDirectory; - - // Threshold below which values are considered 'tiny' and managed using the 'tiny values' strategy - uint64_t TinyValueThreshold = 1024; - - // Threshold above which values are considered 'huge' and managed using the 'huge values' strategy - uint64_t HugeValueThreshold = 1024 * 1024; -}; - -/** Manage a set of IoHash values - */ - -class CasChunkSet -{ -public: - void AddChunkToSet(const IoHash& HashToAdd); - void AddChunksToSet(std::span HashesToAdd); - void RemoveChunksIf(std::function&& Predicate); - void IterateChunks(std::function&& Callback); - [[nodiscard]] inline bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } - [[nodiscard]] inline bool IsEmpty() const { return m_ChunkSet.empty(); } - [[nodiscard]] inline size_t GetSize() const { return m_ChunkSet.size(); } - -private: - // Q: should we protect this with a lock, or is that a higher level concern? - std::unordered_set m_ChunkSet; -}; - -/** Garbage Collection context object - */ - -class GcContext -{ -public: - GcContext(); - ~GcContext(); - - void ContributeCids(std::span Cid); - void ContributeCas(std::span Hash); - -private: - struct GcState; - - std::unique_ptr m_State; -}; - -/** Context object for data scrubbing - * - * Data scrubbing is when we traverse stored data to validate it and - * optionally correct/recover - */ - -class ScrubContext -{ -public: - virtual void ReportBadCasChunks(std::span BadCasChunks); - inline uint64_t ScrubTimestamp() const { return m_ScrubTime; } - inline bool RunRecovery() const { return m_Recover; } - void ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes); - - inline uint64_t ScrubbedChunks() const { return m_ChunkCount; } - inline uint64_t ScrubbedBytes() const { return m_ByteCount; } - -private: - uint64_t m_ScrubTime = GetHifreqTimerValue(); - bool m_Recover = true; - std::atomic m_ChunkCount{0}; - std::atomic m_ByteCount{0}; - CasChunkSet m_BadCas; - CasChunkSet m_BadCid; -}; - -/** Content Addressable Storage interface - - */ - -class CasStore -{ -public: - virtual ~CasStore() = default; - - const CasStoreConfiguration& Config() { return m_Config; } - - struct InsertResult - { - bool New = false; - }; - - virtual void Initialize(const CasStoreConfiguration& Config) = 0; - virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash) = 0; - virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; - virtual void FilterChunks(CasChunkSet& InOutChunks) = 0; - virtual void Flush() = 0; - virtual void Scrub(ScrubContext& Ctx) = 0; - -protected: - CasStoreConfiguration m_Config; - uint64_t m_LastScrubTime = 0; -}; - -ZENCORE_API CasStore* CreateCasStore(); - -void CAS_forcelink(); - -} // namespace zen diff --git a/zenstore/include/zenstore/cas.h b/zenstore/include/zenstore/cas.h new file mode 100644 index 000000000..7f8b5bfa2 --- /dev/null +++ b/zenstore/include/zenstore/cas.h @@ -0,0 +1,129 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenstore.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace zen { + +struct CasStoreConfiguration +{ + // Root directory for CAS store + std::filesystem::path RootDirectory; + + // Threshold below which values are considered 'tiny' and managed using the 'tiny values' strategy + uint64_t TinyValueThreshold = 1024; + + // Threshold above which values are considered 'huge' and managed using the 'huge values' strategy + uint64_t HugeValueThreshold = 1024 * 1024; +}; + +/** Manage a set of IoHash values + */ + +class CasChunkSet +{ +public: + void AddChunkToSet(const IoHash& HashToAdd); + void AddChunksToSet(std::span HashesToAdd); + void RemoveChunksIf(std::function&& Predicate); + void IterateChunks(std::function&& Callback); + [[nodiscard]] inline bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } + [[nodiscard]] inline bool IsEmpty() const { return m_ChunkSet.empty(); } + [[nodiscard]] inline size_t GetSize() const { return m_ChunkSet.size(); } + +private: + // Q: should we protect this with a lock, or is that a higher level concern? + std::unordered_set m_ChunkSet; +}; + +/** Garbage Collection context object + */ + +class GcContext +{ +public: + GcContext(); + ~GcContext(); + + void ContributeCids(std::span Cid); + void ContributeCas(std::span Hash); + +private: + struct GcState; + + std::unique_ptr m_State; +}; + +/** Context object for data scrubbing + * + * Data scrubbing is when we traverse stored data to validate it and + * optionally correct/recover + */ + +class ScrubContext +{ +public: + virtual void ReportBadCasChunks(std::span BadCasChunks); + inline uint64_t ScrubTimestamp() const { return m_ScrubTime; } + inline bool RunRecovery() const { return m_Recover; } + void ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes); + + inline uint64_t ScrubbedChunks() const { return m_ChunkCount; } + inline uint64_t ScrubbedBytes() const { return m_ByteCount; } + +private: + uint64_t m_ScrubTime = GetHifreqTimerValue(); + bool m_Recover = true; + std::atomic m_ChunkCount{0}; + std::atomic m_ByteCount{0}; + CasChunkSet m_BadCas; + CasChunkSet m_BadCid; +}; + +/** Content Addressable Storage interface + + */ + +class CasStore +{ +public: + virtual ~CasStore() = default; + + const CasStoreConfiguration& Config() { return m_Config; } + + struct InsertResult + { + bool New = false; + }; + + virtual void Initialize(const CasStoreConfiguration& Config) = 0; + virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash) = 0; + virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; + virtual void FilterChunks(CasChunkSet& InOutChunks) = 0; + virtual void Flush() = 0; + virtual void Scrub(ScrubContext& Ctx) = 0; + +protected: + CasStoreConfiguration m_Config; + uint64_t m_LastScrubTime = 0; +}; + +ZENCORE_API CasStore* CreateCasStore(); + +void CAS_forcelink(); + +} // namespace zen diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index 8189da87e..1c6ee0bf6 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #if ZEN_PLATFORM_WINDOWS # include diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 26bf2bbde..871ac553e 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp index d852fa64b..cc4a34fdb 100644 --- a/zenstore/zenstore.cpp +++ b/zenstore/zenstore.cpp @@ -2,7 +2,7 @@ #include "zenstore/zenstore.h" -#include +#include #include #include "filecas.h" diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj index eb2ecd02b..9431b7b11 100644 --- a/zenstore/zenstore.vcxproj +++ b/zenstore/zenstore.vcxproj @@ -28,7 +28,7 @@ - + diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters index 8a52c69f6..904de0748 100644 --- a/zenstore/zenstore.vcxproj.filters +++ b/zenstore/zenstore.vcxproj.filters @@ -14,7 +14,7 @@ - + -- cgit v1.2.3