diff options
Diffstat (limited to 'zenstore/CAS.cpp')
| -rw-r--r-- | zenstore/CAS.cpp | 166 |
1 files changed, 112 insertions, 54 deletions
diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index a4bbfa340..86c6eb849 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -5,6 +5,9 @@ #include "compactcas.h" #include "filecas.h" +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> #include <zencore/except.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -14,6 +17,7 @@ #include <zencore/testutils.h> #include <zencore/thread.h> #include <zencore/uid.h> +#include <zenstore/gc.h> #include <gsl/gsl-lite.hpp> @@ -67,34 +71,6 @@ CasChunkSet::IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callba ////////////////////////////////////////////////////////////////////////// -struct GcContext::GcState -{ - CasChunkSet m_CasChunks; - CasChunkSet m_CidChunks; -}; - -GcContext::GcContext() : m_State(std::make_unique<GcState>()) -{ -} - -GcContext::~GcContext() -{ -} - -void -GcContext::ContributeCids(std::span<const IoHash> Cids) -{ - m_State->m_CidChunks.AddChunksToSet(Cids); -} - -void -GcContext::ContributeCas(std::span<const IoHash> Cas) -{ - m_State->m_CasChunks.AddChunksToSet(Cas); -} - -////////////////////////////////////////////////////////////////////////// - void ScrubContext::ReportBadCasChunks(std::span<IoHash> BadCasChunks) { @@ -119,7 +95,7 @@ ScrubContext::ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes) class CasImpl : public CasStore { public: - CasImpl(); + CasImpl(CasGc& Gc); virtual ~CasImpl(); virtual void Initialize(const CasStoreConfiguration& InConfig) override; @@ -128,14 +104,27 @@ public: virtual void FilterChunks(CasChunkSet& InOutChunks) override; virtual void Flush() override; virtual void Scrub(ScrubContext& Ctx) override; + virtual void GarbageCollect(GcContext& GcCtx) override; private: CasContainerStrategy m_TinyStrategy; CasContainerStrategy m_SmallStrategy; FileCasStrategy m_LargeStrategy; + CbObject m_ManifestObject; + + enum class StorageScheme + { + Legacy = 0, + WithCbManifest = 1 + }; + + StorageScheme m_StorageScheme = StorageScheme::Legacy; + + bool OpenOrCreateManifest(); + void UpdateManifest(); }; -CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) +CasImpl::CasImpl(CasGc& Gc) : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config, Gc) { } @@ -155,39 +144,100 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) std::filesystem::create_directories(m_Config.RootDirectory); // Open or create manifest - // - // The manifest is not currently fully implemented. The goal is to - // use it for recovery and configuration + const bool IsNewStore = OpenOrCreateManifest(); + + // Initialize payload storage + + m_LargeStrategy.Initialize(IsNewStore); + m_TinyStrategy.Initialize("tobs", 16, IsNewStore); + m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); +} + +bool +CasImpl::OpenOrCreateManifest() +{ bool IsNewStore = false; - { - std::filesystem::path ManifestPath = m_Config.RootDirectory; - ManifestPath /= ".ucas_root"; + std::filesystem::path ManifestPath = m_Config.RootDirectory; + ManifestPath /= ".ucas_root"; - std::error_code Ec; - BasicFile Marker; - Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); + std::error_code Ec; + BasicFile ManifestFile; + ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); - if (Ec) + bool ManifestIsOk = false; + + if (Ec) + { + if (Ec == std::errc::no_such_file_or_directory) { IsNewStore = true; + } + } + else + { + IoBuffer ManifestBuffer = ManifestFile.ReadAll(); + ManifestFile.Close(); - ExtendableStringBuilder<128> manifest; - manifest.Append("#CAS_ROOT\n"); - manifest.Append("ID="); - zen::Oid id = zen::Oid::NewOid(); - id.ToString(manifest); - - Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); + if (ManifestBuffer.Size() > 0 && ManifestBuffer.Data<uint8_t>()[0] == '#') + { + // Old-style manifest, does not contain any useful information, so we may as well update it + } + else + { + CbObject Manifest{SharedBuffer(ManifestBuffer)}; + CbValidateError ValidationResult = ValidateCompactBinary(ManifestBuffer, CbValidateMode::All); + + if (ValidationResult == CbValidateError::None) + { + if (Manifest["id"]) + { + ManifestIsOk = true; + } + } + else + { + ZEN_ERROR("Store manifest validation failed: {:#x}, will generate new manifest to recover", ValidationResult); + } + + if (ManifestIsOk) + { + m_ManifestObject = std::move(Manifest); + } } } - // Initialize payload storage + if (!ManifestIsOk) + { + UpdateManifest(); + } - m_TinyStrategy.Initialize("tobs", 16, IsNewStore); - m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); + return IsNewStore; +} + +void +CasImpl::UpdateManifest() +{ + if (!m_ManifestObject) + { + CbObjectWriter Cbo; + Cbo << "id" << zen::Oid::NewOid() << "created" << DateTime::Now(); + m_ManifestObject = Cbo.Save(); + } + + // Write manifest to file + + std::filesystem::path ManifestPath = m_Config.RootDirectory; + ManifestPath /= ".ucas_root"; + + // This will throw on failure + + ZEN_TRACE("Writing new manifest to '{}'", ManifestPath); + + BasicFile Marker; + Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); + Marker.Write(m_ManifestObject.GetBuffer(), 0); } CasStore::InsertResult @@ -262,12 +312,18 @@ CasImpl::Scrub(ScrubContext& Ctx) m_LargeStrategy.Scrub(Ctx); } +void +CasImpl::GarbageCollect(GcContext& GcCtx) +{ + m_LargeStrategy.CollectGarbage(GcCtx); +} + ////////////////////////////////////////////////////////////////////////// CasStore* -CreateCasStore() +CreateCasStore(CasGc& Gc) { - return new CasImpl(); + return new CasImpl(Gc); } ////////////////////////////////////////////////////////////////////////// @@ -284,7 +340,9 @@ TEST_CASE("CasStore") CasStoreConfiguration config; config.RootDirectory = TempDir.Path(); - std::unique_ptr<CasStore> Store{CreateCasStore()}; + CasGc Gc; + + std::unique_ptr<CasStore> Store{CreateCasStore(Gc)}; Store->Initialize(config); ScrubContext Ctx; |