From 8e97a98bb73228c6c0bc041d9bf758d2417a62dd Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 19 Oct 2021 13:27:13 +0200 Subject: cas: added structured manifest support --- zenstore/CAS.cpp | 55 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) (limited to 'zenstore/CAS.cpp') diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index 807bba3b3..a43759283 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include @@ -105,6 +108,15 @@ private: CasContainerStrategy m_TinyStrategy; CasContainerStrategy m_SmallStrategy; FileCasStrategy m_LargeStrategy; + CbObject m_ManifestObject; + + enum class StorageScheme + { + Legacy = 0, + WithCbManifest = 1 + }; + + StorageScheme m_StorageScheme = StorageScheme::Legacy; }; CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) @@ -143,16 +155,43 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) if (Ec) { - IsNewStore = true; + if (Ec == std::errc::no_such_file_or_directory) + { + IsNewStore = true; + + CbObjectWriter Cbo; + + Cbo << "id" << zen::Oid::NewOid(); - ExtendableStringBuilder<128> manifest; - manifest.Append("#CAS_ROOT\n"); - manifest.Append("ID="); - zen::Oid id = zen::Oid::NewOid(); - id.ToString(manifest); + Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); + m_ManifestObject = Cbo.Save(); - Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); + Marker.Write(m_ManifestObject.GetBuffer(), 0); + } + } + else + { + IoBuffer ManifestBuffer = Marker.ReadAll(); + + if (ManifestBuffer.Size() > 0 && ManifestBuffer.Data()[0] == '#') + { + // Old-style manifest, does not contain any useful information (so we may as well update it?) + m_StorageScheme = StorageScheme::Legacy; + } + else + { + CbObject Manifest{SharedBuffer(ManifestBuffer)}; + CbValidateError ValidationResult = ValidateCompactBinary(ManifestBuffer, CbValidateMode::All); + + if (ValidationResult == CbValidateError::None) + { + m_ManifestObject = std::move(Manifest); + } + else + { + ZEN_ERROR("Store manifest validation failed: {:#x}", ValidationResult); + } + } } } -- cgit v1.2.3 From 3c817d5bb997a3aaf906be6a5b249a58f3afedfe Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 19 Oct 2021 16:33:27 +0200 Subject: cas: implemented manifest update/upgrade --- zenstore/CAS.cpp | 80 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 24 deletions(-) (limited to 'zenstore/CAS.cpp') diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index a43759283..a71e251e5 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -5,6 +5,9 @@ #include "compactcas.h" #include "filecas.h" +#include +#include +#include #include #include #include @@ -14,9 +17,6 @@ #include #include #include -#include -#include -#include #include @@ -108,15 +108,17 @@ private: CasContainerStrategy m_TinyStrategy; CasContainerStrategy m_SmallStrategy; FileCasStrategy m_LargeStrategy; - CbObject m_ManifestObject; - + CbObject m_ManifestObject; + enum class StorageScheme { - Legacy = 0, + Legacy = 0, WithCbManifest = 1 }; - StorageScheme m_StorageScheme = StorageScheme::Legacy; + StorageScheme m_StorageScheme = StorageScheme::Legacy; + + void UpdateManifest(bool IsNewStore); }; CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) @@ -150,49 +152,55 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) ManifestPath /= ".ucas_root"; std::error_code Ec; - BasicFile Marker; - Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); + BasicFile ManifestFile; + ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); + + bool ManifestIsOk = false; if (Ec) { if (Ec == std::errc::no_such_file_or_directory) { IsNewStore = true; - - CbObjectWriter Cbo; - - Cbo << "id" << zen::Oid::NewOid(); - - Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - m_ManifestObject = Cbo.Save(); - - Marker.Write(m_ManifestObject.GetBuffer(), 0); } } else { - IoBuffer ManifestBuffer = Marker.ReadAll(); + IoBuffer ManifestBuffer = ManifestFile.ReadAll(); + ManifestFile.Close(); if (ManifestBuffer.Size() > 0 && ManifestBuffer.Data()[0] == '#') { - // Old-style manifest, does not contain any useful information (so we may as well update it?) - m_StorageScheme = StorageScheme::Legacy; + // Old-style manifest, does not contain any useful information, so we may as well update it } else { - CbObject Manifest{SharedBuffer(ManifestBuffer)}; + CbObject Manifest{SharedBuffer(ManifestBuffer)}; CbValidateError ValidationResult = ValidateCompactBinary(ManifestBuffer, CbValidateMode::All); if (ValidationResult == CbValidateError::None) { - m_ManifestObject = std::move(Manifest); + if (Manifest["id"]) + { + ManifestIsOk = true; + } } else { - ZEN_ERROR("Store manifest validation failed: {:#x}", ValidationResult); + ZEN_ERROR("Store manifest validation failed: {:#x}, will generate new manifest to recover", ValidationResult); + } + + if (ManifestIsOk) + { + m_ManifestObject = std::move(Manifest); } } } + + if (!ManifestIsOk) + { + UpdateManifest(true); + } } // Initialize payload storage @@ -201,6 +209,30 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); } +void +CasImpl::UpdateManifest(bool IsNewStore) +{ + if (!m_ManifestObject) + { + CbObjectWriter Cbo; + Cbo << "id" << zen::Oid::NewOid() << "created" << DateTime::Now(); + m_ManifestObject = Cbo.Save(); + } + + // Write manifest to file + + std::filesystem::path ManifestPath = m_Config.RootDirectory; + ManifestPath /= ".ucas_root"; + + // This will throw on failure + + ZEN_TRACE("Writing new manifest to '{}'", ManifestPath); + + BasicFile Marker; + Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); + Marker.Write(m_ManifestObject.GetBuffer(), 0); +} + CasStore::InsertResult CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) { -- cgit v1.2.3 From 3079a9e9b10c172bdabd3b68b2d44933f0dfda6f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 19 Oct 2021 18:42:08 +0200 Subject: cas: Factored out OpenOrCreateManifest --- zenstore/CAS.cpp | 94 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 45 deletions(-) (limited to 'zenstore/CAS.cpp') diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index a71e251e5..7ec91cfa5 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -118,6 +118,7 @@ private: StorageScheme m_StorageScheme = StorageScheme::Legacy; + bool OpenOrCreateManifest(); void UpdateManifest(bool IsNewStore); }; @@ -141,72 +142,75 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) std::filesystem::create_directories(m_Config.RootDirectory); // Open or create manifest - // - // The manifest is not currently fully implemented. The goal is to - // use it for recovery and configuration + const bool IsNewStore = OpenOrCreateManifest(); + + // Initialize payload storage + + m_TinyStrategy.Initialize("tobs", 16, IsNewStore); + m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); +} + +bool +CasImpl::OpenOrCreateManifest() +{ bool IsNewStore = false; - { - std::filesystem::path ManifestPath = m_Config.RootDirectory; - ManifestPath /= ".ucas_root"; + std::filesystem::path ManifestPath = m_Config.RootDirectory; + ManifestPath /= ".ucas_root"; - std::error_code Ec; - BasicFile ManifestFile; - ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); + std::error_code Ec; + BasicFile ManifestFile; + ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); - bool ManifestIsOk = false; + bool ManifestIsOk = false; - if (Ec) + if (Ec) + { + if (Ec == std::errc::no_such_file_or_directory) { - if (Ec == std::errc::no_such_file_or_directory) - { - IsNewStore = true; - } + IsNewStore = true; + } + } + else + { + IoBuffer ManifestBuffer = ManifestFile.ReadAll(); + ManifestFile.Close(); + + if (ManifestBuffer.Size() > 0 && ManifestBuffer.Data()[0] == '#') + { + // Old-style manifest, does not contain any useful information, so we may as well update it } else { - IoBuffer ManifestBuffer = ManifestFile.ReadAll(); - ManifestFile.Close(); + CbObject Manifest{SharedBuffer(ManifestBuffer)}; + CbValidateError ValidationResult = ValidateCompactBinary(ManifestBuffer, CbValidateMode::All); - if (ManifestBuffer.Size() > 0 && ManifestBuffer.Data()[0] == '#') + if (ValidationResult == CbValidateError::None) { - // Old-style manifest, does not contain any useful information, so we may as well update it + if (Manifest["id"]) + { + ManifestIsOk = true; + } } else { - CbObject Manifest{SharedBuffer(ManifestBuffer)}; - CbValidateError ValidationResult = ValidateCompactBinary(ManifestBuffer, CbValidateMode::All); - - if (ValidationResult == CbValidateError::None) - { - if (Manifest["id"]) - { - ManifestIsOk = true; - } - } - else - { - ZEN_ERROR("Store manifest validation failed: {:#x}, will generate new manifest to recover", ValidationResult); - } - - if (ManifestIsOk) - { - m_ManifestObject = std::move(Manifest); - } + ZEN_ERROR("Store manifest validation failed: {:#x}, will generate new manifest to recover", ValidationResult); } - } - if (!ManifestIsOk) - { - UpdateManifest(true); + if (ManifestIsOk) + { + m_ManifestObject = std::move(Manifest); + } } } - // Initialize payload storage + if (!ManifestIsOk) + { + UpdateManifest(true); + } - m_TinyStrategy.Initialize("tobs", 16, IsNewStore); - m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); + return IsNewStore; } void -- cgit v1.2.3