Make cas storage an hidden implementation detail of CidStore (#130)v0.1.4-pre6v0.1.4-pre5

- Bumped ZEN_SCHEMA_VERSION
- CasStore no longer a public API, it is hidden behind CidStore
- Moved cas.h from public header folder
- CidStore no longer maps from Cid -> Cas, we store entries in Cas under RawHash
- CasStore now decompresses data to validate content (matching against RawHash)
- CasChunkSet renames to HashKeySet and put in separate header/cpp file
- Disabled "Chunk" command for now as it relied on CAS being exposed as a service
- Changed CAS http service to Cid http server
- Moved "Run" command completely inside ZEN_WITH_EXEC_SERVICES define
- Removed "cas.basic" test
- Uncommented ".exec.basic" test and added return-skip at start of test
- Moved ScrubContext to separate header file
- Renamed CasGC to GcManager
- Cleaned up configuration passing in cas store classes
- Removed CAS stuff from GcContext and clarified naming in class
- Remove migration code

1 files changed, 20 insertions, 70 deletions

diff --git a/zenstore/cas.cpp b/zenstore/cas.cpp
index 0e1d5b242..54e8cb11c 100644
--- a/zenstore/cas.cpp
+++ b/zenstore/cas.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include <zenstore/cas.h>
+#include "cas.h"
 
 #include "compactcas.h"
 #include "filecas.h"
@@ -18,7 +18,9 @@
 #include <zencore/thread.h>
 #include <zencore/trace.h>
 #include <zencore/uid.h>
+#include <zenstore/cidstore.h>
 #include <zenstore/gc.h>
+#include <zenstore/scrubcontext.h>
 
 #include <gsl/gsl-lite.hpp>
 
@@ -30,58 +32,6 @@
 
 namespace zen {
 
-void
-CasChunkSet::AddChunkToSet(const IoHash& HashToAdd)
-{
-	m_ChunkSet.insert(HashToAdd);
-}
-
-void
-CasChunkSet::AddChunksToSet(std::span<const IoHash> HashesToAdd)
-{
-	m_ChunkSet.insert(HashesToAdd.begin(), HashesToAdd.end());
-}
-
-void
-CasChunkSet::RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate)
-{
-	for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;)
-	{
-		if (Predicate(*It))
-		{
-			It = m_ChunkSet.erase(It);
-		}
-		else
-		{
-			++It;
-		}
-	}
-}
-
-void
-CasChunkSet::IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback)
-{
-	for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd; ++It)
-	{
-		Callback(*It);
-	}
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-void
-ScrubContext::ReportBadCasChunks(std::span<IoHash> BadCasChunks)
-{
-	m_BadCas.AddChunksToSet(BadCasChunks);
-}
-
-void
-ScrubContext::ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes)
-{
-	m_ChunkCount.fetch_add(ChunkCount);
-	m_ByteCount.fetch_add(ChunkBytes);
-}
-
 /**
  * CAS store implementation
  *
@@ -93,18 +43,18 @@ ScrubContext::ReportScrubbed(uint64_t ChunkCount, uint64_t ChunkBytes)
 class CasImpl : public CasStore
 {
 public:
-	CasImpl(CasGc& Gc);
+	CasImpl(GcManager& Gc);
 	virtual ~CasImpl();
 
-	virtual void				   Initialize(const CasStoreConfiguration& InConfig) override;
+	virtual void				   Initialize(const CidStoreConfiguration& InConfig) override;
 	virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) override;
 	virtual IoBuffer			   FindChunk(const IoHash& ChunkHash) override;
 	virtual bool				   ContainsChunk(const IoHash& ChunkHash) override;
-	virtual void				   FilterChunks(CasChunkSet& InOutChunks) override;
+	virtual void				   FilterChunks(HashKeySet& InOutChunks) override;
 	virtual void				   Flush() override;
 	virtual void				   Scrub(ScrubContext& Ctx) override;
 	virtual void				   GarbageCollect(GcContext& GcCtx) override;
-	virtual CasStoreSize		   TotalSize() const override;
+	virtual CidStoreSize		   TotalSize() const override;
 
 private:
 	CasContainerStrategy m_TinyStrategy;
@@ -124,7 +74,7 @@ private:
 	void UpdateManifest();
 };
 
-CasImpl::CasImpl(CasGc& Gc) : m_TinyStrategy(m_Config, Gc), m_SmallStrategy(m_Config, Gc), m_LargeStrategy(m_Config, Gc)
+CasImpl::CasImpl(GcManager& Gc) : m_TinyStrategy(Gc), m_SmallStrategy(Gc), m_LargeStrategy(Gc)
 {
 }
 
@@ -133,7 +83,7 @@ CasImpl::~CasImpl()
 }
 
 void
-CasImpl::Initialize(const CasStoreConfiguration& InConfig)
+CasImpl::Initialize(const CidStoreConfiguration& InConfig)
 {
 	m_Config = InConfig;
 
@@ -149,9 +99,9 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
 
 	// Initialize payload storage
 
-	m_LargeStrategy.Initialize(IsNewStore);
-	m_TinyStrategy.Initialize("tobs", 1u << 28, 16, IsNewStore);	 // 256 Mb per block
-	m_SmallStrategy.Initialize("sobs", 1u << 30, 4096, IsNewStore);	 // 1 Gb per block
+	m_LargeStrategy.Initialize(m_Config.RootDirectory, IsNewStore);
+	m_TinyStrategy.Initialize(m_Config.RootDirectory, "tobs", 1u << 28, 16, IsNewStore);	 // 256 Mb per block
+	m_SmallStrategy.Initialize(m_Config.RootDirectory, "sobs", 1u << 30, 4096, IsNewStore);	 // 1 Gb per block
 }
 
 bool
@@ -292,7 +242,7 @@ CasImpl::ContainsChunk(const IoHash& ChunkHash)
 }
 
 void
-CasImpl::FilterChunks(CasChunkSet& InOutChunks)
+CasImpl::FilterChunks(HashKeySet& InOutChunks)
 {
 	m_SmallStrategy.FilterChunks(InOutChunks);
 	m_TinyStrategy.FilterChunks(InOutChunks);
@@ -330,7 +280,7 @@ CasImpl::GarbageCollect(GcContext& GcCtx)
 	m_LargeStrategy.CollectGarbage(GcCtx);
 }
 
-CasStoreSize
+CidStoreSize
 CasImpl::TotalSize() const
 {
 	const uint64_t Tiny	 = m_TinyStrategy.StorageSize().DiskSize;
@@ -343,7 +293,7 @@ CasImpl::TotalSize() const
 //////////////////////////////////////////////////////////////////////////
 
 std::unique_ptr<CasStore>
-CreateCasStore(CasGc& Gc)
+CreateCasStore(GcManager& Gc)
 {
 	return std::make_unique<CasImpl>(Gc);
 }
@@ -359,10 +309,10 @@ TEST_CASE("CasStore")
 {
 	ScopedTemporaryDirectory TempDir;
 
-	CasStoreConfiguration config;
+	CidStoreConfiguration config;
 	config.RootDirectory = TempDir.Path();
 
-	CasGc Gc;
+	GcManager Gc;
 
 	std::unique_ptr<CasStore> Store = CreateCasStore(Gc);
 	Store->Initialize(config);
@@ -382,9 +332,9 @@ TEST_CASE("CasStore")
 	CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2);
 	CHECK(Result2.New);
 
-	CasChunkSet ChunkSet;
-	ChunkSet.AddChunkToSet(Hash1);
-	ChunkSet.AddChunkToSet(Hash2);
+	HashKeySet ChunkSet;
+	ChunkSet.AddHashToSet(Hash1);
+	ChunkSet.AddHashToSet(Hash2);
 
 	Store->FilterChunks(ChunkSet);
 	CHECK(ChunkSet.IsEmpty());