Merge branch 'main' of https://github.com/EpicGames/zen

1 files changed, 74 insertions, 24 deletions

diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index e77c0ed64..a143230d3 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -11,6 +11,7 @@
 #include <zencore/logging.h>
 #include <zencore/memory.h>
 #include <zencore/string.h>
+#include <zencore/testutils.h>
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 
@@ -20,15 +21,23 @@
 #include <functional>
 #include <unordered_map>
 
-struct IUnknown;  // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
-#include <atlfile.h>
-
 //////////////////////////////////////////////////////////////////////////
 
 namespace zen {
 
+void
+ScrubContext::ReportBadChunks(std::span<IoHash> BadChunks)
+{
+	ZEN_UNUSED(BadChunks);
+}
+
 /**
- * Slightly less naive CAS store
+ * CAS store implementation
+ *
+ * Uses a basic strategy of splitting payloads by size, to improve ability to reclaim space
+ * quickly for unused large chunks and to maintain locality for small chunks which are
+ * frequently accessed together.
+ *
  */
 class CasImpl : public CasStore
 {
@@ -41,16 +50,15 @@ public:
 	virtual IoBuffer			   FindChunk(const IoHash& ChunkHash) override;
 	virtual void				   FilterChunks(CasChunkSet& InOutChunks) override;
 	virtual void				   Flush() override;
+	virtual void				   Scrub(ScrubContext& Ctx) override;
 
 private:
-	void PickDefaultDirectory();
-
 	CasContainerStrategy m_TinyStrategy;
 	CasContainerStrategy m_SmallStrategy;
 	FileCasStrategy		 m_LargeStrategy;
 };
 
-CasImpl::CasImpl() : m_TinyStrategy(m_Config, m_Stats), m_SmallStrategy(m_Config, m_Stats), m_LargeStrategy(m_Config, m_Stats)
+CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config)
 {
 }
 
@@ -63,13 +71,16 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
 {
 	m_Config = InConfig;
 
-	ZEN_INFO("initializing CAS pool at {}", m_Config.RootDirectory);
+	ZEN_INFO("initializing CAS pool at '{}'", m_Config.RootDirectory);
 
 	// Ensure root directory exists - create if it doesn't exist already
 
 	std::filesystem::create_directories(m_Config.RootDirectory);
 
 	// Open or create manifest
+	//
+	//  The manifest is not currently fully implemented. The goal is to
+	//  use it for recovery and configuration
 
 	bool IsNewStore = false;
 
@@ -77,23 +88,22 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
 		std::filesystem::path ManifestPath = m_Config.RootDirectory;
 		ManifestPath /= ".ucas_root";
 
-		CAtlFile marker;
-		HRESULT	 hRes = marker.Create(ManifestPath.c_str(), GENERIC_READ, 0, OPEN_EXISTING);
+		std::error_code Ec;
+		BasicFile		Marker;
+		Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec);
 
-		if (FAILED(hRes))
+		if (Ec)
 		{
 			IsNewStore = true;
 
 			ExtendableStringBuilder<128> manifest;
-			manifest.Append("#CAS_ROOT\n");	 // TODO: should write something meaningful here
+			manifest.Append("#CAS_ROOT\n");
 			manifest.Append("ID=");
 			zen::Oid id = zen::Oid::NewOid();
 			id.ToString(manifest);
 
-			hRes = marker.Create(ManifestPath.c_str(), GENERIC_WRITE, 0, CREATE_ALWAYS);
-
-			if (SUCCEEDED(hRes))
-				marker.Write(manifest.c_str(), (DWORD)manifest.Size());
+			Marker.Open(ManifestPath.c_str(), /* IsCreate */ true);
+			Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0);
 		}
 	}
 
@@ -101,6 +111,9 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
 
 	m_TinyStrategy.Initialize("tobs", 16, IsNewStore);
 	m_SmallStrategy.Initialize("sobs", 4096, IsNewStore);
+
+	ScrubContext Ctx;
+	Scrub(Ctx);
 }
 
 CasStore::InsertResult
@@ -160,6 +173,14 @@ CasImpl::Flush()
 	m_LargeStrategy.Flush();
 }
 
+void
+CasImpl::Scrub(ScrubContext& Ctx)
+{
+	m_SmallStrategy.Scrub(Ctx);
+	m_TinyStrategy.Scrub(Ctx);
+	m_LargeStrategy.Scrub(Ctx);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 CasStore*
@@ -173,18 +194,47 @@ CreateCasStore()
 // Testing related code follows...
 //
 
-void
-CAS_forcelink()
-{
-}
-
 TEST_CASE("CasStore")
 {
+	ScopedTemporaryDirectory TempDir;
+
 	zen::CasStoreConfiguration config;
-	config.RootDirectory = "c:\\temp\\test";
+	config.RootDirectory = TempDir.Path();
+
+	std::unique_ptr<zen::CasStore> Store{CreateCasStore()};
+	Store->Initialize(config);
+
+	ScrubContext Ctx;
+	Store->Scrub(Ctx);
+
+	IoBuffer Value1{16};
+	memcpy(Value1.MutableData(), "1234567890123456", 16);
+	IoHash				   Hash1   = IoHash::HashBuffer(Value1.Data(), Value1.Size());
+	CasStore::InsertResult Result1 = Store->InsertChunk(Value1, Hash1);
+	CHECK(Result1.New);
+
+	IoBuffer Value2{16};
+	memcpy(Value2.MutableData(), "ABCDEFGHIJKLMNOP", 16);
+	IoHash				   Hash2   = IoHash::HashBuffer(Value2.Data(), Value2.Size());
+	CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2);
+	CHECK(Result2.New);
 
-	std::unique_ptr<zen::CasStore> store{CreateCasStore()};
-	store->Initialize(config);
+	CasChunkSet ChunkSet;
+	ChunkSet.AddChunk(Hash1);
+	ChunkSet.AddChunk(Hash2);
+
+	Store->FilterChunks(ChunkSet);
+	CHECK(ChunkSet.GetChunkSet().size() == 0);
+
+	IoBuffer Lookup1 = Store->FindChunk(Hash1);
+	CHECK(Lookup1);
+	IoBuffer Lookup2 = Store->FindChunk(Hash2);
+	CHECK(Lookup2);
+}
+
+void
+CAS_forcelink()
+{
 }
 
 }  // namespace zen