Merge branch 'main' into cbpackage-update

5 files changed, 43 insertions, 17 deletions

diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index c05e3703d..e77c0ed64 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -8,13 +8,12 @@
 #include <doctest/doctest.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
+#include <zencore/logging.h>
 #include <zencore/memory.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 
-#include <spdlog/spdlog.h>
-
 #include <gsl/gsl-lite.hpp>
 
 #include <filesystem>
@@ -64,7 +63,7 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
 {
 	m_Config = InConfig;
 
-	spdlog::info("initializing CAS pool at {}", m_Config.RootDirectory);
+	ZEN_INFO("initializing CAS pool at {}", m_Config.RootDirectory);
 
 	// Ensure root directory exists - create if it doesn't exist already
 
diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp
index 169629053..70bcf4669 100644
--- a/zenstore/caslog.cpp
+++ b/zenstore/caslog.cpp
@@ -7,6 +7,7 @@
 #include <zencore/except.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
+#include <zencore/logging.h>
 #include <zencore/memory.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
@@ -14,7 +15,6 @@
 
 #include <xxhash.h>
 
-#include <spdlog/spdlog.h>
 #include <gsl/gsl-lite.hpp>
 
 #include <functional>
@@ -81,7 +81,7 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat
 		if ((0 != memcmp(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic)) || (Header.Checksum != Header.ComputeChecksum()))
 		{
 			// TODO: provide more context!
-			throw std::exception("Mangled log header");
+			throw std::runtime_error("Mangled log header");
 		}
 
 		ULONGLONG Sz;
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 4e5188f1c..e6c7f98ee 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -2,11 +2,12 @@
 
 #include "zenstore/cidstore.h"
 
+#include <zencore/compress.h>
 #include <zencore/filesystem.h>
+#include <zencore/logging.h>
 #include <zenstore/CAS.h>
 #include <zenstore/caslog.h>
 
-#include <spdlog/spdlog.h>
 #include <filesystem>
 
 namespace zen {
@@ -27,10 +28,25 @@ struct CidStore::CidState
 	RwLock						   m_Lock;
 	tsl::robin_map<IoHash, IoHash> m_CidMap;
 
+	CasStore::InsertResult AddChunk(CompressedBuffer& ChunkData)
+	{
+		IoBuffer Payload		= ChunkData.GetCompressed().Flatten().AsIoBuffer();
+		IoHash	 CompressedHash = IoHash::HashBuffer(Payload.Data(), Payload.Size());
+
+		CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, CompressedHash);
+		AddCompressedCid(IoHash::FromBLAKE3(ChunkData.GetRawHash()), CompressedHash);
+
+		return Result;
+	}
+
 	void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
 		m_CidMap.insert_or_assign(DecompressedId, Compressed);
+		// TODO: it's pretty wasteful to log even idempotent updates
+		// however we can't simply use the boolean returned by insert_or_assign
+		// since there's not a 1:1 mapping between compressed and uncompressed
+		// so if we want a last-write-wins policy then we have to log each update
 		m_LogFile.Append({.Uncompressed = DecompressedId, .Compressed = Compressed});
 	}
 
@@ -70,7 +86,7 @@ struct CidStore::CidState
 
 		m_LogFile.Replay([&](const IndexEntry& Ie) { m_CidMap.insert_or_assign(Ie.Uncompressed, Ie.Compressed); });
 
-		spdlog::debug("CID index initialized: {} entries found", m_CidMap.size());
+		ZEN_DEBUG("CID index initialized: {} entries found", m_CidMap.size());
 	}
 
 	void Flush() { m_LogFile.Flush(); }
@@ -87,6 +103,12 @@ CidStore::~CidStore()
 {
 }
 
+CasStore::InsertResult
+CidStore::AddChunk(CompressedBuffer& ChunkData)
+{
+	return m_Impl->AddChunk(ChunkData);
+}
+
 void
 CidStore::AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
 {
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 56d25e729..170f13875 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -5,12 +5,12 @@
 #include <zencore/except.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
+#include <zencore/logging.h>
 #include <zencore/memory.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 
-#include <spdlog/spdlog.h>
 #include <gsl/gsl-lite.hpp>
 
 #include <filesystem>
@@ -84,7 +84,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 
 			if (!Success)
 			{
-				spdlog::warn("Failed to flag temporary payload file for deletion: '{}'", PathFromHandle(FileRef.FileHandle));
+				ZEN_WARN("Failed to flag temporary payload file for deletion: '{}'", PathFromHandle(FileRef.FileHandle));
 			}
 		};
 
@@ -173,9 +173,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 			return CasStore::InsertResult{.New = true};
 		}
 
-		spdlog::warn("rename of CAS payload file failed ('{}'), falling back to regular write for insert of {}",
-					 GetLastErrorAsString(),
-					 ChunkHash);
+		ZEN_WARN("rename of CAS payload file failed ('{}'), falling back to regular write for insert of {}",
+				 GetLastErrorAsString(),
+				 ChunkHash);
 
 		DeletePayloadFileOnClose();
 	}
@@ -224,7 +224,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 
 	if ((hRes != HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) && (hRes != HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND)))
 	{
-		spdlog::warn("Unexpected error code when opening shard file for read: {:#x}", uint32_t(hRes));
+		ZEN_WARN("Unexpected error code when opening shard file for read: {:#x}", uint32_t(hRes));
 	}
 
 	auto InternalCreateFile = [&] { return PayloadFile.Create(ShardedPath.c_str(), GENERIC_WRITE, FILE_SHARE_DELETE, CREATE_ALWAYS); };
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index 2c2b395a5..62d642ad1 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -4,6 +4,7 @@
 
 #include <tsl/robin_map.h>
 #include <zencore/iohash.h>
+#include <zenstore/CAS.h>
 
 namespace std::filesystem {
 class path;
@@ -12,6 +13,7 @@ class path;
 namespace zen {
 
 class CasStore;
+class CompressedBuffer;
 class IoBuffer;
 
 /** Content Store
@@ -29,10 +31,13 @@ public:
 	CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir);
 	~CidStore();
 
-	void	 AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed);
-	IoBuffer FindChunkByCid(const IoHash& DecompressedId);
-	bool	 ContainsChunk(const IoHash& DecompressedId);
-	void	 Flush();
+	CasStore::InsertResult AddChunk(CompressedBuffer& ChunkData);
+	void				   AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed);
+	IoBuffer			   FindChunkByCid(const IoHash& DecompressedId);
+	bool				   ContainsChunk(const IoHash& DecompressedId);
+	void				   Flush();
+
+	// TODO: add batch filter support
 
 private:
 	struct CidState;