Adding zenservice code

4 files changed, 214 insertions, 0 deletions

diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
new file mode 100644
index 000000000..8b9a66e3f
--- /dev/null
+++ b/zenstore/include/zenstore/CAS.h
@@ -0,0 +1,66 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/blake3.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <atomic>
+#include <filesystem>
+#include <memory>
+#include <string>
+
+namespace zen {
+
+struct CasStoreConfiguration
+{
+	// Root directory for CAS store -- if not specified a default folder will be assigned in 'Documents\zen'
+	std::filesystem::path RootDirectory;
+
+	// Threshold below which values are considered 'tiny' and managed using the 'tiny values' strategy
+	uint64_t TinyValueThreshold = 1024;
+
+	// Threshold above which values are considered 'tiny' and managed using the 'huge values' strategy
+	uint64_t HugeValueThreshold = 1024 * 1024;
+};
+
+class CasStore
+{
+public:
+	virtual ~CasStore() = default;
+
+	struct Stats
+	{
+		uint64_t PutBytes = 0;
+		uint64_t PutCount = 0;
+
+		uint64_t GetBytes = 0;
+		uint64_t GetCount = 0;
+	};
+
+	const CasStoreConfiguration& Config() { return m_Config; }
+	const Stats&				 GetStats() const { return m_Stats; }
+
+	struct InsertResult
+	{
+		bool New = false;
+	};
+
+	virtual void		 Initialize(const CasStoreConfiguration& Config)							   = 0;
+	virtual InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash) = 0;
+	virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash)						   = 0;
+	virtual IoBuffer	 FindChunk(const IoHash& ChunkHash)											   = 0;
+
+protected:
+	CasStoreConfiguration m_Config;
+	Stats				  m_Stats;
+};
+
+ZENCORE_API CasStore* CreateCasStore();
+
+void CAS_forcelink();
+
+}  // namespace zen
diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h
new file mode 100644
index 000000000..b318577d7
--- /dev/null
+++ b/zenstore/include/zenstore/caslog.h
@@ -0,0 +1,96 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/iobuffer.h>
+#include <zencore/string.h>
+#include <zencore/thread.h>
+#include <zencore/uid.h>
+#include <zencore/windows.h>
+#include <zenstore/cas.h>
+
+#include <atlfile.h>
+#include <functional>
+
+namespace zen {
+
+class CasLogFile
+{
+public:
+	CasLogFile();
+	~CasLogFile();
+
+	void Open(std::filesystem::path FileName, size_t RecordSize, bool isCreate);
+	void Append(const void* DataPointer, uint64_t DataSize);
+	void Replay(std::function<void(const void*)>&& Handler);
+	void Flush();
+	void Close();
+
+private:
+	struct FileHeader
+	{
+		uint8_t	 Magic[16];
+		uint32_t RecordSize = 0;
+		zen::Oid LogId;
+		uint32_t ValidatedTail = 0;
+		uint32_t Pad[6];
+		uint32_t Checksum = 0;
+
+		static const inline uint8_t MagicSequence[16] = {'.', '-', '=', ' ', 'C', 'A', 'S', 'L', 'O', 'G', 'v', '1', ' ', '=', '-', '.'};
+
+		ZENCORE_API uint32_t ComputeChecksum();
+		void				 Finalize() { Checksum = ComputeChecksum(); }
+	};
+
+	static_assert(sizeof(FileHeader) == 64);
+
+private:
+	CAtlFile   m_File;
+	FileHeader m_Header;
+	size_t	   m_RecordSize	  = 1;
+	uint64_t   m_AppendOffset = 0;
+};
+
+template<typename T>
+class TCasLogFile : public CasLogFile
+{
+public:
+	// This should be called before the Replay() is called to do some basic sanity checking
+	bool Initialize() { return true; }
+
+	void Replay(std::invocable<const T&> auto Handler)
+	{
+		CasLogFile::Replay([&](const void* VoidPtr) {
+			const T& Record = *reinterpret_cast<const T*>(VoidPtr);
+
+			Handler(Record);
+		});
+	}
+
+	void Append(const T& Record) { CasLogFile::Append(&Record, sizeof Record); }
+	void Open(std::filesystem::path FileName, bool IsCreate) { CasLogFile::Open(FileName, sizeof(T), IsCreate); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+//
+// This should go in its own header
+//
+
+class CasBlobFile
+{
+public:
+	void	 Open(std::filesystem::path FileName, bool IsCreate);
+	void	 Read(void* Data, uint64_t Size, uint64_t Offset);
+	void	 Write(const void* Data, uint64_t Size, uint64_t Offset);
+	void	 Flush();
+	uint64_t FileSize();
+	void*	 Handle() { return m_File; }
+	IoBuffer ReadAll();
+
+private:
+	CAtlFile m_File;
+};
+
+}  // namespace zen
diff --git a/zenstore/include/zenstore/gc.h b/zenstore/include/zenstore/gc.h
new file mode 100644
index 000000000..055843547
--- /dev/null
+++ b/zenstore/include/zenstore/gc.h
@@ -0,0 +1,28 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+
+#include <span>
+
+namespace zen {
+
+class CasStore;
+struct IoHash;
+
+class CasGc
+{
+public:
+	CasGc(CasStore& Store);
+	~CasGc();
+
+	void CollectGarbage();
+
+	void OnNewReferences(std::span<IoHash> Hashes);
+
+private:
+	CasStore& m_CasStore;
+};
+
+}  // namespace zen
diff --git a/zenstore/include/zenstore/scrub.h b/zenstore/include/zenstore/scrub.h
new file mode 100644
index 000000000..5a34d4860
--- /dev/null
+++ b/zenstore/include/zenstore/scrub.h
@@ -0,0 +1,24 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+
+#include <span>
+
+namespace zen {
+
+class CasStore;
+struct IoHash;
+
+class CasScrubber
+{
+public:
+	CasScrubber(CasStore& Store);
+	~CasScrubber();
+
+private:
+	CasStore& m_CasStore;
+};
+
+}  // namespace zen