Added CidStore, currently used to track relationships between compressed and uncompressed chunk hashes

This first implementation is in-memory only, persistence is next

7 files changed, 84 insertions, 5 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 317c5641a..43364af1d 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -12,6 +12,7 @@
 #include "structuredcache.h"
 #include "structuredcachestore.h"
 #include "upstream/jupiter.h"
+#include "zenstore/cidstore.h"
 
 #include <spdlog/spdlog.h>
 #include <filesystem>
@@ -20,9 +21,10 @@ namespace zen {
 
 using namespace std::literals;
 
-HttpStructuredCacheService::HttpStructuredCacheService(std::filesystem::path RootPath, zen::CasStore& InStore)
+HttpStructuredCacheService::HttpStructuredCacheService(std::filesystem::path RootPath, zen::CasStore& InStore, zen::CidStore& InCidStore)
 : m_CasStore(InStore)
 , m_CacheStore(InStore, RootPath)
+, m_CidStore(InCidStore)
 {
 	spdlog::info("initializing structured cache at '{}'", RootPath);
 
@@ -144,7 +146,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
 						if (!References.empty())
 						{
 							zen::CbObjectWriter Idx;
-							Idx.BeginArray();
+							Idx.BeginArray("r");
 
 							for (const IoHash& Hash : References)
 							{
@@ -222,7 +224,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
 				// TODO: need to map from uncompressed content address into the storage
 				// (compressed) content address
 
-				zen::IoBuffer Payload = m_CasStore.FindChunk(Ref.PayloadId);
+				zen::IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId);
 
 				if (!Payload)
 				{
@@ -254,6 +256,8 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
 
 					zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash);
 
+					m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+
 					if (Result.New)
 					{
 						return Request.WriteResponse(zen::HttpResponse::Created);
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 761bf3399..63d0a0d52 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -11,6 +11,7 @@
 namespace zen {
 
 class CloudCacheClient;
+class CidStore;
 
 /**
  * New-style cache service. Imposes constraints on keys, supports blobs and
@@ -27,7 +28,7 @@ class CloudCacheClient;
 class HttpStructuredCacheService : public zen::HttpService
 {
 public:
-	HttpStructuredCacheService(std::filesystem::path RootPath, zen::CasStore& InStore);
+	HttpStructuredCacheService(std::filesystem::path RootPath, zen::CasStore& InStore, zen::CidStore& InCidStore);
 	~HttpStructuredCacheService();
 
 	virtual const char* BaseUri() const override;
@@ -47,6 +48,7 @@ private:
 	void			   HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref);
 
 	zen::CasStore&			 m_CasStore;
+	zen::CidStore&			 m_CidStore;
 	ZenCacheStore			 m_CacheStore;
 	RefPtr<CloudCacheClient> m_Cloud;
 };
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 934fd95bc..7142024ea 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -10,6 +10,7 @@
 #include <zencore/timer.h>
 #include <zencore/windows.h>
 #include <zenstore/cas.h>
+#include <zenstore/cidstore.h>
 
 #include <fmt/format.h>
 #include <mimalloc-new-delete.h>
@@ -101,7 +102,7 @@ public:
 		if (ServiceConfig.StructuredCacheEnabled)
 		{
 			spdlog::info("instantiating structured cache service");
-			m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(m_DataRoot / "cache", *m_CasStore));
+			m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(m_DataRoot / "cache", *m_CasStore, m_CidStore));
 		}
 		else
 		{
@@ -207,6 +208,7 @@ private:
 
 	zen::HttpServer									 m_Http;
 	std::unique_ptr<zen::CasStore>					 m_CasStore{zen::CreateCasStore()};
+	zen::CidStore									 m_CidStore{*m_CasStore};
 	zen::CasGc										 m_Gc{*m_CasStore};
 	zen::CasScrubber								 m_Scrubber{*m_CasStore};
 	HttpTestService									 m_TestService;
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
new file mode 100644
index 000000000..6e558e639
--- /dev/null
+++ b/zenstore/cidstore.cpp
@@ -0,0 +1,33 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenstore/cidstore.h"
+#include "zenstore/CAS.h"
+
+namespace zen {
+
+CidStore::CidStore(CasStore& InCasStore) : m_CasStore(InCasStore)
+{
+}
+
+CidStore::~CidStore()
+{
+}
+
+void
+CidStore::AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
+{
+	m_CidMap.insert_or_assign(DecompressedId, Compressed);
+}
+
+IoBuffer
+CidStore::FindChunkByCid(const IoHash& DecompressedId)
+{
+	if (auto It = m_CidMap.find(DecompressedId); It != m_CidMap.end())
+	{
+		return m_CasStore.FindChunk(It->second);
+	}
+
+	return IoBuffer();
+}
+
+}  // namespace zen
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
new file mode 100644
index 000000000..0c8f51740
--- /dev/null
+++ b/zenstore/include/zenstore/cidstore.h
@@ -0,0 +1,34 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <tsl/robin_map.h>
+#include <zencore/iohash.h>
+
+namespace zen {
+
+class CasStore;
+class IoBuffer;
+
+/** Content Store
+ *
+ * Data in the content store is referenced by content identifiers (CIDs), rather than their
+ * literal hash. This is required to map uncompressed hashes to compressed hashes and may
+ * be used to deal with other indirections in the future.
+ *
+ */
+class CidStore
+{
+public:
+	CidStore(CasStore& InCasStore);
+	~CidStore();
+
+	void	 AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed);
+	IoBuffer FindChunkByCid(const IoHash& DecompressedId);
+
+private:
+	CasStore&									   m_CasStore;
+	tsl::robin_map<IoHash, IoHash, IoHash::Hasher> m_CidMap;
+};
+
+}  // namespace zen
diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj
index 5384bf3eb..0e4caa92c 100644
--- a/zenstore/zenstore.vcxproj
+++ b/zenstore/zenstore.vcxproj
@@ -14,6 +14,7 @@
     <ClCompile Include="basicfile.cpp" />
     <ClCompile Include="CAS.cpp" />
     <ClCompile Include="caslog.cpp" />
+    <ClCompile Include="cidstore.cpp" />
     <ClCompile Include="compactcas.cpp" />
     <ClCompile Include="filecas.cpp" />
     <ClCompile Include="gc.cpp" />
@@ -23,6 +24,7 @@
     <ClInclude Include="compactcas.h" />
     <ClInclude Include="filecas.h" />
     <ClInclude Include="include\zenstore\basicfile.h" />
+    <ClInclude Include="include\zenstore\cidstore.h" />
     <ClInclude Include="include\zenstore\gc.h" />
     <ClInclude Include="include\zenstore\scrub.h" />
     <ClInclude Include="include\zenstore\CAS.h" />
diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters
index 8f08ca6cf..3dfb89dbf 100644
--- a/zenstore/zenstore.vcxproj.filters
+++ b/zenstore/zenstore.vcxproj.filters
@@ -8,6 +8,7 @@
     <ClCompile Include="gc.cpp" />
     <ClCompile Include="scrub.cpp" />
     <ClCompile Include="basicfile.cpp" />
+    <ClCompile Include="cidstore.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="compactcas.h" />
@@ -17,5 +18,6 @@
     <ClInclude Include="include\zenstore\gc.h" />
     <ClInclude Include="include\zenstore\scrub.h" />
     <ClInclude Include="include\zenstore\basicfile.h" />
+    <ClInclude Include="include\zenstore\cidstore.h" />
   </ItemGroup>
 </Project>
\ No newline at end of file