Changed to tsl::robin_map

Also added initial logic around attachment indexing (tactical check-in to continue on other computer)

2 files changed, 32 insertions, 11 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index eb3b5d13d..3fd9f5ec4 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -5,6 +5,7 @@
 #include <zencore/except.h>
 #include <zencore/windows.h>
 
+#include <zencore/compactbinary.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
@@ -46,12 +47,10 @@ ZenCacheStore::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCac
 	{
 		Ok = m_DiskLayer.Get(InBucket, HashKey, OutValue);
 
-#if 0  // This would keep file handles open
-		if (ok)
+		if (Ok && (OutValue.Value.Size() <= m_DiskLayerSizeThreshold))
 		{
-			m_memLayer.Put(InBucket, HashKey, OutValue);
+			m_MemLayer.Put(InBucket, HashKey, OutValue);
 		}
-#endif
 	}
 
 	return Ok;
@@ -60,8 +59,23 @@ ZenCacheStore::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCac
 void
 ZenCacheStore::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value)
 {
-	m_MemLayer.Put(InBucket, HashKey, Value);
+	if (Value.Value.Size() <= m_DiskLayerSizeThreshold)
+	{
+		m_MemLayer.Put(InBucket, HashKey, Value);
+	}
+
 	m_DiskLayer.Put(InBucket, HashKey, Value);
+
+	if (Value.IsCompactBinary)
+	{
+		zen::CbObject Cbo(SharedBuffer(Value.Value));
+
+		std::vector<IoHash> ReferencedChunks;
+
+		Cbo.IterateAttachments([&](CbFieldView AttachmentView) { ReferencedChunks.push_back(AttachmentView.AsHash()); });
+
+		// TODO: store references in index
+	}
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -201,9 +215,9 @@ private:
 	void BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey);
 	void PutLargeObject(const zen::IoHash& HashKey, const ZenCacheValue& Value);
 
-	RwLock															   m_IndexLock;
-	std::unordered_map<zen::IoHash, DiskLocation, zen::IoHash::Hasher> m_Index;
-	uint64_t														   m_WriteCursor = 0;
+	RwLock														   m_IndexLock;
+	tsl::robin_map<zen::IoHash, DiskLocation, zen::IoHash::Hasher> m_Index;
+	uint64_t													   m_WriteCursor = 0;
 };
 
 ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
@@ -376,7 +390,7 @@ ZenCacheDiskLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheVa
 	else
 	{
 		// TODO: should check if write is idempotent and bail out if it is?
-		it->second = Loc;
+		it.value() = Loc;
 	}
 
 	m_SlogFile.Append({.Key = HashKey, .Location = Loc});
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 7936f9d84..364644cf7 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -7,6 +7,12 @@
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 #include <zenstore/cas.h>
+
+#pragma warning(push)
+#pragma warning(disable : 4127)
+#include <tsl/robin_map.h>
+#pragma warning(pop)
+
 #include <compare>
 #include <filesystem>
 #include <unordered_map>
@@ -53,8 +59,8 @@ public:
 private:
 	struct CacheBucket
 	{
-		zen::RwLock															m_bucketLock;
-		std::unordered_map<zen::IoHash, zen::IoBuffer, zen::IoHash::Hasher> m_cacheMap;
+		zen::RwLock														m_bucketLock;
+		tsl::robin_map<zen::IoHash, zen::IoBuffer, zen::IoHash::Hasher> m_cacheMap;
 
 		bool Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue);
 		void Put(const zen::IoHash& HashKey, const ZenCacheValue& Value);
@@ -100,6 +106,7 @@ private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
 	ZenCacheDiskLayer	  m_DiskLayer;
+	uint64_t			  m_DiskLayerSizeThreshold = 4 * 1024;
 };
 
 /** Tracks cache entry access, stats and orchestrates cleanup activities