Don't put cache entries into the memory cache on Put, only on Get (#518)

2 files changed, 24 insertions, 21 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index f937f7d4d..a9ac46cab 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -2249,12 +2249,6 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 		EntryFlags |= DiskLocation::kCompressed;
 	}
 
-	uint64_t   PayloadSize	   = Value.Value.GetSize();
-	const bool MemCacheEnabled = (m_Configuration.MemCacheSizeThreshold > 0);
-	IoBuffer   MemCacheBuffer  = (MemCacheEnabled && (PayloadSize <= m_Configuration.MemCacheSizeThreshold))
-									 ? IoBufferBuilder::ReadFromFileMaybe(Value.Value)
-									 : IoBuffer{};
-
 	m_BlockStore.WriteChunk(Value.Value.Data(),
 							Value.Value.Size(),
 							m_Configuration.PayloadAlignment,
@@ -2263,14 +2257,14 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 								m_SlogFile.Append({.Key = HashKey, .Location = Location});
 
 								RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-								PayloadIndex			   EntryIndex = {};
 								if (auto It = m_Index.find(HashKey); It != m_Index.end())
 								{
-									EntryIndex = It.value();
+									PayloadIndex EntryIndex = It.value();
 									ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size()));
 									BucketPayload& Payload = m_Payloads[EntryIndex];
 
 									RemoveMemCachedData(Payload);
+									RemoveMetaData(Payload);
 
 									Payload					  = (BucketPayload{.Location = Location});
 									m_AccessTimes[EntryIndex] = GcClock::TickCount();
@@ -2282,7 +2276,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 								}
 								else
 								{
-									EntryIndex = PayloadIndex(m_Payloads.size());
+									PayloadIndex EntryIndex = PayloadIndex(m_Payloads.size());
 									m_Payloads.emplace_back(BucketPayload{.Location = Location});
 									m_AccessTimes.emplace_back(GcClock::TickCount());
 									if (m_Configuration.EnableReferenceCaching)
@@ -2292,13 +2286,6 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 									}
 									m_Index.insert_or_assign(HashKey, EntryIndex);
 								}
-
-								if (MemCacheBuffer)
-								{
-									BucketPayload& Payload = m_Payloads[EntryIndex];
-									SetMemCachedData(Payload, MemCacheBuffer);
-								}
-								RemoveMetaData(m_Payloads[EntryIndex]);
 							});
 }
 
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 236a0e1d7..cc6fefc76 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -888,13 +888,24 @@ TEST_CASE("z$.size")
 			IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
 			Buffer.SetContentType(ZenContentType::kCbObject);
 
+			std::vector<std::pair<std::string, IoHash>> Keys;
+
 			for (size_t Key = 0; Key < Count; ++Key)
 			{
-				const size_t Bucket = Key % 4;
-				Zcs.Put(fmt::format("test_bucket-{}", Bucket),
-						IoHash::HashBuffer(&Key, sizeof(uint32_t)),
-						ZenCacheValue{.Value = Buffer},
-						{});
+				const size_t Bucket		= Key % 4;
+				std::string	 BucketName = fmt::format("test_bucket-{}", Bucket);
+				IoHash		 Hash		= IoHash::HashBuffer(&Key, sizeof(uint32_t));
+				Zcs.Put(BucketName, Hash, ZenCacheValue{.Value = Buffer}, {});
+				Keys.push_back({BucketName, Hash});
+			}
+			CacheSize = Zcs.StorageSize();
+			CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize);
+			CHECK_EQ(0, CacheSize.MemorySize);
+
+			for (const auto& Key : Keys)
+			{
+				ZenCacheValue _;
+				Zcs.Get(Key.first, Key.second, _);
 			}
 
 			CacheSize = Zcs.StorageSize();
@@ -2359,6 +2370,11 @@ TEST_CASE("z$.newgc.basics")
 		auto Attachments =
 			CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
 		IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
+		{
+			// Do get so it ends up in memcache
+			ZenCacheValue _;
+			Zcs.Get(TearDrinkerBucket, CacheRecord, _);
+		}
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::hours(2));