aboutsummaryrefslogtreecommitdiff
path: root/zenserver/cache
diff options
context:
space:
mode:
Diffstat (limited to 'zenserver/cache')
-rw-r--r--zenserver/cache/structuredcache.cpp15
-rw-r--r--zenserver/cache/structuredcache.h2
-rw-r--r--zenserver/cache/structuredcachestore.cpp57
-rw-r--r--zenserver/cache/structuredcachestore.h34
4 files changed, 61 insertions, 47 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 35cb02cbb..ffb6f563a 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -148,7 +148,6 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams)
//////////////////////////////////////////////////////////////////////////
HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CasStore& InStore,
CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
@@ -157,7 +156,6 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCac
, m_CacheStore(InCacheStore)
, m_StatsService(StatsService)
, m_StatusService(StatusService)
-, m_CasStore(InStore)
, m_CidStore(InCidStore)
, m_UpstreamCache(std::move(UpstreamCache))
{
@@ -194,7 +192,6 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx)
m_LastScrubTime = Ctx.ScrubTimestamp();
- m_CasStore.Scrub(Ctx);
m_CidStore.Scrub(Ctx);
m_CacheStore.Scrub(Ctx);
}
@@ -716,12 +713,8 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
{
if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value)))
{
- Payload = UpstreamResult.Value;
- IoHash ChunkHash = IoHash::HashBuffer(Payload);
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash);
- InUpstreamCache = true;
-
- m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
+ InUpstreamCache = true;
}
else
{
@@ -789,9 +782,7 @@ HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Reques
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Payload ID does not match attachment hash"sv);
}
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash);
-
- m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({})",
Ref.BucketSegment,
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index ad7253f79..68a01becb 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -54,7 +54,6 @@ class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider
{
public:
HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CasStore& InCasStore,
CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
@@ -98,7 +97,6 @@ private:
ZenCacheStore& m_CacheStore;
HttpStatsService& m_StatsService;
HttpStatusService& m_StatusService;
- CasStore& m_CasStore;
CidStore& m_CidStore;
std::unique_ptr<UpstreamCache> m_UpstreamCache;
uint64_t m_LastScrubTime = 0;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index e1d77a088..b120f3955 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -26,6 +26,7 @@
#include <concepts>
#include <filesystem>
+#include <memory_resource>
#include <ranges>
#include <unordered_map>
@@ -40,7 +41,7 @@ namespace zen {
using namespace fmt::literals;
-ZenCacheStore::ZenCacheStore(const std::filesystem::path& RootDir) : m_DiskLayer{RootDir}
+ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcContributor(Gc), m_DiskLayer{RootDir}
{
ZEN_INFO("initializing structured cache at '{}'", RootDir);
CreateDirectories(RootDir);
@@ -89,6 +90,25 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
m_DiskLayer.Put(InBucket, HashKey, Value);
+#if ZEN_USE_REF_TRACKING
+ if (Value.Value.GetContentType() == ZenContentType::kCbObject)
+ {
+ if (ValidateCompactBinary(Value.Value, CbValidateMode::All) == CbValidateError::None)
+ {
+ CbObject Object{SharedBuffer(Value.Value)};
+
+ uint8_t TempBuffer[8 * sizeof(IoHash)];
+ std::pmr::monotonic_buffer_resource Linear{TempBuffer, sizeof TempBuffer};
+ std::pmr::polymorphic_allocator Allocator{&Linear};
+ std::pmr::vector<IoHash> CidReferences{Allocator};
+
+ Object.IterateAttachments([&](CbFieldView Field) { CidReferences.push_back(Field.AsAttachment()); });
+
+ m_Gc.OnNewCidReferences(CidReferences);
+ }
+ }
+#endif
+
if (Value.Value.Size() <= m_DiskLayerSizeThreshold)
{
m_MemLayer.Put(InBucket, HashKey, Value);
@@ -132,10 +152,10 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheStore::GarbageCollect(GcContext& GcCtx)
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
{
- m_DiskLayer.GarbageCollect(GcCtx);
- m_MemLayer.GarbageCollect(GcCtx);
+ m_MemLayer.GatherReferences(GcCtx);
+ m_DiskLayer.GatherReferences(GcCtx);
}
//////////////////////////////////////////////////////////////////////////
@@ -221,13 +241,13 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
+ZenCacheMemoryLayer::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_Lock);
for (auto& Kv : m_Buckets)
{
- Kv.second.GarbageCollect(GcCtx);
+ Kv.second.GatherReferences(GcCtx);
}
}
@@ -253,7 +273,7 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheMemoryLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+ZenCacheMemoryLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
{
// Is it even meaningful to do this? The memory layer shouldn't
// contain anything which is not already in the disk layer
@@ -378,13 +398,12 @@ struct ZenCacheDiskLayer::CacheBucket
void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
static bool Delete(std::filesystem::path BucketDir);
-
- bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(const IoHash& HashKey, const ZenCacheValue& Value);
- void Drop();
- void Flush();
- void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+ void Drop();
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ void GatherReferences(GcContext& GcCtx);
inline bool IsOk() const { return m_IsOk; }
@@ -716,7 +735,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheDiskLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_IndexLock);
@@ -1062,13 +1081,13 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
+ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_Lock);
for (auto& Kv : m_Buckets)
{
- Kv.second.GarbageCollect(GcCtx);
+ Kv.second.GatherReferences(GcCtx);
}
}
@@ -1083,7 +1102,9 @@ TEST_CASE("z$.store")
ScopedTemporaryDirectory TempDir;
- ZenCacheStore Zcs(TempDir.Path() / "cache");
+ CasGc Gc;
+
+ ZenCacheStore Zcs(Gc, TempDir.Path() / "cache");
const int kIterationCount = 100;
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 760f4995c..6beecf78b 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -8,11 +8,11 @@
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <zenstore/cas.h>
+#include <zenstore/gc.h>
-#pragma warning(push)
-#pragma warning(disable : 4127)
+ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
-#pragma warning(pop)
+ZEN_THIRD_PARTY_INCLUDES_END
#include <compare>
#include <filesystem>
@@ -22,6 +22,7 @@ namespace zen {
class WideStringBuilderBase;
class CasStore;
+class CasGc;
/******************************************************************************
@@ -50,6 +51,9 @@ struct ZenCacheValue
Intended for small values which are frequently accessed
+ This should have a better memory management policy to maintain reasonable
+ footprint.
+
*/
class ZenCacheMemoryLayer
{
@@ -61,7 +65,7 @@ public:
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
bool DropBucket(std::string_view Bucket);
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
struct Configuration
{
@@ -87,7 +91,7 @@ private:
bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(const IoHash& HashKey, const ZenCacheValue& Value);
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
private:
uint64_t GetCurrentTimeStamp();
@@ -109,7 +113,7 @@ public:
bool DropBucket(std::string_view Bucket);
void Flush();
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
void DiscoverBuckets();
@@ -124,24 +128,24 @@ private:
std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive
};
-class ZenCacheStore
+class ZenCacheStore : public GcContributor
{
public:
- explicit ZenCacheStore(const std::filesystem::path& RootDir);
+ ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir);
~ZenCacheStore();
- bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
- bool DropBucket(std::string_view Bucket);
- void Flush();
- void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool DropBucket(std::string_view Bucket);
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ virtual void GatherReferences(GcContext& GcCtx) override;
private:
std::filesystem::path m_RootDir;
ZenCacheMemoryLayer m_MemLayer;
ZenCacheDiskLayer m_DiskLayer;
- uint64_t m_DiskLayerSizeThreshold = 4 * 1024;
+ uint64_t m_DiskLayerSizeThreshold = 1 * 1024;
uint64_t m_LastScrubTime = 0;
};
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-29 11:57:04 +0000