aboutsummaryrefslogtreecommitdiff
path: root/zenserver/cache
diff options
context:
space:
mode:
Diffstat (limited to 'zenserver/cache')
-rw-r--r--zenserver/cache/structuredcache.cpp15
-rw-r--r--zenserver/cache/structuredcache.h2
-rw-r--r--zenserver/cache/structuredcachestore.cpp70
-rw-r--r--zenserver/cache/structuredcachestore.h37
4 files changed, 71 insertions, 53 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 35cb02cbb..ffb6f563a 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -148,7 +148,6 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams)
//////////////////////////////////////////////////////////////////////////
HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CasStore& InStore,
CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
@@ -157,7 +156,6 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCac
, m_CacheStore(InCacheStore)
, m_StatsService(StatsService)
, m_StatusService(StatusService)
-, m_CasStore(InStore)
, m_CidStore(InCidStore)
, m_UpstreamCache(std::move(UpstreamCache))
{
@@ -194,7 +192,6 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx)
m_LastScrubTime = Ctx.ScrubTimestamp();
- m_CasStore.Scrub(Ctx);
m_CidStore.Scrub(Ctx);
m_CacheStore.Scrub(Ctx);
}
@@ -716,12 +713,8 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
{
if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value)))
{
- Payload = UpstreamResult.Value;
- IoHash ChunkHash = IoHash::HashBuffer(Payload);
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash);
- InUpstreamCache = true;
-
- m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
+ InUpstreamCache = true;
}
else
{
@@ -789,9 +782,7 @@ HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Reques
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Payload ID does not match attachment hash"sv);
}
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash);
-
- m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({})",
Ref.BucketSegment,
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index ad7253f79..68a01becb 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -54,7 +54,6 @@ class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider
{
public:
HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- CasStore& InCasStore,
CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
@@ -98,7 +97,6 @@ private:
ZenCacheStore& m_CacheStore;
HttpStatsService& m_StatsService;
HttpStatusService& m_StatusService;
- CasStore& m_CasStore;
CidStore& m_CidStore;
std::unique_ptr<UpstreamCache> m_UpstreamCache;
uint64_t m_LastScrubTime = 0;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index e7b840ed8..f964c3102 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -6,6 +6,7 @@
#include <zencore/windows.h>
#include <zencore/compactbinary.h>
+#include <zencore/compactbinaryvalidation.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/iobuffer.h>
@@ -15,9 +16,11 @@
#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
+#include <zenstore/gc.h>
#include <concepts>
#include <filesystem>
+#include <memory_resource>
#include <unordered_map>
ZEN_THIRD_PARTY_INCLUDES_START
@@ -31,7 +34,7 @@ namespace zen {
using namespace fmt::literals;
-ZenCacheStore::ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir}
+ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcContributor(Gc), m_DiskLayer{RootDir}
{
ZEN_INFO("initializing structured cache at '{}'", RootDir);
CreateDirectories(RootDir);
@@ -80,6 +83,25 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
m_DiskLayer.Put(InBucket, HashKey, Value);
+#if ZEN_USE_REF_TRACKING
+ if (Value.Value.GetContentType() == ZenContentType::kCbObject)
+ {
+ if (ValidateCompactBinary(Value.Value, CbValidateMode::All) == CbValidateError::None)
+ {
+ CbObject Object{SharedBuffer(Value.Value)};
+
+ uint8_t TempBuffer[8 * sizeof(IoHash)];
+ std::pmr::monotonic_buffer_resource Linear{TempBuffer, sizeof TempBuffer};
+ std::pmr::polymorphic_allocator Allocator{&Linear};
+ std::pmr::vector<IoHash> CidReferences{Allocator};
+
+ Object.IterateAttachments([&](CbFieldView Field) { CidReferences.push_back(Field.AsAttachment()); });
+
+ m_Gc.OnNewCidReferences(CidReferences);
+ }
+ }
+#endif
+
if (Value.Value.Size() <= m_DiskLayerSizeThreshold)
{
m_MemLayer.Put(InBucket, HashKey, Value);
@@ -123,9 +145,10 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheStore::GarbageCollect(GcContext& GcCtx)
+ZenCacheStore::GatherReferences(GcContext& GcCtx)
{
- ZEN_UNUSED(GcCtx);
+ m_MemLayer.GatherReferences(GcCtx);
+ m_DiskLayer.GatherReferences(GcCtx);
}
//////////////////////////////////////////////////////////////////////////
@@ -211,7 +234,7 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
+ZenCacheMemoryLayer::GatherReferences(GcContext& GcCtx)
{
ZEN_UNUSED(GcCtx);
}
@@ -238,7 +261,7 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheMemoryLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+ZenCacheMemoryLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
{
// Is it even meaningful to do this? The memory layer shouldn't
// contain anything which is not already in the disk layer
@@ -358,23 +381,21 @@ static_assert(sizeof(DiskIndexEntry) == 36);
struct ZenCacheDiskLayer::CacheBucket
{
- CacheBucket(CasStore& Cas);
+ CacheBucket();
~CacheBucket();
void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
static bool Delete(std::filesystem::path BucketDir);
-
- bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(const IoHash& HashKey, const ZenCacheValue& Value);
- void Drop();
- void Flush();
- void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+ void Drop();
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ void GatherReferences(GcContext& GcCtx);
inline bool IsOk() const { return m_IsOk; }
private:
- CasStore& m_CasStore;
std::filesystem::path m_BucketDir;
Oid m_BucketId;
bool m_IsOk = false;
@@ -405,7 +426,7 @@ private:
inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
};
-ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
+ZenCacheDiskLayer::CacheBucket::CacheBucket()
{
}
@@ -702,7 +723,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheDiskLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
{
RwLock::SharedLockScope _(m_IndexLock);
@@ -839,7 +860,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
//////////////////////////////////////////////////////////////////////////
-ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir) : m_RootDir(RootDir), m_CasStore(Cas)
+ZenCacheDiskLayer::ZenCacheDiskLayer(const std::filesystem::path& RootDir) : m_RootDir(RootDir)
{
}
@@ -873,7 +894,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
}
else
{
- auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+ auto It = m_Buckets.try_emplace(std::string(InBucket));
Bucket = &It.first->second;
std::filesystem::path BucketPath = m_RootDir;
@@ -916,7 +937,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
}
else
{
- auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+ auto It = m_Buckets.try_emplace(std::string(InBucket));
Bucket = &It.first->second;
std::filesystem::path bucketPath = m_RootDir;
@@ -972,7 +993,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
}
else
{
- auto InsertResult = m_Buckets.try_emplace(BucketName8, m_CasStore);
+ auto InsertResult = m_Buckets.try_emplace(BucketName8);
std::filesystem::path BucketPath = m_RootDir;
BucketPath /= BucketName8;
@@ -1048,9 +1069,14 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
}
void
-ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
+ZenCacheDiskLayer::GatherReferences(GcContext& GcCtx)
{
- ZEN_UNUSED(GcCtx);
+ RwLock::SharedLockScope _(m_Lock);
+
+ for (auto& Kv : m_Buckets)
+ {
+ Kv.second.GatherReferences(GcCtx);
+ }
}
} // namespace zen
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 4753af627..da3e74126 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -8,11 +8,11 @@
#include <zencore/thread.h>
#include <zencore/uid.h>
#include <zenstore/cas.h>
+#include <zenstore/gc.h>
-#pragma warning(push)
-#pragma warning(disable : 4127)
+ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
-#pragma warning(pop)
+ZEN_THIRD_PARTY_INCLUDES_END
#include <compare>
#include <filesystem>
@@ -22,6 +22,7 @@ namespace zen {
class WideStringBuilderBase;
class CasStore;
+class CasGc;
/******************************************************************************
@@ -50,6 +51,9 @@ struct ZenCacheValue
Intended for small values which are frequently accessed
+ This should have a better memory management policy to maintain reasonable
+ footprint.
+
*/
class ZenCacheMemoryLayer
{
@@ -61,7 +65,7 @@ public:
void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
bool DropBucket(std::string_view Bucket);
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
struct Configuration
{
@@ -87,7 +91,7 @@ private:
bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
void Put(const IoHash& HashKey, const ZenCacheValue& Value);
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
private:
uint64_t GetCurrentTimeStamp();
@@ -101,7 +105,7 @@ private:
class ZenCacheDiskLayer
{
public:
- ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir);
+ ZenCacheDiskLayer(const std::filesystem::path& RootDir);
~ZenCacheDiskLayer();
bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -109,7 +113,7 @@ public:
bool DropBucket(std::string_view Bucket);
void Flush();
void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ void GatherReferences(GcContext& GcCtx);
void DiscoverBuckets();
@@ -119,30 +123,29 @@ private:
*/
struct CacheBucket;
- CasStore& m_CasStore;
std::filesystem::path m_RootDir;
RwLock m_Lock;
std::unordered_map<std::string, CacheBucket> m_Buckets; // TODO: make this case insensitive
};
-class ZenCacheStore
+class ZenCacheStore : public GcContributor
{
public:
- ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir);
+ ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir);
~ZenCacheStore();
- bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
- void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
- bool DropBucket(std::string_view Bucket);
- void Flush();
- void Scrub(ScrubContext& Ctx);
- void GarbageCollect(GcContext& GcCtx);
+ bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+ void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+ bool DropBucket(std::string_view Bucket);
+ void Flush();
+ void Scrub(ScrubContext& Ctx);
+ virtual void GatherReferences(GcContext& GcCtx) override;
private:
std::filesystem::path m_RootDir;
ZenCacheMemoryLayer m_MemLayer;
ZenCacheDiskLayer m_DiskLayer;
- uint64_t m_DiskLayerSizeThreshold = 4 * 1024;
+ uint64_t m_DiskLayerSizeThreshold = 1 * 1024;
uint64_t m_LastScrubTime = 0;
};
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-29 20:08:22 +0000