aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/storage/cache/httpstructuredcache.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/storage/cache/httpstructuredcache.cpp')
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.cpp192
1 files changed, 65 insertions, 127 deletions
diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp
index bbdb03ba4..4d3673e70 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.cpp
+++ b/src/zenserver/storage/cache/httpstructuredcache.cpp
@@ -80,7 +80,8 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach
HttpStatusService& StatusService,
UpstreamCache& UpstreamCache,
const DiskWriteBlocker* InDiskWriteBlocker,
- OpenProcessCache& InOpenProcessCache)
+ OpenProcessCache& InOpenProcessCache,
+ const ILocalRefPolicy* InLocalRefPolicy)
: m_Log(logging::Get("cache"))
, m_CacheStore(InCacheStore)
, m_StatsService(StatsService)
@@ -90,6 +91,7 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach
, m_DiskWriteBlocker(InDiskWriteBlocker)
, m_OpenProcessCache(InOpenProcessCache)
, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker)
+, m_LocalRefPolicy(InLocalRefPolicy)
{
m_StatsService.RegisterHandler("z$", *this);
m_StatusService.RegisterHandler("z$", *this);
@@ -114,6 +116,18 @@ HttpStructuredCacheService::BaseUri() const
return "/z$/";
}
+bool
+HttpStructuredCacheService::AcceptsLocalFileReferences() const
+{
+ return true;
+}
+
+const ILocalRefPolicy*
+HttpStructuredCacheService::GetLocalRefPolicy() const
+{
+ return m_LocalRefPolicy;
+}
+
void
HttpStructuredCacheService::Flush()
{
@@ -423,19 +437,22 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
std::string RecordPath = UrlDecode(Params.GetValue("path"));
- uint32_t ThreadCount = GetHardwareConcurrency();
+ const uint32_t HardwareConcurrency = GetHardwareConcurrency();
+ const uint32_t MaxThreadCount = std::max<uint32_t>(HardwareConcurrency, 16u);
+ uint32_t ThreadCount = HardwareConcurrency;
if (auto Param = Params.GetValue("thread_count"); Param.empty() == false)
{
if (auto Value = ParseInt<uint64_t>(Param))
{
- ThreadCount = gsl::narrow<uint32_t>(Value.value());
+ ThreadCount = gsl::narrow_cast<uint32_t>(std::min<uint64_t>(Value.value(), MaxThreadCount));
}
}
+ ThreadCount = std::clamp<uint32_t>(ThreadCount, 1u, MaxThreadCount);
ZEN_INFO("initiating cache RPC replay using {} threads, from '{}'", ThreadCount, RecordPath);
std::unique_ptr<cache::IRpcRequestReplayer> Replayer(cache::MakeDiskRequestReplayer(RecordPath, false));
- ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount < 1 ? 1 : ThreadCount);
+ ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount);
ZEN_INFO("cache RPC replay COMPLETED");
@@ -543,7 +560,7 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request)
break;
default:
m_CacheStats.BadRequestCount++;
- break;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -693,7 +710,8 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -783,7 +801,8 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -802,7 +821,8 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request,
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -1202,8 +1222,6 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
}
auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) {
- ZEN_UNUSED(PutResult);
-
HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError;
switch (PutResult.Status)
{
@@ -1217,7 +1235,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
if (PutResult.Details)
{
- Request.WriteResponse(ResponseCode, PutResult.Details);
+ return Request.WriteResponse(ResponseCode, PutResult.Details);
}
return Request.WriteResponse(ResponseCode);
};
@@ -1493,7 +1511,8 @@ HttpStructuredCacheService::HandleCacheChunkRequest(HttpServerRequest& Request,
HandlePutCacheChunk(Request, Ref, PolicyFromUrl);
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -1827,113 +1846,12 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st
}
}
-CbObject
-HttpStructuredCacheService::CollectStats()
+void
+HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request)
{
- ZEN_MEMSCOPE(GetCacheHttpTag());
-
CbObjectWriter Cbo;
-
- EmitSnapshot("requests", m_HttpRequests, Cbo);
-
- const uint64_t HitCount = m_CacheStats.HitCount;
- const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
- const uint64_t MissCount = m_CacheStats.MissCount;
- const uint64_t WriteCount = m_CacheStats.WriteCount;
- const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
- struct CidStoreStats StoreStats = m_CidStore.Stats();
- const uint64_t ChunkHitCount = StoreStats.HitCount;
- const uint64_t ChunkMissCount = StoreStats.MissCount;
- const uint64_t ChunkWriteCount = StoreStats.WriteCount;
- const uint64_t TotalCount = HitCount + MissCount;
-
- const uint64_t RpcRequests = m_CacheStats.RpcRequests;
- const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests;
- const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
- const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests;
- const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests;
- const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests;
- const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests;
-
- const CidStoreSize CidSize = m_CidStore.TotalSize();
- const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
-
- Cbo.BeginObject("cache");
- {
- Cbo << "badrequestcount" << BadRequestCount;
- Cbo.BeginObject("rpc");
- Cbo << "count" << RpcRequests;
- Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
- Cbo.BeginObject("records");
- Cbo << "count" << RpcRecordRequests;
- Cbo << "ops" << RpcRecordBatchRequests;
- Cbo.EndObject();
- Cbo.BeginObject("values");
- Cbo << "count" << RpcValueRequests;
- Cbo << "ops" << RpcValueBatchRequests;
- Cbo.EndObject();
- Cbo.BeginObject("chunks");
- Cbo << "count" << RpcChunkRequests;
- Cbo << "ops" << RpcChunkBatchRequests;
- Cbo.EndObject();
- Cbo.EndObject();
-
- Cbo.BeginObject("size");
- {
- Cbo << "disk" << CacheSize.DiskSize;
- Cbo << "memory" << CacheSize.MemorySize;
- }
- Cbo.EndObject();
-
- Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
- Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-
- if (m_UpstreamCache.IsActive())
- {
- Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
- Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
- }
-
- Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
-
- {
- ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
- Cbo.BeginObject("store");
- Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
- << "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
- const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
- Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
- EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
- EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
- Cbo.EndObject();
- }
- }
- Cbo.EndObject();
-
- if (m_UpstreamCache.IsActive())
- {
- EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
- Cbo.BeginObject("upstream");
- {
- m_UpstreamCache.GetStatus(Cbo);
- }
- Cbo.EndObject();
- }
-
- Cbo.BeginObject("cid");
- {
- Cbo.BeginObject("size");
- {
- Cbo << "tiny" << CidSize.TinySize;
- Cbo << "small" << CidSize.SmallSize;
- Cbo << "large" << CidSize.LargeSize;
- Cbo << "total" << CidSize.TotalSize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
-
- return Cbo.Save();
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
}
void
@@ -1944,12 +1862,6 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true";
bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
- if (!ShowCidStoreStats && !ShowCacheStoreStats)
- {
- Request.WriteResponse(HttpResponseCode::OK, CollectStats());
- return;
- }
-
// Full stats with optional detailed store/cid breakdowns
CbObjectWriter Cbo;
@@ -2156,12 +2068,38 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
}
-void
-HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request)
+CbObject
+HttpStructuredCacheService::CollectStats()
{
+ ZEN_TRACE_CPU("HttpStructuredCacheService::Stats");
+ ZEN_MEMSCOPE(GetCacheHttpTag());
+
CbObjectWriter Cbo;
- Cbo << "ok" << true;
- Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+
+ EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+ const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+
+ Cbo.BeginObject("cache");
+ {
+ Cbo.BeginObject("size");
+ {
+ Cbo << "disk" << CacheSize.DiskSize;
+ Cbo << "memory" << CacheSize.MemorySize;
+ }
+ Cbo.EndObject();
+
+ Cbo << "hits" << m_CacheStats.HitCount << "misses" << m_CacheStats.MissCount;
+ }
+ Cbo.EndObject();
+
+ return Cbo.Save();
+}
+
+uint64_t
+HttpStructuredCacheService::GetActivityCounter()
+{
+ return m_HttpRequests.Count();
}
bool
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-01 22:08:12 +0000