More statistics for Cache, Project Store and Cid Store (#405)

Cache: requestcount, badrequestcount, writes
Project Store: requestcount
Cid Store: cidhits, cidmisses, cidwrites

1 files changed, 53 insertions, 5 deletions

diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index ddfd1eeb5..7f1ce18f9 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -395,6 +395,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
 				FilterNamespace = Tokens[1];
 				if (FilterNamespace.empty())
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 				}
 			}
@@ -404,11 +405,13 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
 				FilterNamespace = Tokens[1];
 				if (FilterNamespace.empty())
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 				}
 				FilterBucket = Tokens[2];
 				if (FilterBucket.empty())
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 				}
 			}
@@ -418,21 +421,25 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
 				FilterNamespace = Tokens[1];
 				if (FilterNamespace.empty())
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 				}
 				FilterBucket = Tokens[2];
 				if (FilterBucket.empty())
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 				}
 				FilterValue = Tokens[3];
 				if (FilterValue.empty())
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 				}
 			}
 			break;
 		default:
+			m_CacheStats.BadRequestCount++;
 			return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 	}
 
@@ -596,6 +603,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
 void
 HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
+	m_CacheStats.RequestCount++;
 	metrics::OperationTiming::Scope $(m_HttpRequests);
 
 	std::string_view Key = Request.RelativeUri();
@@ -648,6 +656,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 	HttpRequestData RequestData;
 	if (!HttpRequestParseRelativeUri(Key, RequestData))
 	{
+		m_CacheStats.BadRequestCount++;
 		return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
 	}
 
@@ -733,6 +742,9 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request)
 				return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save());
 			}
 			break;
+		default:
+			m_CacheStats.BadRequestCount++;
+			break;
 	}
 }
 
@@ -1035,6 +1047,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 				if (Success && StoreLocal)
 				{
 					m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue);
+					m_CacheStats.WriteCount++;
 				}
 			}
 			else if (AcceptType == ZenContentType::kCbPackage)
@@ -1109,6 +1122,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 						if (StoreLocal)
 						{
 							m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+							m_CacheStats.WriteCount++;
 						}
 
 						for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
@@ -1203,6 +1217,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 
 	if (!Body || Body.Size() == 0)
 	{
+		m_CacheStats.BadRequestCount++;
 		return Request.WriteResponse(HttpResponseCode::BadRequest);
 	}
 	if (!AreDiskWritesAllowed())
@@ -1228,6 +1243,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		{
 			if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize))
 			{
+				m_CacheStats.BadRequestCount++;
 				return Request.WriteResponse(HttpResponseCode::BadRequest,
 											 HttpContentType::kText,
 											 "Payload is not a valid compressed binary"sv);
@@ -1242,6 +1258,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 						 Ref.BucketSegment,
 						 Ref.HashKey,
 						 {.Value = Body, .RawSize = RawSize, .RawHash = RawHash});
+		m_CacheStats.WriteCount++;
 
 		if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote))
 		{
@@ -1268,11 +1285,13 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 					 Ref.BucketSegment,
 					 Ref.HashKey,
 					 ToString(ContentType));
+			m_CacheStats.BadRequestCount++;
 			return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv);
 		}
 
 		Body.SetContentType(ZenContentType::kCbObject);
 		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		m_CacheStats.WriteCount++;
 
 		CbObjectView		CacheRecord(Body.Data());
 		std::vector<IoHash> ValidAttachments;
@@ -1321,6 +1340,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 					 Ref.BucketSegment,
 					 Ref.HashKey,
 					 ToString(ContentType));
+			m_CacheStats.BadRequestCount++;
 			return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv);
 		}
 		CachePolicy Policy = PolicyFromUrl;
@@ -1365,6 +1385,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 
 		if (Count.Invalid > 0)
 		{
+			m_CacheStats.BadRequestCount++;
 			return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv);
 		}
 
@@ -1372,6 +1393,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
 		CacheValue.Value.SetContentType(ZenContentType::kCbObject);
 		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+		m_CacheStats.WriteCount++;
 
 		for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
 		{
@@ -1408,6 +1430,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 	}
 	else
 	{
+		m_CacheStats.BadRequestCount++;
 		return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Content-Type invalid"sv);
 	}
 }
@@ -1524,6 +1547,7 @@ HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, cons
 
 	if (!Body || Body.Size() == 0)
 	{
+		m_CacheStats.BadRequestCount++;
 		return Request.WriteResponse(HttpResponseCode::BadRequest);
 	}
 	if (!AreDiskWritesAllowed())
@@ -1537,11 +1561,13 @@ HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, cons
 	uint64_t RawSize;
 	if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize))
 	{
+		m_CacheStats.BadRequestCount++;
 		return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"sv);
 	}
 
 	if (RawHash != Ref.ValueContentId)
 	{
+		m_CacheStats.BadRequestCount++;
 		return Request.WriteResponse(HttpResponseCode::BadRequest,
 									 HttpContentType::kText,
 									 "ValueContentId does not match attachment hash"sv);
@@ -1622,6 +1648,7 @@ HttpStructuredCacheService::HandleRpcRequest(const CacheRequestContext& Context,
 	}
 	else
 	{
+		m_CacheStats.BadRequestCount++;
 		return HttpResponseCode::BadRequest;
 	}
 	return HttpResponseCode::OK;
@@ -1704,6 +1731,7 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request)
 				if ((ContentType != HttpContentType::kCbObject && ContentType != HttpContentType::kCbPackage) ||
 					AcceptType != HttpContentType::kCbPackage)
 				{
+					m_CacheStats.BadRequestCount++;
 					return Request.WriteResponse(HttpResponseCode::BadRequest);
 				}
 
@@ -1770,6 +1798,7 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request)
 			}
 			break;
 		default:
+			m_CacheStats.BadRequestCount++;
 			Request.WriteResponse(HttpResponseCode::BadRequest);
 			break;
 	}
@@ -1892,6 +1921,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
 	CacheValue.Value.SetContentType(ZenContentType::kCbObject);
 	m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue);
+	m_CacheStats.WriteCount++;
 
 	for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
 	{
@@ -2072,6 +2102,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 								ZEN_WARN("Skipping invalid chunk in local cache '{}'", Value.ContentId);
 							}
 						}
+
 						if (EnumHasAllFlags(ValuePolicy, CachePolicy::QueryRemote))
 						{
 							NeedUpstreamAttachment = true;
@@ -2152,6 +2183,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 				if (StoreLocal)
 				{
 					m_CacheStore.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
+					m_CacheStats.WriteCount++;
 				}
 				ParseValues(Request);
 				Request.Source = Params.Source;
@@ -2331,6 +2363,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 						RawSize = Chunk.DecodeRawSize();
 					}
 					m_CacheStore.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash});
+					m_CacheStats.WriteCount++;
 					TransferredSize = Chunk.GetCompressedSize();
 				}
 				Succeeded = true;
@@ -2521,6 +2554,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 											 Request.Key.Bucket,
 											 Request.Key.Hash,
 											 ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash});
+							m_CacheStats.WriteCount++;
 						}
 
 						ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}",
@@ -2830,6 +2864,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(const CacheRequestContext&
 				if (StoreLocal)
 				{
 					m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
+					m_CacheStats.WriteCount++;
 				}
 			};
 		m_UpstreamCache.GetCacheRecords(Namespace, UpstreamRecordRequests, std::move(OnCacheRecordGetComplete));
@@ -3009,6 +3044,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(const CacheRequestContext&
 										 Key.Key.Bucket,
 										 Key.Key.Hash,
 										 {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash});
+						m_CacheStats.WriteCount++;
 					}
 				}
 				if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
@@ -3103,10 +3139,17 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
 	EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
 
-	const uint64_t HitCount			= m_CacheStats.HitCount;
-	const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
-	const uint64_t MissCount		= m_CacheStats.MissCount;
-	const uint64_t TotalCount		= HitCount + MissCount;
+	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
+	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
+	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
+	const uint64_t		 RequestCount	  = m_CacheStats.RequestCount;
+	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
+	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
+	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
+	const uint64_t		 ChunkMissCount	  = StoreStats.MissCount;
+	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
+	const uint64_t		 TotalCount		  = HitCount + MissCount;
 
 	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
 	const GcStorageSize CacheSize = m_CacheStore.StorageSize();
@@ -3121,10 +3164,15 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 		Cbo.EndObject();
 
 		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "hits" << HitCount << "misses" << MissCount;
+		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
 		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
 		Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
 		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		Cbo << "requestcount" << RequestCount;
+		Cbo << "badrequestcount" << BadRequestCount;
+
+		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
 	}
 	Cbo.EndObject();