project store stats (#357)

* add basic stats for project store
* stats for BadRequest in ProjectStore::Rpc
* changelog
* group stats
fix ChunkWriteCount when accepting ops

4 files changed, 197 insertions, 37 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 7e7517c33..f52473d99 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -315,6 +315,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 			if (Payload.Size() <= sizeof(RequestHeader))
 			{
+				m_ProjectStats.BadRequestCount++;
 				HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 			}
 
@@ -323,6 +324,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 			if (RequestHdr.Magic != RequestHeader::kMagic)
 			{
+				m_ProjectStats.BadRequestCount++;
 				HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 			}
 
@@ -394,6 +396,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 				memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk));
 				ResponsePtr += sizeof(ResponseChunk);
 			}
+			m_ProjectStats.ChunkHitCount += RequestHdr.ChunkCount;
 			return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs);
 		},
 		HttpVerb::kPost);
@@ -421,6 +424,10 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			}
 			else
 			{
+				if (Result.first == HttpResponseCode::BadRequest)
+				{
+					m_ProjectStats.BadRequestCount++;
+				}
 				ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
 						  ToString(HttpReq.RequestVerb()),
 						  HttpReq.QueryString(),
@@ -448,14 +455,20 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunkInfo(ProjectId, OplogId, ChunkId, ResponsePayload);
 			if (Result.first == HttpResponseCode::OK)
 			{
+				m_ProjectStats.ChunkHitCount++;
 				return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
 			}
 			else if (Result.first == HttpResponseCode::NotFound)
 			{
+				m_ProjectStats.ChunkMissCount++;
 				ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
 			}
 			else
 			{
+				if (Result.first == HttpResponseCode::BadRequest)
+				{
+					m_ProjectStats.BadRequestCount++;
+				}
 				ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
 						  ToString(HttpReq.RequestVerb()),
 						  HttpReq.QueryString(),
@@ -492,6 +505,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 				}
 				else
 				{
+					m_ProjectStats.BadRequestCount++;
 					return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 				}
 			}
@@ -504,6 +518,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 				}
 				else
 				{
+					m_ProjectStats.BadRequestCount++;
 					return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 				}
 			}
@@ -515,15 +530,21 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 				m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk);
 			if (Result.first == HttpResponseCode::OK)
 			{
+				m_ProjectStats.ChunkHitCount++;
 				ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Chunk.GetContentType()));
 				return HttpReq.WriteResponse(HttpResponseCode::OK, Chunk.GetContentType(), Chunk);
 			}
 			else if (Result.first == HttpResponseCode::NotFound)
 			{
+				m_ProjectStats.ChunkMissCount++;
 				ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId);
 			}
 			else
 			{
+				if (Result.first == HttpResponseCode::BadRequest)
+				{
+					m_ProjectStats.BadRequestCount++;
+				}
 				ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
 						  ToString(HttpReq.RequestVerb()),
 						  HttpReq.QueryString(),
@@ -559,14 +580,20 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 						if (Result.first == HttpResponseCode::OK)
 						{
+							m_ProjectStats.ChunkHitCount++;
 							return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
 						}
 						else if (Result.first == HttpResponseCode::NotFound)
 						{
+							m_ProjectStats.ChunkMissCount++;
 							ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid);
 						}
 						else
 						{
+							if (Result.first == HttpResponseCode::BadRequest)
+							{
+								m_ProjectStats.BadRequestCount++;
+							}
 							ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
 									  ToString(HttpReq.RequestVerb()),
 									  HttpReq.QueryString(),
@@ -589,10 +616,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 							m_ProjectStore->PutChunk(ProjectId, OplogId, Cid, RequestType, HttpReq.ReadPayload());
 						if (Result.first == HttpResponseCode::OK || Result.first == HttpResponseCode::Created)
 						{
+							m_ProjectStats.ChunkWriteCount++;
 							return HttpReq.WriteResponse(Result.first);
 						}
 						else
 						{
+							if (Result.first == HttpResponseCode::BadRequest)
+							{
+								m_ProjectStats.BadRequestCount++;
+							}
 							ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
 									  ToString(HttpReq.RequestVerb()),
 									  HttpReq.QueryString(),
@@ -771,37 +803,59 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 					WriteFile(BadPackagePath, Payload);
 
+					m_ProjectStats.BadRequestCount++;
 					return HttpReq.WriteResponse(HttpResponseCode::BadRequest,
 												 HttpContentType::kText,
 												 u8"request body must be a compact binary object or package in legacy format");
 				}
 			}
 
+			m_ProjectStats.ChunkMissCount += MissingChunks.size();
+
 			if (!IsValid)
 			{
-				// TODO: emit diagnostics identifying missing chunks
+				ExtendableStringBuilder<256> ResponseText;
+				ResponseText.Append("Missing chunk references: ");
+
+				bool IsFirst = true;
+				for (const auto& Hash : MissingChunks)
+				{
+					if (IsFirst)
+					{
+						IsFirst = false;
+					}
+					else
+					{
+						ResponseText.Append(", ");
+					}
+					Hash.ToHexString(ResponseText);
+				}
 
-				return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing chunk reference");
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, ResponseText);
 			}
 
 			CbObject Core = Package.GetObject();
 
 			if (!Core["key"sv])
 			{
+				m_ProjectStats.BadRequestCount++;
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
 			}
 
 			// Write core to oplog
 
-			const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Package);
+			size_t		   AttachmentCount = Package.GetAttachments().size();
+			const uint32_t OpLsn		   = Oplog.AppendNewOplogEntry(Package);
 
 			if (OpLsn == ProjectStore::Oplog::kInvalidOp)
 			{
+				m_ProjectStats.BadRequestCount++;
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 			}
+			m_ProjectStats.ChunkWriteCount += AttachmentCount;
 
+			m_ProjectStats.OpWriteCount++;
 			ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString());
-
 			HttpReq.WriteResponse(HttpResponseCode::Created);
 		},
 		HttpVerb::kPost);
@@ -885,17 +939,18 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 								}
 							}
 						});
-
+						m_ProjectStats.OpHitCount++;
 						return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package);
 					}
 					else
 					{
 						// Client cannot accept a package, so we only send the core object
+						m_ProjectStats.OpHitCount++;
 						return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op);
 					}
 				}
 			}
-
+			m_ProjectStats.OpMissCount++;
 			return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 		},
 		HttpVerb::kGet);
@@ -939,6 +994,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						   << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log);
 
 						HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
+
+						m_ProjectStats.OpLogReadCount++;
 					}
 					break;
 
@@ -964,6 +1021,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 							}
 							Project->TouchOplog(OplogId);
 
+							m_ProjectStats.OpLogWriteCount++;
 							ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath);
 
 							return HttpReq.WriteResponse(HttpResponseCode::Created);
@@ -972,6 +1030,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						// I guess this should ultimately be used to execute RPCs but for now, it
 						// does absolutely nothing
 
+						m_ProjectStats.BadRequestCount++;
 						return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 					}
 					break;
@@ -982,6 +1041,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 						Project->DeleteOplog(OplogId);
 
+						m_ProjectStats.OpLogDeleteCount++;
 						return HttpReq.WriteResponse(HttpResponseCode::OK);
 					}
 					break;
@@ -1085,6 +1145,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 								 ProjectFilePath,
 								 ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
 
+						m_ProjectStats.ProjectWriteCount++;
 						HttpReq.WriteResponse(HttpResponseCode::Created);
 					}
 					break;
@@ -1119,6 +1180,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						Response.EndArray();  // oplogs
 
 						HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
+
+						m_ProjectStats.ProjectReadCount++;
 					}
 					break;
 
@@ -1140,6 +1203,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 														 fmt::format("project {} is in use", ProjectId));
 						}
 
+						m_ProjectStats.ProjectDeleteCount++;
 						return HttpReq.WriteResponse(HttpResponseCode::NoContent);
 					}
 					break;
@@ -1167,6 +1231,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			const auto& OplogId	  = Req.GetCapture(2);
 			if (HttpReq.RequestContentType() != HttpContentType::kCbObject)
 			{
+				m_ProjectStats.BadRequestCount++;
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type");
 			}
 			IoBuffer Payload = HttpReq.ReadPayload();
@@ -1177,6 +1242,18 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
 			}
+			else
+			{
+				if (Result.first == HttpResponseCode::BadRequest)
+				{
+					m_ProjectStats.BadRequestCount++;
+				}
+				ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+						  ToString(HttpReq.RequestVerb()),
+						  HttpReq.QueryString(),
+						  static_cast<int>(Result.first),
+						  Result.second);
+			}
 			if (Result.second.empty())
 			{
 				return HttpReq.WriteResponse(Result.first);
@@ -1196,6 +1273,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			const auto&		   OplogId	 = Req.GetCapture(2);
 			if (HttpReq.AcceptContentType() != HttpContentType::kCbObject)
 			{
+				m_ProjectStats.BadRequestCount++;
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type");
 			}
 			IoBuffer Payload = HttpReq.ReadPayload();
@@ -1207,6 +1285,18 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
 			}
+			else
+			{
+				if (Result.first == HttpResponseCode::BadRequest)
+				{
+					m_ProjectStats.BadRequestCount++;
+				}
+				ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+						  ToString(HttpReq.RequestVerb()),
+						  HttpReq.QueryString(),
+						  static_cast<int>(Result.first),
+						  Result.second);
+			}
 			if (Result.second.empty())
 			{
 				return HttpReq.WriteResponse(Result.first);
@@ -1225,7 +1315,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			const auto& OplogId	  = Req.GetCapture(2);
 			IoBuffer	Payload	  = HttpReq.ReadPayload();
 
-			m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
+			bool OkRequest = m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
+			if (!OkRequest)
+			{
+				m_ProjectStats.BadRequestCount++;
+			}
 		},
 		HttpVerb::kPost);
 
@@ -1408,6 +1502,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 			if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
 			{
+				m_ProjectStats.BadRequestCount++;
 				return HttpReq.WriteResponse(
 					HttpResponseCode::BadRequest,
 					HttpContentType::kText,
@@ -1486,6 +1581,36 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 			Cbo << "memory" << StoreSize.MemorySize;
 		}
 		Cbo.EndObject();
+
+		Cbo.BeginObject("project");
+		{
+			Cbo << "readcount" << m_ProjectStats.ProjectReadCount << "writecount" << m_ProjectStats.ProjectWriteCount << "deletecount"
+				<< m_ProjectStats.ProjectDeleteCount;
+		}
+		Cbo.EndObject();
+
+		Cbo.BeginObject("oplog");
+		{
+			Cbo << "readcount" << m_ProjectStats.OpLogReadCount << "writecount" << m_ProjectStats.OpLogWriteCount << "deletecount"
+				<< m_ProjectStats.OpLogDeleteCount;
+		}
+		Cbo.EndObject();
+
+		Cbo.BeginObject("op");
+		{
+			Cbo << "hitcount" << m_ProjectStats.OpHitCount << "misscount" << m_ProjectStats.OpMissCount << "writecount"
+				<< m_ProjectStats.OpWriteCount;
+		}
+		Cbo.EndObject();
+
+		Cbo.BeginObject("chunk");
+		{
+			Cbo << "hitcount" << m_ProjectStats.ChunkHitCount << "misscount" << m_ProjectStats.ChunkMissCount << "writecount"
+				<< m_ProjectStats.ChunkWriteCount;
+		}
+		Cbo.EndObject();
+
+		Cbo << "badrequestcount" << m_ProjectStats.BadRequestCount;
 	}
 	Cbo.EndObject();
 
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index feb7ca82f..5842eb3ec 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -42,6 +42,23 @@ public:
 	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
 
 private:
+	struct ProjectStats
+	{
+		std::atomic_uint64_t ProjectReadCount{};
+		std::atomic_uint64_t ProjectWriteCount{};
+		std::atomic_uint64_t ProjectDeleteCount{};
+		std::atomic_uint64_t OpLogReadCount{};
+		std::atomic_uint64_t OpLogWriteCount{};
+		std::atomic_uint64_t OpLogDeleteCount{};
+		std::atomic_uint64_t OpHitCount{};
+		std::atomic_uint64_t OpMissCount{};
+		std::atomic_uint64_t OpWriteCount{};
+		std::atomic_uint64_t ChunkHitCount{};
+		std::atomic_uint64_t ChunkMissCount{};
+		std::atomic_uint64_t ChunkWriteCount{};
+		std::atomic_uint64_t BadRequestCount{};
+	};
+
 	inline spdlog::logger& Log() { return m_Log; }
 
 	spdlog::logger&	  m_Log;
@@ -50,6 +67,7 @@ private:
 	Ref<ProjectStore> m_ProjectStore;
 	HttpStatsService& m_StatsService;
 	AuthMgr&		  m_AuthMgr;
+	ProjectStats	  m_ProjectStats;
 };
 
-}  // namespace zen
\ No newline at end of file
+}  // namespace zen
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 3a91d7f0c..40fd3d30b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2323,7 +2323,7 @@ ProjectStore::WriteBlock(const std::string_view ProjectId, const std::string_vie
 	return {HttpResponseCode::OK, {}};
 }
 
-void
+bool
 ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 				  const std::string_view ProjectId,
 				  const std::string_view OplogId,
@@ -2346,9 +2346,10 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 				Cb = LoadCompactBinaryFromJson(JsonText).AsObject();
 				if (!Cb)
 				{
-					return HttpReq.WriteResponse(HttpResponseCode::BadRequest,
-												 HttpContentType::kText,
-												 "Content format not supported, expected JSON format");
+					HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+										  HttpContentType::kText,
+										  "Content format not supported, expected JSON format");
+					return false;
 				}
 			}
 			break;
@@ -2356,9 +2357,10 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 			Cb = LoadCompactBinaryObject(Payload);
 			if (!Cb)
 			{
-				return HttpReq.WriteResponse(HttpResponseCode::BadRequest,
-											 HttpContentType::kText,
-											 "Content format not supported, expected compact binary format");
+				HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+									  HttpContentType::kText,
+									  "Content format not supported, expected compact binary format");
+				return false;
 			}
 			break;
 		case HttpContentType::kCbPackage:
@@ -2366,30 +2368,34 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 			Cb		= Package.GetObject();
 			if (!Cb)
 			{
-				return HttpReq.WriteResponse(HttpResponseCode::BadRequest,
-											 HttpContentType::kText,
-											 "Content format not supported, expected package message format");
+				HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+									  HttpContentType::kText,
+									  "Content format not supported, expected package message format");
+				return false;
 			}
 			break;
 		default:
-			return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+			HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type");
+			return false;
 	}
 
 	Ref<ProjectStore::Project> Project = OpenProject(ProjectId);
 	if (!Project)
 	{
-		return HttpReq.WriteResponse(HttpResponseCode::NotFound,
-									 HttpContentType::kText,
-									 fmt::format("Rpc oplog request for unknown project '{}'", ProjectId));
+		HttpReq.WriteResponse(HttpResponseCode::NotFound,
+							  HttpContentType::kText,
+							  fmt::format("Rpc oplog request for unknown project '{}'", ProjectId));
+		return true;
 	}
 	Project->TouchProject();
 
 	ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId);
 	if (!Oplog)
 	{
-		return HttpReq.WriteResponse(HttpResponseCode::NotFound,
-									 HttpContentType::kText,
-									 fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId));
+		HttpReq.WriteResponse(HttpResponseCode::NotFound,
+							  HttpContentType::kText,
+							  fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId));
+		return true;
 	}
 	Project->TouchOplog(OplogId);
 
@@ -2399,23 +2405,28 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 	{
 		if (!AreDiskWritesAllowed())
 		{
-			return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			return true;
 		}
 		std::pair<HttpResponseCode, std::string> Result = Import(*Project.Get(), *Oplog, Cb["params"sv].AsObjectView(), AuthManager);
 		if (Result.second.empty())
 		{
-			return HttpReq.WriteResponse(Result.first);
+			HttpReq.WriteResponse(Result.first);
+			return Result.first != HttpResponseCode::BadRequest;
 		}
-		return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+		HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+		return true;
 	}
 	else if (Method == "export"sv)
 	{
 		std::pair<HttpResponseCode, std::string> Result = Export(*Project.Get(), *Oplog, Cb["params"sv].AsObjectView(), AuthManager);
 		if (Result.second.empty())
 		{
-			return HttpReq.WriteResponse(Result.first);
+			HttpReq.WriteResponse(Result.first);
+			return Result.first != HttpResponseCode::BadRequest;
 		}
-		return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+		HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+		return true;
 	}
 	else if (Method == "getchunks"sv)
 	{
@@ -2439,13 +2450,15 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 			ResponsePackage.SetObject(ResponseWriter.Save());
 		}
 		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, FormatFlags::kDefault);
-		return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
+		HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
+		return true;
 	}
 	else if (Method == "putchunks"sv)
 	{
 		if (!AreDiskWritesAllowed())
 		{
-			return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			return true;
 		}
 		std::span<const CbAttachment> Attachments = Package.GetAttachments();
 		for (const CbAttachment& Attachment : Attachments)
@@ -2454,13 +2467,15 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 			CompressedBuffer Compressed = Attachment.AsCompressedBinary();
 			m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), RawHash, CidStore::InsertMode::kCopyOnly);
 		}
-		return HttpReq.WriteResponse(HttpResponseCode::OK);
+		HttpReq.WriteResponse(HttpResponseCode::OK);
+		return true;
 	}
 	else if (Method == "snapshot"sv)
 	{
 		if (!AreDiskWritesAllowed())
 		{
-			return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			return true;
 		}
 
 		// Snapshot all referenced files. This brings the content of all
@@ -2614,9 +2629,11 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 
 		ZEN_INFO("rewrote {} oplog entries (out of {})", NewOps.size(), OpCount);
 
-		return HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save());
+		HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save());
+		return true;
 	}
-	return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("Unknown rpc method '{}'", Method));
+	HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("Unknown rpc method '{}'", Method));
+	return true;
 }
 
 std::pair<HttpResponseCode, std::string>
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 0d0d73263..eef1b15e0 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -313,7 +313,7 @@ public:
 														const std::string_view OplogId,
 														IoBuffer&&			   Payload);
 
-	void Rpc(HttpServerRequest&		HttpReq,
+	bool Rpc(HttpServerRequest&		HttpReq,
 			 const std::string_view ProjectId,
 			 const std::string_view OplogId,
 			 IoBuffer&&				Payload,