Low disk space detector (#277)

* - Feature: Disk writes are now blocked early and return an insufficient storage error if free disk space falls below the `--low-diskspace-threshold` value
* Never keep an entry in m_ChunkBlocks that points to a nullptr

1 files changed, 80 insertions, 38 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 80e3f27f1..9dd6ddcfc 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1680,6 +1680,7 @@ ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcMa
 , m_Log(logging::Get("project"))
 , m_CidStore(Store)
 , m_ProjectBasePath(BasePath)
+, m_DiskWriteBlocker(Gc.GetDiskWriteBlocker())
 {
 	ZEN_INFO("initializing project store at '{}'", BasePath);
 	// m_Log.set_level(spdlog::level::debug);
@@ -2560,6 +2561,10 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 
 	if (Method == "import")
 	{
+		if (!AreDiskWritesAllowed())
+		{
+			return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+		}
 		std::pair<HttpResponseCode, std::string> Result = Import(*Project.Get(), *Oplog, Cb["params"sv].AsObjectView(), AuthManager);
 		if (Result.second.empty())
 		{
@@ -2602,6 +2607,10 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 	}
 	else if (Method == "putchunks")
 	{
+		if (!AreDiskWritesAllowed())
+		{
+			return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+		}
 		std::span<const CbAttachment> Attachments = Package.GetAttachments();
 		for (const CbAttachment& Attachment : Attachments)
 		{
@@ -2676,6 +2685,12 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
 	return ConvertResult(Result);
 }
 
+bool
+ProjectStore::AreDiskWritesAllowed() const
+{
+	return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed());
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
@@ -2920,7 +2935,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			uint64_t Offset = 0;
 			uint64_t Size	= ~(0ull);
 
-			auto QueryParms = Req.ServerRequest().GetQueryParams();
+			auto QueryParms = HttpReq.GetQueryParams();
 
 			if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false)
 			{
@@ -2987,7 +3002,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			HttpContentType AcceptType	= HttpReq.AcceptContentType();
 			HttpContentType RequestType = HttpReq.RequestContentType();
 
-			switch (Req.ServerRequest().RequestVerb())
+			switch (HttpReq.RequestVerb())
 			{
 				case HttpVerb::kGet:
 					{
@@ -3019,6 +3034,10 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 					}
 				case HttpVerb::kPost:
 					{
+						if (!m_ProjectStore->AreDiskWritesAllowed())
+						{
+							return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+						}
 						std::pair<HttpResponseCode, std::string> Result =
 							m_ProjectStore->PutChunk(ProjectId, OplogId, Cid, RequestType, HttpReq.ReadPayload());
 						if (Result.first == HttpResponseCode::OK || Result.first == HttpResponseCode::Created)
@@ -3107,6 +3126,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		[this](HttpRouterRequest& Req) {
 			HttpServerRequest& HttpReq = Req.ServerRequest();
 
+			if (!m_ProjectStore->AreDiskWritesAllowed())
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			}
+
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
@@ -3252,7 +3276,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 				if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
 				{
 					CbObject& Op = MaybeOp.value();
-					if (Req.ServerRequest().AcceptContentType() == ZenContentType::kCbPackage)
+					if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage)
 					{
 						CbPackage Package;
 						Package.SetObject(Op);
@@ -3319,6 +3343,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 	m_Router.RegisterRoute(
 		"{project}/oplog/{log}",
 		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
@@ -3326,22 +3352,22 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 			if (!Project)
 			{
-				return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
-														 HttpContentType::kText,
-														 fmt::format("project {} not found", ProjectId));
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+											 HttpContentType::kText,
+											 fmt::format("project {} not found", ProjectId));
 			}
 			Project->TouchProject();
 
-			switch (Req.ServerRequest().RequestVerb())
+			switch (HttpReq.RequestVerb())
 			{
 				case HttpVerb::kGet:
 					{
 						ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
 						if (!OplogIt)
 						{
-							return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
-																	 HttpContentType::kText,
-																	 fmt::format("oplog {} not found in project {}", OplogId, ProjectId));
+							return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+														 HttpContentType::kText,
+														 fmt::format("oplog {} not found in project {}", OplogId, ProjectId));
 						}
 						Project->TouchOplog(OplogId);
 
@@ -3352,14 +3378,18 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						   << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount"
 						   << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::max(), Log);
 
-						Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save());
+						HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
 					}
 					break;
 
 				case HttpVerb::kPost:
 					{
+						if (!m_ProjectStore->AreDiskWritesAllowed())
+						{
+							return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+						}
 						std::filesystem::path OplogMarkerPath;
-						if (CbObject Params = Req.ServerRequest().ReadPayloadObject())
+						if (CbObject Params = HttpReq.ReadPayloadObject())
 						{
 							OplogMarkerPath = Params["gcpath"sv].AsString();
 						}
@@ -3370,19 +3400,19 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 							if (!Project->NewOplog(OplogId, OplogMarkerPath))
 							{
 								// TODO: indicate why the operation failed!
-								return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError);
+								return HttpReq.WriteResponse(HttpResponseCode::InternalServerError);
 							}
 							Project->TouchOplog(OplogId);
 
 							ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath);
 
-							return Req.ServerRequest().WriteResponse(HttpResponseCode::Created);
+							return HttpReq.WriteResponse(HttpResponseCode::Created);
 						}
 
 						// I guess this should ultimately be used to execute RPCs but for now, it
 						// does absolutely nothing
 
-						return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+						return HttpReq.WriteResponse(HttpResponseCode::BadRequest);
 					}
 					break;
 
@@ -3392,7 +3422,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 						Project->DeleteOplog(OplogId);
 
-						return Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+						return HttpReq.WriteResponse(HttpResponseCode::OK);
 					}
 					break;
 
@@ -3461,13 +3491,19 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 	m_Router.RegisterRoute(
 		"{project}",
 		[this](HttpRouterRequest& Req) {
-			const std::string ProjectId = Req.GetCapture(1);
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const std::string  ProjectId = Req.GetCapture(1);
 
-			switch (Req.ServerRequest().RequestVerb())
+			switch (HttpReq.RequestVerb())
 			{
 				case HttpVerb::kPost:
 					{
-						IoBuffer		 Payload		 = Req.ServerRequest().ReadPayload();
+						if (!m_ProjectStore->AreDiskWritesAllowed())
+						{
+							return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+						}
+
+						IoBuffer		 Payload		 = HttpReq.ReadPayload();
 						CbObject		 Params			 = LoadCompactBinaryObject(Payload);
 						std::string_view Id				 = Params["id"sv].AsString();
 						std::string_view Root			 = Params["root"sv].AsString();
@@ -3487,7 +3523,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 								 ProjectFilePath,
 								 ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
 
-						Req.ServerRequest().WriteResponse(HttpResponseCode::Created);
+						HttpReq.WriteResponse(HttpResponseCode::Created);
 					}
 					break;
 
@@ -3496,9 +3532,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 						if (!Project)
 						{
-							return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
-																	 HttpContentType::kText,
-																	 fmt::format("project {} not found", ProjectId));
+							return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+														 HttpContentType::kText,
+														 fmt::format("project {} not found", ProjectId));
 						}
 						Project->TouchProject();
 
@@ -3520,7 +3556,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						}
 						Response.EndArray();  // oplogs
 
-						Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
+						HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
 					}
 					break;
 
@@ -3529,20 +3565,20 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 						if (!Project)
 						{
-							return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
-																	 HttpContentType::kText,
-																	 fmt::format("project {} not found", ProjectId));
+							return HttpReq.WriteResponse(HttpResponseCode::NotFound,
+														 HttpContentType::kText,
+														 fmt::format("project {} not found", ProjectId));
 						}
 
 						ZEN_INFO("deleting project '{}'", ProjectId);
 						if (!m_ProjectStore->DeleteProject(ProjectId))
 						{
-							return Req.ServerRequest().WriteResponse(HttpResponseCode::Locked,
-																	 HttpContentType::kText,
-																	 fmt::format("project {} is in use", ProjectId));
+							return HttpReq.WriteResponse(HttpResponseCode::Locked,
+														 HttpContentType::kText,
+														 fmt::format("project {} is in use", ProjectId));
 						}
 
-						return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent);
+						return HttpReq.WriteResponse(HttpResponseCode::NoContent);
 					}
 					break;
 
@@ -3556,14 +3592,20 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/save",
 		[this](HttpRouterRequest& Req) {
-			HttpServerRequest& HttpReq	 = Req.ServerRequest();
-			const auto&		   ProjectId = Req.GetCapture(1);
-			const auto&		   OplogId	 = Req.GetCapture(2);
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+
+			if (!m_ProjectStore->AreDiskWritesAllowed())
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+			}
+
+			const auto& ProjectId = Req.GetCapture(1);
+			const auto& OplogId	  = Req.GetCapture(2);
 			if (HttpReq.RequestContentType() != HttpContentType::kCbObject)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type");
 			}
-			IoBuffer Payload = Req.ServerRequest().ReadPayload();
+			IoBuffer Payload = HttpReq.ReadPayload();
 
 			CbObject								 Response;
 			std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->WriteOplog(ProjectId, OplogId, std::move(Payload), Response);
@@ -3590,11 +3632,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type");
 			}
-			IoBuffer Payload = Req.ServerRequest().ReadPayload();
+			IoBuffer Payload = HttpReq.ReadPayload();
 
 			CbObject								 Response;
 			std::pair<HttpResponseCode, std::string> Result =
-				m_ProjectStore->ReadOplog(ProjectId, OplogId, Req.ServerRequest().GetQueryParams(), Response);
+				m_ProjectStore->ReadOplog(ProjectId, OplogId, HttpReq.GetQueryParams(), Response);
 			if (Result.first == HttpResponseCode::OK)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
@@ -3615,7 +3657,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
-			IoBuffer	Payload	  = Req.ServerRequest().ReadPayload();
+			IoBuffer	Payload	  = HttpReq.ReadPayload();
 
 			m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
 		},