disk layer gc and error/warnings cleanup (#515)

- Improvement: Use GC reserve when writing index/manifest for a disk cache bucket when disk is low when available
- Improvement: Demote errors to warning for issues that are not critical and we handle gracefully
- Improvement: Treat more out of memory errors from windows as Out Of Memory errors

Fixed wrong sizeof() statement for compactcas index (luckily the two structs are of same size)

1 files changed, 41 insertions, 41 deletions

diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 4194c4315..d5d229e42 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -893,7 +893,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 		{
 			RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound),
 								  "Invalid attachment",
-								  fmt::format("Upload requested of unknown attachement '{}'", Needed));
+								  fmt::format("Upload requested of unknown attachment '{}'", Needed));
 			ZEN_ERROR("Failed to upload attachment '{}'. ({}). Reason: '{}'",
 					  Needed,
 					  RemoteResult.GetError(),
@@ -963,10 +963,10 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 					RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound),
 										  fmt::format("Failed to find attachment {}", RawHash),
 										  {});
-					ZEN_ERROR("Failed to save attachment '{}' ({}). Reason: '{}'",
-							  RawHash,
-							  RemoteResult.GetErrorReason(),
-							  RemoteResult.GetError());
+					ZEN_WARN("Failed to save attachment '{}' ({}). Reason: '{}'",
+							 RawHash,
+							 RemoteResult.GetErrorReason(),
+							 RemoteResult.GetError());
 					return;
 				}
 
@@ -975,11 +975,11 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 				if (Result.ErrorCode)
 				{
 					RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
-					ZEN_ERROR("Failed to save attachment '{}', {} ({}). Reason: '{}'",
-							  RawHash,
-							  NiceBytes(Payload.GetSize()),
-							  RemoteResult.GetError(),
-							  RemoteResult.GetErrorReason());
+					ZEN_WARN("Failed to save attachment '{}', {} ({}). Reason: '{}'",
+							 RawHash,
+							 NiceBytes(Payload.GetSize()),
+							 RemoteResult.GetError(),
+							 RemoteResult.GetErrorReason());
 					return;
 				}
 				ZEN_DEBUG("Saved attachment {}, {} in {}",
@@ -1025,11 +1025,11 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			if (Result.ErrorCode)
 			{
 				RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
-				ZEN_ERROR("Failed to save attachment '{}', {} ({}). Reason: '{}'",
-						  RawHash,
-						  NiceBytes(Payload.GetSize()),
-						  RemoteResult.GetError(),
-						  RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to save attachment '{}', {} ({}). Reason: '{}'",
+						 RawHash,
+						 NiceBytes(Payload.GetSize()),
+						 RemoteResult.GetError(),
+						 RemoteResult.GetErrorReason());
 				return;
 			}
 
@@ -1100,10 +1100,10 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			if (Result.ErrorCode)
 			{
 				RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
-				ZEN_ERROR("Failed to save attachments with {} chunks ({}). Reason: '{}'",
-						  Chunks.size(),
-						  RemoteResult.GetError(),
-						  RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to save attachments with {} chunks ({}). Reason: '{}'",
+						 Chunks.size(),
+						 RemoteResult.GetError(),
+						 RemoteResult.GetErrorReason());
 				return;
 			}
 			ZEN_DEBUG("Saved {} bulk attachments in {}",
@@ -1227,7 +1227,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		if (Result.ErrorCode)
 		{
 			RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
-			ZEN_ERROR("Failed to save attachment ({}). Reason: '{}'", RemoteResult.GetErrorReason(), RemoteResult.GetError());
+			ZEN_WARN("Failed to save attachment ({}). Reason: '{}'", RemoteResult.GetErrorReason(), RemoteResult.GetError());
 			return;
 		}
 		ZEN_DEBUG("Saved block {}, {}", BlockHash, NiceBytes(CompressedBlock.GetCompressedSize()));
@@ -1366,10 +1366,10 @@ SaveOplog(CidStore&				 ChunkStore,
 			if (ContainerFinalizeResult.ErrorCode)
 			{
 				RemoteResult.SetError(ContainerFinalizeResult.ErrorCode, ContainerFinalizeResult.Reason, ContainerFinalizeResult.Text);
-				ZEN_ERROR("Failed to finalize oplog container {} ({}). Reason: '{}'",
-						  ContainerSaveResult.RawHash,
-						  RemoteResult.GetError(),
-						  RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to finalize oplog container {} ({}). Reason: '{}'",
+						 ContainerSaveResult.RawHash,
+						 RemoteResult.GetError(),
+						 RemoteResult.GetErrorReason());
 			}
 			ZEN_DEBUG("Finalized container in {}", NiceTimeSpanMs(static_cast<uint64_t>(ContainerFinalizeResult.ElapsedSeconds * 1000)));
 			if (ContainerFinalizeResult.Needs.empty())
@@ -1487,7 +1487,7 @@ SaveOplogContainer(ProjectStore::Oplog&																 Oplog,
 	CbObject SectionObject = LoadCompactBinaryObject(SectionPayload);
 	if (!SectionObject)
 	{
-		ZEN_ERROR("Failed to save oplog container. Reason: '{}'", "Section has unexpected data type");
+		ZEN_WARN("Failed to save oplog container. Reason: '{}'", "Section has unexpected data type");
 		return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest),
 										  Timer.GetElapsedTimeMs() / 1000.500,
 										  "Section has unexpected data type",
@@ -1573,10 +1573,10 @@ LoadOplog(CidStore&			   ChunkStore,
 				if (Result.ErrorCode)
 				{
 					RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
-					ZEN_ERROR("Failed to load attachments with {} chunks ({}). Reason: '{}'",
-							  Chunks.size(),
-							  RemoteResult.GetError(),
-							  RemoteResult.GetErrorReason());
+					ZEN_WARN("Failed to load attachments with {} chunks ({}). Reason: '{}'",
+							 Chunks.size(),
+							 RemoteResult.GetError(),
+							 RemoteResult.GetErrorReason());
 					return;
 				}
 				ZEN_DEBUG("Loaded {} bulk attachments in {}",
@@ -1601,10 +1601,10 @@ LoadOplog(CidStore&			   ChunkStore,
 			if (BlockResult.ErrorCode)
 			{
 				RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text);
-				ZEN_ERROR("Failed to load oplog container, missing attachment {} ({}). Reason: '{}'",
-						  BlockHash,
-						  RemoteResult.GetError(),
-						  RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to load oplog container, missing attachment {} ({}). Reason: '{}'",
+						 BlockHash,
+						 RemoteResult.GetError(),
+						 RemoteResult.GetErrorReason());
 				return;
 			}
 			ZEN_DEBUG("Loaded block attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000)));
@@ -1616,10 +1616,10 @@ LoadOplog(CidStore&			   ChunkStore,
 				RemoteResult.SetError(gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
 									  fmt::format("Invalid format for block {}", BlockHash),
 									  {});
-				ZEN_ERROR("Failed to load oplog container, attachment {} has invalid format ({}). Reason: '{}'",
-						  BlockHash,
-						  RemoteResult.GetError(),
-						  RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to load oplog container, attachment {} has invalid format ({}). Reason: '{}'",
+						 BlockHash,
+						 RemoteResult.GetError(),
+						 RemoteResult.GetErrorReason());
 				return;
 			}
 		});
@@ -1644,10 +1644,10 @@ LoadOplog(CidStore&			   ChunkStore,
 			if (AttachmentResult.ErrorCode)
 			{
 				RemoteResult.SetError(AttachmentResult.ErrorCode, AttachmentResult.Reason, AttachmentResult.Text);
-				ZEN_ERROR("Failed to download attachment {}, reason: '{}', error code: {}",
-						  RawHash,
-						  AttachmentResult.Reason,
-						  AttachmentResult.ErrorCode);
+				ZEN_WARN("Failed to download attachment {}, reason: '{}', error code: {}",
+						 RawHash,
+						 AttachmentResult.Reason,
+						 AttachmentResult.ErrorCode);
 				return;
 			}
 			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000)));