Adjust errors vs warnings messages (#160)

* demote a number of ZEN_ERROR to ZEN_WARN
* changelog

10 files changed, 42 insertions, 41 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ecbb3a108..ee7d6d5f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@
 - Improvement: Frontend: simplified content-type logic
 - Improvement: Improved message indicating no GC is scheduled
 - Improvement: Implement proper GetCacheValues upstream path
+- Improvement: Demote a number of ZEN_ERROR log calls for problems that are recoverable and handled
 - Bugfix: Fixed issue in CbPackage marshaling of local reference
 - Bugfix: Fix crash when switching Zen upstream configured via DNS when one endpoint becomes unresposive
 - Bugfix: Fixed issue where projects would not be discovered via DiscoverProjects due to use of stem() vs filename()
diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp
index e4b85710d..39a363711 100644
--- a/zenhttp/httpasio.cpp
+++ b/zenhttp/httpasio.cpp
@@ -403,7 +403,7 @@ HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused]
 		}
 		else
 		{
-			ZEN_ERROR("on data received ERROR, connection '{}' reason '{}'", m_ConnectionId, Ec.message());
+			ZEN_WARN("on data received ERROR, connection '{}' reason '{}'", m_ConnectionId, Ec.message());
 			return OnError();
 		}
 	}
@@ -441,7 +441,7 @@ HttpServerConnection::OnResponseDataSent(const asio::error_code& Ec, [[maybe_unu
 {
 	if (Ec)
 	{
-		ZEN_ERROR("on data sent ERROR, connection '{}' reason '{}'", m_ConnectionId, Ec.message());
+		ZEN_WARN("on data sent ERROR, connection '{}' reason '{}'", m_ConnectionId, Ec.message());
 		OnError();
 	}
 	else
@@ -987,10 +987,10 @@ struct HttpAcceptor
 		m_Acceptor.async_accept(SocketRef, [this, Socket = std::move(SocketPtr)](const asio::error_code& Ec) mutable {
 			if (Ec)
 			{
-				ZEN_ERROR("asio async_accept, connection failed to '{}:{}' reason '{}'",
-						  m_Acceptor.local_endpoint().address().to_string(),
-						  m_Acceptor.local_endpoint().port(),
-						  Ec.message());
+				ZEN_WARN("asio async_accept, connection failed to '{}:{}' reason '{}'",
+						 m_Acceptor.local_endpoint().address().to_string(),
+						 m_Acceptor.local_endpoint().port(),
+						 Ec.message());
 			}
 			else
 			{
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index 19dba126a..926e6b09f 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -598,10 +598,10 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 
 		CancelThreadpoolIo(Iocp);
 
-		ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}",
-				  GetSystemErrorAsString(SendResult),
-				  HttpReq->pRawUrl,
-				  HttpReq->RequestId);
+		ZEN_WARN("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}",
+				 GetSystemErrorAsString(SendResult),
+				 HttpReq->pRawUrl,
+				 HttpReq->RequestId);
 
 		ErrorCode = MakeErrorCode(SendResult);
 	}
@@ -1134,7 +1134,7 @@ HttpSysTransaction::IssueNextRequest(HttpSysRequestHandler* NewCompletionHandler
 			return true;
 		}
 
-		ZEN_ERROR("IssueRequest() failed: '{}'", ErrorCode.message());
+		ZEN_WARN("IssueRequest() failed: '{}'", ErrorCode.message());
 	}
 	catch (std::exception& Ex)
 	{
@@ -1463,7 +1463,7 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode)
 
 		ErrorCode = MakeErrorCode(HttpApiResult);
 
-		ZEN_ERROR("HttpReceiveHttpRequest failed, error: '{}'", ErrorCode.message());
+		ZEN_WARN("HttpReceiveHttpRequest failed, error: '{}'", ErrorCode.message());
 
 		return;
 	}
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 76a8707c5..e25759192 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1132,7 +1132,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 
 	if (!BadKeys.empty())
 	{
-		ZEN_ERROR("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_BucketDir / m_BucketName);
+		ZEN_WARN("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_BucketDir / m_BucketName);
 
 		if (Ctx.RunRecovery())
 		{
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index e42704ccf..93276f029 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -784,7 +784,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 		}
 		catch (std::exception& ex)
 		{
-			ZEN_ERROR("failed to open oplog '{}' @ '{}': {}", OplogId, OplogBasePath, ex.what());
+			ZEN_WARN("failed to open oplog '{}' @ '{}': {}", OplogId, OplogBasePath, ex.what());
 
 			m_Oplogs.erase(std::string{OplogId});
 		}
@@ -1582,7 +1582,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 				std::filesystem::path BadPackagePath =
 					Oplog.TempPath() / "bad_packages" / fmt::format("session{}_request{}", HttpReq.SessionId(), HttpReq.RequestId());
 
-				ZEN_ERROR("Received malformed package! Saving payload to '{}'", BadPackagePath);
+				ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath);
 
 				zen::WriteFile(BadPackagePath, Payload);
 
@@ -1667,7 +1667,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 										{
 											// Error - malformed object
 
-											ZEN_ERROR("malformed object returned for {}", AttachmentHash);
+											ZEN_WARN("malformed object returned for {}", AttachmentHash);
 										}
 										break;
 
@@ -1680,7 +1680,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 										{
 											// Error - not compressed!
 
-											ZEN_ERROR("invalid compressed binary returned for {}", AttachmentHash);
+											ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash);
 										}
 										break;
 
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index e92dde815..e8f7450fd 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -1497,10 +1497,10 @@ public:
 				{
 					Stats.CacheErrorCount.Increment(1);
 
-					ZEN_ERROR("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'",
-							  Endpoint->GetEndpointInfo().Url,
-							  Result.Error.Reason,
-							  Result.Error.ErrorCode);
+					ZEN_WARN("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'",
+							 Endpoint->GetEndpointInfo().Url,
+							 Result.Error.Reason,
+							 Result.Error.ErrorCode);
 				}
 			}
 		}
@@ -1559,10 +1559,10 @@ public:
 				{
 					Stats.CacheErrorCount.Increment(1);
 
-					ZEN_ERROR("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'",
-							  Endpoint->GetEndpointInfo().Url,
-							  Result.Error.Reason,
-							  Result.Error.ErrorCode);
+					ZEN_WARN("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'",
+							 Endpoint->GetEndpointInfo().Url,
+							 Result.Error.Reason,
+							 Result.Error.ErrorCode);
 				}
 
 				RemainingKeys = std::move(Missing);
@@ -1626,10 +1626,10 @@ public:
 				{
 					Stats.CacheErrorCount.Increment(1);
 
-					ZEN_ERROR("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'",
-							  Endpoint->GetEndpointInfo().Url,
-							  Result.Error.Reason,
-							  Result.Error.ErrorCode);
+					ZEN_WARN("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'",
+							 Endpoint->GetEndpointInfo().Url,
+							 Result.Error.Reason,
+							 Result.Error.ErrorCode);
 				}
 
 				RemainingKeys = std::move(Missing);
@@ -1679,10 +1679,10 @@ public:
 				{
 					Stats.CacheErrorCount.Increment(1);
 
-					ZEN_ERROR("get cache value FAILED, endpoint '{}', reason '{}', error code '{}'",
-							  Endpoint->GetEndpointInfo().Url,
-							  Result.Error.Reason,
-							  Result.Error.ErrorCode);
+					ZEN_WARN("get cache value FAILED, endpoint '{}', reason '{}', error code '{}'",
+							 Endpoint->GetEndpointInfo().Url,
+							 Result.Error.Reason,
+							 Result.Error.ErrorCode);
 				}
 			}
 		}
@@ -1741,10 +1741,10 @@ public:
 				{
 					Stats.CacheErrorCount.Increment(1);
 
-					ZEN_ERROR("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'",
-							  Endpoint->GetEndpointInfo().Url,
-							  Result.Error.Reason,
-							  Result.Error.ErrorCode);
+					ZEN_WARN("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'",
+							 Endpoint->GetEndpointInfo().Url,
+							 Result.Error.Reason,
+							 Result.Error.ErrorCode);
 				}
 
 				RemainingKeys = std::move(Missing);
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 2effa4e68..9e15b0c64 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -668,7 +668,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 		if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All);
 			ValidationResult != CbValidateError::None)
 		{
-			ZEN_ERROR("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult));
+			ZEN_WARN("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult));
 
 			WipeState  = true;
 			WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, uint32_t(ValidationResult));
diff --git a/zenstore/cas.cpp b/zenstore/cas.cpp
index 54e8cb11c..f8fc41341 100644
--- a/zenstore/cas.cpp
+++ b/zenstore/cas.cpp
@@ -148,7 +148,7 @@ CasImpl::OpenOrCreateManifest()
 			}
 			else
 			{
-				ZEN_ERROR("Store manifest validation failed: {:#x}, will generate new manifest to recover", uint32_t(ValidationResult));
+				ZEN_WARN("Store manifest validation failed: {:#x}, will generate new manifest to recover", uint32_t(ValidationResult));
 			}
 
 			if (ManifestIsOk)
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index a7fdfa1f5..519478356 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -327,7 +327,7 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
 
 	if (!BadKeys.empty())
 	{
-		ZEN_ERROR("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_RootDirectory / m_ContainerBaseName);
+		ZEN_WARN("Scrubbing found #{} bad chunks in '{}'", BadKeys.size(), m_RootDirectory / m_ContainerBaseName);
 
 		if (Ctx.RunRecovery())
 		{
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 23e3f4cd8..79867dcfa 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -669,7 +669,7 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
 
 	if (!BadHashes.empty())
 	{
-		ZEN_ERROR("file CAS scrubbing: {} bad chunks found", BadHashes.size());
+		ZEN_WARN("file CAS scrubbing: {} bad chunks found", BadHashes.size());
 
 		if (Ctx.RunRecovery())
 		{