Improved error handling for upstream endpoints.

4 files changed, 79 insertions, 57 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index dc96aecae..e45a26fb9 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -712,7 +712,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
 
 	if (!Payload)
 	{
-		ZEN_DEBUG("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId);
+		ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId, ToString(Request.AcceptContentType()));
 		return Request.WriteResponse(HttpResponseCode::NotFound);
 	}
 
diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h
index 9573a1631..1de417008 100644
--- a/zenserver/upstream/jupiter.h
+++ b/zenserver/upstream/jupiter.h
@@ -47,9 +47,9 @@ struct CloudCacheAccessToken
 struct CloudCacheResult
 {
 	IoBuffer	Response;
-	int64_t		Bytes		   = {};
-	double		ElapsedSeconds = {};
-	int32_t		ErrorCode	   = {};
+	int64_t		Bytes{};
+	double		ElapsedSeconds{};
+	int32_t		ErrorCode{};
 	std::string Reason;
 	bool		Success = false;
 };
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index 03054b542..58a5b1ff3 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -186,16 +186,23 @@ namespace detail {
 					}
 				}
 
-				m_HealthOk = Result.ErrorCode == 0;
-
-				return {.Value			= Result.Response,
-						.Bytes			= Result.Bytes,
-						.ElapsedSeconds = Result.ElapsedSeconds,
-						.Success		= Result.Success};
+				if (Result.ErrorCode == 0)
+				{
+					return {.Value			= Result.Response,
+							.Bytes			= Result.Bytes,
+							.ElapsedSeconds = Result.ElapsedSeconds,
+							.Success		= Result.Success};
+				}
+				else
+				{
+					m_HealthOk = false;
+					return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}};
+				}
 			}
 			catch (std::exception& Err)
 			{
-				return {.Error{.StatusCode = UpstreamStatusCode::Error, .Reason = Err.what()}};
+				m_HealthOk = false;
+				return {.Error{.ErrorCode = -1, .Reason = Err.what()}};
 			}
 		}
 
@@ -206,16 +213,23 @@ namespace detail {
 				CloudCacheSession	   Session(m_Client);
 				const CloudCacheResult Result = Session.GetCompressedBlob(PayloadKey.PayloadId);
 
-				m_HealthOk = Result.ErrorCode == 0;
-
-				return {.Value			= Result.Response,
-						.Bytes			= Result.Bytes,
-						.ElapsedSeconds = Result.ElapsedSeconds,
-						.Success		= Result.Success};
+				if (Result.ErrorCode == 0)
+				{
+					return {.Value			= Result.Response,
+							.Bytes			= Result.Bytes,
+							.ElapsedSeconds = Result.ElapsedSeconds,
+							.Success		= Result.Success};
+				}
+				else
+				{
+					m_HealthOk = false;
+					return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}};
+				}
 			}
 			catch (std::exception& Err)
 			{
-				return {.Error{.StatusCode = UpstreamStatusCode::Error, .Reason = Err.what()}};
+				m_HealthOk = false;
+				return {.Error{.ErrorCode = -1, .Reason = Err.what()}};
 			}
 		}
 
@@ -429,16 +443,23 @@ namespace detail {
 				ZenStructuredCacheSession Session(*m_Client);
 				const ZenCacheResult	  Result = Session.GetCacheRecord(CacheKey.Bucket, CacheKey.Hash, Type);
 
-				m_HealthOk = Result.ErrorCode == 0;
-
-				return {.Value			= Result.Response,
-						.Bytes			= Result.Bytes,
-						.ElapsedSeconds = Result.ElapsedSeconds,
-						.Success		= Result.Success};
+				if (Result.ErrorCode == 0)
+				{
+					return {.Value			= Result.Response,
+							.Bytes			= Result.Bytes,
+							.ElapsedSeconds = Result.ElapsedSeconds,
+							.Success		= Result.Success};
+				}
+				else
+				{
+					m_HealthOk = false;
+					return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}};
+				}
 			}
 			catch (std::exception& Err)
 			{
-				return {.Error{.StatusCode = UpstreamStatusCode::Error, .Reason = Err.what()}};
+				m_HealthOk = false;
+				return {.Error{.ErrorCode = -1, .Reason = Err.what()}};
 			}
 		}
 
@@ -450,16 +471,23 @@ namespace detail {
 				const ZenCacheResult	  Result =
 					Session.GetCachePayload(PayloadKey.CacheKey.Bucket, PayloadKey.CacheKey.Hash, PayloadKey.PayloadId);
 
-				m_HealthOk = Result.ErrorCode == 0;
-
-				return {.Value			= Result.Response,
-						.Bytes			= Result.Bytes,
-						.ElapsedSeconds = Result.ElapsedSeconds,
-						.Success		= Result.Success};
+				if (Result.ErrorCode == 0)
+				{
+					return {.Value			= Result.Response,
+							.Bytes			= Result.Bytes,
+							.ElapsedSeconds = Result.ElapsedSeconds,
+							.Success		= Result.Success};
+				}
+				else
+				{
+					m_HealthOk = false;
+					return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}};
+				}
 			}
 			catch (std::exception& Err)
 			{
-				return {.Error{.StatusCode = UpstreamStatusCode::Error, .Reason = Err.what()}};
+				m_HealthOk = false;
+				return {.Error{.ErrorCode = -1, .Reason = Err.what()}};
 			}
 		}
 
@@ -575,7 +603,7 @@ namespace detail {
 
 struct UpstreamStats
 {
-	static constexpr uint64_t MaxSampleCount = 100ull;
+	static constexpr uint64_t MaxSampleCount = 1000ull;
 
 	UpstreamStats(bool Enabled) : m_Enabled(Enabled) {}
 
diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h
index a6b1e9784..08f379b11 100644
--- a/zenserver/upstream/upstreamcache.h
+++ b/zenserver/upstream/upstreamcache.h
@@ -45,35 +45,29 @@ struct UpstreamCacheOptions
 	bool				 StatsEnabled  = false;
 };
 
-enum class UpstreamStatusCode : uint8_t
-{
-	Ok,
-	Error
-};
-
 struct UpstreamError
 {
-	UpstreamStatusCode StatusCode = UpstreamStatusCode::Ok;
-	std::string		   Reason;
+	int32_t		ErrorCode{};
+	std::string Reason{};
 
-	explicit operator bool() const { return StatusCode != UpstreamStatusCode::Ok; }
+	explicit operator bool() const { return ErrorCode != 0; }
 };
 
 struct GetUpstreamCacheResult
 {
 	IoBuffer	  Value;
-	UpstreamError Error;
-	int64_t		  Bytes			 = {};
-	double		  ElapsedSeconds = {};
-	bool		  Success		 = false;
+	UpstreamError Error{};
+	int64_t		  Bytes{};
+	double		  ElapsedSeconds{};
+	bool		  Success = false;
 };
 
 struct PutUpstreamCacheResult
 {
 	std::string Reason;
-	int64_t		Bytes		   = {};
-	double		ElapsedSeconds = {};
-	bool		Success		   = false;
+	int64_t		Bytes{};
+	double		ElapsedSeconds{};
+	bool		Success = false;
 };
 
 struct UpstreamEndpointHealth
@@ -84,14 +78,14 @@ struct UpstreamEndpointHealth
 
 struct UpstreamEndpointStats
 {
-	std::atomic_uint64_t HitCount	 = {};
-	std::atomic_uint64_t MissCount	 = {};
-	std::atomic_uint64_t UpCount	 = {};
-	std::atomic_uint64_t ErrorCount	 = {};
-	std::atomic<double>	 UpBytes	 = {};
-	std::atomic<double>	 DownBytes	 = {};
-	std::atomic<double>	 SecondsUp	 = {};
-	std::atomic<double>	 SecondsDown = {};
+	std::atomic_uint64_t HitCount{};
+	std::atomic_uint64_t MissCount{};
+	std::atomic_uint64_t UpCount{};
+	std::atomic_uint64_t ErrorCount{};
+	std::atomic<double>	 UpBytes{};
+	std::atomic<double>	 DownBytes{};
+	std::atomic<double>	 SecondsUp{};
+	std::atomic<double>	 SecondsDown{};
 };
 
 /**