1 files changed, 49 insertions, 14 deletions

diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp
index d62ccccb6..d6e24cbe3 100644
--- a/src/zenserver/diag/otlphttp.cpp
+++ b/src/zenserver/diag/otlphttp.cpp
@@ -10,11 +10,18 @@
 #include <protozero/buffer_string.hpp>
 #include <protozero/pbf_builder.hpp>
 
+#include <cstdio>
+
 #if ZEN_WITH_OTEL
 
 namespace zen::logging {
 
 //////////////////////////////////////////////////////////////////////////
+//
+// Important note: in general we cannot use ZEN_WARN/ZEN_ERROR etc in this
+// file as it could cause recursive logging calls when we attempt to log
+// errors from the OTLP HTTP client itself.
+//
 
 OtelHttpProtobufSink::OtelHttpProtobufSink(const std::string_view& Uri) : m_OtelHttp(Uri)
 {
@@ -36,14 +43,44 @@ OtelHttpProtobufSink::~OtelHttpProtobufSink()
 }
 
 void
+OtelHttpProtobufSink::CheckPostResult(const HttpClient::Response& Result, const char* Endpoint) noexcept
+{
+	if (!Result.IsSuccess())
+	{
+		uint32_t PrevFailures = m_ConsecutivePostFailures.fetch_add(1);
+		if (PrevFailures < kMaxReportedFailures)
+		{
+			fprintf(stderr, "OtelHttpProtobufSink: %s\n", Result.ErrorMessage(Endpoint).c_str());
+			if (PrevFailures + 1 == kMaxReportedFailures)
+			{
+				fprintf(stderr, "OtelHttpProtobufSink: suppressing further export errors\n");
+			}
+		}
+	}
+	else
+	{
+		m_ConsecutivePostFailures.store(0);
+	}
+}
+
+void
 OtelHttpProtobufSink::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans)
 {
-	std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans);
+	try
+	{
+		std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans);
+
+		IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+		Payload.SetContentType(ZenContentType::kProtobuf);
 
-	IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
-	Payload.SetContentType(ZenContentType::kProtobuf);
+		HttpClient::Response Result = m_OtelHttp.Post("/v1/traces", Payload);
 
-	auto Result = m_OtelHttp.Post("/v1/traces", Payload);
+		CheckPostResult(Result, "POST /v1/traces");
+	}
+	catch (const std::exception& Ex)
+	{
+		fprintf(stderr, "OtelHttpProtobufSink: exception exporting traces: %s\n", Ex.what());
+	}
 }
 
 void
@@ -53,28 +90,26 @@ OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std::
 }
 
 void
-OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg)
+OtelHttpProtobufSink::Log(const LogMessage& Msg)
 {
+	try
 	{
 		std::string Data = m_Encoder.FormatOtelProtobuf(Msg);
 
 		IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
 		Payload.SetContentType(ZenContentType::kProtobuf);
 
-		auto Result = m_OtelHttp.Post("/v1/logs", Payload);
-	}
+		HttpClient::Response Result = m_OtelHttp.Post("/v1/logs", Payload);
 
+		CheckPostResult(Result, "POST /v1/logs");
+	}
+	catch (const std::exception& Ex)
 	{
-		std::string Data = m_Encoder.FormatOtelMetrics();
-
-		IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
-		Payload.SetContentType(ZenContentType::kProtobuf);
-
-		auto Result = m_OtelHttp.Post("/v1/metrics", Payload);
+		fprintf(stderr, "OtelHttpProtobufSink: exception exporting logs: %s\n", Ex.what());
 	}
 }
 void
-OtelHttpProtobufSink::flush()
+OtelHttpProtobufSink::Flush()
 {
 }