Merge branch 'main' into fd-fix-binary-json

4 files changed, 71 insertions, 48 deletions

diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp
index 1a9892d5c..8f7befc80 100644
--- a/src/zenhttp/auth/authmgr.cpp
+++ b/src/zenhttp/auth/authmgr.cpp
@@ -379,7 +379,7 @@ private:
 				AuthState.EndArray();
 			}
 
-			std::filesystem::create_directories(m_Config.RootDirectory);
+			CreateDirectories(m_Config.RootDirectory);
 
 			std::optional<std::string> Reason;
 
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 30711a432..f3baf37ce 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -45,6 +45,7 @@ namespace detail {
 		TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {}
 		~TempPayloadFile()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Close");
 			try
 			{
 				if (m_FileHandle)
@@ -87,6 +88,7 @@ namespace detail {
 
 		std::error_code Open(const std::filesystem::path& TempFolderPath)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Open");
 			ZEN_ASSERT(m_FileHandle == nullptr);
 
 			std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) |
@@ -131,6 +133,7 @@ namespace detail {
 
 		std::error_code Write(std::string_view DataString)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Write");
 			const uint8_t* DataPtr	= (const uint8_t*)DataString.data();
 			size_t		   DataSize = DataString.size();
 			if (DataSize >= CacheBufferSize)
@@ -165,6 +168,7 @@ namespace detail {
 
 		IoBuffer DetachToIoBuffer()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::DetachToIoBuffer");
 			if (std::error_code Ec = Flush(); Ec)
 			{
 				ThrowSystemError(Ec.value(), Ec.message());
@@ -180,6 +184,7 @@ namespace detail {
 
 		IoBuffer BorrowIoBuffer()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::BorrowIoBuffer");
 			if (std::error_code Ec = Flush(); Ec)
 			{
 				ThrowSystemError(Ec.value(), Ec.message());
@@ -193,6 +198,7 @@ namespace detail {
 		uint64_t GetSize() const { return m_WriteOffset; }
 		void	 ResetWritePos(uint64_t WriteOffset)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::ResetWritePos");
 			Flush();
 			m_WriteOffset = WriteOffset;
 		}
@@ -200,6 +206,7 @@ namespace detail {
 	private:
 		std::error_code Flush()
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::Flush");
 			if (m_CacheBufferOffset == 0)
 			{
 				return {};
@@ -211,6 +218,7 @@ namespace detail {
 
 		std::error_code AppendData(const void* Data, uint64_t Size)
 		{
+			ZEN_TRACE_CPU("TempPayloadFile::AppendData");
 			ZEN_ASSERT(m_FileHandle != nullptr);
 			const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
 
@@ -314,7 +322,11 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe
 	const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code);
 	if (HttpResponse.error)
 	{
-		ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse);
+		if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT &&
+			HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED)
+		{
+			ZEN_WARN("HttpClient client failure (session: {}): {}", SessionId, HttpResponse);
+		}
 
 		// Client side failure code
 		return HttpClient::Response{
@@ -376,6 +388,7 @@ ShouldRetry(const cpr::Response& Response)
 static bool
 ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile>& PayloadFile)
 {
+	ZEN_TRACE_CPU("ValidatePayload");
 	IoBuffer ResponseBuffer = (Response.text.empty() && PayloadFile) ? PayloadFile->BorrowIoBuffer()
 																	 : IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size());
 
@@ -535,12 +548,14 @@ struct HttpClient::Impl : public RefCounted
 		inline cpr::Session* operator->() const { return CprSession; }
 		inline cpr::Response Get()
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Get");
 			cpr::Response Result = CprSession->Get();
 			ZEN_TRACE("GET {}", Result);
 			return Result;
 		}
 		inline cpr::Response Download(cpr::WriteCallback&& Write, std::optional<cpr::HeaderCallback>&& Header = {})
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Download");
 			if (Header)
 			{
 				CprSession->SetHeaderCallback(std::move(Header.value()));
@@ -553,12 +568,14 @@ struct HttpClient::Impl : public RefCounted
 		}
 		inline cpr::Response Head()
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Head");
 			cpr::Response Result = CprSession->Head();
 			ZEN_TRACE("HEAD {}", Result);
 			return Result;
 		}
 		inline cpr::Response Put(std::optional<cpr::ReadCallback>&& Read = {})
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Put");
 			if (Read)
 			{
 				CprSession->SetReadCallback(std::move(Read.value()));
@@ -570,6 +587,7 @@ struct HttpClient::Impl : public RefCounted
 		}
 		inline cpr::Response Post(std::optional<cpr::ReadCallback>&& Read = {})
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Post");
 			if (Read)
 			{
 				CprSession->SetReadCallback(std::move(Read.value()));
@@ -581,6 +599,7 @@ struct HttpClient::Impl : public RefCounted
 		}
 		inline cpr::Response Delete()
 		{
+			ZEN_TRACE_CPU("HttpClient::Impl::Delete");
 			cpr::Response Result = CprSession->Delete();
 			ZEN_TRACE("DELETE {}", Result);
 			return Result;
@@ -620,6 +639,7 @@ HttpClient::Impl::Impl(LoggerRef Log) : m_Log(Log)
 
 HttpClient::Impl::~Impl()
 {
+	ZEN_TRACE_CPU("HttpClient::Impl::~Impl");
 	m_SessionLock.WithExclusiveLock([&] {
 		for (auto CprSession : m_Sessions)
 		{
@@ -638,6 +658,7 @@ HttpClient::Impl::AllocSession(const std::string_view				BaseUrl,
 							   const std::string_view				SessionId,
 							   std::optional<HttpClientAccessToken> AccessToken)
 {
+	ZEN_TRACE_CPU("HttpClient::Impl::AllocSession");
 	cpr::Session* CprSession = nullptr;
 	m_SessionLock.WithExclusiveLock([&] {
 		if (!m_Sessions.empty())
@@ -694,6 +715,7 @@ HttpClient::Impl::AllocSession(const std::string_view				BaseUrl,
 void
 HttpClient::Impl::ReleaseSession(cpr::Session* CprSession)
 {
+	ZEN_TRACE_CPU("HttpClient::Impl::ReleaseSession");
 	CprSession->SetUrl({});
 	CprSession->SetHeader({});
 	CprSession->SetBody({});
@@ -718,6 +740,7 @@ HttpClient::~HttpClient()
 bool
 HttpClient::Authenticate()
 {
+	ZEN_TRACE_CPU("HttpClient::Authenticate");
 	std::optional<HttpClientAccessToken> Token = GetAccessToken();
 	if (!Token)
 	{
@@ -729,6 +752,7 @@ HttpClient::Authenticate()
 const std::optional<HttpClientAccessToken>
 HttpClient::GetAccessToken()
 {
+	ZEN_TRACE_CPU("HttpClient::GetAccessToken");
 	if (!m_ConnectionSettings.AccessTokenProvider.has_value())
 	{
 		return {};
diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h
index 74da9ab05..05a23d675 100644
--- a/src/zenhttp/include/zenhttp/formatters.h
+++ b/src/zenhttp/include/zenhttp/formatters.h
@@ -73,9 +73,11 @@ struct fmt::formatter<cpr::Response>
 		if (zen::IsHttpSuccessCode(Response.status_code))
 		{
 			return fmt::format_to(Ctx.out(),
-								  "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}",
+								  "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}",
 								  Response.url.str(),
 								  Response.status_code,
+								  Response.error.message,
+								  int(Response.error.code),
 								  Response.uploaded_bytes,
 								  Response.downloaded_bytes,
 								  NiceResponseTime.c_str());
@@ -92,29 +94,35 @@ struct fmt::formatter<cpr::Response>
 				zen::ExtendableStringBuilder<256> Sb;
 				std::string_view				  Json = Obj.ToJson(Sb).ToView();
 
-				return fmt::format_to(Ctx.out(),
-									  "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
-									  Response.url.str(),
-									  Response.status_code,
-									  Response.uploaded_bytes,
-									  Response.downloaded_bytes,
-									  NiceResponseTime.c_str(),
-									  Json,
-									  Response.reason);
+				return fmt::format_to(
+					Ctx.out(),
+					"Url: {}, Status: {}, Error: '{}' ({}). Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
+					Response.url.str(),
+					Response.status_code,
+					Response.error.message,
+					int(Response.error.code),
+					Response.uploaded_bytes,
+					Response.downloaded_bytes,
+					NiceResponseTime.c_str(),
+					Json,
+					Response.reason);
 			}
 			else
 			{
 				zen::BodyLogFormatter Body(Response.text);
 
-				return fmt::format_to(Ctx.out(),
-									  "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
-									  Response.url.str(),
-									  Response.status_code,
-									  Response.uploaded_bytes,
-									  Response.downloaded_bytes,
-									  NiceResponseTime.c_str(),
-									  Body.GetText(),
-									  Response.reason);
+				return fmt::format_to(
+					Ctx.out(),
+					"Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'",
+					Response.url.str(),
+					Response.status_code,
+					Response.error.message,
+					int(Response.error.code),
+					Response.uploaded_bytes,
+					Response.downloaded_bytes,
+					NiceResponseTime.c_str(),
+					Body.GetText(),
+					Response.reason);
 			}
 		}
 	}
diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp
index ae80851e4..9d423ecbc 100644
--- a/src/zenhttp/packageformat.cpp
+++ b/src/zenhttp/packageformat.cpp
@@ -279,11 +279,10 @@ FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* Tar
 		{
 			IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer();
 			ZEN_ASSERT(ObjIoBuffer.GetSize() > 0);
-			ResponseBuffers.emplace_back(std::move(ObjIoBuffer));
-
 			*AttachmentInfo++ = {.PayloadSize	 = ObjIoBuffer.Size(),
 								 .Flags			 = CbAttachmentEntry::kIsObject,
 								 .AttachmentHash = Attachment.GetHash()};
+			ResponseBuffers.emplace_back(std::move(ObjIoBuffer));
 		}
 		else if (const CompositeBuffer& AttachmentBinary = Attachment.AsCompositeBinary())
 		{
@@ -500,30 +499,25 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 		{
 			if (Entry.Flags & CbAttachmentEntry::kIsObject)
 			{
+				CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
+				if (!CompBuf)
+				{
+					// First payload is always a compact binary object
+					MalformedAttachments.push_back(
+						std::make_pair(i,
+									   fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}",
+												   AttachmentBuffer.GetSize(),
+												   Entry.AttachmentHash)));
+				}
+				CbObject AttachmentObject = LoadCompactBinaryObject(std::move(CompBuf));
 				if (i == 0)
 				{
-					CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
-					if (CompBuf)
-					{
-						Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf)));
-					}
-					else
-					{
-						// First payload is always a compact binary object
-						MalformedAttachments.push_back(
-							std::make_pair(i,
-										   fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}",
-													   AttachmentBuffer.GetSize(),
-													   Entry.AttachmentHash)));
-					}
+					// First payload is always a compact binary object
+					Package.SetObject(AttachmentObject);
 				}
 				else
 				{
-					MalformedAttachments.push_back(std::make_pair(
-						i,
-						fmt::format("Invalid format, compressed object attachments are not currently supported (size {}) for {}",
-									AttachmentBuffer.GetSize(),
-									Entry.AttachmentHash)));
+					Attachments.emplace_back(CbAttachment(AttachmentObject, Entry.AttachmentHash));
 				}
 			}
 			else
@@ -547,17 +541,14 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 		{
 			if (Entry.Flags & CbAttachmentEntry::kIsObject)
 			{
+				CbObject AttachmentObject = LoadCompactBinaryObject(AttachmentBuffer);
 				if (i == 0)
 				{
-					Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer));
+					Package.SetObject(AttachmentObject);
 				}
 				else
 				{
-					MalformedAttachments.push_back(
-						std::make_pair(i,
-									   fmt::format("Invalid format, object attachments are not currently supported (size {}) for {}",
-												   AttachmentBuffer.GetSize(),
-												   Entry.AttachmentHash)));
+					Attachments.emplace_back(CbAttachment(AttachmentObject, Entry.AttachmentHash));
 				}
 			}
 			else if (AttachmentSize > 0)