add validation of payload responses in http client (#240)

if response payload does not validate properly do a retry if applicable

1 files changed, 292 insertions, 220 deletions

diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 584433f79..87097ca45 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -35,6 +35,234 @@ namespace zen {
 
 using namespace std::literals;
 
+namespace detail {
+
+	static std::atomic_uint32_t TempFileBaseIndex;
+
+	class TempPayloadFile
+	{
+	public:
+		TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {}
+		~TempPayloadFile()
+		{
+			try
+			{
+				if (m_FileHandle)
+				{
+#if ZEN_PLATFORM_WINDOWS
+					// Mark file for deletion when final handle is closed
+					FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE};
+
+					SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
+					BOOL Success = CloseHandle(m_FileHandle);
+#else
+					std::error_code		  Ec;
+					std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle, Ec);
+					if (Ec)
+					{
+						ZEN_WARN("Error reported on get file path from handle {} for temp payload unlink operation, reason '{}'",
+								 m_FileHandle,
+								 Ec.message());
+					}
+					else
+					{
+						unlink(FilePath.c_str());
+					}
+					int	 Fd		 = int(uintptr_t(m_FileHandle));
+					bool Success = (close(Fd) == 0);
+#endif
+					if (!Success)
+					{
+						ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString());
+					}
+
+					m_FileHandle = nullptr;
+				}
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_ERROR("Failed deleting temp file {}. Reason '{}'", m_FileHandle, Ex.what());
+			}
+		}
+
+		std::error_code Open(const std::filesystem::path& TempFolderPath)
+		{
+			ZEN_ASSERT(m_FileHandle == nullptr);
+
+			std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) |
+									 detail::TempFileBaseIndex.fetch_add(1);
+
+			std::filesystem::path FileName = TempFolderPath / fmt::to_string(TmpIndex);
+#if ZEN_PLATFORM_WINDOWS
+			LPCWSTR				  lpFileName			= FileName.c_str();
+			const DWORD			  dwDesiredAccess		= (GENERIC_READ | GENERIC_WRITE | DELETE);
+			const DWORD			  dwShareMode			= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
+			LPSECURITY_ATTRIBUTES lpSecurityAttributes	= nullptr;
+			const DWORD			  dwCreationDisposition = CREATE_ALWAYS;
+			const DWORD			  dwFlagsAndAttributes	= FILE_ATTRIBUTE_NORMAL;
+			const HANDLE		  hTemplateFile			= nullptr;
+			const HANDLE		  FileHandle			= CreateFile(lpFileName,
+												 dwDesiredAccess,
+												 dwShareMode,
+												 lpSecurityAttributes,
+												 dwCreationDisposition,
+												 dwFlagsAndAttributes,
+												 hTemplateFile);
+
+			if (FileHandle == INVALID_HANDLE_VALUE)
+			{
+				return MakeErrorCodeFromLastError();
+			}
+#else	// ZEN_PLATFORM_WINDOWS
+			int OpenFlags = O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC;
+			int Fd = open(FileName.c_str(), OpenFlags, 0666);
+			if (Fd < 0)
+			{
+				return MakeErrorCodeFromLastError();
+			}
+			fchmod(Fd, 0666);
+
+			void* FileHandle = (void*)(uintptr_t(Fd));
+#endif	// ZEN_PLATFORM_WINDOWS
+			m_FileHandle = FileHandle;
+
+			return {};
+		}
+
+		std::error_code Write(std::string_view DataString)
+		{
+			const uint8_t* DataPtr	= (const uint8_t*)DataString.data();
+			size_t		   DataSize = DataString.size();
+			if (DataSize >= CacheBufferSize)
+			{
+				std::error_code Ec = Flush();
+				if (Ec)
+				{
+					return Ec;
+				}
+				return AppendData(DataPtr, DataSize);
+			}
+			size_t CopySize = Min(DataSize, CacheBufferSize - m_CacheBufferOffset);
+			memcpy(&m_CacheBuffer[m_CacheBufferOffset], DataPtr, CopySize);
+			m_CacheBufferOffset += CopySize;
+			DataSize -= CopySize;
+			if (m_CacheBufferOffset == CacheBufferSize)
+			{
+				AppendData(m_CacheBuffer, CacheBufferSize);
+				if (DataSize > 0)
+				{
+					ZEN_ASSERT(DataSize < CacheBufferSize);
+					memcpy(m_CacheBuffer, DataPtr + CopySize, DataSize);
+				}
+				m_CacheBufferOffset = DataSize;
+			}
+			else
+			{
+				ZEN_ASSERT(DataSize == 0);
+			}
+			return {};
+		}
+
+		IoBuffer DetachToIoBuffer()
+		{
+			if (std::error_code Ec = Flush(); Ec)
+			{
+				ThrowSystemError(Ec.value(), Ec.message());
+			}
+			ZEN_ASSERT(m_FileHandle != nullptr);
+			void*	 FileHandle = m_FileHandle;
+			IoBuffer Buffer(IoBuffer::File, FileHandle, 0, m_WriteOffset, /*IsWholeFile*/ true);
+			Buffer.SetDeleteOnClose(true);
+			m_FileHandle  = 0;
+			m_WriteOffset = 0;
+			return Buffer;
+		}
+
+		IoBuffer BorrowIoBuffer()
+		{
+			if (std::error_code Ec = Flush(); Ec)
+			{
+				ThrowSystemError(Ec.value(), Ec.message());
+			}
+			ZEN_ASSERT(m_FileHandle != nullptr);
+			void*	 FileHandle = m_FileHandle;
+			IoBuffer Buffer(IoBuffer::BorrowedFile, FileHandle, 0, m_WriteOffset);
+			return Buffer;
+		}
+
+		uint64_t GetSize() const { return m_WriteOffset; }
+		void	 ResetWritePos(uint64_t WriteOffset)
+		{
+			Flush();
+			m_WriteOffset = WriteOffset;
+		}
+
+	private:
+		std::error_code Flush()
+		{
+			if (m_CacheBufferOffset == 0)
+			{
+				return {};
+			}
+			std::error_code Res = AppendData(m_CacheBuffer, m_CacheBufferOffset);
+			m_CacheBufferOffset = 0;
+			return Res;
+		}
+
+		std::error_code AppendData(const void* Data, uint64_t Size)
+		{
+			ZEN_ASSERT(m_FileHandle != nullptr);
+			const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+
+			while (Size)
+			{
+				const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize);
+				uint64_t	   NumberOfBytesWritten = 0;
+#if ZEN_PLATFORM_WINDOWS
+				OVERLAPPED Ovl{};
+
+				Ovl.Offset	   = DWORD(m_WriteOffset & 0xffff'ffffu);
+				Ovl.OffsetHigh = DWORD(m_WriteOffset >> 32);
+
+				DWORD dwNumberOfBytesWritten = 0;
+
+				BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
+				if (Success)
+				{
+					NumberOfBytesWritten = static_cast<uint64_t>(dwNumberOfBytesWritten);
+				}
+#else
+				static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
+				int Fd = int(uintptr_t(m_FileHandle));
+				int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, m_WriteOffset);
+				bool Success = (BytesWritten > 0);
+				if (Success)
+				{
+					NumberOfBytesWritten = static_cast<uint64_t>(BytesWritten);
+				}
+#endif
+
+				if (!Success)
+				{
+					return MakeErrorCodeFromLastError();
+				}
+
+				Size -= NumberOfBytesWritten;
+				m_WriteOffset += NumberOfBytesWritten;
+				Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesWritten;
+			}
+			return {};
+		}
+
+		void*					  m_FileHandle;
+		std::uint64_t			  m_WriteOffset;
+		static constexpr uint64_t CacheBufferSize = 512u * 1024u;
+		uint8_t					  m_CacheBuffer[CacheBufferSize];
+		std::uint64_t			  m_CacheBufferOffset = 0;
+	};
+
+}  // namespace detail
+
 //////////////////////////////////////////////////////////////////////////
 //
 // CPR helpers
@@ -144,6 +372,43 @@ ShouldRetry(const cpr::Response& Response)
 	}
 };
 
+static bool
+ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile>& PayloadFile)
+{
+	if (IsHttpSuccessCode(Response.status_code))
+	{
+		if (auto ContentType = Response.header.find("Content-Type");
+			ContentType != Response.header.end() && ContentType->second == "application/x-ue-comp")
+		{
+			IoBuffer ResponseBuffer = (Response.text.empty() && PayloadFile)
+										  ? PayloadFile->BorrowIoBuffer()
+										  : IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size());
+			IoHash	 RawHash;
+			uint64_t RawSize;
+			if (!CompressedBuffer::ValidateCompressedHeader(ResponseBuffer, RawHash, RawSize))
+			{
+				Response.error = cpr::Error(/*CURLE_READ_ERROR*/ 26, "Compressed binary failed validateion");
+				return false;
+			}
+		}
+		else if (auto JupiterHash = Response.header.find("X-Jupiter-IoHash"); JupiterHash != Response.header.end())
+		{
+			IoHash ExpectedPayloadHash;
+			if (IoHash::TryParse(JupiterHash->second, ExpectedPayloadHash))
+			{
+				IoBuffer ResponseBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size());
+				IoHash	 PayloadHash = IoHash::HashBuffer(ResponseBuffer);
+				if (PayloadHash != ExpectedPayloadHash)
+				{
+					Response.error = cpr::Error(/*CURLE_READ_ERROR*/ 26, "Compressed binary failed validateion");
+					return false;
+				}
+			}
+		}
+	}
+	return true;
+}
+
 static cpr::Response
 DoWithRetry(std::function<cpr::Response()>&& Func, uint8_t RetryCount)
 {
@@ -159,6 +424,28 @@ DoWithRetry(std::function<cpr::Response()>&& Func, uint8_t RetryCount)
 	return Result;
 }
 
+static cpr::Response
+DoWithRetry(std::function<cpr::Response()>&& Func, std::unique_ptr<detail::TempPayloadFile>& PayloadFile, uint8_t RetryCount)
+{
+	uint8_t		  Attempt = 0;
+	cpr::Response Result  = Func();
+	while (Attempt < RetryCount)
+	{
+		if (!ShouldRetry(Result))
+		{
+			if (Result.error || ValidatePayload(Result, PayloadFile))
+			{
+				break;
+			}
+		}
+		Sleep(100 * (Attempt + 1));
+		Attempt++;
+		ZEN_INFO("{} Attempt {}/{}", CommonResponse(std::move(Result)).ErrorMessage("Retry"), Attempt, RetryCount + 1);
+		Result = Func();
+	}
+	return Result;
+}
+
 static std::pair<std::string, std::string>
 HeaderContentType(ZenContentType ContentType)
 {
@@ -347,222 +634,6 @@ HttpClient::Impl::ReleaseSession(cpr::Session* CprSession)
 	m_SessionLock.WithExclusiveLock([&] { m_Sessions.push_back(CprSession); });
 }
 
-namespace detail {
-
-	static std::atomic_uint32_t TempFileBaseIndex;
-
-}  // namespace detail
-
-class TempPayloadFile
-{
-public:
-	TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {}
-	~TempPayloadFile()
-	{
-		try
-		{
-			if (m_FileHandle)
-			{
-#if ZEN_PLATFORM_WINDOWS
-				// Mark file for deletion when final handle is closed
-				FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE};
-
-				SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
-				BOOL Success = CloseHandle(m_FileHandle);
-#else
-				std::error_code		  Ec;
-				std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle, Ec);
-				if (Ec)
-				{
-					ZEN_WARN("Error reported on get file path from handle {} for temp payload unlink operation, reason '{}'",
-							 m_FileHandle,
-							 Ec.message());
-				}
-				else
-				{
-					unlink(FilePath.c_str());
-				}
-				int	 Fd		 = int(uintptr_t(m_FileHandle));
-				bool Success = (close(Fd) == 0);
-#endif
-				if (!Success)
-				{
-					ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString());
-				}
-
-				m_FileHandle = nullptr;
-			}
-		}
-		catch (const std::exception& Ex)
-		{
-			ZEN_ERROR("Failed deleting temp file {}. Reason '{}'", m_FileHandle, Ex.what());
-		}
-	}
-
-	std::error_code Open(const std::filesystem::path& TempFolderPath)
-	{
-		ZEN_ASSERT(m_FileHandle == nullptr);
-
-		std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) |
-								 detail::TempFileBaseIndex.fetch_add(1);
-
-		std::filesystem::path FileName = TempFolderPath / fmt::to_string(TmpIndex);
-#if ZEN_PLATFORM_WINDOWS
-		LPCWSTR				  lpFileName			= FileName.c_str();
-		const DWORD			  dwDesiredAccess		= (GENERIC_READ | GENERIC_WRITE | DELETE);
-		const DWORD			  dwShareMode			= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
-		LPSECURITY_ATTRIBUTES lpSecurityAttributes	= nullptr;
-		const DWORD			  dwCreationDisposition = CREATE_ALWAYS;
-		const DWORD			  dwFlagsAndAttributes	= FILE_ATTRIBUTE_NORMAL;
-		const HANDLE		  hTemplateFile			= nullptr;
-		const HANDLE		  FileHandle			= CreateFile(lpFileName,
-											 dwDesiredAccess,
-											 dwShareMode,
-											 lpSecurityAttributes,
-											 dwCreationDisposition,
-											 dwFlagsAndAttributes,
-											 hTemplateFile);
-
-		if (FileHandle == INVALID_HANDLE_VALUE)
-		{
-			return MakeErrorCodeFromLastError();
-		}
-#else	// ZEN_PLATFORM_WINDOWS
-		int OpenFlags = O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC;
-		int Fd = open(FileName.c_str(), OpenFlags, 0666);
-		if (Fd < 0)
-		{
-			return MakeErrorCodeFromLastError();
-		}
-		fchmod(Fd, 0666);
-
-		void* FileHandle = (void*)(uintptr_t(Fd));
-#endif	// ZEN_PLATFORM_WINDOWS
-		m_FileHandle = FileHandle;
-
-		return {};
-	}
-
-	std::error_code Write(std::string_view DataString)
-	{
-		const uint8_t* DataPtr	= (const uint8_t*)DataString.data();
-		size_t		   DataSize = DataString.size();
-		if (DataSize >= CacheBufferSize)
-		{
-			std::error_code Ec = Flush();
-			if (Ec)
-			{
-				return Ec;
-			}
-			return AppendData(DataPtr, DataSize);
-		}
-		size_t CopySize = Min(DataSize, CacheBufferSize - m_CacheBufferOffset);
-		memcpy(&m_CacheBuffer[m_CacheBufferOffset], DataPtr, CopySize);
-		m_CacheBufferOffset += CopySize;
-		DataSize -= CopySize;
-		if (m_CacheBufferOffset == CacheBufferSize)
-		{
-			AppendData(m_CacheBuffer, CacheBufferSize);
-			if (DataSize > 0)
-			{
-				ZEN_ASSERT(DataSize < CacheBufferSize);
-				memcpy(m_CacheBuffer, DataPtr + CopySize, DataSize);
-			}
-			m_CacheBufferOffset = DataSize;
-		}
-		else
-		{
-			ZEN_ASSERT(DataSize == 0);
-		}
-		return {};
-	}
-
-	IoBuffer DetachToIoBuffer()
-	{
-		if (std::error_code Ec = Flush(); Ec)
-		{
-			ThrowSystemError(Ec.value(), Ec.message());
-		}
-		ZEN_ASSERT(m_FileHandle != nullptr);
-		void*	 FileHandle = m_FileHandle;
-		IoBuffer Buffer(IoBuffer::File, FileHandle, 0, m_WriteOffset, /*IsWholeFile*/ true);
-		Buffer.SetDeleteOnClose(true);
-		m_FileHandle  = 0;
-		m_WriteOffset = 0;
-		return Buffer;
-	}
-
-	uint64_t GetSize() const { return m_WriteOffset; }
-	void	 ResetWritePos(uint64_t WriteOffset)
-	{
-		Flush();
-		m_WriteOffset = WriteOffset;
-	}
-
-private:
-	std::error_code Flush()
-	{
-		if (m_CacheBufferOffset == 0)
-		{
-			return {};
-		}
-		std::error_code Res = AppendData(m_CacheBuffer, m_CacheBufferOffset);
-		m_CacheBufferOffset = 0;
-		return Res;
-	}
-
-	std::error_code AppendData(const void* Data, uint64_t Size)
-	{
-		ZEN_ASSERT(m_FileHandle != nullptr);
-		const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
-
-		while (Size)
-		{
-			const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize);
-			uint64_t	   NumberOfBytesWritten = 0;
-#if ZEN_PLATFORM_WINDOWS
-			OVERLAPPED Ovl{};
-
-			Ovl.Offset	   = DWORD(m_WriteOffset & 0xffff'ffffu);
-			Ovl.OffsetHigh = DWORD(m_WriteOffset >> 32);
-
-			DWORD dwNumberOfBytesWritten = 0;
-
-			BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
-			if (Success)
-			{
-				NumberOfBytesWritten = static_cast<uint64_t>(dwNumberOfBytesWritten);
-			}
-#else
-			static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-			int Fd = int(uintptr_t(m_FileHandle));
-			int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, m_WriteOffset);
-			bool Success = (BytesWritten > 0);
-			if (Success)
-			{
-				NumberOfBytesWritten = static_cast<uint64_t>(BytesWritten);
-			}
-#endif
-
-			if (!Success)
-			{
-				return MakeErrorCodeFromLastError();
-			}
-
-			Size -= NumberOfBytesWritten;
-			m_WriteOffset += NumberOfBytesWritten;
-			Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesWritten;
-		}
-		return {};
-	}
-
-	void*					  m_FileHandle;
-	std::uint64_t			  m_WriteOffset;
-	static constexpr uint64_t CacheBufferSize = 512u * 1024u;
-	uint8_t					  m_CacheBuffer[CacheBufferSize];
-	std::uint64_t			  m_CacheBufferOffset = 0;
-};
-
 //////////////////////////////////////////////////////////////////////////
 
 HttpClient::HttpClient(std::string_view BaseUri, const HttpClientSettings& Connectionsettings)
@@ -932,9 +1003,9 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
 {
 	ZEN_TRACE_CPU("HttpClient::Download");
 
-	std::string						 PayloadString;
-	std::unique_ptr<TempPayloadFile> PayloadFile;
-	cpr::Response					 Response = DoWithRetry(
+	std::string								 PayloadString;
+	std::unique_ptr<detail::TempPayloadFile> PayloadFile;
+	cpr::Response							 Response = DoWithRetry(
 		   [&]() {
 			   auto GetHeader = [&](std::string header) -> std::pair<std::string, std::string> {
 				   size_t DelimiterPos = header.find(':');
@@ -986,7 +1057,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
 						   {
 							   if (ContentSize.value() > 1024 * 1024)
 							   {
-								   PayloadFile = std::make_unique<TempPayloadFile>();
+								   PayloadFile = std::make_unique<detail::TempPayloadFile>();
 								   std::error_code Ec = PayloadFile->Open(TempFolderPath);
 								   if (Ec)
 								   {
@@ -1133,6 +1204,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
 			   }
 			   return Response;
 		   },
+		   PayloadFile,
 		   m_ConnectionSettings.RetryCount);
 
 	return CommonResponse(std::move(Response), PayloadFile ? PayloadFile->DetachToIoBuffer() : IoBuffer{});