improvements and infrastructure for upcoming builds api command line (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api

1 files changed, 23 insertions, 4 deletions

diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 211a5d05c..bb15a6fce 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -449,12 +449,27 @@ ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile
 }
 
 static cpr::Response
-DoWithRetry(std::string_view SessionId, std::function<cpr::Response()>&& Func, uint8_t RetryCount)
+DoWithRetry(
+	std::string_view							 SessionId,
+	std::function<cpr::Response()>&&			 Func,
+	uint8_t										 RetryCount,
+	std::function<bool(cpr::Response& Result)>&& Validate = [](cpr::Response&) { return true; })
 {
 	uint8_t		  Attempt = 0;
 	cpr::Response Result  = Func();
-	while (Attempt < RetryCount && ShouldRetry(Result))
+	while (Attempt < RetryCount)
 	{
+		if (!ShouldRetry(Result))
+		{
+			if (Result.error || !IsHttpSuccessCode(Result.status_code))
+			{
+				break;
+			}
+			if (Validate(Result))
+			{
+				break;
+			}
+		}
 		Sleep(100 * (Attempt + 1));
 		Attempt++;
 		ZEN_INFO("{} Attempt {}/{}", CommonResponse(SessionId, std::move(Result)).ErrorMessage("Retry"), Attempt, RetryCount + 1);
@@ -881,7 +896,11 @@ HttpClient::Get(std::string_view Url, const KeyValueMap& AdditionalHeader, const
 					m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken());
 				return Sess.Get();
 			},
-			m_ConnectionSettings.RetryCount));
+			m_ConnectionSettings.RetryCount,
+			[](cpr::Response& Result) {
+				std::unique_ptr<detail::TempPayloadFile> NoTempFile;
+				return ValidatePayload(Result, NoTempFile);
+			}));
 }
 
 HttpClient::Response
@@ -1247,7 +1266,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
 							   {
 								   uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length();
 
-								   std::string Range = fmt::format("bytes={}-{}", DownloadedSize, ContentLength.value());
+								   std::string Range = fmt::format("bytes={}-{}", DownloadedSize, DownloadedSize + ContentLength.value() - 1);
 								   if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end())
 								   {
 									   if (RangeIt->second == Range)