Add `response.text` to output in log when jupiter request fails (#354)

* Add `response.text` to output in log when jupiter request fails
* only hash jupiter ref once and keep that
* Increase request timeout when uploading to Jupiter to 2 min (to handle very large attachments)
* changelog

3 files changed, 72 insertions, 36 deletions

diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index b1a68c6d2..a7bf25ad6 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -55,7 +55,7 @@ public:
 			}
 		}
 
-		SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, IoHash::HashBuffer(Payload)};
+		SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash};
 		if (Result.ErrorCode)
 		{
 			Result.Reason = fmt::format("Failed saving oplog container to {}/{}/{}/{}. Reason: '{}'",
@@ -252,7 +252,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options)
 	CloudCacheClientOptions ClientOptions{.Name			  = "Remote store"sv,
 										  .ServiceUrl	  = Url,
 										  .ConnectTimeout = std::chrono::milliseconds(2000),
-										  .Timeout		  = std::chrono::milliseconds(60000),
+										  .Timeout		  = std::chrono::milliseconds(120000),
 										  .AssumeHttp2	  = Options.AssumeHttp2};
 	// 1) Access token as parameter in request
 	// 2) Environment variable (different win vs linux/mac)
diff --git a/src/zenserver/upstream/jupiter.cpp b/src/zenserver/upstream/jupiter.cpp
index 3ff8525d1..2137523b2 100644
--- a/src/zenserver/upstream/jupiter.cpp
+++ b/src/zenserver/upstream/jupiter.cpp
@@ -145,7 +145,8 @@ CloudCacheSession::GetRef(std::string_view Namespace, std::string_view BucketId,
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -154,7 +155,8 @@ CloudCacheSession::GetRef(std::string_view Namespace, std::string_view BucketId,
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -192,7 +194,8 @@ CloudCacheSession::GetBlob(std::string_view Namespace, const IoHash& Key)
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -201,7 +204,8 @@ CloudCacheSession::GetBlob(std::string_view Namespace, const IoHash& Key)
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -241,7 +245,8 @@ CloudCacheSession::GetCompressedBlob(std::string_view Namespace, const IoHash& K
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -250,7 +255,8 @@ CloudCacheSession::GetCompressedBlob(std::string_view Namespace, const IoHash& K
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -290,7 +296,8 @@ CloudCacheSession::GetInlineBlob(std::string_view Namespace, std::string_view Bu
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -299,7 +306,8 @@ CloudCacheSession::GetInlineBlob(std::string_view Namespace, std::string_view Bu
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	if (auto It = Response.header.find("X-Jupiter-InlinePayloadHash"); It != Response.header.end())
@@ -349,7 +357,8 @@ CloudCacheSession::GetObject(std::string_view Namespace, const IoHash& Key)
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -358,7 +367,8 @@ CloudCacheSession::GetObject(std::string_view Namespace, const IoHash& Key)
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	return Result;
@@ -400,6 +410,7 @@ CloudCacheSession::PutRef(std::string_view Namespace, std::string_view BucketId,
 				Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value()));
 			}
 		}
+		Result.RawHash = Hash;
 	}
 	else
 	{
@@ -415,7 +426,8 @@ CloudCacheSession::PutRef(std::string_view Namespace, std::string_view BucketId,
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -426,7 +438,8 @@ CloudCacheSession::PutRef(std::string_view Namespace, std::string_view BucketId,
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	return Result;
@@ -482,7 +495,8 @@ CloudCacheSession::FinalizeRef(std::string_view Namespace, std::string_view Buck
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -493,7 +507,8 @@ CloudCacheSession::FinalizeRef(std::string_view Namespace, std::string_view Buck
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	return Result;
@@ -531,7 +546,8 @@ CloudCacheSession::PutBlob(std::string_view Namespace, const IoHash& Key, IoBuff
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -541,7 +557,8 @@ CloudCacheSession::PutBlob(std::string_view Namespace, const IoHash& Key, IoBuff
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -595,7 +612,8 @@ CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& K
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -605,7 +623,8 @@ CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& K
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -651,7 +670,8 @@ CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& K
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -661,7 +681,8 @@ CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& K
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -698,7 +719,8 @@ CloudCacheSession::PutObject(std::string_view Namespace, const IoHash& Key, IoBu
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -708,7 +730,8 @@ CloudCacheSession::PutObject(std::string_view Namespace, const IoHash& Key, IoBu
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -743,7 +766,8 @@ CloudCacheSession::RefExists(std::string_view Namespace, std::string_view Bucket
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -751,7 +775,8 @@ CloudCacheSession::RefExists(std::string_view Namespace, std::string_view Bucket
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -797,7 +822,8 @@ CloudCacheSession::GetObjectReferences(std::string_view Namespace, const IoHash&
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -806,7 +832,8 @@ CloudCacheSession::GetObjectReferences(std::string_view Namespace, const IoHash&
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	return Result;
@@ -879,7 +906,8 @@ CloudCacheSession::PostComputeTasks(IoBuffer TasksData)
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -888,7 +916,8 @@ CloudCacheSession::PostComputeTasks(IoBuffer TasksData)
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -930,7 +959,8 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -939,7 +969,8 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	return Result;
@@ -1000,7 +1031,8 @@ CloudCacheSession::CacheTypeExists(std::string_view Namespace, std::string_view
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -1009,7 +1041,8 @@ CloudCacheSession::CacheTypeExists(std::string_view Namespace, std::string_view
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 	return Result;
 }
@@ -1063,7 +1096,8 @@ CloudCacheSession::CacheTypeExists(std::string_view Namespace, std::string_view
 			"Response.reason: '{}', "
 			"Response.error.code: {}, "
 			"Response.error.message: '{}', "
-			"Resonse.raw_header: '{}'",
+			"Resonse.raw_header: '{}'"
+			"Resonse.text: '{}'",
 			Response.elapsed,
 			Uri,
 			AccessToken.Value.substr(0, 6),
@@ -1072,7 +1106,8 @@ CloudCacheSession::CacheTypeExists(std::string_view Namespace, std::string_view
 			Response.reason,
 			gsl::narrow<int>(Response.error.code),
 			Response.error.message,
-			Response.raw_header);
+			Response.raw_header,
+			Response.text);
 	}
 
 	return Result;
diff --git a/src/zenserver/upstream/jupiter.h b/src/zenserver/upstream/jupiter.h
index 3d965e846..cabb61a1f 100644
--- a/src/zenserver/upstream/jupiter.h
+++ b/src/zenserver/upstream/jupiter.h
@@ -64,6 +64,7 @@ struct CloudCacheResult
 struct PutRefResult : CloudCacheResult
 {
 	std::vector<IoHash> Needs;
+	IoHash				RawHash;
 };
 
 struct FinalizeRefResult : CloudCacheResult