De/more upstream details (#168)v0.1.6-pre9v0.1.6-pre10

* Pass along endpoint info for each upstream request
* Add more timing details in log
* more log details for single item upstream fetch
* DISABLEDQUERY over SKIP

1 files changed, 48 insertions, 37 deletions

diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h
index 108c097da..695c06b32 100644
--- a/zenserver/upstream/upstreamcache.h
+++ b/zenserver/upstream/upstreamcache.h
@@ -53,15 +53,27 @@ struct UpstreamError
 	explicit operator bool() const { return ErrorCode != 0; }
 };
 
+struct UpstreamEndpointInfo
+{
+	std::string Name;
+	std::string Url;
+};
+
 struct GetUpstreamCacheResult
 {
-	IoBuffer	  Value;
 	UpstreamError Error{};
 	int64_t		  Bytes{};
 	double		  ElapsedSeconds{};
 	bool		  Success = false;
 };
 
+struct GetUpstreamCacheSingleResult
+{
+	GetUpstreamCacheResult		Status;
+	IoBuffer					Value;
+	const UpstreamEndpointInfo* Source = nullptr;
+};
+
 struct PutUpstreamCacheResult
 {
 	std::string Reason;
@@ -72,32 +84,35 @@ struct PutUpstreamCacheResult
 
 struct CacheRecordGetCompleteParams
 {
-	CacheKeyRequest&	Request;
-	const CbObjectView& Record;
-	const CbPackage&	Package;
-	double				ElapsedSeconds{};
+	CacheKeyRequest&			Request;
+	const CbObjectView&			Record;
+	const CbPackage&			Package;
+	double						ElapsedSeconds{};
+	const UpstreamEndpointInfo* Source = nullptr;
 };
 
 using OnCacheRecordGetComplete = std::function<void(CacheRecordGetCompleteParams&&)>;
 
 struct CacheValueGetCompleteParams
 {
-	CacheValueRequest& Request;
-	IoHash			   RawHash;
-	uint64_t		   RawSize;
-	IoBuffer		   Value;
-	double			   ElapsedSeconds{};
+	CacheValueRequest&			Request;
+	IoHash						RawHash;
+	uint64_t					RawSize;
+	IoBuffer					Value;
+	double						ElapsedSeconds{};
+	const UpstreamEndpointInfo* Source = nullptr;
 };
 
 using OnCacheValueGetComplete = std::function<void(CacheValueGetCompleteParams&&)>;
 
 struct CacheChunkGetCompleteParams
 {
-	CacheChunkRequest& Request;
-	IoHash			   RawHash;
-	uint64_t		   RawSize;
-	IoBuffer		   Value;
-	double			   ElapsedSeconds{};
+	CacheChunkRequest&			Request;
+	IoHash						RawHash;
+	uint64_t					RawSize;
+	IoBuffer					Value;
+	double						ElapsedSeconds{};
+	const UpstreamEndpointInfo* Source = nullptr;
 };
 
 using OnCacheChunksGetComplete = std::function<void(CacheChunkGetCompleteParams&&)>;
@@ -156,12 +171,6 @@ struct UpstreamEndpointStatus
 	UpstreamEndpointState State;
 };
 
-struct UpstreamEndpointInfo
-{
-	std::string Name;
-	std::string Url;
-};
-
 /**
  * The upstream endpoint is responsible for handling upload/downloading of cache records.
  */
@@ -177,19 +186,19 @@ public:
 	virtual UpstreamEndpointState  GetState()  = 0;
 	virtual UpstreamEndpointStatus GetStatus() = 0;
 
-	virtual GetUpstreamCacheResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0;
-	virtual GetUpstreamCacheResult GetCacheRecords(std::string_view			   Namespace,
-												   std::span<CacheKeyRequest*> Requests,
-												   OnCacheRecordGetComplete&&  OnComplete)									 = 0;
+	virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0;
+	virtual GetUpstreamCacheResult		 GetCacheRecords(std::string_view			 Namespace,
+														 std::span<CacheKeyRequest*> Requests,
+														 OnCacheRecordGetComplete&&	 OnComplete)									   = 0;
 
 	virtual GetUpstreamCacheResult GetCacheValues(std::string_view				Namespace,
 												  std::span<CacheValueRequest*> CacheValueRequests,
 												  OnCacheValueGetComplete&&		OnComplete) = 0;
 
-	virtual GetUpstreamCacheResult GetCacheChunk(std::string_view Namespace, const CacheKey& CacheKey, const IoHash& PayloadId) = 0;
-	virtual GetUpstreamCacheResult GetCacheChunks(std::string_view				Namespace,
-												  std::span<CacheChunkRequest*> CacheChunkRequests,
-												  OnCacheChunksGetComplete&&	OnComplete)										= 0;
+	virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, const CacheKey& CacheKey, const IoHash& PayloadId) = 0;
+	virtual GetUpstreamCacheResult		 GetCacheChunks(std::string_view			  Namespace,
+														std::span<CacheChunkRequest*> CacheChunkRequests,
+														OnCacheChunksGetComplete&&	  OnComplete)										  = 0;
 
 	virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord,
 												  IoBuffer					 RecordValue,
@@ -217,19 +226,21 @@ public:
 	virtual void RegisterEndpoint(std::unique_ptr<UpstreamEndpoint> Endpoint)  = 0;
 	virtual void IterateEndpoints(std::function<bool(UpstreamEndpoint&)>&& Fn) = 0;
 
-	virtual GetUpstreamCacheResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0;
-	virtual void				   GetCacheRecords(std::string_view			   Namespace,
-												   std::span<CacheKeyRequest*> Requests,
-												   OnCacheRecordGetComplete&&  OnComplete)									 = 0;
+	virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0;
+	virtual void						 GetCacheRecords(std::string_view			 Namespace,
+														 std::span<CacheKeyRequest*> Requests,
+														 OnCacheRecordGetComplete&&	 OnComplete)									   = 0;
 
 	virtual void GetCacheValues(std::string_view			  Namespace,
 								std::span<CacheValueRequest*> CacheValueRequests,
 								OnCacheValueGetComplete&&	  OnComplete) = 0;
 
-	virtual GetUpstreamCacheResult GetCacheChunk(std::string_view Namespace, const CacheKey& CacheKey, const IoHash& ValueContentId) = 0;
-	virtual void				   GetCacheChunks(std::string_view				Namespace,
-												  std::span<CacheChunkRequest*> CacheChunkRequests,
-												  OnCacheChunksGetComplete&&	OnComplete)											 = 0;
+	virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace,
+													   const CacheKey&	CacheKey,
+													   const IoHash&	ValueContentId)		   = 0;
+	virtual void						 GetCacheChunks(std::string_view			  Namespace,
+														std::span<CacheChunkRequest*> CacheChunkRequests,
+														OnCacheChunksGetComplete&&	  OnComplete) = 0;
 
 	virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) = 0;