Support file reference in package message (#184)

* Fix packed message parsing for absolute path
* Always enable are sharing when opening files as IoBuffers.
* Allow control over sending partial files as localfile ref
* Check "AcceptFlags" field in RPC message for allowing localfile ref in reply
* make oplog entry add operations ZEN_DEBUG level logs
* changelog

1 files changed, 26 insertions, 8 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 0e2462a4a..c5beef4b3 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -1214,7 +1214,7 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req
 
 	if (AcceptMagic == kCbPkgMagic)
 	{
-		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResponse);
+		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResponse, FormatFlags::kDefault);
 		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
 	}
 	else
@@ -1316,7 +1316,9 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
 
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv);
 
-	uint32_t AcceptMagic = RpcRequest["Accept"sv].AsUInt32();
+	uint32_t		 AcceptMagic		 = RpcRequest["Accept"sv].AsUInt32();
+	RpcAcceptOptions AcceptFlags		 = static_cast<RpcAcceptOptions>(RpcRequest["AcceptFlags"sv].AsUInt16(0u));
+	bool			 AllowFileReferences = EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences);
 
 	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
 
@@ -1649,7 +1651,9 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
 
 	if (AcceptMagic == kCbPkgMagic)
 	{
-		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
+		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(
+			ResponsePackage,
+			AllowFileReferences ? FormatFlags::kAllowLocalReferences | FormatFlags::kDenyPartialLocalReferences : FormatFlags::kDefault);
 		HttpRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
 	}
 	else
@@ -1771,7 +1775,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
 
 	if (AcceptMagic == kCbPkgMagic)
 	{
-		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResponse);
+		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResponse, FormatFlags::kDefault);
 		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
 	}
 	else
@@ -1792,7 +1796,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
 
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheValues"sv);
 
-	uint32_t AcceptMagic = RpcRequest["Accept"sv].AsUInt32();
+	uint32_t		 AcceptMagic		 = RpcRequest["Accept"sv].AsUInt32();
+	RpcAcceptOptions AcceptFlags		 = static_cast<RpcAcceptOptions>(RpcRequest["AcceptFlags"sv].AsUInt16(0u));
+	bool			 AllowFileReferences = EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences);
 
 	CbObjectView			   Params		 = RpcRequest["Params"sv].AsObjectView();
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"sv].AsString();
@@ -1977,7 +1983,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
 
 	if (AcceptMagic == kCbPkgMagic)
 	{
-		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResponse);
+		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(
+			RpcResponse,
+			AllowFileReferences ? FormatFlags::kAllowLocalReferences | FormatFlags::kDenyPartialLocalReferences : FormatFlags::kDefault);
 		HttpRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
 	}
 	else
@@ -2043,9 +2051,11 @@ HttpStructuredCacheService::HandleRpcGetCacheChunks(zen::HttpServerRequest& Http
 	std::vector<ChunkRequest*>		ValueRequests;	 // The ChunkRequests that are requesting a Value Key
 	std::vector<CacheChunkRequest*> UpstreamChunks;	 // ChunkRequests that we need to send to the upstream
 	uint32_t						AcceptMagic = 0;
+	uint16_t						AcceptFlags = 0;
 
 	// Parse requests from the CompactBinary body of the RpcRequest and divide it into RecordRequests and ValueRequests
 	if (!ParseGetCacheChunksRequest(AcceptMagic,
+									AcceptFlags,
 									Namespace,
 									RecordKeys,
 									Records,
@@ -2069,11 +2079,12 @@ HttpStructuredCacheService::HandleRpcGetCacheChunks(zen::HttpServerRequest& Http
 	GetUpstreamCacheChunks(Namespace, UpstreamChunks, RequestKeys, Requests);
 
 	// Send the payload and descriptive data about each chunk to the client
-	WriteGetCacheChunksResponse(AcceptMagic, Namespace, Requests, HttpRequest);
+	WriteGetCacheChunksResponse(AcceptMagic, AcceptFlags, Namespace, Requests, HttpRequest);
 }
 
 bool
 HttpStructuredCacheService::ParseGetCacheChunksRequest(uint32_t&								  AcceptMagic,
+													   uint16_t&								  AcceptFlags,
 													   std::string&								  Namespace,
 													   std::vector<CacheKeyRequest>&			  RecordKeys,
 													   std::vector<cache::detail::RecordBody>&	  Records,
@@ -2088,6 +2099,7 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(uint32_t&								  Accept
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheChunks"sv);
 
 	AcceptMagic = RpcRequest["Accept"sv].AsUInt32();
+	AcceptFlags = RpcRequest["AcceptFlags"sv].AsUInt16(0u);
 
 	CbObjectView	 Params			   = RpcRequest["Params"sv].AsObjectView();
 	std::string_view DefaultPolicyText = Params["DefaultPolicy"sv].AsString();
@@ -2440,12 +2452,16 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view							 Names
 
 void
 HttpStructuredCacheService::WriteGetCacheChunksResponse(uint32_t								  AcceptMagic,
+														uint16_t								  AcceptFlags,
 														std::string_view						  Namespace,
 														std::vector<cache::detail::ChunkRequest>& Requests,
 														zen::HttpServerRequest&					  HttpRequest)
 {
 	using namespace cache::detail;
 
+	RpcAcceptOptions AcceptOptions		 = static_cast<RpcAcceptOptions>(AcceptFlags);
+	bool			 AllowFileReferences = EnumHasAllFlags(AcceptOptions, RpcAcceptOptions::kAllowLocalReferences);
+
 	CbPackage	   RpcResponse;
 	CbObjectWriter Writer;
 
@@ -2505,7 +2521,9 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(uint32_t								  Accept
 
 	if (AcceptMagic == kCbPkgMagic)
 	{
-		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResponse);
+		CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(
+			RpcResponse,
+			AllowFileReferences ? FormatFlags::kAllowLocalReferences | FormatFlags::kDenyPartialLocalReferences : FormatFlags::kDefault);
 		HttpRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
 	}
 	else