Implemented new URI addressing scheme for the Zen cache endpoints, and prepared for additional indexing capabilities

1 files changed, 137 insertions, 18 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 2704bedc3..728cfaded 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
 #include <zencore/fmtutils.h>
 #include <zencore/httpserver.h>
 #include <zencore/timer.h>
@@ -53,6 +55,21 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
 		return Request.WriteResponse(zen::HttpResponse::BadRequest);  // invalid URL
 	}
 
+	if (Ref.PayloadId == IoHash::Zero)
+	{
+		return HandleCacheRecordRequest(Request, Ref);
+	}
+	else
+	{
+		return HandleCachePayloadRequest(Request, Ref);
+	}
+
+	return;
+}
+
+void
+HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref)
+{
 	switch (auto Verb = Request.RequestVerb())
 	{
 		using enum zen::HttpVerb;
@@ -65,20 +82,15 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
 
 				if (!Success)
 				{
-					Request.WriteResponse(zen::HttpResponse::NotFound);
+					return Request.WriteResponse(zen::HttpResponse::NotFound);
 				}
-				else
+
+				if (Verb == kHead)
 				{
-					if (Verb == kHead)
-					{
-						Request.SetSuppressResponseBody();
-						Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value);
-					}
-					else
-					{
-						Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value);
-					}
+					Request.SetSuppressResponseBody();
 				}
+
+				return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Value.Value);
 			}
 			break;
 
@@ -91,21 +103,55 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
 
 					HttpContentType ContentType = Request.RequestContentType();
 
+					bool IsCompactBinary;
+
 					switch (ContentType)
 					{
 						case HttpContentType::kUnknownContentType:
 						case HttpContentType::kBinary:
-							Value.IsCompactBinary = false;
+							IsCompactBinary = false;
 							break;
 
 						case HttpContentType::kCbObject:
-							Value.IsCompactBinary = true;
+							IsCompactBinary = true;
 							break;
 
 						default:
 							return Request.WriteResponse(zen::HttpResponse::BadRequest);
 					}
 
+					// Compute index data
+
+					if (IsCompactBinary)
+					{
+						// Validate payload before accessing it
+						zen::CbValidateError ValidationResult =
+							zen::ValidateCompactBinary(MemoryView(Body.Data(), Body.Size()), zen::CbValidateMode::All);
+
+						if (ValidationResult != CbValidateError::None)
+						{
+							// TODO: add details in response
+							return Request.WriteResponse(HttpResponse::BadRequest);
+						}
+
+						// Extract data for index
+						zen::CbObjectView Cbo(Body.Data());
+
+						int ReferenceCount = 0;
+
+						zen::CbObjectWriter Idx;
+						Idx.BeginArray();
+
+						Cbo.IterateAttachments([&](CbFieldView AttachmentView) {
+							Idx.AddHash(AttachmentView.AsHash());
+							++ReferenceCount;
+						});
+
+						Idx.EndArray();
+
+						// TODO: store references in index
+					}
+
 					m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, Value);
 
 					// This is currently synchronous for simplicity and debuggability but should be
@@ -152,7 +198,76 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
 	}
 }
 
-[[nodiscard]] bool
+void
+HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref)
+{
+	// Note: the URL references the uncompressed payload hash - so this maintains the mapping
+	// from uncompressed CAS identity to the stored payload hash
+	//
+	// this is a PITA but a consequence of the fact that the client side code is not able to
+	// address data by compressed hash
+
+	switch (auto Verb = Request.RequestVerb())
+	{
+		using enum zen::HttpVerb;
+
+		case kHead:
+		case kGet:
+			{
+				// TODO: need to map from uncompressed content address into the storage 
+				// (compressed) content address
+
+				zen::IoBuffer Payload = m_CasStore.FindChunk(Ref.PayloadId);
+
+				if (!Payload)
+				{
+					return Request.WriteResponse(zen::HttpResponse::NotFound);
+				}
+
+				if (Verb == kHead)
+				{
+					Request.SetSuppressResponseBody();
+				}
+
+				return Request.WriteResponse(zen::HttpResponse::OK, zen::HttpContentType::kBinary, Payload);
+			}
+			break;
+
+		case kPut:
+			{
+				if (zen::IoBuffer Body = Request.ReadPayload())
+				{
+					zen::IoHash ChunkHash = zen::IoHash::HashMemory(Body);
+
+					if (ChunkHash != Ref.PayloadId)
+					{
+						// the URL and data hashes don't match!
+						return Request.WriteResponse(HttpResponse::BadRequest);
+					}
+
+					zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash);
+
+					if (Result.New)
+					{
+						return Request.WriteResponse(zen::HttpResponse::Created);
+					}
+					else
+					{
+						return Request.WriteResponse(zen::HttpResponse::OK);
+					}
+				}
+			}
+			break;
+
+		case kPost:
+			break;
+
+		default:
+			break;
+	}
+}
+
+bool
 HttpStructuredCacheService::ValidateUri(zen::HttpServerRequest& Request, CacheRef& OutRef)
 {
 	std::string_view			Key				  = Request.RelativeUri();
@@ -184,20 +299,24 @@ HttpStructuredCacheService::ValidateUri(zen::HttpServerRequest& Request, CacheRe
 		PayloadSegment = Key.substr(PayloadSplitOffset + 1);
 	}
 
-	if (HashSegment.size() != (2 * sizeof OutRef.HashKey.Hash))
+	if (HashSegment.size() != zen::IoHash::StringLength)
 	{
 		return false;
 	}
 
-	if (!PayloadSegment.empty() && PayloadSegment.size() != 24)
+	if (!PayloadSegment.empty() && PayloadSegment.size() == zen::IoHash::StringLength)
 	{
-		OutRef.PayloadId = zen::Oid::FromHexString(PayloadSegment);
+		const bool IsOk = zen::ParseHexBytes(PayloadSegment.data(), PayloadSegment.size(), OutRef.PayloadId.Hash);
 
-		if (!OutRef.PayloadId)
+		if (!IsOk)
 		{
 			return false;
 		}
 	}
+	else
+	{
+		OutRef.PayloadId = zen::IoHash::Zero;
+	}
 
 	const bool IsOk = zen::ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash);