non memory copy compressed range (#13)

* Add CompressedBuffer::GetRange that references source data rather than make a memory copy
* Use Compressed.CopyRange in project store GetChunkRange
* docs for CompressedBuffer::CopyRange and CompressedBuffer::GetRange

1 files changed, 30 insertions, 26 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index cfa53c080..e4a39e55f 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2647,7 +2647,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 							uint64_t			   Offset,
 							uint64_t			   Size,
 							ZenContentType		   AcceptType,
-							IoBuffer&			   OutChunk)
+							CompositeBuffer&	   OutChunk,
+							ZenContentType&		   OutContentType)
 {
 	if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
 	{
@@ -2656,7 +2657,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 
 	const Oid Obj = Oid::FromHexString(ChunkId);
 
-	return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk);
+	return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType);
 }
 
 std::pair<HttpResponseCode, std::string>
@@ -2666,7 +2667,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 							uint64_t			   Offset,
 							uint64_t			   Size,
 							ZenContentType		   AcceptType,
-							IoBuffer&			   OutChunk)
+							CompositeBuffer&	   OutChunk,
+							ZenContentType&		   OutContentType)
 {
 	bool IsOffset = Offset != 0 || Size != ~(0ull);
 
@@ -2690,10 +2692,9 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 		return {HttpResponseCode::NotFound, {}};
 	}
 
-	OutChunk					= Chunk;
-	HttpContentType ContentType = Chunk.GetContentType();
+	OutContentType = Chunk.GetContentType();
 
-	if (Chunk.GetContentType() == HttpContentType::kCompressedBinary)
+	if (OutContentType == ZenContentType::kCompressedBinary)
 	{
 		IoHash			 RawHash;
 		uint64_t		 RawSize;
@@ -2702,46 +2703,47 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 
 		if (IsOffset)
 		{
-			if ((Offset + Size) > RawSize)
+			if (Size == ~(0ull) || (Offset + Size) > RawSize)
 			{
 				Size = RawSize - Offset;
 			}
 
-			if (AcceptType == HttpContentType::kBinary)
+			if (AcceptType == ZenContentType::kBinary)
 			{
-				OutChunk = Compressed.Decompress(Offset, Size).AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kBinary);
+				OutChunk	   = CompositeBuffer(Compressed.Decompress(Offset, Size));
+				OutContentType = ZenContentType::kBinary;
 			}
 			else
 			{
 				// Value will be a range of compressed blocks that covers the requested range
 				// The client will have to compensate for any offsets that do not land on an even block size multiple
-				OutChunk = Compressed.CopyRange(Offset, Size).GetCompressed().Flatten().AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kCompressedBinary);
+				OutChunk = Compressed.GetRange(Offset, Size).GetCompressed();
 			}
 		}
 		else
 		{
-			if (AcceptType == HttpContentType::kBinary)
+			if (AcceptType == ZenContentType::kBinary)
 			{
-				OutChunk = Compressed.Decompress().AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kBinary);
+				OutChunk = Compressed.DecompressToComposite();
 			}
 			else
 			{
-				OutChunk = Compressed.GetCompressed().Flatten().AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kCompressedBinary);
+				OutChunk	   = Compressed.GetCompressed();
+				OutContentType = ZenContentType::kCompressedBinary;
 			}
 		}
 	}
 	else if (IsOffset)
 	{
-		if ((Offset + Size) > Chunk.GetSize())
+		if (Size == ~(0ull) || (Offset + Size) > Chunk.GetSize())
 		{
 			Size = Chunk.GetSize() - Offset;
 		}
-		OutChunk = IoBuffer(std::move(Chunk), Offset, Size);
-		OutChunk.SetContentType(ContentType);
+		OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size)));
+	}
+	else
+	{
+		OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk)));
 	}
 
 	return {HttpResponseCode::OK, {}};
@@ -4428,7 +4430,8 @@ TEST_CASE("project.store.partial.read")
 		CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize());
 	}
 
-	IoBuffer ChunkResult;
+	CompositeBuffer ChunkResult;
+	HttpContentType ContentType;
 	CHECK(ProjectStore
 			  .GetChunkRange("proj1"sv,
 							 "oplog1"sv,
@@ -4436,13 +4439,14 @@ TEST_CASE("project.store.partial.read")
 							 0,
 							 ~0ull,
 							 HttpContentType::kCompressedBinary,
-							 ChunkResult)
+							 ChunkResult,
+							 ContentType)
 			  .first == HttpResponseCode::OK);
 	CHECK(ChunkResult);
 	CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
 		  Attachments[OpIds[2]][1].second.DecodeRawSize());
 
-	IoBuffer PartialChunkResult;
+	CompositeBuffer PartialChunkResult;
 	CHECK(ProjectStore
 			  .GetChunkRange("proj1"sv,
 							 "oplog1"sv,
@@ -4450,13 +4454,13 @@ TEST_CASE("project.store.partial.read")
 							 5,
 							 1773,
 							 HttpContentType::kCompressedBinary,
-							 PartialChunkResult)
+							 PartialChunkResult,
+							 ContentType)
 			  .first == HttpResponseCode::OK);
 	CHECK(PartialChunkResult);
 	IoHash			 PartialRawHash;
 	uint64_t		 PartialRawSize;
-	CompressedBuffer PartialCompressedResult =
-		CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult), PartialRawHash, PartialRawSize);
+	CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize);
 	CHECK(PartialRawSize >= 1773);
 
 	uint64_t	   RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);