non memory copy compressed range (#13)

* Add CompressedBuffer::GetRange that references source data rather than make a memory copy
* Use Compressed.CopyRange in project store GetChunkRange
* docs for CompressedBuffer::CopyRange and CompressedBuffer::GetRange

4 files changed, 53 insertions, 44 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 0ba49cf8a..bc71e2fa0 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -768,14 +768,15 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req)
 
 	HttpContentType AcceptType = HttpReq.AcceptContentType();
 
-	IoBuffer								 Chunk;
+	CompositeBuffer							 Chunk;
+	HttpContentType							 ContentType;
 	std::pair<HttpResponseCode, std::string> Result =
-		m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk);
+		m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk, ContentType);
 	if (Result.first == HttpResponseCode::OK)
 	{
 		m_ProjectStats.ChunkHitCount++;
-		ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Chunk.GetContentType()));
-		return HttpReq.WriteResponse(HttpResponseCode::OK, Chunk.GetContentType(), Chunk);
+		ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(ContentType));
+		return HttpReq.WriteResponse(HttpResponseCode::OK, ContentType, Chunk);
 	}
 	else if (Result.first == HttpResponseCode::NotFound)
 	{
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index cfa53c080..e4a39e55f 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2647,7 +2647,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 							uint64_t			   Offset,
 							uint64_t			   Size,
 							ZenContentType		   AcceptType,
-							IoBuffer&			   OutChunk)
+							CompositeBuffer&	   OutChunk,
+							ZenContentType&		   OutContentType)
 {
 	if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
 	{
@@ -2656,7 +2657,7 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 
 	const Oid Obj = Oid::FromHexString(ChunkId);
 
-	return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk);
+	return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk, OutContentType);
 }
 
 std::pair<HttpResponseCode, std::string>
@@ -2666,7 +2667,8 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 							uint64_t			   Offset,
 							uint64_t			   Size,
 							ZenContentType		   AcceptType,
-							IoBuffer&			   OutChunk)
+							CompositeBuffer&	   OutChunk,
+							ZenContentType&		   OutContentType)
 {
 	bool IsOffset = Offset != 0 || Size != ~(0ull);
 
@@ -2690,10 +2692,9 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 		return {HttpResponseCode::NotFound, {}};
 	}
 
-	OutChunk					= Chunk;
-	HttpContentType ContentType = Chunk.GetContentType();
+	OutContentType = Chunk.GetContentType();
 
-	if (Chunk.GetContentType() == HttpContentType::kCompressedBinary)
+	if (OutContentType == ZenContentType::kCompressedBinary)
 	{
 		IoHash			 RawHash;
 		uint64_t		 RawSize;
@@ -2702,46 +2703,47 @@ ProjectStore::GetChunkRange(const std::string_view ProjectId,
 
 		if (IsOffset)
 		{
-			if ((Offset + Size) > RawSize)
+			if (Size == ~(0ull) || (Offset + Size) > RawSize)
 			{
 				Size = RawSize - Offset;
 			}
 
-			if (AcceptType == HttpContentType::kBinary)
+			if (AcceptType == ZenContentType::kBinary)
 			{
-				OutChunk = Compressed.Decompress(Offset, Size).AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kBinary);
+				OutChunk	   = CompositeBuffer(Compressed.Decompress(Offset, Size));
+				OutContentType = ZenContentType::kBinary;
 			}
 			else
 			{
 				// Value will be a range of compressed blocks that covers the requested range
 				// The client will have to compensate for any offsets that do not land on an even block size multiple
-				OutChunk = Compressed.CopyRange(Offset, Size).GetCompressed().Flatten().AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kCompressedBinary);
+				OutChunk = Compressed.GetRange(Offset, Size).GetCompressed();
 			}
 		}
 		else
 		{
-			if (AcceptType == HttpContentType::kBinary)
+			if (AcceptType == ZenContentType::kBinary)
 			{
-				OutChunk = Compressed.Decompress().AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kBinary);
+				OutChunk = Compressed.DecompressToComposite();
 			}
 			else
 			{
-				OutChunk = Compressed.GetCompressed().Flatten().AsIoBuffer();
-				OutChunk.SetContentType(HttpContentType::kCompressedBinary);
+				OutChunk	   = Compressed.GetCompressed();
+				OutContentType = ZenContentType::kCompressedBinary;
 			}
 		}
 	}
 	else if (IsOffset)
 	{
-		if ((Offset + Size) > Chunk.GetSize())
+		if (Size == ~(0ull) || (Offset + Size) > Chunk.GetSize())
 		{
 			Size = Chunk.GetSize() - Offset;
 		}
-		OutChunk = IoBuffer(std::move(Chunk), Offset, Size);
-		OutChunk.SetContentType(ContentType);
+		OutChunk = CompositeBuffer(SharedBuffer(IoBuffer(std::move(Chunk), Offset, Size)));
+	}
+	else
+	{
+		OutChunk = CompositeBuffer(SharedBuffer(std::move(Chunk)));
 	}
 
 	return {HttpResponseCode::OK, {}};
@@ -4428,7 +4430,8 @@ TEST_CASE("project.store.partial.read")
 		CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize());
 	}
 
-	IoBuffer ChunkResult;
+	CompositeBuffer ChunkResult;
+	HttpContentType ContentType;
 	CHECK(ProjectStore
 			  .GetChunkRange("proj1"sv,
 							 "oplog1"sv,
@@ -4436,13 +4439,14 @@ TEST_CASE("project.store.partial.read")
 							 0,
 							 ~0ull,
 							 HttpContentType::kCompressedBinary,
-							 ChunkResult)
+							 ChunkResult,
+							 ContentType)
 			  .first == HttpResponseCode::OK);
 	CHECK(ChunkResult);
 	CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
 		  Attachments[OpIds[2]][1].second.DecodeRawSize());
 
-	IoBuffer PartialChunkResult;
+	CompositeBuffer PartialChunkResult;
 	CHECK(ProjectStore
 			  .GetChunkRange("proj1"sv,
 							 "oplog1"sv,
@@ -4450,13 +4454,13 @@ TEST_CASE("project.store.partial.read")
 							 5,
 							 1773,
 							 HttpContentType::kCompressedBinary,
-							 PartialChunkResult)
+							 PartialChunkResult,
+							 ContentType)
 			  .first == HttpResponseCode::OK);
 	CHECK(PartialChunkResult);
 	IoHash			 PartialRawHash;
 	uint64_t		 PartialRawSize;
-	CompressedBuffer PartialCompressedResult =
-		CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult), PartialRawHash, PartialRawSize);
+	CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(PartialChunkResult, PartialRawHash, PartialRawSize);
 	CHECK(PartialRawSize >= 1773);
 
 	uint64_t	   RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index d8c053649..eda336150 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -333,14 +333,16 @@ public:
 														   uint64_t				  Offset,
 														   uint64_t				  Size,
 														   ZenContentType		  AcceptType,
-														   IoBuffer&			  OutChunk);
+														   CompositeBuffer&		  OutChunk,
+														   ZenContentType&		  OutContentType);
 	std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId,
 														   const std::string_view OplogId,
 														   const std::string_view ChunkId,
 														   uint64_t				  Offset,
 														   uint64_t				  Size,
 														   ZenContentType		  AcceptType,
-														   IoBuffer&			  OutChunk);
+														   CompositeBuffer&		  OutChunk,
+														   ZenContentType&		  OutContentType);
 	std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId,
 													  const std::string_view OplogId,
 													  const std::string_view Cid,
diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp
index f528b2620..5ef89ee77 100644
--- a/src/zenserver/vfs/vfsimpl.cpp
+++ b/src/zenserver/vfs/vfsimpl.cpp
@@ -38,21 +38,23 @@ VfsOplogDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t
 void
 VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount)
 {
-	IoBuffer ChunkBuffer;
-	auto	 Result =
-		m_ProjectStore->GetChunkRange(m_ProjectId, m_OplogId, ChunkId, 0, ~0ull, ZenContentType::kCompressedBinary, /* out */ ChunkBuffer);
+	CompositeBuffer ChunkBuffer;
+	ZenContentType	ContentType;
+	auto			Result = m_ProjectStore->GetChunkRange(m_ProjectId,
+												   m_OplogId,
+												   ChunkId,
+												   0,
+												   ~0ull,
+												   ZenContentType::kCompressedBinary,
+												   /* out */ ChunkBuffer,
+												   /* out */ ContentType);
 
 	if (Result.first == HttpResponseCode::OK)
 	{
-		const uint8_t* SourceBuffer			= reinterpret_cast<const uint8_t*>(ChunkBuffer.GetData());
-		uint64_t	   AvailableBufferBytes = ChunkBuffer.GetSize();
-
-		ZEN_ASSERT(AvailableBufferBytes >= ByteOffset);
-		AvailableBufferBytes -= ByteOffset;
-		SourceBuffer += ByteOffset;
-
-		ZEN_ASSERT(AvailableBufferBytes >= ByteCount);
-		memcpy(Buffer, SourceBuffer, ByteCount);
+		ZEN_ASSERT(ChunkBuffer.GetSize() >= ByteOffset);
+		ZEN_ASSERT(ChunkBuffer.GetSize() - ByteOffset >= ByteCount);
+		MutableMemoryView Target(Buffer, ByteCount);
+		ChunkBuffer.CopyTo(Target, ByteOffset);
 	}
 }