Removed MemoryOutStream, MemoryInStream

BinaryWriter/BinaryReader now implements memory buffer functionality which previously needed two chained instances of a Buffer/Reader. This was originally expected to be an abstraction for file and other stream access but this is not going to be useful so may as well collapse the functionality.

This also eliminates the need for stack-aware ref-counting which is the real reason for wanting to get rid of this code. This was a very old experimental feature which turned out to be a bad idea.

This also removes the /cas/batch endpoint

6 files changed, 52 insertions, 251 deletions

diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index fbdcd24e9..88757d47f 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -83,8 +83,7 @@ CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash)
 bool
 CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator)
 {
-	MemoryInStream InStream(InBuffer.Data(), InBuffer.Size());
-	BinaryReader   Reader(InStream);
+	BinaryReader Reader(InBuffer.Data(), InBuffer.Size());
 
 	return TryLoad(Reader, Allocator);
 }
@@ -427,8 +426,7 @@ CbPackage::GatherAttachments(const CbObject& Value, AttachmentResolver Resolver)
 bool
 CbPackage::TryLoad(IoBuffer InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper)
 {
-	MemoryInStream InStream(InBuffer.Data(), InBuffer.Size());
-	BinaryReader   Reader(InStream);
+	BinaryReader Reader(InBuffer.Data(), InBuffer.Size());
 
 	return TryLoad(Reader, Allocator, Mapper);
 }
@@ -670,8 +668,7 @@ namespace legacy {
 
 	bool TryLoadCbPackage(CbPackage& Package, IoBuffer InBuffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper)
 	{
-		MemoryInStream InStream(InBuffer.Data(), InBuffer.Size());
-		BinaryReader   Reader(InStream);
+		BinaryReader Reader(InBuffer.Data(), InBuffer.Size());
 
 		return TryLoadCbPackage(Package, Reader, Allocator, Mapper);
 	}
@@ -764,24 +761,22 @@ TEST_CASE("usonpackage")
 		Attachment.Save(Writer);
 		CbFieldIterator Fields = Writer.Save();
 
-		MemoryOutStream WriteStream;
-		BinaryWriter	StreamWriter{WriteStream};
+		BinaryWriter StreamWriter;
 		Attachment.Save(StreamWriter);
 
-		CHECK(MakeMemoryView(WriteStream).EqualBytes(Fields.GetRangeBuffer().GetView()));
-		CHECK(ValidateCompactBinaryRange(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None);
-		CHECK(ValidateObjectAttachment(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None);
+		CHECK(MakeMemoryView(StreamWriter).EqualBytes(Fields.GetRangeBuffer().GetView()));
+		CHECK(ValidateCompactBinaryRange(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None);
+		CHECK(ValidateObjectAttachment(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None);
 
 		CbAttachment FromFields;
 		FromFields.TryLoad(Fields);
 		CHECK(!bool(Fields));
 		CHECK(FromFields == Attachment);
 
-		CbAttachment   FromArchive;
-		MemoryInStream InStream(MakeMemoryView(WriteStream));
-		BinaryReader   Reader(InStream);
+		CbAttachment FromArchive;
+		BinaryReader Reader(MakeMemoryView(StreamWriter));
 		FromArchive.TryLoad(Reader);
-		CHECK(Reader.CurrentOffset() == InStream.Size());
+		CHECK(Reader.CurrentOffset() == Reader.Size());
 		CHECK(FromArchive == Attachment);
 	};
 
@@ -956,9 +951,8 @@ TEST_CASE("usonpackage.serialization")
 		Package.Save(Writer);
 		CbFieldIterator Fields = Writer.Save();
 
-		MemoryOutStream MemStream;
-		BinaryWriter	WriteAr(MemStream);
-		Package.Save(WriteAr);
+		BinaryWriter MemStream;
+		Package.Save(MemStream);
 
 		CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView()));
 		CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None);
@@ -969,11 +963,10 @@ TEST_CASE("usonpackage.serialization")
 		CHECK_FALSE(bool(Fields));
 		CHECK(FromFields == Package);
 
-		CbPackage	   FromArchive;
-		MemoryInStream ReadMemStream(MakeMemoryView(MemStream));
-		BinaryReader   ReadAr(ReadMemStream);
+		CbPackage	 FromArchive;
+		BinaryReader ReadAr(MakeMemoryView(MemStream));
 		FromArchive.TryLoad(ReadAr);
-		CHECK(ReadAr.CurrentOffset() == ReadMemStream.Size());
+		CHECK(ReadAr.CurrentOffset() == ReadAr.Size());
 		CHECK(FromArchive == Package);
 	};
 
@@ -1195,12 +1188,10 @@ TEST_CASE("usonpackage.serialization")
 		CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinary().GetView()));
 		CHECK(Level4Attachment->GetHash() == Level4Hash);
 
-		MemoryOutStream WriteStream;
-		BinaryWriter	WriteAr(WriteStream);
-		Writer.Save(WriteAr);
-		CbPackage	   FromArchive;
-		MemoryInStream ReadStream(MakeMemoryView(WriteStream));
-		BinaryReader   ReadAr(ReadStream);
+		BinaryWriter WriteStream;
+		Writer.Save(WriteStream);
+		CbPackage	 FromArchive;
+		BinaryReader ReadAr(MakeMemoryView(WriteStream));
 		FromArchive.TryLoad(ReadAr);
 
 		Writer.Reset();
diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h
index bab349068..9d1a7628c 100644
--- a/zencore/include/zencore/stream.h
+++ b/zencore/include/zencore/stream.h
@@ -5,85 +5,49 @@
 #include "zencore.h"
 
 #include <zencore/memory.h>
-#include <zencore/refcount.h>
 #include <zencore/thread.h>
 
-#include <string_view>
 #include <vector>
 
 namespace zen {
 
 /**
- * Stream which writes into a growing memory buffer
- */
-class MemoryOutStream : public RefCounted
-{
-public:
-	MemoryOutStream()  = default;
-	~MemoryOutStream() = default;
-
-	void				  Write(const void* DataPtr, size_t ByteCount, uint64_t Offset);
-	void				  Flush();
-	inline const uint8_t* Data() const { return m_Buffer.data(); }
-	inline const uint8_t* GetData() const { return m_Buffer.data(); }
-	inline uint64_t		  Size() const { return m_Buffer.size(); }
-	inline uint64_t		  GetSize() const { return m_Buffer.size(); }
-
-private:
-	RwLock				 m_Lock;
-	std::vector<uint8_t> m_Buffer;
-};
-
-inline MemoryView
-MakeMemoryView(const MemoryOutStream& Stream)
-{
-	return MemoryView(Stream.Data(), Stream.Size());
-}
-
-/**
  * Binary stream writer
  */
 
 class BinaryWriter
 {
 public:
-	inline BinaryWriter(MemoryOutStream& Stream) : m_Stream(&Stream) {}
-	~BinaryWriter() = default;
+	inline BinaryWriter() = default;
+	~BinaryWriter()		  = default;
 
 	inline void Write(const void* DataPtr, size_t ByteCount)
 	{
-		m_Stream->Write(DataPtr, ByteCount, m_Offset);
+		Write(DataPtr, ByteCount, m_Offset);
 		m_Offset += ByteCount;
 	}
 
 	inline uint64_t CurrentOffset() const { return m_Offset; }
 
-private:
-	RefPtr<MemoryOutStream> m_Stream;
-	uint64_t				m_Offset = 0;
-};
-
-/**
- * Stream which reads from a memory buffer
- */
-class MemoryInStream : public RefCounted
-{
-public:
-	MemoryInStream(const void* Buffer, size_t Size);
-	MemoryInStream(MemoryView View) : MemoryInStream(View.GetData(), View.GetSize()) {}
-	~MemoryInStream() = default;
-
-	void				  Read(void* DataPtr, size_t ByteCount, uint64_t ReadOffset);
-	uint64_t			  Size() const { return m_Buffer.size(); }
-	uint64_t			  GetSize() const { return Size(); }
 	inline const uint8_t* Data() const { return m_Buffer.data(); }
 	inline const uint8_t* GetData() const { return m_Buffer.data(); }
+	inline uint64_t		  Size() const { return m_Buffer.size(); }
+	inline uint64_t		  GetSize() const { return m_Buffer.size(); }
 
 private:
 	RwLock				 m_Lock;
 	std::vector<uint8_t> m_Buffer;
+	uint64_t			 m_Offset = 0;
+
+	void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset);
 };
 
+inline MemoryView
+MakeMemoryView(const BinaryWriter& Stream)
+{
+	return MemoryView(Stream.Data(), Stream.Size());
+}
+
 /**
  * Binary stream reader
  */
@@ -91,90 +55,29 @@ private:
 class BinaryReader
 {
 public:
-	inline BinaryReader(MemoryInStream& Stream) : m_Stream(&Stream) {}
-	~BinaryReader() = default;
-
-	inline void Read(void* DataPtr, size_t ByteCount)
-	{
-		m_Stream->Read(DataPtr, ByteCount, m_Offset);
-		m_Offset += ByteCount;
-	}
-
-	void Seek(uint64_t Offset)
+	inline BinaryReader(const void* Buffer, uint64_t Size) : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer)), m_BufferSize(Size) {}
+	inline BinaryReader(MemoryView Buffer)
+	: m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer.GetData()))
+	, m_BufferSize(Buffer.GetSize())
 	{
-		ZEN_ASSERT(Offset <= m_Stream->Size());
-		m_Offset = Offset;
 	}
 
-	void Skip(uint64_t SkipOffset)
+	inline void Read(void* DataPtr, size_t ByteCount)
 	{
-		ZEN_ASSERT((m_Offset + SkipOffset) <= m_Stream->Size());
-		m_Offset += SkipOffset;
+		memcpy(DataPtr, m_BufferBase + m_Offset, ByteCount);
+		m_Offset += ByteCount;
 	}
 
+	inline uint64_t Size() const { return m_BufferSize; }
+	inline uint64_t GetSize() const { return Size(); }
 	inline uint64_t CurrentOffset() const { return m_Offset; }
-	inline uint64_t AvailableBytes() const { return m_Stream->Size() - m_Offset; }
 
 private:
-	RefPtr<MemoryInStream> m_Stream;
-	uint64_t			   m_Offset = 0;
+	const uint8_t* m_BufferBase;
+	uint64_t	   m_BufferSize;
+	uint64_t	   m_Offset = 0;
 };
 
-inline BinaryReader&
-operator>>(BinaryReader& Reader, bool& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, int8_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, int16_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, int32_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, int64_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, uint8_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, uint16_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, uint32_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-inline BinaryReader&
-operator>>(BinaryReader& Reader, uint64_t& Value)
-{
-	Reader.Read(&Value, sizeof Value);
-	return Reader;
-}
-
 void stream_forcelink();  // internal
 
 }  // namespace zen
diff --git a/zencore/include/zencore/streamutil.h b/zencore/include/zencore/streamutil.h
deleted file mode 100644
index d9b6b5167..000000000
--- a/zencore/include/zencore/streamutil.h
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <fmt/format.h>
-#include <zencore/string.h>
-#include <string>
-#include <string_view>
-
-#include "blake3.h"
-#include "iohash.h"
-#include "sha1.h"
-#include "stream.h"
-
-namespace zen {
-
-}  // namespace zen
-
-//////////////////////////////////////////////////////////////////////////
-
-template<>
-struct fmt::formatter<zen::IoHash>
-{
-	constexpr auto parse(format_parse_context& ctx)
-	{
-		// Parse the presentation format and store it in the formatter:
-		auto it = ctx.begin(), end = ctx.end();
-
-		// Check if reached the end of the range:
-		if (it != end && *it != '}')
-			throw format_error("invalid format");
-
-		// Return an iterator past the end of the parsed range:
-		return it;
-	}
-
-	template<typename FormatContext>
-	auto format(const zen::IoHash& h, FormatContext& ctx)
-	{
-		zen::ExtendableStringBuilder<48> String;
-		h.ToHexString(String);
-		return format_to(ctx.out(), std::string_view(String));
-	}
-};
-
-template<>
-struct fmt::formatter<zen::BLAKE3>
-{
-	constexpr auto parse(format_parse_context& ctx)
-	{
-		// Parse the presentation format and store it in the formatter:
-		auto it = ctx.begin(), end = ctx.end();
-
-		// Check if reached the end of the range:
-		if (it != end && *it != '}')
-			throw format_error("invalid format");
-
-		// Return an iterator past the end of the parsed range:
-		return it;
-	}
-
-	template<typename FormatContext>
-	auto format(const zen::BLAKE3& h, FormatContext& ctx)
-	{
-		zen::ExtendableStringBuilder<80> String;
-		h.ToHexString(String);
-		return format_to(ctx.out(), std::string_view(String));
-	}
-};
diff --git a/zencore/stream.cpp b/zencore/stream.cpp
index b9a88de66..aa9705764 100644
--- a/zencore/stream.cpp
+++ b/zencore/stream.cpp
@@ -10,41 +10,19 @@
 
 namespace zen {
 
-MemoryInStream::MemoryInStream(const void* buffer, size_t size)
-: m_Buffer(reinterpret_cast<const uint8_t*>(buffer), reinterpret_cast<const uint8_t*>(buffer) + size)
-{
-}
-
 void
-MemoryInStream::Read(void* buffer, size_t byteCount, uint64_t offset)
+BinaryWriter::Write(const void* data, size_t ByteCount, uint64_t Offset)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
 
-	const size_t needEnd = offset + byteCount;
-
-	if (needEnd > m_Buffer.size())
-		throw std::runtime_error("read past end of file!");	 // TODO: better exception
+	const size_t NeedEnd = Offset + ByteCount;
 
-	memcpy(buffer, m_Buffer.data() + offset, byteCount);
-}
+	if (NeedEnd > m_Buffer.size())
+	{
+		m_Buffer.resize(NeedEnd);
+	}
 
-void
-MemoryOutStream::Write(const void* data, size_t byteCount, uint64_t offset)
-{
-	RwLock::ExclusiveLockScope _(m_Lock);
-
-	const size_t needEnd = offset + byteCount;
-
-	if (needEnd > m_Buffer.size())
-		m_Buffer.resize(needEnd);
-
-	memcpy(m_Buffer.data() + offset, data, byteCount);
-}
-
-void
-MemoryOutStream::Flush()
-{
-	// No-op
+	memcpy(m_Buffer.data() + Offset, data, ByteCount);
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj
index 947ae9f10..421802d3f 100644
--- a/zencore/zencore.vcxproj
+++ b/zencore/zencore.vcxproj
@@ -141,7 +141,6 @@
     <ClInclude Include="include\zencore\sharedbuffer.h" />
     <ClInclude Include="include\zencore\stats.h" />
     <ClInclude Include="include\zencore\stream.h" />
-    <ClInclude Include="include\zencore\streamutil.h" />
     <ClInclude Include="include\zencore\string.h" />
     <ClInclude Include="include\zencore\targetver.h" />
     <ClInclude Include="include\zencore\testing.h" />
diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters
index e3d1f6c67..e701e9354 100644
--- a/zencore/zencore.vcxproj.filters
+++ b/zencore/zencore.vcxproj.filters
@@ -12,7 +12,6 @@
     <ClInclude Include="include\zencore\timer.h" />
     <ClInclude Include="include\zencore\thread.h" />
     <ClInclude Include="include\zencore\string.h" />
-    <ClInclude Include="include\zencore\streamutil.h" />
     <ClInclude Include="include\zencore\stream.h" />
     <ClInclude Include="include\zencore\stats.h" />
     <ClInclude Include="include\zencore\blake3.h" />