Merge branch 'main' into use-catch2

8 files changed, 441 insertions, 9 deletions

diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index aa87fb4cb..375a97fc5 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -3,6 +3,7 @@
 #include "zencore/compactbinary.h"
 
 #include <zencore/base64.h>
+#include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinaryvalidation.h>
 #include <zencore/compactbinaryvalue.h>
 #include <zencore/compress.h>
@@ -22,10 +23,9 @@
 #	include <time.h>
 #endif
 
-#if ZEN_WITH_TESTS
-#	include <json11.hpp>
-#	include <zencore/compactbinarybuilder.h>
-#endif
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <json11.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
@@ -1715,6 +1715,225 @@ CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder)
 
 //////////////////////////////////////////////////////////////////////////
 
+class CbJsonReader
+{
+public:
+	static CbFieldIterator Read(std::string_view JsonText, std::string& Error)
+	{
+		using namespace json11;
+
+		const Json Json = Json::parse(std::string(JsonText), Error);
+
+		if (Error.empty())
+		{
+			CbWriter Writer;
+			if (ReadField(Writer, Json, std::string_view(), Error))
+			{
+				return Writer.Save();
+			}
+		}
+
+		return CbFieldIterator();
+	}
+
+private:
+	static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error)
+	{
+		using namespace json11;
+
+		switch (Json.type())
+		{
+			case Json::Type::OBJECT:
+				{
+					if (FieldName.empty())
+					{
+						Writer.BeginObject();
+					}
+					else
+					{
+						Writer.BeginObject(FieldName);
+					}
+
+					for (const auto& Kv : Json.object_items())
+					{
+						const std::string&	Name = Kv.first;
+						const json11::Json& Item = Kv.second;
+
+						if (ReadField(Writer, Item, Name, Error) == false)
+						{
+							return false;
+						}
+					}
+
+					Writer.EndObject();
+				}
+				break;
+			case Json::Type::ARRAY:
+				{
+					if (FieldName.empty())
+					{
+						Writer.BeginArray();
+					}
+					else
+					{
+						Writer.BeginArray(FieldName);
+					}
+
+					for (const json11::Json& Item : Json.array_items())
+					{
+						if (ReadField(Writer, Item, std::string_view(), Error) == false)
+						{
+							return false;
+						}
+					}
+
+					Writer.EndArray();
+				}
+				break;
+			case Json::Type::NUL:
+				{
+					if (FieldName.empty())
+					{
+						Writer.AddNull();
+					}
+					else
+					{
+						Writer.AddNull(FieldName);
+					}
+				}
+				break;
+			case Json::Type::BOOL:
+				{
+					if (FieldName.empty())
+					{
+						Writer.AddBool(Json.bool_value());
+					}
+					else
+					{
+						Writer.AddBool(FieldName, Json.bool_value());
+					}
+				}
+				break;
+			case Json::Type::NUMBER:
+				{
+					if (FieldName.empty())
+					{
+						Writer.AddFloat(Json.number_value());
+					}
+					else
+					{
+						Writer.AddFloat(FieldName, Json.number_value());
+					}
+				}
+				break;
+			case Json::Type::STRING:
+				{
+					Oid Id;
+					if (TryParseObjectId(Json.string_value(), Id))
+					{
+						if (FieldName.empty())
+						{
+							Writer.AddObjectId(Id);
+						}
+						else
+						{
+							Writer.AddObjectId(FieldName, Id);
+						}
+
+						return true;
+					}
+
+					IoHash Hash;
+					if (TryParseIoHash(Json.string_value(), Hash))
+					{
+						if (FieldName.empty())
+						{
+							Writer.AddHash(Hash);
+						}
+						else
+						{
+							Writer.AddHash(FieldName, Hash);
+						}
+
+						return true;
+					}
+
+					if (FieldName.empty())
+					{
+						Writer.AddString(Json.string_value());
+					}
+					else
+					{
+						Writer.AddString(FieldName, Json.string_value());
+					}
+				}
+				break;
+			default:
+				break;
+		}
+
+		return true;
+	}
+
+	static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF");
+
+	static bool TryParseObjectId(std::string_view Str, Oid& Id)
+	{
+		using namespace std::literals;
+
+		if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet))
+		{
+			Id = Oid::FromHexString(Str);
+			return true;
+		}
+
+		if (Str.starts_with("0x"sv))
+		{
+			return TryParseObjectId(Str.substr(2), Id);
+		}
+
+		return false;
+	}
+
+	static bool TryParseIoHash(std::string_view Str, IoHash& Hash)
+	{
+		using namespace std::literals;
+
+		if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet))
+		{
+			Hash = IoHash::FromHexString(Str);
+			return true;
+		}
+
+		if (Str.starts_with("0x"sv))
+		{
+			return TryParseIoHash(Str.substr(2), Hash);
+		}
+
+		return false;
+	}
+};
+
+CbFieldIterator
+LoadCompactBinaryFromJson(std::string_view Json, std::string& Error)
+{
+	if (Json.empty() == false)
+	{
+		return CbJsonReader::Read(Json, Error);
+	}
+
+	return CbFieldIterator();
+}
+
+CbFieldIterator
+LoadCompactBinaryFromJson(std::string_view Json)
+{
+	std::string Error;
+	return LoadCompactBinaryFromJson(Json, Error);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 #if ZEN_WITH_TESTS
 void
 uson_forcelink()
@@ -1970,6 +2189,107 @@ TEST_CASE("uson.datetime")
 		CHECK_EQ(D72.GetSecond(), 10);
 	}
 }
+
+TEST_CASE("json.uson")
+{
+	using namespace std::literals;
+	using namespace json11;
+
+	SUBCASE("empty")
+	{
+		CbFieldIterator It = LoadCompactBinaryFromJson(""sv);
+		CHECK(It.HasValue() == false);
+	}
+
+	SUBCASE("object")
+	{
+		const Json JsonObject = Json::object{{"Null", nullptr},
+											 {"String", "Value1"},
+											 {"Bool", true},
+											 {"Number", 46.2},
+											 {"Array", Json::array{1, 2, 3}},
+											 {"Object",
+											  Json::object{
+												  {"String", "Value2"},
+											  }}};
+
+		CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["Null"].IsNull());
+		CHECK(Cb["String"].AsString() == "Value1"sv);
+		CHECK(Cb["Bool"].AsBool());
+		CHECK(Cb["Number"].AsDouble() == 46.2);
+		CHECK(Cb["Object"].IsObject());
+		CbObjectView Object = Cb["Object"].AsObjectView();
+		CHECK(Object["String"].AsString() == "Value2"sv);
+	}
+
+	SUBCASE("array")
+	{
+		const Json JsonArray = Json::array{42, 43, 44};
+		CbArray	   Cb		 = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray();
+
+		auto It = Cb.CreateIterator();
+		CHECK((*It).AsDouble() == 42);
+		It++;
+		CHECK((*It).AsDouble() == 43);
+		It++;
+		CHECK((*It).AsDouble() == 44);
+	}
+
+	SUBCASE("objectid")
+	{
+		const Oid& Id = Oid::NewOid();
+
+		StringBuilder<64> Sb;
+		Id.ToString(Sb);
+
+		Json	 JsonObject = Json::object{{"value", Sb.ToString()}};
+		CbObject Cb			= LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsObjectId());
+		CHECK(Cb["value"sv].AsObjectId() == Id);
+
+		Sb.Reset();
+		Sb << "0x";
+		Id.ToString(Sb);
+
+		JsonObject = Json::object{{"value", Sb.ToString()}};
+		Cb		   = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsObjectId());
+		CHECK(Cb["value"sv].AsObjectId() == Id);
+	}
+
+	SUBCASE("iohash")
+	{
+		const uint8_t Data[] = {
+			1,
+			2,
+			3,
+			4,
+			5,
+			6,
+			7,
+			8,
+			9,
+		};
+
+		const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data));
+
+		Json	 JsonObject = Json::object{{"value", Hash.ToHexString()}};
+		CbObject Cb			= LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsHash());
+		CHECK(Cb["value"sv].AsHash() == Hash);
+
+		JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}};
+		Cb		   = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsHash());
+		CHECK(Cb["value"sv].AsHash() == Hash);
+	}
+}
 #endif
 
 }  // namespace zen
diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp
index 262f4e7d3..d4ccd434d 100644
--- a/zencore/compactbinarybuilder.cpp
+++ b/zencore/compactbinarybuilder.cpp
@@ -436,9 +436,10 @@ CbWriter::AddNull()
 void
 CbWriter::AddBinary(const void* const Value, const uint64_t Size)
 {
+	const size_t SizeByteCount = MeasureVarUInt(Size);
+	Data.reserve(Data.size() + 1 + SizeByteCount + Size);
 	BeginField();
-	const uint32_t SizeByteCount = MeasureVarUInt(Size);
-	const int64_t  SizeOffset	 = Data.size();
+	const size_t SizeOffset = Data.size();
 	Data.resize(Data.size() + SizeByteCount);
 	WriteVarUInt(Size, Data.data() + SizeOffset);
 	Data.insert(Data.end(), static_cast<const uint8_t*>(Value), static_cast<const uint8_t*>(Value) + Size);
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 437741161..bd85f5a11 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -1022,6 +1022,40 @@ MaximizeOpenFileCount()
 #endif
 }
 
+void
+GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent)
+{
+	FileSystemTraversal Traversal;
+	struct Visitor : public FileSystemTraversal::TreeVisitor
+	{
+		Visitor(uint8_t Flags, DirectoryContent& OutContent) : Flags(Flags), Content(OutContent) {}
+
+		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
+							   [[maybe_unused]] const path_view&			 File,
+							   [[maybe_unused]] uint64_t					 FileSize) override
+		{
+			if (Flags & DirectoryContent::IncludeFilesFlag)
+			{
+				Content.Files.push_back(Parent / File);
+			}
+		}
+
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		{
+			if (Flags & DirectoryContent::IncludeDirsFlag)
+			{
+				Content.Directories.push_back(Parent / DirectoryName);
+			}
+			return (Flags & DirectoryContent::RecursiveFlag) != 0;
+		}
+
+		const uint8_t	  Flags;
+		DirectoryContent& Content;
+	} Visit(Flags, OutContent);
+
+	Traversal.TraverseFileSystem(RootDir, Visit);
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index 19f1597dc..eba4a1694 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -1405,6 +1405,12 @@ ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload);
 ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload);
 ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload);
 
+/**
+ * Load a compact binary from JSON.
+ */
+ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error);
+ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json);
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /**
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index a6e76eaa0..6d07a79b4 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -169,6 +169,17 @@ public:
 	void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
 };
 
+struct DirectoryContent
+{
+	static const uint8_t			   IncludeDirsFlag	= 1u << 0;
+	static const uint8_t			   IncludeFilesFlag = 1u << 1;
+	static const uint8_t			   RecursiveFlag	= 1u << 2;
+	std::vector<std::filesystem::path> Files;
+	std::vector<std::filesystem::path> Directories;
+};
+
+void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, DirectoryContent& OutContent);
+
 //////////////////////////////////////////////////////////////////////////
 
 void filesystem_forcelink();  // internal
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index bc8cfdc0f..5d9daa1c7 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -276,12 +276,14 @@ struct IoBufferExtendedCore : public IoBufferCore
 
 	void Materialize() const;
 	bool GetFileReference(IoBufferFileReference& OutRef) const;
+	void MarkAsDeleteOnClose();
 
 private:
 	void*		  m_FileHandle	  = nullptr;
 	uint64_t	  m_FileOffset	  = 0;
 	mutable void* m_MmapHandle	  = nullptr;
 	mutable void* m_MappedPointer = nullptr;
+	bool		  m_DeleteOnClose = false;
 };
 
 inline IoBufferExtendedCore*
@@ -377,6 +379,7 @@ public:
 	inline void							SetContentType(ZenContentType ContentType) { m_Core->SetContentType(ContentType); }
 	[[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); }
 	[[nodiscard]] ZENCORE_API bool		GetFileReference(IoBufferFileReference& OutRef) const;
+	void								MarkAsDeleteOnClose();
 
 	inline MemoryView		 GetView() const { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); }
 	inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Core->MutableDataPointer(), m_Core->DataBytes()); }
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index 027730063..92f567dae 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -9,6 +9,7 @@
 #include <string.h>
 #include <charconv>
 #include <codecvt>
+#include <compare>
 #include <concepts>
 #include <optional>
 #include <span>
@@ -795,6 +796,21 @@ StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length = -1)
 #endif
 }
 
+/**
+ * @brief
+ * Helper function to implement case sensitive spaceship operator for strings.
+ * MacOS clang version we use does not implement <=> for std::string
+ * @param Lhs string
+ * @param Rhs string
+ * @return std::strong_ordering indicating relationship between Lhs and Rhs
+ */
+inline auto
+caseSensitiveCompareStrings(const std::string& Lhs, const std::string& Rhs)
+{
+	int r = Lhs.compare(Rhs);
+	return r == 0 ? std::strong_ordering::equal : r < 0 ? std::strong_ordering::less : std::strong_ordering::greater;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 /**
@@ -999,7 +1015,7 @@ public:
 	static constexpr bool HasOnly(const StringType& Str, AsciiSet Set)
 	{
 		auto End = Str.data() + Str.size();
-		return FindFirst<EInclude::Members>(Set, GetData(Str), End) == End;
+		return FindFirst<EInclude::Members>(Set, Str.data(), End) == End;
 	}
 
 private:
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index 8a3ab8427..c4b7f7bdf 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -211,6 +211,18 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
 
 	if (LocalFlags & kOwnsFile)
 	{
+		if (m_DeleteOnClose)
+		{
+#if ZEN_PLATFORM_WINDOWS
+			// Mark file for deletion when final handle is closed
+			FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE};
+
+			SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi);
+#else
+			std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle);
+			unlink(FilePath.c_str());
+#endif
+		}
 #if ZEN_PLATFORM_WINDOWS
 		BOOL Success = CloseHandle(m_FileHandle);
 #else
@@ -226,7 +238,18 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
 	m_DataPtr = nullptr;
 }
 
-static RwLock g_MappingLock;
+static constexpr size_t MappingLockCount = 64;
+static_assert(IsPow2(MappingLockCount), "MappingLockCount must be power of two");
+
+static RwLock g_MappingLocks[MappingLockCount];
+
+static RwLock&
+MappingLockForInstance(const IoBufferExtendedCore* instance)
+{
+	intptr_t base		= (intptr_t)instance;
+	size_t	 lock_index = ((base >> 8) ^ (base >> 16)) & (MappingLockCount - 1u);
+	return g_MappingLocks[lock_index];
+}
 
 void
 IoBufferExtendedCore::Materialize() const
@@ -237,7 +260,7 @@ IoBufferExtendedCore::Materialize() const
 	if (m_Flags.load(std::memory_order_acquire) & kIsMaterialized)
 		return;
 
-	RwLock::ExclusiveLockScope _(g_MappingLock);
+	RwLock::ExclusiveLockScope _(MappingLockForInstance(this));
 
 	// Someone could have gotten here first
 	// We can use memory_order_relaxed on this load because the mutex has already provided the fence
@@ -287,6 +310,9 @@ IoBufferExtendedCore::Materialize() const
 
 	if (MappedBase == nullptr)
 	{
+#if ZEN_PLATFORM_WINDOWS
+		CloseHandle(NewMmapHandle);
+#endif	// ZEN_PLATFORM_WINDOWS
 		throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()),
 								fmt::format("MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'",
 											MapOffset,
@@ -316,6 +342,12 @@ IoBufferExtendedCore::GetFileReference(IoBufferFileReference& OutRef) const
 	return true;
 }
 
+void
+IoBufferExtendedCore::MarkAsDeleteOnClose()
+{
+	m_DeleteOnClose = true;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize))
@@ -378,6 +410,15 @@ IoBuffer::GetFileReference(IoBufferFileReference& OutRef) const
 	return false;
 }
 
+void
+IoBuffer::MarkAsDeleteOnClose()
+{
+	if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore())
+	{
+		ExtCore->MarkAsDeleteOnClose();
+	}
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 IoBuffer