Merge remote-tracking branch 'origin/main' into de/cache-with-block-store

5 files changed, 344 insertions, 9 deletions

diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index ffc1da10c..a51253989 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -3,6 +3,7 @@
 #include "zencore/compactbinary.h"
 
 #include <zencore/base64.h>
+#include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinaryvalidation.h>
 #include <zencore/compactbinaryvalue.h>
 #include <zencore/compress.h>
@@ -22,10 +23,9 @@
 #	include <time.h>
 #endif
 
-#if ZEN_WITH_TESTS
-#	include <json11.hpp>
-#	include <zencore/compactbinarybuilder.h>
-#endif
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <json11.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
@@ -1715,6 +1715,225 @@ CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder)
 
 //////////////////////////////////////////////////////////////////////////
 
+class CbJsonReader
+{
+public:
+	static CbFieldIterator Read(std::string_view JsonText, std::string& Error)
+	{
+		using namespace json11;
+
+		const Json Json = Json::parse(std::string(JsonText), Error);
+
+		if (Error.empty())
+		{
+			CbWriter Writer;
+			if (ReadField(Writer, Json, std::string_view(), Error))
+			{
+				return Writer.Save();
+			}
+		}
+
+		return CbFieldIterator();
+	}
+
+private:
+	static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error)
+	{
+		using namespace json11;
+
+		switch (Json.type())
+		{
+			case Json::Type::OBJECT:
+				{
+					if (FieldName.empty())
+					{
+						Writer.BeginObject();
+					}
+					else
+					{
+						Writer.BeginObject(FieldName);
+					}
+
+					for (const auto& Kv : Json.object_items())
+					{
+						const std::string&	Name = Kv.first;
+						const json11::Json& Item = Kv.second;
+
+						if (ReadField(Writer, Item, Name, Error) == false)
+						{
+							return false;
+						}
+					}
+
+					Writer.EndObject();
+				}
+				break;
+			case Json::Type::ARRAY:
+				{
+					if (FieldName.empty())
+					{
+						Writer.BeginArray();
+					}
+					else
+					{
+						Writer.BeginArray(FieldName);
+					}
+
+					for (const json11::Json& Item : Json.array_items())
+					{
+						if (ReadField(Writer, Item, std::string_view(), Error) == false)
+						{
+							return false;
+						}
+					}
+
+					Writer.EndArray();
+				}
+				break;
+			case Json::Type::NUL:
+				{
+					if (FieldName.empty())
+					{
+						Writer.AddNull();
+					}
+					else
+					{
+						Writer.AddNull(FieldName);
+					}
+				}
+				break;
+			case Json::Type::BOOL:
+				{
+					if (FieldName.empty())
+					{
+						Writer.AddBool(Json.bool_value());
+					}
+					else
+					{
+						Writer.AddBool(FieldName, Json.bool_value());
+					}
+				}
+				break;
+			case Json::Type::NUMBER:
+				{
+					if (FieldName.empty())
+					{
+						Writer.AddFloat(Json.number_value());
+					}
+					else
+					{
+						Writer.AddFloat(FieldName, Json.number_value());
+					}
+				}
+				break;
+			case Json::Type::STRING:
+				{
+					Oid Id;
+					if (TryParseObjectId(Json.string_value(), Id))
+					{
+						if (FieldName.empty())
+						{
+							Writer.AddObjectId(Id);
+						}
+						else
+						{
+							Writer.AddObjectId(FieldName, Id);
+						}
+
+						return true;
+					}
+
+					IoHash Hash;
+					if (TryParseIoHash(Json.string_value(), Hash))
+					{
+						if (FieldName.empty())
+						{
+							Writer.AddHash(Hash);
+						}
+						else
+						{
+							Writer.AddHash(FieldName, Hash);
+						}
+
+						return true;
+					}
+
+					if (FieldName.empty())
+					{
+						Writer.AddString(Json.string_value());
+					}
+					else
+					{
+						Writer.AddString(FieldName, Json.string_value());
+					}
+				}
+				break;
+			default:
+				break;
+		}
+
+		return true;
+	}
+
+	static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF");
+
+	static bool TryParseObjectId(std::string_view Str, Oid& Id)
+	{
+		using namespace std::literals;
+
+		if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet))
+		{
+			Id = Oid::FromHexString(Str);
+			return true;
+		}
+
+		if (Str.starts_with("0x"sv))
+		{
+			return TryParseObjectId(Str.substr(2), Id);
+		}
+
+		return false;
+	}
+
+	static bool TryParseIoHash(std::string_view Str, IoHash& Hash)
+	{
+		using namespace std::literals;
+
+		if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet))
+		{
+			Hash = IoHash::FromHexString(Str);
+			return true;
+		}
+
+		if (Str.starts_with("0x"sv))
+		{
+			return TryParseIoHash(Str.substr(2), Hash);
+		}
+
+		return false;
+	}
+};
+
+CbFieldIterator
+LoadCompactBinaryFromJson(std::string_view Json, std::string& Error)
+{
+	if (Json.empty() == false)
+	{
+		return CbJsonReader::Read(Json, Error);
+	}
+
+	return CbFieldIterator();
+}
+
+CbFieldIterator
+LoadCompactBinaryFromJson(std::string_view Json)
+{
+	std::string Error;
+	return LoadCompactBinaryFromJson(Json, Error);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 #if ZEN_WITH_TESTS
 void
 uson_forcelink()
@@ -1970,6 +2189,107 @@ TEST_CASE("uson.datetime")
 		CHECK_EQ(D72.GetSecond(), 10);
 	}
 }
+
+TEST_CASE("json.uson")
+{
+	using namespace std::literals;
+	using namespace json11;
+
+	SUBCASE("empty")
+	{
+		CbFieldIterator It = LoadCompactBinaryFromJson(""sv);
+		CHECK(It.HasValue() == false);
+	}
+
+	SUBCASE("object")
+	{
+		const Json JsonObject = Json::object{{"Null", nullptr},
+											 {"String", "Value1"},
+											 {"Bool", true},
+											 {"Number", 46.2},
+											 {"Array", Json::array{1, 2, 3}},
+											 {"Object",
+											  Json::object{
+												  {"String", "Value2"},
+											  }}};
+
+		CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["Null"].IsNull());
+		CHECK(Cb["String"].AsString() == "Value1"sv);
+		CHECK(Cb["Bool"].AsBool());
+		CHECK(Cb["Number"].AsDouble() == 46.2);
+		CHECK(Cb["Object"].IsObject());
+		CbObjectView Object = Cb["Object"].AsObjectView();
+		CHECK(Object["String"].AsString() == "Value2"sv);
+	}
+
+	SUBCASE("array")
+	{
+		const Json JsonArray = Json::array{42, 43, 44};
+		CbArray	   Cb		 = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray();
+
+		auto It = Cb.CreateIterator();
+		CHECK((*It).AsDouble() == 42);
+		It++;
+		CHECK((*It).AsDouble() == 43);
+		It++;
+		CHECK((*It).AsDouble() == 44);
+	}
+
+	SUBCASE("objectid")
+	{
+		const Oid& Id = Oid::NewOid();
+
+		StringBuilder<64> Sb;
+		Id.ToString(Sb);
+
+		Json	 JsonObject = Json::object{{"value", Sb.ToString()}};
+		CbObject Cb			= LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsObjectId());
+		CHECK(Cb["value"sv].AsObjectId() == Id);
+
+		Sb.Reset();
+		Sb << "0x";
+		Id.ToString(Sb);
+
+		JsonObject = Json::object{{"value", Sb.ToString()}};
+		Cb		   = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsObjectId());
+		CHECK(Cb["value"sv].AsObjectId() == Id);
+	}
+
+	SUBCASE("iohash")
+	{
+		const uint8_t Data[] = {
+			1,
+			2,
+			3,
+			4,
+			5,
+			6,
+			7,
+			8,
+			9,
+		};
+
+		const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data));
+
+		Json	 JsonObject = Json::object{{"value", Hash.ToHexString()}};
+		CbObject Cb			= LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsHash());
+		CHECK(Cb["value"sv].AsHash() == Hash);
+
+		JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}};
+		Cb		   = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject();
+
+		CHECK(Cb["value"sv].IsHash());
+		CHECK(Cb["value"sv].AsHash() == Hash);
+	}
+}
 #endif
 
 }  // namespace zen
diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp
index 5111504e1..1d2ba45df 100644
--- a/zencore/compactbinarybuilder.cpp
+++ b/zencore/compactbinarybuilder.cpp
@@ -436,9 +436,10 @@ CbWriter::AddNull()
 void
 CbWriter::AddBinary(const void* const Value, const uint64_t Size)
 {
+	const size_t SizeByteCount = MeasureVarUInt(Size);
+	Data.reserve(Data.size() + 1 + SizeByteCount + Size);
 	BeginField();
-	const uint32_t SizeByteCount = MeasureVarUInt(Size);
-	const int64_t  SizeOffset	 = Data.size();
+	const size_t SizeOffset = Data.size();
 	Data.resize(Data.size() + SizeByteCount);
 	WriteVarUInt(Size, Data.data() + SizeOffset);
 	Data.insert(Data.end(), static_cast<const uint8_t*>(Value), static_cast<const uint8_t*>(Value) + Size);
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index 19f1597dc..eba4a1694 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -1405,6 +1405,12 @@ ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload);
 ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload);
 ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload);
 
+/**
+ * Load a compact binary from JSON.
+ */
+ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error);
+ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json);
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /**
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index 027730063..012ee73ee 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -999,7 +999,7 @@ public:
 	static constexpr bool HasOnly(const StringType& Str, AsciiSet Set)
 	{
 		auto End = Str.data() + Str.size();
-		return FindFirst<EInclude::Members>(Set, GetData(Str), End) == End;
+		return FindFirst<EInclude::Members>(Set, Str.data(), End) == End;
 	}
 
 private:
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index 8a3ab8427..c069aa0f1 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -226,7 +226,15 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
 	m_DataPtr = nullptr;
 }
 
-static RwLock g_MappingLock;
+static RwLock g_MappingLock[0x40];
+
+static RwLock&
+MappingLockForInstance(const IoBufferExtendedCore* instance)
+{
+	intptr_t base		= (intptr_t)instance;
+	size_t	 lock_index = ((base >> 8) ^ (base >> 16)) & 0x3f;
+	return g_MappingLock[lock_index];
+}
 
 void
 IoBufferExtendedCore::Materialize() const
@@ -237,7 +245,7 @@ IoBufferExtendedCore::Materialize() const
 	if (m_Flags.load(std::memory_order_acquire) & kIsMaterialized)
 		return;
 
-	RwLock::ExclusiveLockScope _(g_MappingLock);
+	RwLock::ExclusiveLockScope _(MappingLockForInstance(this));
 
 	// Someone could have gotten here first
 	// We can use memory_order_relaxed on this load because the mutex has already provided the fence