gc history log (#519)

- Feature: Writes a `gc.log` with settings and detailed result after each GC execution (version 2 only)
- Break out file name rotate to allow access for gclog
- CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder)

4 files changed, 215 insertions, 18 deletions

diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp
index 416b49f62..5e8ce22ed 100644
--- a/src/zencore/compactbinary.cpp
+++ b/src/zencore/compactbinary.cpp
@@ -1472,6 +1472,40 @@ class CbJsonWriter
 public:
 	explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; }
 
+	void BeginObject()
+	{
+		Builder << '{';
+		NewLineAndIndent << '\t';
+		NeedsNewLine = true;
+	}
+
+	void EndObject()
+	{
+		NewLineAndIndent.RemoveSuffix(1);
+		if (NeedsComma)
+		{
+			WriteOptionalNewLine();
+		}
+		Builder << '}';
+	}
+
+	void BeginArray()
+	{
+		Builder << '[';
+		NewLineAndIndent << '\t';
+		NeedsNewLine = true;
+	}
+
+	void EndArray()
+	{
+		NewLineAndIndent.RemoveSuffix(1);
+		if (NeedsComma)
+		{
+			WriteOptionalNewLine();
+		}
+		Builder << ']';
+	}
+
 	void WriteField(CbFieldView Field)
 	{
 		using namespace std::literals;
@@ -1493,37 +1527,23 @@ public:
 			case CbFieldType::Object:
 			case CbFieldType::UniformObject:
 				{
-					Builder << '{';
-					NewLineAndIndent << '\t';
-					NeedsNewLine = true;
+					BeginObject();
 					for (CbFieldView It : Field)
 					{
 						WriteField(It);
 					}
-					NewLineAndIndent.RemoveSuffix(1);
-					if (NeedsComma)
-					{
-						WriteOptionalNewLine();
-					}
-					Builder << '}';
+					EndObject();
 				}
 				break;
 			case CbFieldType::Array:
 			case CbFieldType::UniformArray:
 				{
-					Builder << '[';
-					NewLineAndIndent << '\t';
-					NeedsNewLine = true;
+					BeginArray();
 					for (CbFieldView It : Field)
 					{
 						WriteField(It);
 					}
-					NewLineAndIndent.RemoveSuffix(1);
-					if (NeedsComma)
-					{
-						WriteOptionalNewLine();
-					}
-					Builder << ']';
+					EndArray();
 				}
 				break;
 			case CbFieldType::Binary:
@@ -1744,6 +1764,62 @@ CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder)
 	Writer.WriteField(Array.AsFieldView());
 }
 
+void
+CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder)
+{
+	std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data);
+	CbJsonWriter			 Writer(InBuilder);
+	if (!Fields.empty())
+	{
+		if (Fields.size() == 1)
+		{
+			Writer.WriteField(Fields[0]);
+			return;
+		}
+		bool UseTopLevelObject = Fields[0].HasName();
+		if (UseTopLevelObject)
+		{
+			Writer.BeginObject();
+		}
+		else
+		{
+			Writer.BeginArray();
+		}
+		for (const CbFieldView& Field : Fields)
+		{
+			Writer.WriteField(Field);
+		}
+		if (UseTopLevelObject)
+		{
+			Writer.EndObject();
+		}
+		else
+		{
+			Writer.EndArray();
+		}
+	}
+}
+
+std::vector<CbFieldView>
+ReadCompactBinaryStream(MemoryView Data)
+{
+	std::vector<CbFieldView> Result;
+	const uint8_t*			 Buffer = reinterpret_cast<const uint8_t*>(Data.GetData());
+	uint64_t				 Offset = 0;
+	const uint64_t			 Size	= Data.GetSize();
+	while (Offset < Size)
+	{
+		if (ValidateCompactBinary(MemoryView(Buffer + Offset, Size - Offset), CbValidateMode::Default) != CbValidateError::None)
+		{
+			break;
+		}
+		CbFieldView Field(Buffer + Offset);
+		Offset += Field.GetSize();
+		Result.emplace_back(std::move(Field));
+	}
+	return Result;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 class CbJsonReader
@@ -2136,6 +2212,8 @@ TEST_CASE("uson.null")
 
 TEST_CASE("uson.json")
 {
+	using namespace std::literals;
+
 	SUBCASE("string")
 	{
 		CbObjectWriter Writer;
@@ -2208,6 +2286,52 @@ TEST_CASE("uson.json")
 		CHECK(FloatValue == 0);
 		CHECK(DoubleValue == 0);
 	}
+
+	SUBCASE("stream")
+	{
+		const auto MakeObject = [&](std::string_view Name, const std::vector<int>& Fields) -> CbObject {
+			CbWriter Writer;
+			Writer.SetName(Name);
+			Writer.BeginObject();
+			for (const auto& Field : Fields)
+			{
+				Writer.AddInteger(fmt::format("{}", Field), Field);
+			}
+			Writer.EndObject();
+			return Writer.Save().AsObject();
+		};
+
+		std::vector<uint8_t> Buffer;
+
+		auto AppendToBuffer = [&](const void* Data, size_t Count) {
+			const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data);
+			Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count);
+		};
+
+		auto Append = [&](const CbFieldView& Field) {
+			Field.WriteToStream([&](const void* Data, size_t Count) {
+				const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data);
+				Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count);
+			});
+		};
+
+		CbObject DataObjects[] = {MakeObject("Empty object"sv, {}),
+								  MakeObject("OneField object"sv, {5}),
+								  MakeObject("TwoField object"sv, {-5, 999}),
+								  MakeObject("ThreeField object"sv, {1, 2, -129})};
+		for (const CbObject& Object : DataObjects)
+		{
+			Object.AsField().WriteToStream(AppendToBuffer);
+		}
+
+		ExtendableStringBuilder<128> Sb;
+		CompactBinaryToJson(MemoryView(Buffer.data(), Buffer.size()), Sb);
+		std::string	 JsonText = Sb.ToString().c_str();
+		std::string	 JsonError;
+		json11::Json Json			  = json11::Json::parse(JsonText, JsonError);
+		std::string	 ParsedJsonString = Json.dump();
+		CHECK(!ParsedJsonString.empty());
+	}
 }
 
 TEST_CASE("uson.datetime")
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index b64c9973e..06cda7382 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -40,6 +40,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #	include <unistd.h>
 #endif
 
+#include <fmt/format.h>
 #include <filesystem>
 #include <gsl/gsl-lite.hpp>
 
@@ -1396,6 +1397,69 @@ GetEnvVariable(std::string_view VariableName)
 	return "";
 }
 
+std::error_code
+RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles)
+{
+	const std::filesystem::path BasePath(Filename.parent_path());
+	const std::string			Stem(Filename.stem().string());
+	const std::string			Extension(Filename.extension().string());
+
+	std::error_code Result;
+
+	auto GetFileName = [&](size_t Index) -> std::filesystem::path {
+		if (Index == 0)
+		{
+			return BasePath / (Stem + Extension);
+		}
+		return BasePath / fmt::format("{}.{}{}", Stem, Index, Extension);
+	};
+
+	auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool {
+		bool Exists = std::filesystem::exists(Path, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		if (!Exists)
+		{
+			return true;
+		}
+		uintmax_t Size = std::filesystem::file_size(Path, Ec);
+		if (Ec)
+		{
+			return false;
+		}
+		return Size == 0;
+	};
+
+	bool BaseIsEmpty = IsEmpty(GetFileName(0), Result);
+	if (Result)
+	{
+		return Result;
+	}
+	if (!BaseIsEmpty)
+	{
+		// We try our best to rotate the logs, if we fail we fail and will try to open the base log file anyway
+		for (auto i = MaxFiles; i > 0; i--)
+		{
+			std::filesystem::path src = GetFileName(i - 1);
+			if (!std::filesystem::exists(src))
+			{
+				continue;
+			}
+			std::error_code		  DummyEc;
+			std::filesystem::path target = GetFileName(i);
+			if (std::filesystem::exists(target, DummyEc))
+			{
+				std::filesystem::remove(target, DummyEc);
+			}
+			std::filesystem::rename(src, target, DummyEc);
+		}
+	}
+
+	return Result;
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h
index 66e621a8a..cb032e34a 100644
--- a/src/zencore/include/zencore/compactbinary.h
+++ b/src/zencore/include/zencore/compactbinary.h
@@ -1470,6 +1470,13 @@ end(CbFieldView&)
 	return CbFieldViewIterator();
 }
 
+/**
+ * Serialize serialized compact binary blob to jaons. It must be 0 to n fields with including type for each field
+ */
+ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder);
+
+ZENCORE_API std::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data);
+
 void uson_forcelink();	// internal
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 075188993..22eb40e45 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -211,6 +211,8 @@ void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, Di
 
 std::string GetEnvVariable(std::string_view VariableName);
 
+std::error_code RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles);
+
 //////////////////////////////////////////////////////////////////////////
 
 void filesystem_forcelink();  // internal