Merge remote-tracking branch 'origin/main' into de/zen-service-command

10 files changed, 213 insertions, 27 deletions

diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp
index 9b06d4575..b22f2ec1f 100644
--- a/src/zencore/callstack.cpp
+++ b/src/zencore/callstack.cpp
@@ -179,19 +179,26 @@ FormatCallstack(const CallstackFrames* Callstack, StringBuilderBase& SB, std::st
 	bool First = true;
 	for (const std::string& Symbol : GetFrameSymbols(Callstack))
 	{
-		if (!First)
+		try
 		{
-			SB.Append("\n");
-		}
-		else
-		{
-			First = false;
+			if (!First)
+			{
+				SB.Append("\n");
+			}
+			else
+			{
+				First = false;
+			}
+			if (!Prefix.empty())
+			{
+				SB.Append(Prefix);
+			}
+			SB.Append(Symbol);
 		}
-		if (!Prefix.empty())
+		catch (const std::exception&)
 		{
-			SB.Append(Prefix);
+			break;
 		}
-		SB.Append(Symbol);
 	}
 }
 
diff --git a/src/zencore/compactbinaryfile.cpp b/src/zencore/compactbinaryfile.cpp
index f2121a0bd..1526c21d5 100644
--- a/src/zencore/compactbinaryfile.cpp
+++ b/src/zencore/compactbinaryfile.cpp
@@ -19,7 +19,7 @@ LoadCompactBinaryObject(const std::filesystem::path& FilePath)
 
 	IoBuffer ObjectBuffer = ObjectFile.Flatten();
 
-	if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::All); Result == CbValidateError::None)
+	if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::Default); Result == CbValidateError::None)
 	{
 		CbObject	 Object	  = LoadCompactBinaryObject(ObjectBuffer);
 		const IoHash WorkerId = IoHash::HashBuffer(ObjectBuffer);
diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp
index 68ed09549..02f22ba4d 100644
--- a/src/zencore/compactbinaryjson.cpp
+++ b/src/zencore/compactbinaryjson.cpp
@@ -22,7 +22,10 @@ namespace zen {
 class CbJsonWriter
 {
 public:
-	explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; }
+	explicit CbJsonWriter(StringBuilderBase& InBuilder, bool AddTypeComment) : Builder(InBuilder), m_AddTypeComment(AddTypeComment)
+	{
+		NewLineAndIndent << LINE_TERMINATOR_ANSI;
+	}
 
 	void BeginObject()
 	{
@@ -74,11 +77,32 @@ public:
 		switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
 		{
 			case CbFieldType::Null:
+				if (m_AddTypeComment)
+				{
+					Builder << "[Null] ";
+				}
 				Builder << "null"sv;
 				break;
 			case CbFieldType::Object:
+				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[Object] ";
+					}
+					BeginObject();
+					for (CbFieldView It : Field)
+					{
+						WriteField(It);
+					}
+					EndObject();
+				}
+				break;
 			case CbFieldType::UniformObject:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[UniformObject] ";
+					}
 					BeginObject();
 					for (CbFieldView It : Field)
 					{
@@ -88,8 +112,25 @@ public:
 				}
 				break;
 			case CbFieldType::Array:
+				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[Array] ";
+					}
+					BeginArray();
+					for (CbFieldView It : Field)
+					{
+						WriteField(It);
+					}
+					EndArray();
+				}
+				break;
 			case CbFieldType::UniformArray:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[UniformArray] ";
+					}
 					BeginArray();
 					for (CbFieldView It : Field)
 					{
@@ -99,19 +140,39 @@ public:
 				}
 				break;
 			case CbFieldType::Binary:
+				if (m_AddTypeComment)
+				{
+					Builder << "[Binary] ";
+				}
 				AppendBase64String(Accessor.AsBinary());
 				break;
 			case CbFieldType::String:
+				if (m_AddTypeComment)
+				{
+					Builder << "[String] ";
+				}
 				AppendQuotedString(Accessor.AsU8String());
 				break;
 			case CbFieldType::IntegerPositive:
+				if (m_AddTypeComment)
+				{
+					Builder << "[IntegerPositive] ";
+				}
 				Builder << Accessor.AsIntegerPositive();
 				break;
 			case CbFieldType::IntegerNegative:
+				if (m_AddTypeComment)
+				{
+					Builder << "[IntegerNegative] ";
+				}
 				Builder << Accessor.AsIntegerNegative();
 				break;
 			case CbFieldType::Float32:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[Float32] ";
+					}
 					const float Value = Accessor.AsFloat32();
 					if (std::isfinite(Value))
 					{
@@ -125,6 +186,10 @@ public:
 				break;
 			case CbFieldType::Float64:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[Float64] ";
+					}
 					const double Value = Accessor.AsFloat64();
 					if (std::isfinite(Value))
 					{
@@ -137,14 +202,36 @@ public:
 				}
 				break;
 			case CbFieldType::BoolFalse:
+				if (m_AddTypeComment)
+				{
+					Builder << "[BoolFalse] ";
+				}
 				Builder << "false"sv;
 				break;
 			case CbFieldType::BoolTrue:
+				if (m_AddTypeComment)
+				{
+					Builder << "[BoolTrue] ";
+				}
 				Builder << "true"sv;
 				break;
 			case CbFieldType::ObjectAttachment:
+				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[ObjectAttachment] ";
+					}
+					Builder << '"';
+					Accessor.AsAttachment().ToHexString(Builder);
+					Builder << '"';
+				}
+				break;
 			case CbFieldType::BinaryAttachment:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[BinaryAttachment] ";
+					}
 					Builder << '"';
 					Accessor.AsAttachment().ToHexString(Builder);
 					Builder << '"';
@@ -152,6 +239,10 @@ public:
 				break;
 			case CbFieldType::Hash:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[Hash] ";
+					}
 					Builder << '"';
 					Accessor.AsHash().ToHexString(Builder);
 					Builder << '"';
@@ -159,16 +250,28 @@ public:
 				break;
 			case CbFieldType::Uuid:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[Uuid] ";
+					}
 					Builder << '"';
 					Accessor.AsUuid().ToString(Builder);
 					Builder << '"';
 				}
 				break;
 			case CbFieldType::DateTime:
+				if (m_AddTypeComment)
+				{
+					Builder << "[DateTime] ";
+				}
 				Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"';
 				break;
 			case CbFieldType::TimeSpan:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[TimeSpan] ";
+					}
 					const TimeSpan Span(Accessor.AsTimeSpanTicks());
 					if (Span.GetDays() == 0)
 					{
@@ -181,12 +284,20 @@ public:
 					break;
 				}
 			case CbFieldType::ObjectId:
+				if (m_AddTypeComment)
+				{
+					Builder << "[ObjectId] ";
+				}
 				Builder << '"';
 				Accessor.AsObjectId().ToString(Builder);
 				Builder << '"';
 				break;
 			case CbFieldType::CustomById:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[CustomById] ";
+					}
 					CbCustomById Custom = Accessor.AsCustomById();
 					Builder << "{ \"Id\": ";
 					Builder << Custom.Id;
@@ -197,6 +308,10 @@ public:
 				}
 			case CbFieldType::CustomByName:
 				{
+					if (m_AddTypeComment)
+					{
+						Builder << "[CustomByName] ";
+					}
 					CbCustomByName Custom = Accessor.AsCustomByName();
 					Builder << "{ \"Name\": ";
 					AppendQuotedString(Custom.Name);
@@ -299,29 +414,30 @@ private:
 private:
 	StringBuilderBase&			Builder;
 	ExtendableStringBuilder<32> NewLineAndIndent;
+	const bool					m_AddTypeComment;
 	bool						NeedsComma{false};
 	bool						NeedsNewLine{false};
 };
 
 void
-CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder)
+CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment)
 {
-	CbJsonWriter Writer(Builder);
+	CbJsonWriter Writer(Builder, AddTypeComment);
 	Writer.WriteField(Object.AsFieldView());
 }
 
 void
 CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder)
 {
-	CbJsonWriter Writer(Builder);
+	CbJsonWriter Writer(Builder, /*AddTypeComment*/ false);
 	Writer.WriteField(Array.AsFieldView());
 }
 
 void
-CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder)
+CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment)
 {
 	std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data);
-	CbJsonWriter			 Writer(InBuilder);
+	CbJsonWriter			 Writer(InBuilder, AddTypeComment);
 	if (!Fields.empty())
 	{
 		if (Fields.size() == 1)
diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp
index 833649b88..d7292f405 100644
--- a/src/zencore/compactbinaryvalidation.cpp
+++ b/src/zencore/compactbinaryvalidation.cpp
@@ -705,10 +705,11 @@ ToString(const CbValidateError Error)
 
 	ExtendableStringBuilder<128> Out;
 
-	auto AppendFlag = [&, IsFirst = false](std::string_view FlagString) {
+	auto AppendFlag = [&, IsFirst = true](std::string_view FlagString) mutable {
 		if (!IsFirst)
 			Out.Append('|');
 		Out.Append(FlagString);
+		IsFirst = false;
 	};
 
 #define _ENUM_CASE(V)                               \
@@ -737,7 +738,11 @@ ToString(const CbValidateError Error)
 
 #undef _ENUM_CASE
 
-	return "Error";
+	if (Out.Size() == 0)
+	{
+		return "Error";
+	}
+	return Out.ToString();
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h
index 0fdb56d67..82ca055ab 100644
--- a/src/zencore/include/zencore/compactbinary.h
+++ b/src/zencore/include/zencore/compactbinary.h
@@ -996,7 +996,7 @@ private:
 /**
  * Serialize a compact binary object to JSON.
  */
-ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder);
+ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment = false);
 /**
  * Serialize a compact binary object to YAML.
  */
@@ -1520,7 +1520,7 @@ end(CbFieldView&)
 /**
  * Serialize serialized compact binary blob to JSON. It must be 0 to n fields with including type for each field
  */
-ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder);
+ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment = false);
 
 /**
  * Serialize serialized compact binary blob to YAML. It must be 0 to n fields with including type for each field
diff --git a/src/zencore/include/zencore/jobqueue.h b/src/zencore/include/zencore/jobqueue.h
index d5ec6255a..470ed3fc6 100644
--- a/src/zencore/include/zencore/jobqueue.h
+++ b/src/zencore/include/zencore/jobqueue.h
@@ -28,6 +28,16 @@ public:
 	virtual void ReportProgress(std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount) = 0;
 };
 
+class JobError : public std::runtime_error
+{
+public:
+	using _Mybase = runtime_error;
+
+	JobError(const std::string& Message, int ReturnCode) : _Mybase(Message), m_ReturnCode(ReturnCode) {}
+
+	const int m_ReturnCode = 0;
+};
+
 class JobQueue
 {
 public:
@@ -73,6 +83,7 @@ public:
 		std::chrono::system_clock::time_point StartTime;
 		std::chrono::system_clock::time_point EndTime;
 		int									  WorkerThreadId;
+		int									  ReturnCode;
 	};
 
 	// Will only respond once when status is Complete or Aborted
diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp
index 8e9a37a27..be9b39e7a 100644
--- a/src/zencore/iobuffer.cpp
+++ b/src/zencore/iobuffer.cpp
@@ -429,7 +429,25 @@ IoBufferExtendedCore::SetDeleteOnClose(bool DeleteOnClose)
 
 //////////////////////////////////////////////////////////////////////////
 
-RefPtr<IoBufferCore> IoBuffer::NullBufferCore(new IoBufferCore);
+static IoBufferCore*
+GetNullBufferCore()
+{
+	// This is safe from a threading standpoint since the first call is non-threaded (during static init) and for the following
+	// calls Core is never nullptr
+	// We do this workaround since we don't want to call new (IoBufferCore) at static initializers
+	// Calling new during static initialize causes problem with memtracing since the flags are not set up correctly yet
+
+	static IoBufferCore	 NullBufferCore;
+	static IoBufferCore* Core = nullptr;
+	if (Core == nullptr)
+	{
+		Core = &NullBufferCore;
+		Core->AddRef();	 // Make sure we never deallocate it as it is a static instance
+	}
+	return Core;
+}
+
+RefPtr<IoBufferCore> IoBuffer::NullBufferCore(GetNullBufferCore());
 
 IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize))
 {
diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp
index b97484458..5d727b69c 100644
--- a/src/zencore/jobqueue.cpp
+++ b/src/zencore/jobqueue.cpp
@@ -50,6 +50,7 @@ public:
 		JobClock::Tick	 StartTick;
 		JobClock::Tick	 EndTick;
 		int				 WorkerThreadId;
+		int				 ReturnCode;
 
 		virtual bool IsCancelled() const override { return CancelFlag.load(); }
 		virtual void ReportMessage(std::string_view Message) override { Queue->ReportMessage(Id, Message); }
@@ -101,6 +102,7 @@ public:
 		NewJob->StartTick	   = JobClock::Never();
 		NewJob->EndTick		   = JobClock::Never();
 		NewJob->WorkerThreadId = 0;
+		NewJob->ReturnCode	   = -1;
 
 		ZEN_DEBUG("Scheduling background job {}:'{}'", NewJob->Id.Id, NewJob->Name);
 		QueueLock.WithExclusiveLock([&]() { QueuedJobs.emplace_back(std::move(NewJob)); });
@@ -274,7 +276,8 @@ public:
 				.CreateTime		= JobClock::TimePointFromTick(Job.CreateTick),
 				.StartTime		= JobClock::TimePointFromTick(Job.StartTick),
 				.EndTime		= JobClock::TimePointFromTick(Job.EndTick),
-				.WorkerThreadId = Job.WorkerThreadId};
+				.WorkerThreadId = Job.WorkerThreadId,
+				.ReturnCode		= Job.ReturnCode};
 		};
 
 		std::optional<JobDetails> Result;
@@ -365,6 +368,7 @@ public:
 			ZEN_DEBUG("Executing background job {}:'{}'", CurrentJob->Id.Id, CurrentJob->Name);
 			CurrentJob->Callback(*CurrentJob);
 			ZEN_DEBUG("Completed background job {}:'{}'", CurrentJob->Id.Id, CurrentJob->Name);
+			CurrentJob->ReturnCode = 0;
 			QueueLock.WithExclusiveLock([&]() {
 				CurrentJob->EndTick		   = JobClock::Now();
 				CurrentJob->WorkerThreadId = 0;
@@ -383,6 +387,22 @@ public:
 				AbortedJobs.insert_or_assign(CurrentJob->Id.Id, std::move(CurrentJob));
 			});
 		}
+		catch (const JobError& Ex)
+		{
+			ZEN_DEBUG("Background job {}:'{}' failed. Reason: '{}'. Return code {}",
+					  CurrentJob->Id.Id,
+					  CurrentJob->Name,
+					  Ex.what(),
+					  Ex.m_ReturnCode);
+			QueueLock.WithExclusiveLock([&]() {
+				CurrentJob->State.AbortReason = Ex.what();
+				CurrentJob->EndTick			  = JobClock::Now();
+				CurrentJob->WorkerThreadId	  = 0;
+				CurrentJob->ReturnCode		  = Ex.m_ReturnCode;
+				RunningJobs.erase(CurrentJob->Id.Id);
+				AbortedJobs.insert_or_assign(CurrentJob->Id.Id, std::move(CurrentJob));
+			});
+		}
 		catch (const std::exception& Ex)
 		{
 			ZEN_DEBUG("Background job {}:'{}' aborted. Reason: '{}'", CurrentJob->Id.Id, CurrentJob->Name, Ex.what());
diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp
index fef5c28a4..b8ec85a4a 100644
--- a/src/zencore/thread.cpp
+++ b/src/zencore/thread.cpp
@@ -80,8 +80,10 @@ SetNameInternal(DWORD thread_id, const char* name)
 void
 SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName)
 {
-	StringBuilder<256> ThreadNameZ;
-	ThreadNameZ << ThreadName;
+	constexpr std::string_view::size_type  MaxThreadNameLength = 255;
+	std::string_view					   LimitedThreadName   = ThreadName.substr(0, MaxThreadNameLength);
+	StringBuilder<MaxThreadNameLength + 1> ThreadNameZ;
+	ThreadNameZ << LimitedThreadName;
 	const int ThreadId = GetCurrentThreadId();
 
 #if ZEN_WITH_TRACE
@@ -95,8 +97,8 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName)
 
 	if (SetThreadDescriptionFunc)
 	{
-		WideStringBuilder<256> ThreadNameW;
-		Utf8ToWide(ThreadName, ThreadNameW);
+		WideStringBuilder<MaxThreadNameLength + 1> ThreadNameW;
+		Utf8ToWide(LimitedThreadName, ThreadNameW);
 
 		SetThreadDescriptionFunc(::GetCurrentThread(), ThreadNameW.c_str());
 	}
diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp
index 82d28c0e3..51e06ae14 100644
--- a/src/zencore/zencore.cpp
+++ b/src/zencore/zencore.cpp
@@ -124,7 +124,14 @@ AssertImpl::ExecAssert(const char* Filename, int LineNumber, const char* Functio
 	AssertImpl* AssertImpl = CurrentAssertImpl;
 	while (AssertImpl)
 	{
-		AssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg, Callstack);
+		try
+		{
+			AssertImpl->OnAssert(Filename, LineNumber, FunctionName, Msg, Callstack);
+		}
+		catch (const std::exception&)
+		{
+			// Just keep exception silent - we don't want exception thrown from assert callbacks
+		}
 		AssertImpl = AssertImpl->NextAssertImpl;
 	}
 	ThrowAssertException(Filename, LineNumber, FunctionName, Msg, Callstack);