aboutsummaryrefslogtreecommitdiff
path: root/src/zencore/include
diff options
context:
space:
mode:
Diffstat (limited to 'src/zencore/include')
-rw-r--r--src/zencore/include/zencore/guardvalue.h40
-rw-r--r--src/zencore/include/zencore/iobuffer.h6
-rw-r--r--src/zencore/include/zencore/memory.h11
-rw-r--r--src/zencore/include/zencore/memory/fmalloc.h103
-rw-r--r--src/zencore/include/zencore/memory/llm.h31
-rw-r--r--src/zencore/include/zencore/memory/mallocansi.h31
-rw-r--r--src/zencore/include/zencore/memory/mallocmimalloc.h36
-rw-r--r--src/zencore/include/zencore/memory/mallocrpmalloc.h37
-rw-r--r--src/zencore/include/zencore/memory/mallocstomp.h100
-rw-r--r--src/zencore/include/zencore/memory/memory.h78
-rw-r--r--src/zencore/include/zencore/memory/memorytrace.h251
-rw-r--r--src/zencore/include/zencore/memory/newdelete.h155
-rw-r--r--src/zencore/include/zencore/memory/tagtrace.h93
-rw-r--r--src/zencore/include/zencore/string.h24
-rw-r--r--src/zencore/include/zencore/trace.h4
15 files changed, 990 insertions, 10 deletions
diff --git a/src/zencore/include/zencore/guardvalue.h b/src/zencore/include/zencore/guardvalue.h
new file mode 100644
index 000000000..5419e882a
--- /dev/null
+++ b/src/zencore/include/zencore/guardvalue.h
@@ -0,0 +1,40 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+namespace zen {
+
+/**
+ * exception-safe guard around saving/restoring a value.
+ * Commonly used to make sure a value is restored
+ * even if the code early outs in the future.
+ * Usage:
+ * TGuardValue<bool> GuardSomeBool(bSomeBool, false); // Sets bSomeBool to false, and restores it in dtor.
+ */
+template<typename RefType, typename AssignedType = RefType>
+struct TGuardValue
+{
+ [[nodiscard]] TGuardValue(RefType& ReferenceValue, const AssignedType& NewValue)
+ : RefValue(ReferenceValue)
+ , OriginalValue(ReferenceValue)
+ {
+ RefValue = NewValue;
+ }
+ ~TGuardValue() { RefValue = OriginalValue; }
+
+ /**
+ * Provides read-only access to the original value of the data being tracked by this struct
+ *
+ * @return a const reference to the original data value
+ */
+ const AssignedType& GetOriginalValue() const { return OriginalValue; }
+
+ TGuardValue& operator=(const TGuardValue&) = delete;
+ TGuardValue(const TGuardValue&) = delete;
+
+private:
+ RefType& RefValue;
+ AssignedType OriginalValue;
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h
index 493b7375e..93a27ea58 100644
--- a/src/zencore/include/zencore/iobuffer.h
+++ b/src/zencore/include/zencore/iobuffer.h
@@ -99,6 +99,11 @@ public:
ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment);
ZENCORE_API ~IoBufferCore();
+ void* operator new(size_t Size);
+ void operator delete(void* Ptr);
+ void* operator new[](size_t Size) = delete;
+ void operator delete[](void* Ptr) = delete;
+
// Reference counting
inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); }
@@ -244,7 +249,6 @@ protected:
kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore
kIsMaterialized = 1 << 3, // Data pointers are valid
kIsWholeFile = 1 << 5, // References an entire file
- kIoBufferAlloc = 1 << 6, // Using IoBuffer allocator
kIsOwnedByThis = 1 << 7,
// Note that we have some extended flags defined below
diff --git a/src/zencore/include/zencore/memory.h b/src/zencore/include/zencore/memory.h
index fdea1a5f1..8361ab9d8 100644
--- a/src/zencore/include/zencore/memory.h
+++ b/src/zencore/include/zencore/memory.h
@@ -22,17 +22,10 @@ template<typename T>
concept ContiguousRange = true;
#endif
-struct MemoryView;
-
-class Memory
-{
-public:
- ZENCORE_API static void* Alloc(size_t Size, size_t Alignment = sizeof(void*));
- ZENCORE_API static void Free(void* Ptr);
-};
-
//////////////////////////////////////////////////////////////////////////
+struct MemoryView;
+
struct MutableMemoryView
{
MutableMemoryView() = default;
diff --git a/src/zencore/include/zencore/memory/fmalloc.h b/src/zencore/include/zencore/memory/fmalloc.h
new file mode 100644
index 000000000..aeb05b651
--- /dev/null
+++ b/src/zencore/include/zencore/memory/fmalloc.h
@@ -0,0 +1,103 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/zenbase.h>
+
+namespace zen {
+
+enum
+{
+ DEFAULT_ALIGNMENT = 0
+};
+
+/**
+ * Inherit from FUseSystemMallocForNew if you want your objects to be placed in memory
+ * alloced by the system malloc routines, bypassing GMalloc. This is e.g. used by FMalloc
+ * itself.
+ */
+class FUseSystemMallocForNew
+{
+public:
+ void* operator new(size_t Size);
+ void operator delete(void* Ptr);
+ void* operator new[](size_t Size);
+ void operator delete[](void* Ptr);
+};
+
+/** Memory allocator abstraction
+ */
+
+class FMalloc : public FUseSystemMallocForNew
+{
+public:
+ /**
+ * Malloc
+ */
+ virtual void* Malloc(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT) = 0;
+
+ /**
+ * TryMalloc - like Malloc(), but may return a nullptr result if the allocation
+ * request cannot be satisfied.
+ */
+ virtual void* TryMalloc(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+
+ /**
+ * Realloc
+ */
+ virtual void* Realloc(void* Original, size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT) = 0;
+
+ /**
+ * TryRealloc - like Realloc(), but may return a nullptr if the allocation
+ * request cannot be satisfied. Note that in this case the memory
+ * pointed to by Original will still be valid
+ */
+ virtual void* TryRealloc(void* Original, size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+
+ /**
+ * Free
+ */
+ virtual void Free(void* Original) = 0;
+
+ /**
+ * Malloc zeroed memory
+ */
+ virtual void* MallocZeroed(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+
+ /**
+ * TryMallocZeroed - like MallocZeroed(), but may return a nullptr result if the allocation
+ * request cannot be satisfied.
+ */
+ virtual void* TryMallocZeroed(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+
+ /**
+ * For some allocators this will return the actual size that should be requested to eliminate
+ * internal fragmentation. The return value will always be >= Count. This can be used to grow
+ * and shrink containers to optimal sizes.
+ * This call is always fast and threadsafe with no locking.
+ */
+ virtual size_t QuantizeSize(size_t Count, uint32_t Alignment);
+
+ /**
+ * If possible determine the size of the memory allocated at the given address
+ *
+ * @param Original - Pointer to memory we are checking the size of
+ * @param SizeOut - If possible, this value is set to the size of the passed in pointer
+ * @return true if succeeded
+ */
+ virtual bool GetAllocationSize(void* Original, size_t& SizeOut);
+
+ /**
+ * Notifies the malloc implementation that initialization of all allocators in GMalloc is complete, so it's safe to initialize any extra
+ * features that require "regular" allocations
+ */
+ virtual void OnMallocInitialized();
+
+ virtual void Trim(bool bTrimThreadCaches);
+
+ virtual void OutOfMemory(size_t Size, uint32_t Alignment);
+};
+
+extern FMalloc* GMalloc; /* Memory allocator */
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/memory/llm.h b/src/zencore/include/zencore/memory/llm.h
new file mode 100644
index 000000000..4f1c9de77
--- /dev/null
+++ b/src/zencore/include/zencore/memory/llm.h
@@ -0,0 +1,31 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/zenbase.h>
+#include <zencore/memory/tagtrace.h>
+
+namespace zen {
+
+// clang-format off
+#define LLM_ENUM_GENERIC_TAGS(macro) \
+ macro(Untagged, "Untagged", -1) \
+ macro(ProgramSize, "ProgramSize", -1) \
+ macro(Metrics, "Metrics", -1) \
+ macro(Logging, "Logging", -1) \
+ macro(IoBuffer, "IoBuffer", -1) \
+ macro(IoBufferMemory, "IoMemory", ELLMTag::IoBuffer) \
+ macro(IoBufferCore, "IoCore", ELLMTag::IoBuffer)
+
+// clang-format on
+
+enum class ELLMTag : uint8_t
+{
+#define LLM_ENUM(Enum, Str, Parent) Enum,
+ LLM_ENUM_GENERIC_TAGS(LLM_ENUM)
+#undef LLM_ENUM
+
+ GenericTagCount
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/memory/mallocansi.h b/src/zencore/include/zencore/memory/mallocansi.h
new file mode 100644
index 000000000..510695c8c
--- /dev/null
+++ b/src/zencore/include/zencore/memory/mallocansi.h
@@ -0,0 +1,31 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "fmalloc.h"
+#include "memory.h"
+
+namespace zen {
+
+void* AnsiMalloc(size_t Size, uint32_t Alignment);
+void* AnsiRealloc(void* Ptr, size_t NewSize, uint32_t Alignment);
+void AnsiFree(void* Ptr);
+
+//
+// ANSI C memory allocator.
+//
+
+class FMallocAnsi final : public FMalloc
+{
+public:
+ FMallocAnsi();
+
+ virtual void* Malloc(size_t Size, uint32_t Alignment) override;
+ virtual void* TryMalloc(size_t Size, uint32_t Alignment) override;
+ virtual void* Realloc(void* Ptr, size_t NewSize, uint32_t Alignment) override;
+ virtual void* TryRealloc(void* Ptr, size_t NewSize, uint32_t Alignment) override;
+ virtual void Free(void* Ptr) override;
+ virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override;
+};
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/memory/mallocmimalloc.h b/src/zencore/include/zencore/memory/mallocmimalloc.h
new file mode 100644
index 000000000..759eeb4a6
--- /dev/null
+++ b/src/zencore/include/zencore/memory/mallocmimalloc.h
@@ -0,0 +1,36 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/memory/fmalloc.h>
+
+#if ZEN_USE_MIMALLOC
+# define ZEN_MIMALLOC_ENABLED 1
+#endif
+
+#if !defined(ZEN_MIMALLOC_ENABLED)
+# define ZEN_MIMALLOC_ENABLED 0
+#endif
+
+#if ZEN_MIMALLOC_ENABLED
+
+namespace zen {
+
+class FMallocMimalloc final : public FMalloc
+{
+public:
+ FMallocMimalloc();
+ virtual void* Malloc(size_t Size, uint32_t Alignment) override;
+ virtual void* TryMalloc(size_t Size, uint32_t Alignment) override;
+ virtual void* Realloc(void* Ptr, size_t NewSize, uint32_t Alignment) override;
+ virtual void* TryRealloc(void* Ptr, size_t NewSize, uint32_t Alignment) override;
+ virtual void Free(void* Ptr) override;
+ virtual void* MallocZeroed(size_t Count, uint32_t Alignment) override;
+ virtual void* TryMallocZeroed(size_t Count, uint32_t Alignment) override;
+ virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override;
+ virtual void Trim(bool bTrimThreadCaches) override;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zencore/include/zencore/memory/mallocrpmalloc.h b/src/zencore/include/zencore/memory/mallocrpmalloc.h
new file mode 100644
index 000000000..be2627b2d
--- /dev/null
+++ b/src/zencore/include/zencore/memory/mallocrpmalloc.h
@@ -0,0 +1,37 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/memory/fmalloc.h>
+
+#if ZEN_USE_RPMALLOC
+# define ZEN_RPMALLOC_ENABLED 1
+#endif
+
+#if !defined(ZEN_RPMALLOC_ENABLED)
+# define ZEN_RPMALLOC_ENABLED 0
+#endif
+
+#if ZEN_RPMALLOC_ENABLED
+
+namespace zen {
+
+class FMallocRpmalloc final : public FMalloc
+{
+public:
+ FMallocRpmalloc();
+ ~FMallocRpmalloc();
+ virtual void* Malloc(size_t Size, uint32_t Alignment) override;
+ virtual void* TryMalloc(size_t Size, uint32_t Alignment) override;
+ virtual void* Realloc(void* Ptr, size_t NewSize, uint32_t Alignment) override;
+ virtual void* TryRealloc(void* Ptr, size_t NewSize, uint32_t Alignment) override;
+ virtual void Free(void* Ptr) override;
+ virtual void* MallocZeroed(size_t Count, uint32_t Alignment) override;
+ virtual void* TryMallocZeroed(size_t Count, uint32_t Alignment) override;
+ virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override;
+ virtual void Trim(bool bTrimThreadCaches) override;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zencore/include/zencore/memory/mallocstomp.h b/src/zencore/include/zencore/memory/mallocstomp.h
new file mode 100644
index 000000000..5d83868bb
--- /dev/null
+++ b/src/zencore/include/zencore/memory/mallocstomp.h
@@ -0,0 +1,100 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/zenbase.h>
+
+#if ZEN_PLATFORM_WINDOWS
+# define ZEN_WITH_MALLOC_STOMP 1
+#endif
+
+#ifndef ZEN_WITH_MALLOC_STOMP
+# define ZEN_WITH_MALLOC_STOMP 0
+#endif
+
+/**
+ * Stomp memory allocator support should be enabled in Core.Build.cs.
+ * Run-time validation should be enabled using '-stompmalloc' command line argument.
+ */
+
+#if ZEN_WITH_MALLOC_STOMP
+
+# include <zencore/memory/fmalloc.h>
+# include <zencore/thread.h>
+
+namespace zen {
+
+/**
+ * Stomp memory allocator. It helps find the following errors:
+ * - Read or writes off the end of an allocation.
+ * - Read or writes off the beginning of an allocation.
+ * - Read or writes after freeing an allocation.
+ */
+class FMallocStomp final : public FMalloc
+{
+ struct FAllocationData;
+
+ const size_t PageSize;
+
+ /** If it is set to true, instead of focusing on overruns the allocator will focus on underruns. */
+ const bool bUseUnderrunMode;
+ RwLock Lock;
+
+ uintptr_t VirtualAddressCursor = 0;
+ size_t VirtualAddressMax = 0;
+ static constexpr size_t VirtualAddressBlockSize = 1 * 1024 * 1024 * 1024; // 1 GB blocks
+
+public:
+ // FMalloc interface.
+ explicit FMallocStomp(const bool InUseUnderrunMode = false);
+
+ /**
+ * Allocates a block of a given number of bytes of memory with the required alignment.
+ * In the process it allocates as many pages as necessary plus one that will be protected
+ * making it unaccessible and causing an exception. The actual allocation will be pushed
+ * to the end of the last valid unprotected page. To deal with underrun errors a sentinel
+ * is added right before the allocation in page which is checked on free.
+ *
+ * @param Size Size in bytes of the memory block to allocate.
+ * @param Alignment Alignment in bytes of the memory block to allocate.
+ * @return A pointer to the beginning of the memory block.
+ */
+ virtual void* Malloc(size_t Size, uint32_t Alignment) override;
+
+ virtual void* TryMalloc(size_t Size, uint32_t Alignment) override;
+
+ /**
+ * Changes the size of the memory block pointed to by OldPtr.
+ * The function may move the memory block to a new location.
+ *
+ * @param OldPtr Pointer to a memory block previously allocated with Malloc.
+ * @param NewSize New size in bytes for the memory block.
+ * @param Alignment Alignment in bytes for the reallocation.
+ * @return A pointer to the reallocated memory block, which may be either the same as ptr or a new location.
+ */
+ virtual void* Realloc(void* InPtr, size_t NewSize, uint32_t Alignment) override;
+
+ virtual void* TryRealloc(void* InPtr, size_t NewSize, uint32_t Alignment) override;
+
+ /**
+ * Frees a memory allocation and verifies the sentinel in the process.
+ *
+ * @param InPtr Pointer of the data to free.
+ */
+ virtual void Free(void* InPtr) override;
+
+ /**
+ * If possible determine the size of the memory allocated at the given address.
+ * This will included all the pages that were allocated so it will be far more
+ * than what's set on the FAllocationData.
+ *
+ * @param Original - Pointer to memory we are checking the size of
+ * @param SizeOut - If possible, this value is set to the size of the passed in pointer
+ * @return true if succeeded
+ */
+ virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override;
+};
+
+} // namespace zen
+
+#endif // WITH_MALLOC_STOMP
diff --git a/src/zencore/include/zencore/memory/memory.h b/src/zencore/include/zencore/memory/memory.h
new file mode 100644
index 000000000..2fc20def6
--- /dev/null
+++ b/src/zencore/include/zencore/memory/memory.h
@@ -0,0 +1,78 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <stdlib.h>
+#include <zencore/memory/fmalloc.h>
+
+#define UE_ALLOCATION_FUNCTION(...)
+
+namespace zen {
+
+/**
+ * Corresponds to UE-side FMemory implementation
+ */
+
+class Memory
+{
+public:
+ static void Initialize();
+
+ //
+ // C style memory allocation stubs that fall back to C runtime
+ //
+ UE_ALLOCATION_FUNCTION(1) static void* SystemMalloc(size_t Size);
+ static void SystemFree(void* Ptr);
+
+ //
+ // C style memory allocation stubs.
+ //
+
+ static inline void* Alloc(size_t Size, size_t Alignment = sizeof(void*)) { return Malloc(Size, uint32_t(Alignment)); }
+
+ UE_ALLOCATION_FUNCTION(1, 2) static inline void* Malloc(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+ UE_ALLOCATION_FUNCTION(2, 3) static inline void* Realloc(void* Original, size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+ static inline void Free(void* Original);
+ static inline size_t GetAllocSize(void* Original);
+
+ UE_ALLOCATION_FUNCTION(1, 2) static inline void* MallocZeroed(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT);
+
+private:
+ static void GCreateMalloc();
+};
+
+inline void*
+Memory::Malloc(size_t Count, uint32_t Alignment)
+{
+ return GMalloc->TryMalloc(Count, Alignment);
+}
+
+inline void*
+Memory::Realloc(void* Original, size_t Count, uint32_t Alignment)
+{
+ return GMalloc->TryRealloc(Original, Count, Alignment);
+}
+
+inline void
+Memory::Free(void* Original)
+{
+ if (Original)
+ {
+ GMalloc->Free(Original);
+ }
+}
+
+inline size_t
+Memory::GetAllocSize(void* Original)
+{
+ size_t Size = 0;
+ return GMalloc->GetAllocationSize(Original, Size) ? Size : 0;
+}
+
+inline void*
+Memory::MallocZeroed(size_t Count, uint32_t Alignment)
+{
+ return GMalloc->TryMallocZeroed(Count, Alignment);
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/memory/memorytrace.h b/src/zencore/include/zencore/memory/memorytrace.h
new file mode 100644
index 000000000..d1ab1f914
--- /dev/null
+++ b/src/zencore/include/zencore/memory/memorytrace.h
@@ -0,0 +1,251 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
+
+#include <zencore/enumflags.h>
+#include <zencore/trace.h>
+
+#if !defined(UE_MEMORY_TRACE_AVAILABLE)
+# define UE_MEMORY_TRACE_AVAILABLE 0
+#endif
+
+#if !defined(UE_MEMORY_TRACE_LATE_INIT)
+# define UE_MEMORY_TRACE_LATE_INIT 0
+#endif
+
+#if !defined(PLATFORM_USES_FIXED_GMalloc_CLASS)
+# define PLATFORM_USES_FIXED_GMalloc_CLASS 0
+#endif
+
+#if !defined(UE_MEMORY_TRACE_ENABLED) && UE_TRACE_ENABLED
+# if UE_MEMORY_TRACE_AVAILABLE
+# define UE_MEMORY_TRACE_ENABLED ZEN_WITH_MEMTRACK
+# endif
+#endif
+
+#if !defined(UE_MEMORY_TRACE_ENABLED)
+# define UE_MEMORY_TRACE_ENABLED 0
+#endif
+
+namespace zen {
+
+////////////////////////////////////////////////////////////////////////////////
+typedef uint32_t HeapId;
+
+////////////////////////////////////////////////////////////////////////////////
+enum EMemoryTraceRootHeap : uint8_t
+{
+ SystemMemory, // RAM
+ VideoMemory, // VRAM
+ EndHardcoded = VideoMemory,
+ EndReserved = 15
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// These values are traced. Do not modify existing values in order to maintain
+// compatibility.
+enum class EMemoryTraceHeapFlags : uint16_t
+{
+ None = 0,
+ Root = 1 << 0,
+ NeverFrees = 1 << 1, // The heap doesn't free (e.g. linear allocator)
+};
+ENUM_CLASS_FLAGS(EMemoryTraceHeapFlags);
+
+////////////////////////////////////////////////////////////////////////////////
+// These values are traced. Do not modify existing values in order to maintain
+// compatibility.
+enum class EMemoryTraceHeapAllocationFlags : uint8_t
+{
+ None = 0,
+ Heap = 1 << 0, // Is a heap, can be used to unmark alloc as heap.
+ Swap = 2 << 0, // Is a swap page
+};
+ENUM_CLASS_FLAGS(EMemoryTraceHeapAllocationFlags);
+
+////////////////////////////////////////////////////////////////////////////////
+enum class EMemoryTraceSwapOperation : uint8
+{
+ PageOut = 0, // Paged out to swap
+ PageIn = 1, // Read from swap via page fault
+ FreeInSwap = 2, // Freed while being paged out in swap
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Internal options for early initialization of memory tracing systems. Exposed
+// here due to visibility in platform implementations.
+enum class EMemoryTraceInit : uint8
+{
+ Disabled = 0,
+ AllocEvents = 1 << 0,
+ Callstacks = 1 << 1,
+ Tags = 1 << 2,
+ Full = AllocEvents | Callstacks | Tags,
+ Light = AllocEvents | Tags,
+};
+
+ENUM_CLASS_FLAGS(EMemoryTraceInit);
+
+////////////////////////////////////////////////////////////////////////////////
+#if UE_MEMORY_TRACE_ENABLED
+
+# define UE_MEMORY_TRACE(x) x
+
+UE_TRACE_CHANNEL_EXTERN(MemAllocChannel);
+
+////////////////////////////////////////////////////////////////////////////////
+class FMalloc* MemoryTrace_Create(class FMalloc* InMalloc);
+void MemoryTrace_Initialize();
+void MemoryTrace_Shutdown();
+
+/**
+ * Register a new heap specification (name). Use the returned value when marking heaps.
+ * @param ParentId Heap id of parent heap.
+ * @param Name Descriptive name of the heap.
+ * @param Flags Properties of this heap. See \ref EMemoryTraceHeapFlags
+ * @return Heap id to use when allocating memory
+ */
+HeapId MemoryTrace_HeapSpec(HeapId ParentId, const char16_t* Name, EMemoryTraceHeapFlags Flags = EMemoryTraceHeapFlags::None);
+
+/**
+ * Register a new root heap specification (name). Use the returned value as parent to other heaps.
+ * @param Name Descriptive name of the root heap.
+ * @param Flags Properties of the this root heap. See \ref EMemoryTraceHeapFlags
+ * @return Heap id to use when allocating memory
+ */
+HeapId MemoryTrace_RootHeapSpec(const char16_t* Name, EMemoryTraceHeapFlags Flags = EMemoryTraceHeapFlags::None);
+
+/**
+ * Mark a traced allocation as being a heap.
+ * @param Address Address of the allocation
+ * @param Heap Heap id, see /ref MemoryTrace_HeapSpec. If no specific heap spec has been created the correct root heap needs to be given.
+ * @param Flags Additional properties of the heap allocation. Note that \ref EMemoryTraceHeapAllocationFlags::Heap is implicit.
+ * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id.
+ */
+void MemoryTrace_MarkAllocAsHeap(uint64 Address,
+ HeapId Heap,
+ EMemoryTraceHeapAllocationFlags Flags = EMemoryTraceHeapAllocationFlags::None,
+ uint32 ExternalCallstackId = 0);
+
+/**
+ * Unmark an allocation as a heap. When an allocation that has previously been used as a heap is reused as a regular
+ * allocation.
+ * @param Address Address of the allocation
+ * @param Heap Heap id
+ * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id.
+ */
+void MemoryTrace_UnmarkAllocAsHeap(uint64 Address, HeapId Heap, uint32 ExternalCallstackId = 0);
+
+/**
+ * Trace an allocation event.
+ * @param Address Address of allocation
+ * @param Size Size of allocation
+ * @param Alignment Alignment of the allocation
+ * @param RootHeap Which root heap this belongs to (system memory, video memory etc)
+ * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id.
+ */
+void MemoryTrace_Alloc(uint64 Address,
+ uint64 Size,
+ uint32 Alignment,
+ HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory,
+ uint32 ExternalCallstackId = 0);
+
+/**
+ * Trace a free event.
+ * @param Address Address of the allocation being freed
+ * @param RootHeap Which root heap this belongs to (system memory, video memory etc)
+ * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id.
+ */
+void MemoryTrace_Free(uint64 Address, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32 ExternalCallstackId = 0);
+
+/**
+ * Trace a free related to a reallocation event.
+ * @param Address Address of the allocation being freed
+ * @param RootHeap Which root heap this belongs to (system memory, video memory etc)
+ * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id.
+ */
+void MemoryTrace_ReallocFree(uint64 Address, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32 ExternalCallstackId = 0);
+
+/** Trace an allocation related to a reallocation event.
+ * @param Address Address of allocation
+ * @param NewSize Size of allocation
+ * @param Alignment Alignment of the allocation
+ * @param RootHeap Which root heap this belongs to (system memory, video memory etc)
+ * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id.
+ */
+void MemoryTrace_ReallocAlloc(uint64 Address,
+ uint64 NewSize,
+ uint32 Alignment,
+ HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory,
+ uint32 ExternalCallstackId = 0);
+
+/** Trace a swap operation. Only available for system memory root heap (EMemoryTraceRootHeap::SystemMemory).
+ * @param PageAddress Page address for operation, in case of PageIn can be address of the page fault (not aligned to page boundary).
+ * @param SwapOperation Which swap operation is happening to the address.
+ * @param CompressedSize Compressed size of the page for page out operation.
+ * @param CallstackId CallstackId to use, if 0 to ignore (will not use current callstack id).
+ */
+void MemoryTrace_SwapOp(uint64 PageAddress, EMemoryTraceSwapOperation SwapOperation, uint32 CompressedSize = 0, uint32 CallstackId = 0);
+
+////////////////////////////////////////////////////////////////////////////////
+#else // UE_MEMORY_TRACE_ENABLED
+
+# define UE_MEMORY_TRACE(x)
+inline HeapId
+MemoryTrace_RootHeapSpec(const char16_t* /*Name*/, EMemoryTraceHeapFlags /* Flags = EMemoryTraceHeapFlags::None */)
+{
+ return ~0u;
+};
+inline HeapId
+MemoryTrace_HeapSpec(HeapId /*ParentId*/, const char16_t* /*Name*/, EMemoryTraceHeapFlags /* Flags = EMemoryTraceHeapFlags::None */)
+{
+ return ~0u;
+}
+inline void
+MemoryTrace_MarkAllocAsHeap(uint64_t /*Address*/, HeapId /*Heap*/)
+{
+}
+inline void
+MemoryTrace_UnmarkAllocAsHeap(uint64_t /*Address*/, HeapId /*Heap*/)
+{
+}
+inline void
+MemoryTrace_Alloc(uint64_t /*Address*/,
+ uint64_t /*Size*/,
+ uint32_t /*Alignment*/,
+ HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory,
+ uint32_t ExternalCallstackId = 0)
+{
+ ZEN_UNUSED(RootHeap, ExternalCallstackId);
+}
+inline void
+MemoryTrace_Free(uint64_t /*Address*/, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32_t ExternalCallstackId = 0)
+{
+ ZEN_UNUSED(RootHeap, ExternalCallstackId);
+}
+inline void
+MemoryTrace_ReallocFree(uint64_t /*Address*/, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32_t ExternalCallstackId = 0)
+{
+ ZEN_UNUSED(RootHeap, ExternalCallstackId);
+}
+inline void
+MemoryTrace_ReallocAlloc(uint64_t /*Address*/,
+ uint64_t /*NewSize*/,
+ uint32_t /*Alignment*/,
+ HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory,
+ uint32_t ExternalCallstackId = 0)
+{
+ ZEN_UNUSED(RootHeap, ExternalCallstackId);
+}
+inline void
+MemoryTrace_SwapOp(uint64_t /*PageAddress*/,
+ EMemoryTraceSwapOperation /*SwapOperation*/,
+ uint32_t CompressedSize = 0,
+ uint32_t CallstackId = 0)
+{
+ ZEN_UNUSED(CompressedSize, CallstackId);
+}
+
+#endif // UE_MEMORY_TRACE_ENABLED
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/memory/newdelete.h b/src/zencore/include/zencore/memory/newdelete.h
new file mode 100644
index 000000000..d22c8604f
--- /dev/null
+++ b/src/zencore/include/zencore/memory/newdelete.h
@@ -0,0 +1,155 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/zenbase.h>
+#include <new>
+
+#if defined(_MSC_VER)
+# if (_MSC_VER >= 1900) && !defined(__EDG__)
+# define ZEN_RESTRICT __declspec(allocator) __declspec(restrict)
+# else
+# define ZEN_RESTRICT __declspec(restrict)
+# endif
+#else
+# define ZEN_RESTRICT
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+[[nodiscard]] ZEN_RESTRICT void* zen_new(size_t size);
+[[nodiscard]] ZEN_RESTRICT void* zen_new_aligned(size_t size, size_t alignment);
+[[nodiscard]] ZEN_RESTRICT void* zen_new_nothrow(size_t size) noexcept;
+[[nodiscard]] ZEN_RESTRICT void* zen_new_aligned_nothrow(size_t size, size_t alignment) noexcept;
+
+void zen_free(void* p) noexcept;
+void zen_free_size(void* p, size_t size) noexcept;
+void zen_free_size_aligned(void* p, size_t size, size_t alignment) noexcept;
+void zen_free_aligned(void* p, size_t alignment) noexcept;
+
+//////////////////////////////////////////////////////////////////////////
+
+#if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)
+# define zen_decl_new(n) [[nodiscard]] _VCRT_ALLOCATOR _Ret_notnull_ _Post_writable_byte_size_(n)
+# define zen_decl_new_nothrow(n) [[nodiscard]] _VCRT_ALLOCATOR _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)
+#else
+# define zen_decl_new(n) [[nodiscard]]
+# define zen_decl_new_nothrow(n) [[nodiscard]]
+#endif
+
+void
+operator delete(void* p) noexcept
+{
+ zen_free(p);
+}
+
+void
+operator delete[](void* p) noexcept
+{
+ zen_free(p);
+}
+
+void
+operator delete(void* p, const std::nothrow_t&) noexcept
+{
+ zen_free(p);
+}
+
+void
+operator delete[](void* p, const std::nothrow_t&) noexcept
+{
+ zen_free(p);
+}
+
+zen_decl_new(n) void*
+operator new(std::size_t n) noexcept(false)
+{
+ return zen_new(n);
+}
+
+zen_decl_new(n) void*
+operator new[](std::size_t n) noexcept(false)
+{
+ return zen_new(n);
+}
+
+zen_decl_new_nothrow(n) void*
+operator new(std::size_t n, const std::nothrow_t& tag) noexcept
+{
+ (void)(tag);
+ return zen_new_nothrow(n);
+}
+
+zen_decl_new_nothrow(n) void*
+operator new[](std::size_t n, const std::nothrow_t& tag) noexcept
+{
+ (void)(tag);
+ return zen_new_nothrow(n);
+}
+
+#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
+void
+operator delete(void* p, std::size_t n) noexcept
+{
+ zen_free_size(p, n);
+};
+void
+operator delete[](void* p, std::size_t n) noexcept
+{
+ zen_free_size(p, n);
+};
+#endif
+
+#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
+void
+operator delete(void* p, std::align_val_t al) noexcept
+{
+ zen_free_aligned(p, static_cast<size_t>(al));
+}
+void
+operator delete[](void* p, std::align_val_t al) noexcept
+{
+ zen_free_aligned(p, static_cast<size_t>(al));
+}
+void
+operator delete(void* p, std::size_t n, std::align_val_t al) noexcept
+{
+ zen_free_size_aligned(p, n, static_cast<size_t>(al));
+};
+void
+operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept
+{
+ zen_free_size_aligned(p, n, static_cast<size_t>(al));
+};
+void
+operator delete(void* p, std::align_val_t al, const std::nothrow_t&) noexcept
+{
+ zen_free_aligned(p, static_cast<size_t>(al));
+}
+void
+operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept
+{
+ zen_free_aligned(p, static_cast<size_t>(al));
+}
+
+void*
+operator new(std::size_t n, std::align_val_t al) noexcept(false)
+{
+ return zen_new_aligned(n, static_cast<size_t>(al));
+}
+void*
+operator new[](std::size_t n, std::align_val_t al) noexcept(false)
+{
+ return zen_new_aligned(n, static_cast<size_t>(al));
+}
+void*
+operator new(std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept
+{
+ return zen_new_aligned_nothrow(n, static_cast<size_t>(al));
+}
+void*
+operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept
+{
+ return zen_new_aligned_nothrow(n, static_cast<size_t>(al));
+}
+#endif
diff --git a/src/zencore/include/zencore/memory/tagtrace.h b/src/zencore/include/zencore/memory/tagtrace.h
new file mode 100644
index 000000000..f51b21466
--- /dev/null
+++ b/src/zencore/include/zencore/memory/tagtrace.h
@@ -0,0 +1,93 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
+
+#include <zenbase/zenbase.h>
+#include <zencore/trace.h>
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+enum class ELLMTag : uint8_t;
+
+int32_t MemoryTrace_AnnounceCustomTag(int32_t Tag, int32_t ParentTag, const char* Display);
+int32_t MemoryTrace_GetActiveTag();
+
+inline constexpr int32_t TRACE_TAG = 257;
+
+} // namespace zen
+
+////////////////////////////////////////////////////////////////////////////////
+#if !defined(UE_MEMORY_TAGS_TRACE_ENABLED)
+# define UE_MEMORY_TAGS_TRACE_ENABLED 1
+#endif
+
+#if UE_MEMORY_TAGS_TRACE_ENABLED && UE_TRACE_ENABLED
+
+namespace zen {
+////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Used to associate any allocation within this scope to a given tag.
+ *
+ * We need to be able to convert the three types of inputs to LLM scopes:
+ * - ELLMTag, an uint8 with fixed categories. There are three sub ranges
+ Generic tags, platform and project tags.
+ * - FName, free form string, for example a specific asset.
+ * - TagData, an opaque pointer from LLM.
+ *
+ */
+class FMemScope
+{
+public:
+ FMemScope(); // Used with SetTagAndActivate
+ FMemScope(int32_t InTag, bool bShouldActivate = true);
+ FMemScope(ELLMTag InTag, bool bShouldActivate = true);
+ ~FMemScope();
+
+private:
+ void ActivateScope(int32_t InTag);
+ UE::Trace::Private::FScopedLogScope Inner;
+ int32_t PrevTag;
+};
+
+/**
+ * A scope that activates in case no existing scope is active.
+ */
+template<typename TagType>
+class FDefaultMemScope : public FMemScope
+{
+public:
+ FDefaultMemScope(TagType InTag) : FMemScope(InTag, MemoryTrace_GetActiveTag() == 0) {}
+};
+
+/**
+ * Used order to keep the tag for memory that is being reallocated.
+ */
+class FMemScopePtr
+{
+public:
+ FMemScopePtr(uint64_t InPtr);
+ ~FMemScopePtr();
+
+private:
+ UE::Trace::Private::FScopedLogScope Inner;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+# define UE_MEMSCOPE(InTag) FMemScope PREPROCESSOR_JOIN(MemScope, __LINE__)(InTag);
+# define UE_MEMSCOPE_PTR(InPtr) FMemScopePtr PREPROCESSOR_JOIN(MemPtrScope, __LINE__)((uint64)InPtr);
+# define UE_MEMSCOPE_DEFAULT(InTag) FDefaultMemScope PREPROCESSOR_JOIN(MemScope, __LINE__)(InTag);
+# define UE_MEMSCOPE_UNINITIALIZED(Line) FMemScope PREPROCESSOR_JOIN(MemScope, Line);
+
+#else // UE_MEMORY_TAGS_TRACE_ENABLED
+
+////////////////////////////////////////////////////////////////////////////////
+# define UE_MEMSCOPE(...)
+# define UE_MEMSCOPE_PTR(...)
+# define UE_MEMSCOPE_DEFAULT(...)
+# define UE_MEMSCOPE_UNINITIALIZED(...)
+# define UE_MEMSCOPE_ACTIVATE(...)
+
+#endif // UE_MEMORY_TAGS_TRACE_ENABLED
+}
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index b10b6a2ba..e2ef1c1a0 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -51,6 +51,30 @@ StringLength(const wchar_t* str)
return wcslen(str);
}
+inline bool
+StringCompare(const char16_t* s1, const char16_t* s2)
+{
+ char16_t c1, c2;
+
+ while ((c1 = *s1) == (c2 = *s2))
+ {
+ if (c1 == 0)
+ {
+ return 0;
+ }
+
+ ++s1;
+ ++s2;
+ }
+ return uint16_t(c1) - uint16_t(c2);
+}
+
+inline bool
+StringEquals(const char16_t* s1, const char16_t* s2)
+{
+ return StringCompare(s1, s2) == 0;
+}
+
inline size_t
StringLength(const char16_t* str)
{
diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h
index 89e4b76bf..2ca2b7c81 100644
--- a/src/zencore/include/zencore/trace.h
+++ b/src/zencore/include/zencore/trace.h
@@ -19,6 +19,8 @@ ZEN_THIRD_PARTY_INCLUDES_END
#define ZEN_TRACE_CPU(x) TRACE_CPU_SCOPE(x)
#define ZEN_TRACE_CPU_FLUSH(x) TRACE_CPU_SCOPE(x, trace::CpuScopeFlags::CpuFlush)
+namespace zen {
+
enum class TraceType
{
File,
@@ -32,6 +34,8 @@ bool IsTracing();
void TraceStart(std::string_view ProgramName, const char* HostOrPath, TraceType Type);
bool TraceStop();
+}
+
#else
#define ZEN_TRACE_CPU(x)