diff options
Diffstat (limited to 'src/zencore/include')
| -rw-r--r-- | src/zencore/include/zencore/guardvalue.h | 40 | ||||
| -rw-r--r-- | src/zencore/include/zencore/iobuffer.h | 6 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory.h | 11 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/fmalloc.h | 103 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/llm.h | 31 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/mallocansi.h | 31 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/mallocmimalloc.h | 36 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/mallocrpmalloc.h | 37 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/mallocstomp.h | 100 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/memory.h | 78 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/memorytrace.h | 251 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/newdelete.h | 155 | ||||
| -rw-r--r-- | src/zencore/include/zencore/memory/tagtrace.h | 93 | ||||
| -rw-r--r-- | src/zencore/include/zencore/string.h | 24 | ||||
| -rw-r--r-- | src/zencore/include/zencore/trace.h | 4 |
15 files changed, 990 insertions, 10 deletions
diff --git a/src/zencore/include/zencore/guardvalue.h b/src/zencore/include/zencore/guardvalue.h new file mode 100644 index 000000000..5419e882a --- /dev/null +++ b/src/zencore/include/zencore/guardvalue.h @@ -0,0 +1,40 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +namespace zen { + +/** + * exception-safe guard around saving/restoring a value. + * Commonly used to make sure a value is restored + * even if the code early outs in the future. + * Usage: + * TGuardValue<bool> GuardSomeBool(bSomeBool, false); // Sets bSomeBool to false, and restores it in dtor. + */ +template<typename RefType, typename AssignedType = RefType> +struct TGuardValue +{ + [[nodiscard]] TGuardValue(RefType& ReferenceValue, const AssignedType& NewValue) + : RefValue(ReferenceValue) + , OriginalValue(ReferenceValue) + { + RefValue = NewValue; + } + ~TGuardValue() { RefValue = OriginalValue; } + + /** + * Provides read-only access to the original value of the data being tracked by this struct + * + * @return a const reference to the original data value + */ + const AssignedType& GetOriginalValue() const { return OriginalValue; } + + TGuardValue& operator=(const TGuardValue&) = delete; + TGuardValue(const TGuardValue&) = delete; + +private: + RefType& RefValue; + AssignedType OriginalValue; +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h index 493b7375e..93a27ea58 100644 --- a/src/zencore/include/zencore/iobuffer.h +++ b/src/zencore/include/zencore/iobuffer.h @@ -99,6 +99,11 @@ public: ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment); ZENCORE_API ~IoBufferCore(); + void* operator new(size_t Size); + void operator delete(void* Ptr); + void* operator new[](size_t Size) = delete; + void operator delete[](void* Ptr) = delete; + // Reference counting inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); } @@ -244,7 +249,6 @@ protected: kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore kIsMaterialized = 1 << 3, // Data pointers are valid kIsWholeFile = 1 << 5, // References an entire file - kIoBufferAlloc = 1 << 6, // Using IoBuffer allocator kIsOwnedByThis = 1 << 7, // Note that we have some extended flags defined below diff --git a/src/zencore/include/zencore/memory.h b/src/zencore/include/zencore/memory.h index fdea1a5f1..8361ab9d8 100644 --- a/src/zencore/include/zencore/memory.h +++ b/src/zencore/include/zencore/memory.h @@ -22,17 +22,10 @@ template<typename T> concept ContiguousRange = true; #endif -struct MemoryView; - -class Memory -{ -public: - ZENCORE_API static void* Alloc(size_t Size, size_t Alignment = sizeof(void*)); - ZENCORE_API static void Free(void* Ptr); -}; - ////////////////////////////////////////////////////////////////////////// +struct MemoryView; + struct MutableMemoryView { MutableMemoryView() = default; diff --git a/src/zencore/include/zencore/memory/fmalloc.h b/src/zencore/include/zencore/memory/fmalloc.h new file mode 100644 index 000000000..aeb05b651 --- /dev/null +++ b/src/zencore/include/zencore/memory/fmalloc.h @@ -0,0 +1,103 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/zenbase.h> + +namespace zen { + +enum +{ + DEFAULT_ALIGNMENT = 0 +}; + +/** + * Inherit from FUseSystemMallocForNew if you want your objects to be placed in memory + * alloced by the system malloc routines, bypassing GMalloc. This is e.g. used by FMalloc + * itself. + */ +class FUseSystemMallocForNew +{ +public: + void* operator new(size_t Size); + void operator delete(void* Ptr); + void* operator new[](size_t Size); + void operator delete[](void* Ptr); +}; + +/** Memory allocator abstraction + */ + +class FMalloc : public FUseSystemMallocForNew +{ +public: + /** + * Malloc + */ + virtual void* Malloc(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT) = 0; + + /** + * TryMalloc - like Malloc(), but may return a nullptr result if the allocation + * request cannot be satisfied. + */ + virtual void* TryMalloc(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + + /** + * Realloc + */ + virtual void* Realloc(void* Original, size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT) = 0; + + /** + * TryRealloc - like Realloc(), but may return a nullptr if the allocation + * request cannot be satisfied. Note that in this case the memory + * pointed to by Original will still be valid + */ + virtual void* TryRealloc(void* Original, size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + + /** + * Free + */ + virtual void Free(void* Original) = 0; + + /** + * Malloc zeroed memory + */ + virtual void* MallocZeroed(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + + /** + * TryMallocZeroed - like MallocZeroed(), but may return a nullptr result if the allocation + * request cannot be satisfied. + */ + virtual void* TryMallocZeroed(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + + /** + * For some allocators this will return the actual size that should be requested to eliminate + * internal fragmentation. The return value will always be >= Count. This can be used to grow + * and shrink containers to optimal sizes. + * This call is always fast and threadsafe with no locking. + */ + virtual size_t QuantizeSize(size_t Count, uint32_t Alignment); + + /** + * If possible determine the size of the memory allocated at the given address + * + * @param Original - Pointer to memory we are checking the size of + * @param SizeOut - If possible, this value is set to the size of the passed in pointer + * @return true if succeeded + */ + virtual bool GetAllocationSize(void* Original, size_t& SizeOut); + + /** + * Notifies the malloc implementation that initialization of all allocators in GMalloc is complete, so it's safe to initialize any extra + * features that require "regular" allocations + */ + virtual void OnMallocInitialized(); + + virtual void Trim(bool bTrimThreadCaches); + + virtual void OutOfMemory(size_t Size, uint32_t Alignment); +}; + +extern FMalloc* GMalloc; /* Memory allocator */ + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/llm.h b/src/zencore/include/zencore/memory/llm.h new file mode 100644 index 000000000..4f1c9de77 --- /dev/null +++ b/src/zencore/include/zencore/memory/llm.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/zenbase.h> +#include <zencore/memory/tagtrace.h> + +namespace zen { + +// clang-format off +#define LLM_ENUM_GENERIC_TAGS(macro) \ + macro(Untagged, "Untagged", -1) \ + macro(ProgramSize, "ProgramSize", -1) \ + macro(Metrics, "Metrics", -1) \ + macro(Logging, "Logging", -1) \ + macro(IoBuffer, "IoBuffer", -1) \ + macro(IoBufferMemory, "IoMemory", ELLMTag::IoBuffer) \ + macro(IoBufferCore, "IoCore", ELLMTag::IoBuffer) + +// clang-format on + +enum class ELLMTag : uint8_t +{ +#define LLM_ENUM(Enum, Str, Parent) Enum, + LLM_ENUM_GENERIC_TAGS(LLM_ENUM) +#undef LLM_ENUM + + GenericTagCount +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/mallocansi.h b/src/zencore/include/zencore/memory/mallocansi.h new file mode 100644 index 000000000..510695c8c --- /dev/null +++ b/src/zencore/include/zencore/memory/mallocansi.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "fmalloc.h" +#include "memory.h" + +namespace zen { + +void* AnsiMalloc(size_t Size, uint32_t Alignment); +void* AnsiRealloc(void* Ptr, size_t NewSize, uint32_t Alignment); +void AnsiFree(void* Ptr); + +// +// ANSI C memory allocator. +// + +class FMallocAnsi final : public FMalloc +{ +public: + FMallocAnsi(); + + virtual void* Malloc(size_t Size, uint32_t Alignment) override; + virtual void* TryMalloc(size_t Size, uint32_t Alignment) override; + virtual void* Realloc(void* Ptr, size_t NewSize, uint32_t Alignment) override; + virtual void* TryRealloc(void* Ptr, size_t NewSize, uint32_t Alignment) override; + virtual void Free(void* Ptr) override; + virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override; +}; + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/mallocmimalloc.h b/src/zencore/include/zencore/memory/mallocmimalloc.h new file mode 100644 index 000000000..759eeb4a6 --- /dev/null +++ b/src/zencore/include/zencore/memory/mallocmimalloc.h @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/memory/fmalloc.h> + +#if ZEN_USE_MIMALLOC +# define ZEN_MIMALLOC_ENABLED 1 +#endif + +#if !defined(ZEN_MIMALLOC_ENABLED) +# define ZEN_MIMALLOC_ENABLED 0 +#endif + +#if ZEN_MIMALLOC_ENABLED + +namespace zen { + +class FMallocMimalloc final : public FMalloc +{ +public: + FMallocMimalloc(); + virtual void* Malloc(size_t Size, uint32_t Alignment) override; + virtual void* TryMalloc(size_t Size, uint32_t Alignment) override; + virtual void* Realloc(void* Ptr, size_t NewSize, uint32_t Alignment) override; + virtual void* TryRealloc(void* Ptr, size_t NewSize, uint32_t Alignment) override; + virtual void Free(void* Ptr) override; + virtual void* MallocZeroed(size_t Count, uint32_t Alignment) override; + virtual void* TryMallocZeroed(size_t Count, uint32_t Alignment) override; + virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override; + virtual void Trim(bool bTrimThreadCaches) override; +}; + +} // namespace zen + +#endif diff --git a/src/zencore/include/zencore/memory/mallocrpmalloc.h b/src/zencore/include/zencore/memory/mallocrpmalloc.h new file mode 100644 index 000000000..be2627b2d --- /dev/null +++ b/src/zencore/include/zencore/memory/mallocrpmalloc.h @@ -0,0 +1,37 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/memory/fmalloc.h> + +#if ZEN_USE_RPMALLOC +# define ZEN_RPMALLOC_ENABLED 1 +#endif + +#if !defined(ZEN_RPMALLOC_ENABLED) +# define ZEN_RPMALLOC_ENABLED 0 +#endif + +#if ZEN_RPMALLOC_ENABLED + +namespace zen { + +class FMallocRpmalloc final : public FMalloc +{ +public: + FMallocRpmalloc(); + ~FMallocRpmalloc(); + virtual void* Malloc(size_t Size, uint32_t Alignment) override; + virtual void* TryMalloc(size_t Size, uint32_t Alignment) override; + virtual void* Realloc(void* Ptr, size_t NewSize, uint32_t Alignment) override; + virtual void* TryRealloc(void* Ptr, size_t NewSize, uint32_t Alignment) override; + virtual void Free(void* Ptr) override; + virtual void* MallocZeroed(size_t Count, uint32_t Alignment) override; + virtual void* TryMallocZeroed(size_t Count, uint32_t Alignment) override; + virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override; + virtual void Trim(bool bTrimThreadCaches) override; +}; + +} // namespace zen + +#endif diff --git a/src/zencore/include/zencore/memory/mallocstomp.h b/src/zencore/include/zencore/memory/mallocstomp.h new file mode 100644 index 000000000..5d83868bb --- /dev/null +++ b/src/zencore/include/zencore/memory/mallocstomp.h @@ -0,0 +1,100 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/zenbase.h> + +#if ZEN_PLATFORM_WINDOWS +# define ZEN_WITH_MALLOC_STOMP 1 +#endif + +#ifndef ZEN_WITH_MALLOC_STOMP +# define ZEN_WITH_MALLOC_STOMP 0 +#endif + +/** + * Stomp memory allocator support should be enabled in Core.Build.cs. + * Run-time validation should be enabled using '-stompmalloc' command line argument. + */ + +#if ZEN_WITH_MALLOC_STOMP + +# include <zencore/memory/fmalloc.h> +# include <zencore/thread.h> + +namespace zen { + +/** + * Stomp memory allocator. It helps find the following errors: + * - Read or writes off the end of an allocation. + * - Read or writes off the beginning of an allocation. + * - Read or writes after freeing an allocation. + */ +class FMallocStomp final : public FMalloc +{ + struct FAllocationData; + + const size_t PageSize; + + /** If it is set to true, instead of focusing on overruns the allocator will focus on underruns. */ + const bool bUseUnderrunMode; + RwLock Lock; + + uintptr_t VirtualAddressCursor = 0; + size_t VirtualAddressMax = 0; + static constexpr size_t VirtualAddressBlockSize = 1 * 1024 * 1024 * 1024; // 1 GB blocks + +public: + // FMalloc interface. + explicit FMallocStomp(const bool InUseUnderrunMode = false); + + /** + * Allocates a block of a given number of bytes of memory with the required alignment. + * In the process it allocates as many pages as necessary plus one that will be protected + * making it unaccessible and causing an exception. The actual allocation will be pushed + * to the end of the last valid unprotected page. To deal with underrun errors a sentinel + * is added right before the allocation in page which is checked on free. + * + * @param Size Size in bytes of the memory block to allocate. + * @param Alignment Alignment in bytes of the memory block to allocate. + * @return A pointer to the beginning of the memory block. + */ + virtual void* Malloc(size_t Size, uint32_t Alignment) override; + + virtual void* TryMalloc(size_t Size, uint32_t Alignment) override; + + /** + * Changes the size of the memory block pointed to by OldPtr. + * The function may move the memory block to a new location. + * + * @param OldPtr Pointer to a memory block previously allocated with Malloc. + * @param NewSize New size in bytes for the memory block. + * @param Alignment Alignment in bytes for the reallocation. + * @return A pointer to the reallocated memory block, which may be either the same as ptr or a new location. + */ + virtual void* Realloc(void* InPtr, size_t NewSize, uint32_t Alignment) override; + + virtual void* TryRealloc(void* InPtr, size_t NewSize, uint32_t Alignment) override; + + /** + * Frees a memory allocation and verifies the sentinel in the process. + * + * @param InPtr Pointer of the data to free. + */ + virtual void Free(void* InPtr) override; + + /** + * If possible determine the size of the memory allocated at the given address. + * This will included all the pages that were allocated so it will be far more + * than what's set on the FAllocationData. + * + * @param Original - Pointer to memory we are checking the size of + * @param SizeOut - If possible, this value is set to the size of the passed in pointer + * @return true if succeeded + */ + virtual bool GetAllocationSize(void* Original, size_t& SizeOut) override; +}; + +} // namespace zen + +#endif // WITH_MALLOC_STOMP diff --git a/src/zencore/include/zencore/memory/memory.h b/src/zencore/include/zencore/memory/memory.h new file mode 100644 index 000000000..2fc20def6 --- /dev/null +++ b/src/zencore/include/zencore/memory/memory.h @@ -0,0 +1,78 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <stdlib.h> +#include <zencore/memory/fmalloc.h> + +#define UE_ALLOCATION_FUNCTION(...) + +namespace zen { + +/** + * Corresponds to UE-side FMemory implementation + */ + +class Memory +{ +public: + static void Initialize(); + + // + // C style memory allocation stubs that fall back to C runtime + // + UE_ALLOCATION_FUNCTION(1) static void* SystemMalloc(size_t Size); + static void SystemFree(void* Ptr); + + // + // C style memory allocation stubs. + // + + static inline void* Alloc(size_t Size, size_t Alignment = sizeof(void*)) { return Malloc(Size, uint32_t(Alignment)); } + + UE_ALLOCATION_FUNCTION(1, 2) static inline void* Malloc(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + UE_ALLOCATION_FUNCTION(2, 3) static inline void* Realloc(void* Original, size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + static inline void Free(void* Original); + static inline size_t GetAllocSize(void* Original); + + UE_ALLOCATION_FUNCTION(1, 2) static inline void* MallocZeroed(size_t Count, uint32_t Alignment = DEFAULT_ALIGNMENT); + +private: + static void GCreateMalloc(); +}; + +inline void* +Memory::Malloc(size_t Count, uint32_t Alignment) +{ + return GMalloc->TryMalloc(Count, Alignment); +} + +inline void* +Memory::Realloc(void* Original, size_t Count, uint32_t Alignment) +{ + return GMalloc->TryRealloc(Original, Count, Alignment); +} + +inline void +Memory::Free(void* Original) +{ + if (Original) + { + GMalloc->Free(Original); + } +} + +inline size_t +Memory::GetAllocSize(void* Original) +{ + size_t Size = 0; + return GMalloc->GetAllocationSize(Original, Size) ? Size : 0; +} + +inline void* +Memory::MallocZeroed(size_t Count, uint32_t Alignment) +{ + return GMalloc->TryMallocZeroed(Count, Alignment); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/memorytrace.h b/src/zencore/include/zencore/memory/memorytrace.h new file mode 100644 index 000000000..d1ab1f914 --- /dev/null +++ b/src/zencore/include/zencore/memory/memorytrace.h @@ -0,0 +1,251 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#pragma once + +#include <zencore/enumflags.h> +#include <zencore/trace.h> + +#if !defined(UE_MEMORY_TRACE_AVAILABLE) +# define UE_MEMORY_TRACE_AVAILABLE 0 +#endif + +#if !defined(UE_MEMORY_TRACE_LATE_INIT) +# define UE_MEMORY_TRACE_LATE_INIT 0 +#endif + +#if !defined(PLATFORM_USES_FIXED_GMalloc_CLASS) +# define PLATFORM_USES_FIXED_GMalloc_CLASS 0 +#endif + +#if !defined(UE_MEMORY_TRACE_ENABLED) && UE_TRACE_ENABLED +# if UE_MEMORY_TRACE_AVAILABLE +# define UE_MEMORY_TRACE_ENABLED ZEN_WITH_MEMTRACK +# endif +#endif + +#if !defined(UE_MEMORY_TRACE_ENABLED) +# define UE_MEMORY_TRACE_ENABLED 0 +#endif + +namespace zen { + +//////////////////////////////////////////////////////////////////////////////// +typedef uint32_t HeapId; + +//////////////////////////////////////////////////////////////////////////////// +enum EMemoryTraceRootHeap : uint8_t +{ + SystemMemory, // RAM + VideoMemory, // VRAM + EndHardcoded = VideoMemory, + EndReserved = 15 +}; + +//////////////////////////////////////////////////////////////////////////////// +// These values are traced. Do not modify existing values in order to maintain +// compatibility. +enum class EMemoryTraceHeapFlags : uint16_t +{ + None = 0, + Root = 1 << 0, + NeverFrees = 1 << 1, // The heap doesn't free (e.g. linear allocator) +}; +ENUM_CLASS_FLAGS(EMemoryTraceHeapFlags); + +//////////////////////////////////////////////////////////////////////////////// +// These values are traced. Do not modify existing values in order to maintain +// compatibility. +enum class EMemoryTraceHeapAllocationFlags : uint8_t +{ + None = 0, + Heap = 1 << 0, // Is a heap, can be used to unmark alloc as heap. + Swap = 2 << 0, // Is a swap page +}; +ENUM_CLASS_FLAGS(EMemoryTraceHeapAllocationFlags); + +//////////////////////////////////////////////////////////////////////////////// +enum class EMemoryTraceSwapOperation : uint8 +{ + PageOut = 0, // Paged out to swap + PageIn = 1, // Read from swap via page fault + FreeInSwap = 2, // Freed while being paged out in swap +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Internal options for early initialization of memory tracing systems. Exposed +// here due to visibility in platform implementations. +enum class EMemoryTraceInit : uint8 +{ + Disabled = 0, + AllocEvents = 1 << 0, + Callstacks = 1 << 1, + Tags = 1 << 2, + Full = AllocEvents | Callstacks | Tags, + Light = AllocEvents | Tags, +}; + +ENUM_CLASS_FLAGS(EMemoryTraceInit); + +//////////////////////////////////////////////////////////////////////////////// +#if UE_MEMORY_TRACE_ENABLED + +# define UE_MEMORY_TRACE(x) x + +UE_TRACE_CHANNEL_EXTERN(MemAllocChannel); + +//////////////////////////////////////////////////////////////////////////////// +class FMalloc* MemoryTrace_Create(class FMalloc* InMalloc); +void MemoryTrace_Initialize(); +void MemoryTrace_Shutdown(); + +/** + * Register a new heap specification (name). Use the returned value when marking heaps. + * @param ParentId Heap id of parent heap. + * @param Name Descriptive name of the heap. + * @param Flags Properties of this heap. See \ref EMemoryTraceHeapFlags + * @return Heap id to use when allocating memory + */ +HeapId MemoryTrace_HeapSpec(HeapId ParentId, const char16_t* Name, EMemoryTraceHeapFlags Flags = EMemoryTraceHeapFlags::None); + +/** + * Register a new root heap specification (name). Use the returned value as parent to other heaps. + * @param Name Descriptive name of the root heap. + * @param Flags Properties of the this root heap. See \ref EMemoryTraceHeapFlags + * @return Heap id to use when allocating memory + */ +HeapId MemoryTrace_RootHeapSpec(const char16_t* Name, EMemoryTraceHeapFlags Flags = EMemoryTraceHeapFlags::None); + +/** + * Mark a traced allocation as being a heap. + * @param Address Address of the allocation + * @param Heap Heap id, see /ref MemoryTrace_HeapSpec. If no specific heap spec has been created the correct root heap needs to be given. + * @param Flags Additional properties of the heap allocation. Note that \ref EMemoryTraceHeapAllocationFlags::Heap is implicit. + * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id. + */ +void MemoryTrace_MarkAllocAsHeap(uint64 Address, + HeapId Heap, + EMemoryTraceHeapAllocationFlags Flags = EMemoryTraceHeapAllocationFlags::None, + uint32 ExternalCallstackId = 0); + +/** + * Unmark an allocation as a heap. When an allocation that has previously been used as a heap is reused as a regular + * allocation. + * @param Address Address of the allocation + * @param Heap Heap id + * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id. + */ +void MemoryTrace_UnmarkAllocAsHeap(uint64 Address, HeapId Heap, uint32 ExternalCallstackId = 0); + +/** + * Trace an allocation event. + * @param Address Address of allocation + * @param Size Size of allocation + * @param Alignment Alignment of the allocation + * @param RootHeap Which root heap this belongs to (system memory, video memory etc) + * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id. + */ +void MemoryTrace_Alloc(uint64 Address, + uint64 Size, + uint32 Alignment, + HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, + uint32 ExternalCallstackId = 0); + +/** + * Trace a free event. + * @param Address Address of the allocation being freed + * @param RootHeap Which root heap this belongs to (system memory, video memory etc) + * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id. + */ +void MemoryTrace_Free(uint64 Address, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32 ExternalCallstackId = 0); + +/** + * Trace a free related to a reallocation event. + * @param Address Address of the allocation being freed + * @param RootHeap Which root heap this belongs to (system memory, video memory etc) + * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id. + */ +void MemoryTrace_ReallocFree(uint64 Address, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32 ExternalCallstackId = 0); + +/** Trace an allocation related to a reallocation event. + * @param Address Address of allocation + * @param NewSize Size of allocation + * @param Alignment Alignment of the allocation + * @param RootHeap Which root heap this belongs to (system memory, video memory etc) + * @param ExternalCallstackId CallstackId to use, if 0 will use current callstack id. + */ +void MemoryTrace_ReallocAlloc(uint64 Address, + uint64 NewSize, + uint32 Alignment, + HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, + uint32 ExternalCallstackId = 0); + +/** Trace a swap operation. Only available for system memory root heap (EMemoryTraceRootHeap::SystemMemory). + * @param PageAddress Page address for operation, in case of PageIn can be address of the page fault (not aligned to page boundary). + * @param SwapOperation Which swap operation is happening to the address. + * @param CompressedSize Compressed size of the page for page out operation. + * @param CallstackId CallstackId to use, if 0 to ignore (will not use current callstack id). + */ +void MemoryTrace_SwapOp(uint64 PageAddress, EMemoryTraceSwapOperation SwapOperation, uint32 CompressedSize = 0, uint32 CallstackId = 0); + +//////////////////////////////////////////////////////////////////////////////// +#else // UE_MEMORY_TRACE_ENABLED + +# define UE_MEMORY_TRACE(x) +inline HeapId +MemoryTrace_RootHeapSpec(const char16_t* /*Name*/, EMemoryTraceHeapFlags /* Flags = EMemoryTraceHeapFlags::None */) +{ + return ~0u; +}; +inline HeapId +MemoryTrace_HeapSpec(HeapId /*ParentId*/, const char16_t* /*Name*/, EMemoryTraceHeapFlags /* Flags = EMemoryTraceHeapFlags::None */) +{ + return ~0u; +} +inline void +MemoryTrace_MarkAllocAsHeap(uint64_t /*Address*/, HeapId /*Heap*/) +{ +} +inline void +MemoryTrace_UnmarkAllocAsHeap(uint64_t /*Address*/, HeapId /*Heap*/) +{ +} +inline void +MemoryTrace_Alloc(uint64_t /*Address*/, + uint64_t /*Size*/, + uint32_t /*Alignment*/, + HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, + uint32_t ExternalCallstackId = 0) +{ + ZEN_UNUSED(RootHeap, ExternalCallstackId); +} +inline void +MemoryTrace_Free(uint64_t /*Address*/, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32_t ExternalCallstackId = 0) +{ + ZEN_UNUSED(RootHeap, ExternalCallstackId); +} +inline void +MemoryTrace_ReallocFree(uint64_t /*Address*/, HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, uint32_t ExternalCallstackId = 0) +{ + ZEN_UNUSED(RootHeap, ExternalCallstackId); +} +inline void +MemoryTrace_ReallocAlloc(uint64_t /*Address*/, + uint64_t /*NewSize*/, + uint32_t /*Alignment*/, + HeapId RootHeap = EMemoryTraceRootHeap::SystemMemory, + uint32_t ExternalCallstackId = 0) +{ + ZEN_UNUSED(RootHeap, ExternalCallstackId); +} +inline void +MemoryTrace_SwapOp(uint64_t /*PageAddress*/, + EMemoryTraceSwapOperation /*SwapOperation*/, + uint32_t CompressedSize = 0, + uint32_t CallstackId = 0) +{ + ZEN_UNUSED(CompressedSize, CallstackId); +} + +#endif // UE_MEMORY_TRACE_ENABLED + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/newdelete.h b/src/zencore/include/zencore/memory/newdelete.h new file mode 100644 index 000000000..d22c8604f --- /dev/null +++ b/src/zencore/include/zencore/memory/newdelete.h @@ -0,0 +1,155 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/zenbase.h> +#include <new> + +#if defined(_MSC_VER) +# if (_MSC_VER >= 1900) && !defined(__EDG__) +# define ZEN_RESTRICT __declspec(allocator) __declspec(restrict) +# else +# define ZEN_RESTRICT __declspec(restrict) +# endif +#else +# define ZEN_RESTRICT +#endif + +////////////////////////////////////////////////////////////////////////// + +[[nodiscard]] ZEN_RESTRICT void* zen_new(size_t size); +[[nodiscard]] ZEN_RESTRICT void* zen_new_aligned(size_t size, size_t alignment); +[[nodiscard]] ZEN_RESTRICT void* zen_new_nothrow(size_t size) noexcept; +[[nodiscard]] ZEN_RESTRICT void* zen_new_aligned_nothrow(size_t size, size_t alignment) noexcept; + +void zen_free(void* p) noexcept; +void zen_free_size(void* p, size_t size) noexcept; +void zen_free_size_aligned(void* p, size_t size, size_t alignment) noexcept; +void zen_free_aligned(void* p, size_t alignment) noexcept; + +////////////////////////////////////////////////////////////////////////// + +#if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_) +# define zen_decl_new(n) [[nodiscard]] _VCRT_ALLOCATOR _Ret_notnull_ _Post_writable_byte_size_(n) +# define zen_decl_new_nothrow(n) [[nodiscard]] _VCRT_ALLOCATOR _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n) +#else +# define zen_decl_new(n) [[nodiscard]] +# define zen_decl_new_nothrow(n) [[nodiscard]] +#endif + +void +operator delete(void* p) noexcept +{ + zen_free(p); +} + +void +operator delete[](void* p) noexcept +{ + zen_free(p); +} + +void +operator delete(void* p, const std::nothrow_t&) noexcept +{ + zen_free(p); +} + +void +operator delete[](void* p, const std::nothrow_t&) noexcept +{ + zen_free(p); +} + +zen_decl_new(n) void* +operator new(std::size_t n) noexcept(false) +{ + return zen_new(n); +} + +zen_decl_new(n) void* +operator new[](std::size_t n) noexcept(false) +{ + return zen_new(n); +} + +zen_decl_new_nothrow(n) void* +operator new(std::size_t n, const std::nothrow_t& tag) noexcept +{ + (void)(tag); + return zen_new_nothrow(n); +} + +zen_decl_new_nothrow(n) void* +operator new[](std::size_t n, const std::nothrow_t& tag) noexcept +{ + (void)(tag); + return zen_new_nothrow(n); +} + +#if (__cplusplus >= 201402L || _MSC_VER >= 1916) +void +operator delete(void* p, std::size_t n) noexcept +{ + zen_free_size(p, n); +}; +void +operator delete[](void* p, std::size_t n) noexcept +{ + zen_free_size(p, n); +}; +#endif + +#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) +void +operator delete(void* p, std::align_val_t al) noexcept +{ + zen_free_aligned(p, static_cast<size_t>(al)); +} +void +operator delete[](void* p, std::align_val_t al) noexcept +{ + zen_free_aligned(p, static_cast<size_t>(al)); +} +void +operator delete(void* p, std::size_t n, std::align_val_t al) noexcept +{ + zen_free_size_aligned(p, n, static_cast<size_t>(al)); +}; +void +operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept +{ + zen_free_size_aligned(p, n, static_cast<size_t>(al)); +}; +void +operator delete(void* p, std::align_val_t al, const std::nothrow_t&) noexcept +{ + zen_free_aligned(p, static_cast<size_t>(al)); +} +void +operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept +{ + zen_free_aligned(p, static_cast<size_t>(al)); +} + +void* +operator new(std::size_t n, std::align_val_t al) noexcept(false) +{ + return zen_new_aligned(n, static_cast<size_t>(al)); +} +void* +operator new[](std::size_t n, std::align_val_t al) noexcept(false) +{ + return zen_new_aligned(n, static_cast<size_t>(al)); +} +void* +operator new(std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept +{ + return zen_new_aligned_nothrow(n, static_cast<size_t>(al)); +} +void* +operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept +{ + return zen_new_aligned_nothrow(n, static_cast<size_t>(al)); +} +#endif diff --git a/src/zencore/include/zencore/memory/tagtrace.h b/src/zencore/include/zencore/memory/tagtrace.h new file mode 100644 index 000000000..f51b21466 --- /dev/null +++ b/src/zencore/include/zencore/memory/tagtrace.h @@ -0,0 +1,93 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#pragma once + +#include <zenbase/zenbase.h> +#include <zencore/trace.h> + +//////////////////////////////////////////////////////////////////////////////// + +namespace zen { + +enum class ELLMTag : uint8_t; + +int32_t MemoryTrace_AnnounceCustomTag(int32_t Tag, int32_t ParentTag, const char* Display); +int32_t MemoryTrace_GetActiveTag(); + +inline constexpr int32_t TRACE_TAG = 257; + +} // namespace zen + +//////////////////////////////////////////////////////////////////////////////// +#if !defined(UE_MEMORY_TAGS_TRACE_ENABLED) +# define UE_MEMORY_TAGS_TRACE_ENABLED 1 +#endif + +#if UE_MEMORY_TAGS_TRACE_ENABLED && UE_TRACE_ENABLED + +namespace zen { +//////////////////////////////////////////////////////////////////////////////// + +/** + * Used to associate any allocation within this scope to a given tag. + * + * We need to be able to convert the three types of inputs to LLM scopes: + * - ELLMTag, an uint8 with fixed categories. There are three sub ranges + Generic tags, platform and project tags. + * - FName, free form string, for example a specific asset. + * - TagData, an opaque pointer from LLM. + * + */ +class FMemScope +{ +public: + FMemScope(); // Used with SetTagAndActivate + FMemScope(int32_t InTag, bool bShouldActivate = true); + FMemScope(ELLMTag InTag, bool bShouldActivate = true); + ~FMemScope(); + +private: + void ActivateScope(int32_t InTag); + UE::Trace::Private::FScopedLogScope Inner; + int32_t PrevTag; +}; + +/** + * A scope that activates in case no existing scope is active. + */ +template<typename TagType> +class FDefaultMemScope : public FMemScope +{ +public: + FDefaultMemScope(TagType InTag) : FMemScope(InTag, MemoryTrace_GetActiveTag() == 0) {} +}; + +/** + * Used order to keep the tag for memory that is being reallocated. + */ +class FMemScopePtr +{ +public: + FMemScopePtr(uint64_t InPtr); + ~FMemScopePtr(); + +private: + UE::Trace::Private::FScopedLogScope Inner; +}; + +//////////////////////////////////////////////////////////////////////////////// +# define UE_MEMSCOPE(InTag) FMemScope PREPROCESSOR_JOIN(MemScope, __LINE__)(InTag); +# define UE_MEMSCOPE_PTR(InPtr) FMemScopePtr PREPROCESSOR_JOIN(MemPtrScope, __LINE__)((uint64)InPtr); +# define UE_MEMSCOPE_DEFAULT(InTag) FDefaultMemScope PREPROCESSOR_JOIN(MemScope, __LINE__)(InTag); +# define UE_MEMSCOPE_UNINITIALIZED(Line) FMemScope PREPROCESSOR_JOIN(MemScope, Line); + +#else // UE_MEMORY_TAGS_TRACE_ENABLED + +//////////////////////////////////////////////////////////////////////////////// +# define UE_MEMSCOPE(...) +# define UE_MEMSCOPE_PTR(...) +# define UE_MEMSCOPE_DEFAULT(...) +# define UE_MEMSCOPE_UNINITIALIZED(...) +# define UE_MEMSCOPE_ACTIVATE(...) + +#endif // UE_MEMORY_TAGS_TRACE_ENABLED +} diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index b10b6a2ba..e2ef1c1a0 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -51,6 +51,30 @@ StringLength(const wchar_t* str) return wcslen(str); } +inline bool +StringCompare(const char16_t* s1, const char16_t* s2) +{ + char16_t c1, c2; + + while ((c1 = *s1) == (c2 = *s2)) + { + if (c1 == 0) + { + return 0; + } + + ++s1; + ++s2; + } + return uint16_t(c1) - uint16_t(c2); +} + +inline bool +StringEquals(const char16_t* s1, const char16_t* s2) +{ + return StringCompare(s1, s2) == 0; +} + inline size_t StringLength(const char16_t* str) { diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h index 89e4b76bf..2ca2b7c81 100644 --- a/src/zencore/include/zencore/trace.h +++ b/src/zencore/include/zencore/trace.h @@ -19,6 +19,8 @@ ZEN_THIRD_PARTY_INCLUDES_END #define ZEN_TRACE_CPU(x) TRACE_CPU_SCOPE(x) #define ZEN_TRACE_CPU_FLUSH(x) TRACE_CPU_SCOPE(x, trace::CpuScopeFlags::CpuFlush) +namespace zen { + enum class TraceType { File, @@ -32,6 +34,8 @@ bool IsTracing(); void TraceStart(std::string_view ProgramName, const char* HostOrPath, TraceType Type); bool TraceStop(); +} + #else #define ZEN_TRACE_CPU(x) |