diff options
| author | Dan Engelbrecht <[email protected]> | 2026-04-11 11:00:24 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-04-11 11:00:24 +0200 |
| commit | 170cc57780bb1337e2348c94e8d78f41ec15f804 (patch) | |
| tree | d2fbc75c24eeec6174b749ce0919cef37991ffe8 | |
| parent | HTTP range responses (RFC 7233) - httpobjectstore (#928) (diff) | |
| download | zen-170cc57780bb1337e2348c94e8d78f41ec15f804.tar.xz zen-170cc57780bb1337e2348c94e8d78f41ec15f804.zip | |
update rpmalloc and tweak for commit/decommit churn (#934)
- Improvement: Updated rpmalloc to develop branch commit feb43aee0d4d (2025-10-26), which fixes `VirtualAlloc(MEM_COMMIT)` failures being silently ignored under memory pressure
- Improvement: Increased rpmalloc page decommit thresholds to reduce commit/decommit churn under high allocation turnover
| -rw-r--r-- | CHANGELOG.md | 2 | ||||
| -rw-r--r-- | thirdparty/VERSIONS.md | 1 | ||||
| -rw-r--r-- | thirdparty/rpmalloc/rpmalloc.c | 221 | ||||
| -rw-r--r-- | thirdparty/rpmalloc/rpmalloc.h | 84 | ||||
| -rw-r--r-- | thirdparty/xmake.lua | 6 |
5 files changed, 226 insertions, 88 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 2beee79d4..4830dc6f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ - Single-byte ranges (`bytes=N-N`) are now correctly accepted (were previously rejected) - Range byte positions widened from 32-bit to 64-bit; RFC 7233 imposes no size limit on byte range values - Build store binary GET requests with a Range header now return 206 Partial Content with `Content-Range` (previously returned 200 OK without it) +- Improvement: Updated rpmalloc to develop branch commit 262c698d7019 (2026-04-10), which fixes memory ordering on weak architectures and avoids assert on mmap failure with callback +- Improvement: Increased rpmalloc page decommit thresholds to reduce commit/decommit churn under high allocation turnover - Bugfix: Added logic to shared memory instance state management to ensure unclean shutdown followed by restart with identical pid doesn't lead to errors. Particularly likely to happen when running on k8s ## 5.8.3 diff --git a/thirdparty/VERSIONS.md b/thirdparty/VERSIONS.md index 38a1415d3..9b3dcd103 100644 --- a/thirdparty/VERSIONS.md +++ b/thirdparty/VERSIONS.md @@ -19,6 +19,7 @@ dependency. * doctest - v2.4.12 from https://github.com/doctest/doctest/releases/download/v2.4.12/doctest.h * fmt - v12.0.0 from https://github.com/fmtlib/fmt/archive/refs/tags/12.0.0.tar.gz * robin-map - v1.4.0 from https://github.com/Tessil/robin-map/archive/refs/tags/v1.4.0.tar.gz +* rpmalloc - 1.5.0-dev (develop branch commit 262c698d7019, 2026-04-10) from https://github.com/mjansson/rpmalloc (`global_page_free_overflow` and `global_page_free_retain` manually tweaked) * ryml - v0.5.0 from https://github.com/biojppm/rapidyaml (note that there are submodules here which have also been fetched, after stripping all `.git` metadata, for future updates it's probably easier to just grab the .zip/.tar.gz since it includes all submodules) * sol2 - v3.5.0 from https://github.com/ThePhD/sol2/archive/refs/tags/v3.5.0.tar.gz (single/single.py generates the headers) * spdlog - v1.16.0 from https://github.com/gabime/spdlog/releases/tag/v1.16.0.zip diff --git a/thirdparty/rpmalloc/rpmalloc.c b/thirdparty/rpmalloc/rpmalloc.c index 08cefe6dd..b8fe16a0a 100644 --- a/thirdparty/rpmalloc/rpmalloc.c +++ b/thirdparty/rpmalloc/rpmalloc.c @@ -57,6 +57,9 @@ #endif #if PLATFORM_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif #include <windows.h> #include <fibersapi.h> static DWORD fls_key; @@ -184,6 +187,12 @@ madvise(caddr_t, size_t, int); #define SPAN_SIZE (256 * 1024 * 1024) #define SPAN_MASK (~((uintptr_t)(SPAN_SIZE - 1))) +#if ENABLE_VALIDATE_ARGS +//! Maximum allocation size to avoid integer overflow +#undef MAX_ALLOC_SIZE +#define MAX_ALLOC_SIZE (((size_t)-1) - SPAN_SIZE) +#endif + //////////// /// /// Utility macros @@ -258,13 +267,13 @@ static inline size_t rpmalloc_clz(uintptr_t x) { #if ARCH_64BIT #if defined(_MSC_VER) && !defined(__clang__) - return (size_t)_lzcnt_u64(x); + return (size_t)__lzcnt64(x); #else return (size_t)__builtin_clzll(x); #endif #else #if defined(_MSC_VER) && !defined(__clang__) - return (size_t)_lzcnt_u32(x); + return (size_t)__lzcnt32(x); #else return (size_t)__builtin_clzl(x); #endif @@ -279,9 +288,9 @@ wait_spin(void) { #else _mm_pause(); #endif -#elif defined(__x86_64__) || defined(__i386__) +#elif (defined(__x86_64__) || defined(__i386__)) && !defined(_M_ARM64EC) __asm__ volatile("pause" ::: "memory"); -#elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH >= 7) +#elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(_M_ARM64EC) __asm__ volatile("yield" ::: "memory"); #elif defined(__powerpc__) || defined(__powerpc64__) // No idea if ever been compiled in such archs but ... as precaution @@ -468,6 +477,9 @@ struct heap_t { uint32_t offset; //! Memory map size size_t mapped_size; +#if RPMALLOC_HEAP_STATISTICS + struct rpmalloc_heap_statistics_t stats; +#endif }; _Static_assert(sizeof(page_t) <= PAGE_HEADER_SIZE, "Invalid page header size"); @@ -530,10 +542,10 @@ static const size_class_t global_size_class[SIZE_CLASS_COUNT] = { LCLASS(262144), LCLASS(327680), LCLASS(393216), LCLASS(458752), LCLASS(524288)}; //! Threshold number of pages for when free pages are decommitted -static uint32_t global_page_free_overflow[4] = {16, 8, 2, 0}; +static uint32_t global_page_free_overflow[4] = {64, 16, 4, 0}; //! Number of pages to retain when free page threshold overflows -static uint32_t global_page_free_retain[4] = {4, 2, 1, 0}; +static uint32_t global_page_free_retain[4] = {16, 4, 2, 0}; //! OS huge page support static int os_huge_pages; @@ -719,6 +731,8 @@ os_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) { // page to avoid saturating the OS commit limit #if ENABLE_DECOMMIT DWORD do_commit = 0; + if (global_config.disable_decommit) + do_commit = MEM_COMMIT; #else DWORD do_commit = MEM_COMMIT; #endif @@ -788,35 +802,29 @@ os_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) { page_mapped_current, memory_order_relaxed, memory_order_relaxed)) break; } -#if ENABLE_DECOMMIT - size_t page_active_current = - atomic_fetch_add_explicit(&global_statistics.page_active, page_count, memory_order_relaxed) + page_count; - size_t page_active_peak = atomic_load_explicit(&global_statistics.page_active_peak, memory_order_relaxed); - while (page_active_current > page_active_peak) { - if (atomic_compare_exchange_weak_explicit(&global_statistics.page_active_peak, &page_active_peak, - page_active_current, memory_order_relaxed, memory_order_relaxed)) - break; - } -#endif #endif return ptr; } -static void +static int os_mcommit(void* address, size_t size) { #if ENABLE_DECOMMIT - if (global_config.disable_decommit) - return; + if (global_config.disable_decommit) { + return 0; + } #if PLATFORM_WINDOWS if (!VirtualAlloc(address, size, MEM_COMMIT, PAGE_READWRITE)) { + if (global_memory_interface->map_fail_callback && global_memory_interface->map_fail_callback(size)) + return os_mcommit(address, size); rpmalloc_assert(0, "Failed to commit virtual memory block"); + return 1; } #else - /* - if (mprotect(address, size, PROT_READ | PROT_WRITE)) { - rpmalloc_assert(0, "Failed to commit virtual memory block"); - } - */ + /* + if (mprotect(address, size, PROT_READ | PROT_WRITE)) { + rpmalloc_assert(0, "Failed to commit virtual memory block"); + } + */ #endif #if ENABLE_STATISTICS size_t page_count = size / global_config.page_size; @@ -833,23 +841,25 @@ os_mcommit(void* address, size_t size) { #endif (void)sizeof(address); (void)sizeof(size); + return 0; } -static void +static int os_mdecommit(void* address, size_t size) { #if ENABLE_DECOMMIT if (global_config.disable_decommit) - return; + return 1; #if PLATFORM_WINDOWS if (!VirtualFree(address, size, MEM_DECOMMIT)) { rpmalloc_assert(0, "Failed to decommit virtual memory block"); + return 1; } #else - /* - if (mprotect(address, size, PROT_NONE)) { - rpmalloc_assert(0, "Failed to decommit virtual memory block"); - } - */ + /* + if (mprotect(address, size, PROT_NONE)) { + rpmalloc_assert(0, "Failed to decommit virtual memory block"); + } + */ #if defined(MADV_DONTNEED) if (madvise(address, size, MADV_DONTNEED)) { #elif defined(MADV_FREE_REUSABLE) @@ -865,6 +875,7 @@ os_mdecommit(void* address, size_t size) { if (posix_madvise(address, size, POSIX_MADV_DONTNEED)) { #endif rpmalloc_assert(0, "Failed to decommit virtual memory block"); + return 1; } #endif #if ENABLE_STATISTICS @@ -879,6 +890,7 @@ os_mdecommit(void* address, size_t size) { (void)sizeof(address); (void)sizeof(size); #endif + return 0; } static void @@ -986,19 +998,29 @@ page_decommit_memory_pages(page_t* page) { return; void* extra_page = pointer_offset(page, global_config.page_size); size_t extra_page_size = page_get_size(page) - global_config.page_size; - global_memory_interface->memory_decommit(extra_page, extra_page_size); + if (global_memory_interface->memory_decommit(extra_page, extra_page_size) != 0) + return; +#if RPMALLOC_HEAP_STATISTICS && ENABLE_DECOMMIT + if (page->heap) + page->heap->stats.committed_size -= extra_page_size; +#endif page->is_decommitted = 1; } -static inline void +static inline int page_commit_memory_pages(page_t* page) { if (!page->is_decommitted) - return; + return 0; void* extra_page = pointer_offset(page, global_config.page_size); size_t extra_page_size = page_get_size(page) - global_config.page_size; - global_memory_interface->memory_commit(extra_page, extra_page_size); + if (global_memory_interface->memory_commit(extra_page, extra_page_size) != 0) + return 1; page->is_decommitted = 0; #if ENABLE_DECOMMIT +#if RPMALLOC_HEAP_STATISTICS + if (page->heap) + page->heap->stats.committed_size += extra_page_size; +#endif #if !defined(__APPLE__) // When page is recommitted, the blocks in the second memory page and forward // will be zeroed out by OS - take advantage in zalloc/calloc calls and make sure @@ -1008,6 +1030,7 @@ page_commit_memory_pages(page_t* page) { page->is_zero = 1; #endif #endif + return 0; } static void @@ -1090,7 +1113,7 @@ static NOINLINE void page_adopt_thread_free_block_list(page_t* page) { if (page->local_free) return; - unsigned long long thread_free = atomic_load_explicit(&page->thread_free, memory_order_acquire); + unsigned long long thread_free = atomic_load_explicit(&page->thread_free, memory_order_relaxed); if (thread_free != 0) { // Other threads can only replace with another valid list head, this will never change to 0 in other threads while (!atomic_compare_exchange_weak_explicit(&page->thread_free, &thread_free, 0, memory_order_acquire, @@ -1243,8 +1266,13 @@ span_allocate_page(span_t* span) { #if ENABLE_DECOMMIT // The first page is always committed on initial span map of memory - if (span->page_initialized) - global_memory_interface->memory_commit(page, span->page_size); + if (span->page_initialized) { + if (global_memory_interface->memory_commit(page, span->page_size) != 0) + return 0; +#if RPMALLOC_HEAP_STATISTICS + heap->stats.committed_size += span->page_size; +#endif + } #endif ++span->page_initialized; @@ -1268,6 +1296,16 @@ span_allocate_page(span_t* span) { static NOINLINE void span_deallocate_block(span_t* span, page_t* page, void* block) { if (UNEXPECTED(page->page_type == PAGE_HUGE)) { +#if RPMALLOC_HEAP_STATISTICS + if (span->heap) { + span->heap->stats.mapped_size -= span->mapped_size; +#if ENABLE_DECOMMIT + span->heap->stats.committed_size -= span->page_count * span->page_size; +#else + span->heap->stats.committed_size -= mapped_size; +#endif + } +#endif global_memory_interface->memory_unmap(span, span->offset, span->mapped_size); return; } @@ -1303,6 +1341,16 @@ block_deallocate(block_t* block) { page_t* page = span_get_page_from_block(span, block); const int is_thread_local = page_is_thread_heap(page); +#if RPMALLOC_HEAP_STATISTICS + heap_t* heap = span->heap; + if (heap) { + if (span->page_type <= PAGE_LARGE) + heap->stats.allocated_size -= page->block_size; + else + heap->stats.allocated_size -= ((size_t)span->page_size * (size_t)span->page_count); + } +#endif + // Optimized path for thread local free with non-huge block in page // that has no aligned blocks if (EXPECTED(is_thread_local != 0)) { @@ -1373,7 +1421,8 @@ heap_allocate_new(void) { size_t mapped_size = 0; block_t* block = global_memory_interface->memory_map(heap_size, 0, &offset, &mapped_size); #if ENABLE_DECOMMIT - global_memory_interface->memory_commit(block, heap_size); + if (global_memory_interface->memory_commit(block, heap_size) != 0) + return 0; #endif heap_t* heap = heap_initialize((void*)block); heap->offset = (uint32_t)offset; @@ -1442,7 +1491,7 @@ heap_page_free_decommit(heap_t* heap, uint32_t page_type, uint32_t page_retain_c } } -static inline void +static inline int heap_make_free_page_available(heap_t* heap, uint32_t size_class, page_t* page) { page->size_class = size_class; page->block_size = global_size_class[size_class].block_size; @@ -1463,8 +1512,9 @@ heap_make_free_page_available(heap_t* heap, uint32_t size_class, page_t* page) { if (head) head->prev = page; heap->page_available[size_class] = page; - if (page->is_decommitted) - page_commit_memory_pages(page); + if (page->is_decommitted != 0) + return page_commit_memory_pages(page); + return 0; } //! Find or allocate a span for the given page type with the given size class @@ -1478,6 +1528,9 @@ heap_get_span(heap_t* heap, page_type_t page_type) { size_t offset = 0; size_t mapped_size = 0; span_t* span = global_memory_interface->memory_map(SPAN_SIZE, SPAN_SIZE, &offset, &mapped_size); +#if RPMALLOC_HEAP_STATISTICS + heap->stats.mapped_size += mapped_size; +#endif if (EXPECTED(span != 0)) { uint32_t page_count = 0; uint32_t page_size = 0; @@ -1496,7 +1549,15 @@ heap_get_span(heap_t* heap, page_type_t page_type) { page_address_mask = LARGE_PAGE_MASK; } #if ENABLE_DECOMMIT - global_memory_interface->memory_commit(span, page_size); + if (global_memory_interface->memory_commit(span, page_size) != 0) + return 0; +#endif +#if RPMALLOC_HEAP_STATISTICS +#if ENABLE_DECOMMIT + heap->stats.committed_size += page_size; +#else + heap->stats.committed_size += mapped_size; +#endif #endif span->heap = heap; span->page_type = page_type; @@ -1523,9 +1584,9 @@ heap_get_page_generic(heap_t* heap, uint32_t size_class) { page_type_t page_type = get_page_type(size_class); // Check if there is a free page from multithreaded deallocations - uintptr_t block_mt = atomic_load_explicit(&heap->thread_free[page_type], memory_order_acquire); + uintptr_t block_mt = atomic_load_explicit(&heap->thread_free[page_type], memory_order_relaxed); if (UNEXPECTED(block_mt != 0)) { - while (!atomic_compare_exchange_weak_explicit(&heap->thread_free[page_type], &block_mt, 0, memory_order_release, + while (!atomic_compare_exchange_weak_explicit(&heap->thread_free[page_type], &block_mt, 0, memory_order_acquire, memory_order_relaxed)) { wait_spin(); } @@ -1547,7 +1608,8 @@ heap_get_page_generic(heap_t* heap, uint32_t size_class) { rpmalloc_assert(heap->page_free_commit_count[page_type] > 0, "Free committed page count out of sync"); --heap->page_free_commit_count[page_type]; } - heap_make_free_page_available(heap, size_class, page); + if (heap_make_free_page_available(heap, size_class, page) != 0) + return 0; return page; } rpmalloc_assert(heap->page_free_commit_count[page_type] == 0, "Free committed page count out of sync"); @@ -1565,7 +1627,8 @@ heap_get_page_generic(heap_t* heap, uint32_t size_class) { span_t* span = heap_get_span(heap, page_type); if (EXPECTED(span != 0)) { page = span_allocate_page(span); - heap_make_free_page_available(page->heap, size_class, page); + if (heap_make_free_page_available(page->heap, size_class, page) != 0) + return 0; } return page; @@ -1604,6 +1667,7 @@ heap_allocate_block_small_to_large(heap_t* heap, uint32_t size_class, unsigned i static NOINLINE RPMALLOC_ALLOCATOR void* heap_allocate_block_huge(heap_t* heap, size_t size, unsigned int zero) { if (heap->id == 0) { + // Thread has not yet initialized, assign heap and try again rpmalloc_initialize(0); heap = get_thread_heap(); } @@ -1614,7 +1678,16 @@ heap_allocate_block_huge(heap_t* heap, size_t size, unsigned int zero) { if (block) { span_t* span = block; #if ENABLE_DECOMMIT - global_memory_interface->memory_commit(span, alloc_size); + if (global_memory_interface->memory_commit(span, alloc_size) != 0) + return 0; +#endif +#if RPMALLOC_HEAP_STATISTICS + heap->stats.mapped_size += mapped_size; +#if ENABLE_DECOMMIT + heap->stats.committed_size += alloc_size; +#else + heap->stats.committed_size += mapped_size; +#endif #endif span->heap = heap; span->page_type = PAGE_HUGE; @@ -1635,6 +1708,9 @@ heap_allocate_block_huge(heap_t* heap, size_t size, unsigned int zero) { void* ptr = pointer_offset(block, SPAN_HEADER_SIZE); if (zero) memset(ptr, 0, size); +#if RPMALLOC_HEAP_STATISTICS + heap->stats.allocated_size += size; +#endif return ptr; } return 0; @@ -1644,6 +1720,10 @@ static RPMALLOC_ALLOCATOR NOINLINE void* heap_allocate_block_generic(heap_t* heap, size_t size, unsigned int zero) { uint32_t size_class = get_size_class(size); if (EXPECTED(size_class < SIZE_CLASS_COUNT)) { +#if RPMALLOC_HEAP_STATISTICS + heap->stats.allocated_size += global_size_class[size_class].block_size; +#endif + block_t* block = heap_pop_local_free(heap, size_class); if (EXPECTED(block != 0)) { // Fast track with small block available in heap level local free list @@ -1668,6 +1748,9 @@ heap_allocate_block(heap_t* heap, size_t size, unsigned int zero) { // Fast track with small block available in heap level local free list if (zero) memset(block, 0, global_size_class[size_class].block_size); +#if RPMALLOC_HEAP_STATISTICS + heap->stats.allocated_size += global_size_class[size_class].block_size; +#endif return block; } } @@ -1901,7 +1984,7 @@ rprealloc(void* ptr, size_t size) { extern RPMALLOC_ALLOCATOR void* rpaligned_realloc(void* ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) { #if ENABLE_VALIDATE_ARGS - if ((size + alignment < size) || (alignment > _memory_page_size)) { + if ((size + alignment < size) || (alignment > SMALL_PAGE_SIZE)) { errno = EINVAL; return 0; } @@ -2210,6 +2293,21 @@ rpmalloc_dump_statistics(void* file) { #endif } +void +rpmalloc_global_statistics(rpmalloc_global_statistics_t* stats) { +#if ENABLE_STATISTICS + stats->mapped = global_config.page_size * atomic_load_explicit(&global_statistics.page_mapped, memory_order_relaxed); + stats->mapped_peak = global_config.page_size * atomic_load_explicit(&global_statistics.page_mapped_peak, memory_order_relaxed); + stats->committed = global_config.page_size * atomic_load_explicit(&global_statistics.page_commit, memory_order_relaxed); + stats->decommitted = global_config.page_size * atomic_load_explicit(&global_statistics.page_decommit, memory_order_relaxed); + stats->active = global_config.page_size * atomic_load_explicit(&global_statistics.page_active, memory_order_relaxed); + stats->active_peak = global_config.page_size * atomic_load_explicit(&global_statistics.page_active_peak, memory_order_relaxed); + stats->heap_count = atomic_load_explicit(&global_statistics.heap_count, memory_order_relaxed); +#else + memset(stats, 0, sizeof(rpmalloc_global_statistics_t)); +#endif +} + #if RPMALLOC_FIRST_CLASS_HEAPS rpmalloc_heap_t* @@ -2253,6 +2351,17 @@ rpmalloc_heap_aligned_alloc(rpmalloc_heap_t* heap, size_t alignment, size_t size } RPMALLOC_ALLOCATOR void* +rpmalloc_heap_aligned_zalloc(rpmalloc_heap_t* heap, size_t alignment, size_t size) { +#if ENABLE_VALIDATE_ARGS + if (size >= MAX_ALLOC_SIZE) { + errno = EINVAL; + return 0; + } +#endif + return heap_allocate_block_aligned(heap, alignment, size, 1); +} + +RPMALLOC_ALLOCATOR void* rpmalloc_heap_calloc(rpmalloc_heap_t* heap, size_t num, size_t size) { size_t total; #if ENABLE_VALIDATE_ARGS @@ -2312,7 +2421,7 @@ rpmalloc_heap_realloc(rpmalloc_heap_t* heap, void* ptr, size_t size, unsigned in RPMALLOC_ALLOCATOR void* rpmalloc_heap_aligned_realloc(rpmalloc_heap_t* heap, void* ptr, size_t alignment, size_t size, unsigned int flags) { #if ENABLE_VALIDATE_ARGS - if ((size + alignment < size) || (alignment > _memory_page_size)) { + if ((size + alignment < size) || (alignment > SMALL_PAGE_SIZE)) { errno = EINVAL; return 0; } @@ -2332,6 +2441,18 @@ rpmalloc_heap_free_all(rpmalloc_heap_t* heap) { heap_free_all(heap); } +struct rpmalloc_heap_statistics_t +rpmalloc_heap_statistics(rpmalloc_heap_t* heap) { +#if RPMALLOC_HEAP_STATISTICS + if (heap) { + return heap->stats; + } +#endif + (void)sizeof(heap); + struct rpmalloc_heap_statistics_t stats = {0}; + return stats; +} + extern inline void rpmalloc_heap_thread_set_current(rpmalloc_heap_t* heap) { heap_t* prev_heap = get_thread_heap(); diff --git a/thirdparty/rpmalloc/rpmalloc.h b/thirdparty/rpmalloc/rpmalloc.h index d11292fb1..ea7d18e23 100644 --- a/thirdparty/rpmalloc/rpmalloc.h +++ b/thirdparty/rpmalloc/rpmalloc.h @@ -54,11 +54,16 @@ extern "C" { #define RPMALLOC_MAX_ALIGNMENT (256 * 1024) -//! Define RPMALLOC_FIRST_CLASS_HEAPS to enable heap based API (rpmalloc_heap_* functions). +//! Define RPMALLOC_FIRST_CLASS_HEAPS to non-zero to enable heap based API (rpmalloc_heap_* functions). #ifndef RPMALLOC_FIRST_CLASS_HEAPS #define RPMALLOC_FIRST_CLASS_HEAPS 0 #endif +//! Define RPMALLOC_HEAP_STATISTICS to non-zero to enable first class heap statistics gathering. +#ifndef RPMALLOC_HEAP_STATISTICS +#define RPMALLOC_HEAP_STATISTICS 0 +#endif + //! Flag to rpaligned_realloc to not preserve content in reallocation #define RPMALLOC_NO_PRESERVE 1 //! Flag to rpaligned_realloc to fail and return null pointer if grow cannot be done in-place, @@ -72,18 +77,16 @@ typedef struct rpmalloc_global_statistics_t { size_t mapped; //! Peak amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_STATISTICS=1) size_t mapped_peak; - //! Current amount of memory in global caches for small and medium sizes (<32KiB) - size_t cached; - //! Current amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB by - //! default (only if ENABLE_STATISTICS=1) - size_t huge_alloc; - //! Peak amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB by default - //! (only if ENABLE_STATISTICS=1) - size_t huge_alloc_peak; - //! Total amount of memory mapped since initialization (only if ENABLE_STATISTICS=1) - size_t mapped_total; - //! Total amount of memory unmapped since initialization (only if ENABLE_STATISTICS=1) - size_t unmapped_total; + //! Running counter of total amount of memory committed (only if ENABLE_STATISTICS=1) + size_t committed; + //! Running counter of total amount of memory decommitted (only if ENABLE_STATISTICS=1) + size_t decommitted; + //! Current amount of virtual memory active and committed (only if ENABLE_STATISTICS=1) + size_t active; + //! Peak amount of virtual memory active and committed (only if ENABLE_STATISTICS=1) + size_t active_peak; + //! Current heap count (only if ENABLE_STATISTICS=1) + size_t heap_count; } rpmalloc_global_statistics_t; typedef struct rpmalloc_thread_statistics_t { @@ -147,10 +150,10 @@ typedef struct rpmalloc_interface_t { //! set a memory_unmap function or else the default implementation will be used for both. This function must be //! thread safe, it can be called by multiple threads simultaneously. void* (*memory_map)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size); - //! Commit a range of memory pages - void (*memory_commit)(void* address, size_t size); - //! Decommit a range of memory pages - void (*memory_decommit)(void* address, size_t size); + //! Commit a range of memory pages. Return non-zero if the operation failed and the address range could not be committed. + int (*memory_commit)(void* address, size_t size); + //! Decommit a range of memory pages. Return non-zero if the operation failed and the address range could not be decommitted. + int (*memory_decommit)(void* address, size_t size); //! Unmap the memory pages starting at address and spanning the given number of bytes. If you set a memory_unmap //! function, you must also set a memory_map function or else the default implementation will be used for both. This //! function must be thread safe, it can be called by multiple threads simultaneously. @@ -260,44 +263,38 @@ rprealloc(void* ptr, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_S //! Reallocate the given block to at least the given size and alignment, // with optional control flags (see RPMALLOC_NO_PRESERVE). // Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB) +// and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpaligned_realloc(void* ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3); //! Allocate a memory block of at least the given size and alignment. // Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB) +// and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); //! Allocate a memory block of at least the given size and alignment. // Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB) +// and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpaligned_zalloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); //! Allocate a memory block of at least the given size and alignment, and zero initialize it. // Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB) +// and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpaligned_calloc(size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3); //! Allocate a memory block of at least the given size and alignment. // Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB) +// and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); //! Allocate a memory block of at least the given size and alignment. // Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB) +// and should ideally be less than memory page size. RPMALLOC_EXPORT int rpposix_memalign(void** memptr, size_t alignment, size_t size); @@ -336,12 +333,18 @@ rpmalloc_heap_alloc(rpmalloc_heap_t* heap, size_t size) RPMALLOC_ATTRIB_MALLOC R //! Allocate a memory block of at least the given size using the given heap. The returned // block will have the requested alignment. Alignment must be a power of two and a multiple of sizeof(void*), -// and should ideally be less than memory page size. A caveat of rpmalloc -// internals is that this must also be strictly less than the span size (default 64KiB). +// and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpmalloc_heap_aligned_alloc(rpmalloc_heap_t* heap, size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3); +//! Allocate a zero initialized memory block of at least the given size using the given heap. The returned +// block will have the requested alignment. Alignment must be a power of two and a multiple of sizeof(void*), +// and should ideally be less than memory page size. +RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* +rpmalloc_heap_aligned_zalloc(rpmalloc_heap_t* heap, size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC + RPMALLOC_ATTRIB_ALLOC_SIZE(3); + //! Allocate a memory block of at least the given size using the given heap and zero initialize it. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpmalloc_heap_calloc(rpmalloc_heap_t* heap, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC @@ -349,8 +352,7 @@ rpmalloc_heap_calloc(rpmalloc_heap_t* heap, size_t num, size_t size) RPMALLOC_AT //! Allocate a memory block of at least the given size using the given heap and zero initialize it. The returned // block will have the requested alignment. Alignment must either be zero, or a power of two and a multiple of -// sizeof(void*), and should ideally be less than memory page size. A caveat of rpmalloc internals is that this must -// also be strictly less than the span size (default 64KiB). +// sizeof(void*), and should ideally be less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpmalloc_heap_aligned_calloc(rpmalloc_heap_t* heap, size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(3, 4); @@ -364,8 +366,7 @@ rpmalloc_heap_realloc(rpmalloc_heap_t* heap, void* ptr, size_t size, unsigned in //! Reallocate the given block to at least the given size. The memory block MUST be allocated // by the same heap given to this function. The returned block will have the requested alignment. // Alignment must be either zero, or a power of two and a multiple of sizeof(void*), and should ideally be -// less than memory page size. A caveat of rpmalloc internals is that this must also be strictly less than -// the span size (default 64KiB). +// less than memory page size. RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* rpmalloc_heap_aligned_realloc(rpmalloc_heap_t* heap, void* ptr, size_t alignment, size_t size, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(4); @@ -379,6 +380,19 @@ rpmalloc_heap_free(rpmalloc_heap_t* heap, void* ptr); RPMALLOC_EXPORT void rpmalloc_heap_free_all(rpmalloc_heap_t* heap); +struct rpmalloc_heap_statistics_t { + // Number of bytes allocated + size_t allocated_size; + // Number of bytes committed + size_t committed_size; + // Number of bytes mapped + size_t mapped_size; +}; + +//! Get heap statistics (if enabled in build) +RPMALLOC_EXPORT struct rpmalloc_heap_statistics_t +rpmalloc_heap_statistics(rpmalloc_heap_t* heap); + //! Set the given heap as the current heap for the calling thread. A heap MUST only be current heap // for a single thread, a heap can never be shared between multiple threads. The previous // current heap for the calling thread is released to be reused by other threads. diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index 1f5902fdf..ea861fc55 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -32,9 +32,9 @@ target('ue-trace') add_includedirs("trace", {public=true}) add_headerfiles("trace/**.h") --- rpmalloc 1.5.0-dev.20250810 --- Vendored from develop branch commit 6b34d956911b (2025-08-10) --- https://github.com/mjansson/rpmalloc/commit/6b34d956911b +-- rpmalloc 1.5.0-dev.20251026 +-- Vendored from develop branch commit feb43aee0d4d (2025-10-26) +-- https://github.com/mjansson/rpmalloc/commit/feb43aee0d4d target('rpmalloc') set_kind("static") set_group('thirdparty') |