Skip to content
6 changes: 4 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -660,7 +660,8 @@ if(MI_USE_CXX)
set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c PROPERTIES LANGUAGE CXX )
# the white-box tests call internal functions, whose names are mangled now; the heap tests use C11 atomics, which MSVC lacks in C mode
set_source_files_properties(test/test-theap-sentinel.c test/test-purge-holes.c test/test-commit-fail.c
test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c PROPERTIES LANGUAGE CXX )
test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c
test/test-abandoned-lazy.c PROPERTIES LANGUAGE CXX )
endif()


Expand Down Expand Up @@ -877,7 +878,7 @@ if (MI_BUILD_TESTS)
enable_testing()

# static link tests
set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN))
list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first
endif()
Expand Down Expand Up @@ -923,6 +924,7 @@ if (MI_BUILD_TESTS)

# with no scavenger to hand off to, `mi_on_thread_idle_start` must do the work inline instead
add_test(NAME test-park-handoff-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $<TARGET_FILE:mimalloc-test-park-handoff>)
add_test(NAME test-heap-release-mt-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $<TARGET_FILE:mimalloc-test-heap-release-mt>)
# and sweep every park, so the reclaim path is hit as hard as possible
add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $<TARGET_FILE:mimalloc-test-park-handoff>)

Expand Down
2 changes: 2 additions & 0 deletions include/mimalloc/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,7 @@ void _mi_arenas_abandoned_page_free(mi_page_t* page, mi_theap_t* curren
void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap);
void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
void _mi_arena_pages_free(mi_arena_pages_t* arena_pages);
size_t mi_arenas_get_count(mi_subproc_t* subproc);
uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index);

Expand Down Expand Up @@ -356,6 +357,7 @@ extern "C" {
#endif
extern mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_thread_theaps_done;
extern mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_heap_delete_claim;
extern mi_decl_export _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated;
extern mi_decl_export volatile long mi_debug_fail_os_commit_after;
#ifdef __cplusplus
}
Expand Down
10 changes: 10 additions & 0 deletions include/mimalloc/prim-tls.h
Original file line number Diff line number Diff line change
Expand Up @@ -149,13 +149,23 @@ static inline void* mi_prim_tls_slot(size_t slot) {
#else
return mi_prim_thread_pointer()[slot];
#endif
#elif defined(__GNUC__) || defined(__clang__)
// The slot lives in the thread's control block, which the OS reuses for a later thread, so the
// exiting thread's last store (`_mi_thread_done`) and a new thread's first load touch the same
// address from different threads. A relaxed atomic is the same single load/store instruction and
// states that this is by design (and keeps TSAN quiet about it).
return __atomic_load_n(&mi_prim_thread_pointer()[slot], __ATOMIC_RELAXED);
#else
return mi_prim_thread_pointer()[slot];
#endif
}

static inline void mi_prim_tls_slot_set(size_t slot, void* value) {
#if defined(__GNUC__) || defined(__clang__)
__atomic_store_n(&mi_prim_thread_pointer()[slot], value, __ATOMIC_RELAXED);
#else
mi_prim_thread_pointer()[slot] = value;
#endif
}
#endif

Expand Down
10 changes: 8 additions & 2 deletions include/mimalloc/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -628,6 +628,7 @@ typedef struct mi_heap_s {
mi_lock_t theaps_lock; // lock for the theaps list operations

_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap
_Atomic(uintptr_t) releasing; // set when `mi_heap_delete`/`mi_heap_destroy` starts: its pages are abandoned unmapped
mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena
mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)

Expand Down Expand Up @@ -756,8 +757,13 @@ typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in

struct mi_arena_pages_s {
mi_bitmap_t* pages; // all registered pages (abandoned and owned)
mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)
// followed by the bitmaps (whose siz`es depend on the arena size)
// Abandoned pages per size bin (a set bit means the start of the page). Each bitmap is
// allocated the first time a page of that bin is abandoned (`mi_arena_pages_abandoned_ensure`);
// NULL means no page of that bin was ever abandoned. Eagerly laying out all MI_ARENA_BIN_COUNT
// of them cost a page fault per bitmap (one header write each, on its own OS page) for every
// heap that touched an arena, and most heaps never abandon a page.
_Atomic(mi_bitmap_t*) pages_abandoned[MI_ARENA_BIN_COUNT];
// followed by the `pages` bitmap (whose size depends on the arena size)
};


Expand Down
Loading
Loading