From 6a7b171d0081e8a3bab3e7889aa4bd07f77d106f Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 8 Jan 2025 10:26:57 +0000 Subject: [PATCH 1/6] Adding support for building using BB version of mmtk-julia --- Make.inc | 7 +++ deps/checksums/mmtk_julia | 2 + deps/mmtk/mmtk.h | 113 +++++++++++++++++++++++++++++++++++++ deps/mmtk/mmtkMutator.h | 114 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 deps/mmtk/mmtk.h create mode 100644 deps/mmtk/mmtkMutator.h diff --git a/Make.inc b/Make.inc index 16e238c6f0683..7af7c59fdf7c8 100644 --- a/Make.inc +++ b/Make.inc @@ -1426,6 +1426,13 @@ $(error "Attempting to build OpenBLAS or SuiteSparse without a functioning fortr endif endif +ifneq (${MMTK_PLAN},None) +# We still need some .h files from mmtk when +# building Julia using the BinaryBuilder +ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1) +MMTK_API_INC=$(JULIAHOME)/deps/mmtk/ +endif +endif # OS specific stuff diff --git a/deps/checksums/mmtk_julia b/deps/checksums/mmtk_julia index 979ab79e52207..b3179627d2dc3 100644 --- a/deps/checksums/mmtk_julia +++ b/deps/checksums/mmtk_julia @@ -4,3 +4,5 @@ mmtk_julia-f07d66aafc86af84ea988b35335acc9bbc770fa1.tar.gz/md5/38afb5db6d8c55413 mmtk_julia-f07d66aafc86af84ea988b35335acc9bbc770fa1.tar.gz/sha512/78525582a46a6baf8d33df7b622e55cf244439afcd7192ba55489c1bc18393d1237d2903d517c610484bf9e2a7338ad31435a9cbf70889d6bcf87c40cec829e5 mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/md5/631b204574da7062802dac501a4b711f mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/sha512/daaed59d08fc49621479ed638dea0aac0cba123986e486571447e8e21e9a098776ce2e87fbd92ddea276782fc44621f23d40fa213296b28e1d4480553c7de4f7 +mmtk_julia.v0.30.2+0.x86_64-linux-gnu.tar.gz/md5/0b8d667927b7ae181245dd1aba467dcb +mmtk_julia.v0.30.2+0.x86_64-linux-gnu.tar.gz/sha512/df45a7b148efcbf8fe24b51899bdc78f052cd96bcf43ff8e96f4ca02851589f4d634f879e2ae23e98917ba235a533a13cf0cb092bfa4fc2a48e260ae1efc007d diff --git a/deps/mmtk/mmtk.h b/deps/mmtk/mmtk.h new file mode 100644 index 0000000000000..c88204a769c41 --- /dev/null +++ b/deps/mmtk/mmtk.h @@ -0,0 +1,113 @@ +#ifndef MMTK_H +#define MMTK_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* MMTk_Mutator; +typedef void* MMTk_TraceLocal; +typedef void (*ProcessSlotFn)(void* closure, void* slot); +typedef void (*ProcessOffsetSlotFn)(void* closure, void* slot, int offset); + +typedef struct { + void** ptr; + size_t cap; +} RootsWorkBuffer; + +typedef struct { + RootsWorkBuffer (*report_slots_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + void* data; +} RootsWorkClosure; + +/** + * Allocation + */ +extern MMTk_Mutator mmtk_bind_mutator(void *tls, int tid); +extern void mmtk_post_bind_mutator(MMTk_Mutator mutator, MMTk_Mutator original_mutator); +extern void mmtk_destroy_mutator(MMTk_Mutator mutator); + +extern void* mmtk_alloc(MMTk_Mutator mutator, size_t size, + size_t align, size_t offset, int allocator); + +extern void* mmtk_alloc_large(MMTk_Mutator mutator, size_t size, + size_t align, size_t offset, int allocator); + +extern void mmtk_post_alloc(MMTk_Mutator mutator, void* refer, + size_t bytes, int allocator); + +extern bool mmtk_is_live_object(void* ref); +extern bool mmtk_is_mapped_object(void* ref); +extern bool mmtk_is_mapped_address(void* addr); +extern int mmtk_object_is_managed_by_mmtk(void* addr); +extern void mmtk_runtime_panic(void); +extern void mmtk_unreachable(void); +extern unsigned char mmtk_pin_object(void* obj); +extern bool mmtk_is_pinned(void* obj); +extern const char* get_mmtk_version(void); + +extern void mmtk_set_vm_space(void* addr, size_t size); +extern void mmtk_immortal_region_post_alloc(void* addr, size_t size); + +// Write barriers +extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* src_addr, void* dst_obj, void* dst_addr, size_t size); +extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target); +extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target); +extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; + +extern _Atomic(uintptr_t) JULIA_MALLOC_BYTES; + +/** + * Misc + */ +extern void mmtk_gc_init(uintptr_t min_heap_size, uintptr_t max_heap_size, uintptr_t n_gcthreads, uintptr_t header_size, uintptr_t tag); +extern bool mmtk_will_never_move(void* object); +extern bool mmtk_process(char* name, char* value); +extern void mmtk_scan_region(void); +extern void mmtk_handle_user_collection_request(void *tls, uint8_t collection); +extern void mmtk_initialize_collection(void* tls); +extern void mmtk_start_control_collector(void *tls); +extern void mmtk_start_worker(void *tls, void* worker, void* mmtk); +extern void mmtk_process_julia_obj(void* addr); +extern void mmtk_register_finalizer(void* obj, void* function, bool is_ptr); +extern void mmtk_run_finalizers_for_obj(void* obj); +extern void mmtk_run_finalizers(bool at_exit); +extern void mmtk_gc_poll(void *tls); +extern void mmtk_julia_copy_stack_check(int copy_stack); +extern void* mmtk_get_possibly_forwarded(void* object); +extern void mmtk_block_thread_for_gc(void); +extern void* mmtk_new_mutator_iterator(void); +extern void* mmtk_get_next_mutator_tls(void*); +extern void* mmtk_close_mutator_iterator(void*); + + +/** + * VM Accounting + */ +extern size_t mmtk_free_bytes(void); +extern size_t mmtk_total_bytes(void); +extern size_t mmtk_used_bytes(void); +extern void* mmtk_starting_heap_address(void); +extern void* mmtk_last_heap_address(void); + +/** + * Reference Processing + */ +extern void mmtk_add_weak_candidate(void* ref); +extern void mmtk_add_soft_candidate(void* ref); +extern void mmtk_add_phantom_candidate(void* ref); + +extern void mmtk_harness_begin(void *tls); +extern void mmtk_harness_end(void); + +#ifdef __cplusplus +} +#endif + +#endif // MMTK_H diff --git a/deps/mmtk/mmtkMutator.h b/deps/mmtk/mmtkMutator.h new file mode 100644 index 0000000000000..2f513406353f2 --- /dev/null +++ b/deps/mmtk/mmtkMutator.h @@ -0,0 +1,114 @@ +#ifndef MMTK_JULIA_MMTK_MUTATOR_H +#define MMTK_JULIA_MMTK_MUTATOR_H + +// mmtk_julia_types.h refers to the types in this file. +// So if this file is updated, make sure you regenerate Rust types for mmtk_julia_types.h. + +enum Allocator { + AllocatorDefault = 0, + AllocatorImmortal = 1, + AllocatorLos = 2, + AllocatorCode = 3, + AllocatorReadOnly = 4, +}; + +typedef struct { + void* data; + void* vtable; +} RustDynPtr; + +// These constants should match the constants defind in mmtk::util::alloc::allocators +#define MAX_BUMP_ALLOCATORS 6 +#define MAX_LARGE_OBJECT_ALLOCATORS 2 +#define MAX_MALLOC_ALLOCATORS 1 +#define MAX_IMMIX_ALLOCATORS 1 +#define MAX_FREE_LIST_ALLOCATORS 2 +#define MAX_MARK_COMPACT_ALLOCATORS 1 + +// The following types should have the same layout as the types with the same name in MMTk core (Rust) + +typedef struct { + void* tls; + void* cursor; + void* limit; + RustDynPtr space; + void* context; +} BumpAllocator; + +typedef struct { + void* tls; + void* space; + void* context; +} LargeObjectAllocator; + +typedef struct { + void* tls; + void* cursor; + void* limit; + void* immix_space; + void* context; + uint8_t hot; + uint8_t copy; + void* large_cursor; + void* large_limit; + uint8_t request_for_large; + uint8_t _align[7]; + uint8_t line_opt_tag; + uintptr_t line_opt; +} ImmixAllocator; + +typedef struct { + void* Address; +} FLBlock; + +typedef struct { + FLBlock first; + FLBlock last; + size_t size; + char lock; +} FLBlockList; + +typedef struct { + void* tls; + void* space; + void* context; + FLBlockList* available_blocks; + FLBlockList* available_blocks_stress; + FLBlockList* unswept_blocks; + FLBlockList* consumed_blocks; +} FreeListAllocator; + +typedef struct { + void* tls; + void* space; + void* context; +} MMTkMallocAllocator; // Prefix with MMTk to avoid name clash + +typedef struct { + BumpAllocator bump_allocator; +} MarkCompactAllocator; + +typedef struct { + BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS]; + LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS]; + MMTkMallocAllocator malloc[MAX_MALLOC_ALLOCATORS]; + ImmixAllocator immix[MAX_IMMIX_ALLOCATORS]; + FreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS]; + MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS]; +} Allocators; + +typedef struct { + void* allocator_mapping; + void* space_mapping; + RustDynPtr prepare_func; + RustDynPtr release_func; +} MutatorConfig; + +typedef struct { + Allocators allocators; + RustDynPtr barrier; + void* mutator_tls; + RustDynPtr plan; + MutatorConfig config; +} MMTkMutatorContext; +#endif // MMTK_MUTATOR_HPP From 148d7f9a6334f5d54875d33db20af4b41a188b1c Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Fri, 17 Jan 2025 00:47:25 +0000 Subject: [PATCH 2/6] Updating BB version and adding .h files to BB artifact --- Make.inc | 7 --- deps/checksums/mmtk_julia | 9 ++- deps/mmtk/mmtk.h | 113 ------------------------------------- deps/mmtk/mmtkMutator.h | 114 -------------------------------------- deps/mmtk_julia.version | 4 +- 5 files changed, 6 insertions(+), 241 deletions(-) delete mode 100644 deps/mmtk/mmtk.h delete mode 100644 deps/mmtk/mmtkMutator.h diff --git a/Make.inc b/Make.inc index 7af7c59fdf7c8..16e238c6f0683 100644 --- a/Make.inc +++ b/Make.inc @@ -1426,13 +1426,6 @@ $(error "Attempting to build OpenBLAS or SuiteSparse without a functioning fortr endif endif -ifneq (${MMTK_PLAN},None) -# We still need some .h files from mmtk when -# building Julia using the BinaryBuilder -ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1) -MMTK_API_INC=$(JULIAHOME)/deps/mmtk/ -endif -endif # OS specific stuff diff --git a/deps/checksums/mmtk_julia b/deps/checksums/mmtk_julia index b3179627d2dc3..3297dda790cb4 100644 --- a/deps/checksums/mmtk_julia +++ b/deps/checksums/mmtk_julia @@ -1,8 +1,7 @@ -mmtk_julia-b69acf5af7a7dd97c1cc6fd99f7c2d51b477f214.tar.gz/md5/1911cf084d26c48e2ed58af3d268b4b6 -mmtk_julia-b69acf5af7a7dd97c1cc6fd99f7c2d51b477f214.tar.gz/sha512/75beab54398989c46b62e714b242cf6705d88d220f40c21e494e0f29161437f5fbe9ba05b543d2353a1ad76f4239ac4025b476be0be864649f310f14935289fe -mmtk_julia-f07d66aafc86af84ea988b35335acc9bbc770fa1.tar.gz/md5/38afb5db6d8c55413a4ec96aefa2ebb4 mmtk_julia-f07d66aafc86af84ea988b35335acc9bbc770fa1.tar.gz/sha512/78525582a46a6baf8d33df7b622e55cf244439afcd7192ba55489c1bc18393d1237d2903d517c610484bf9e2a7338ad31435a9cbf70889d6bcf87c40cec829e5 mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/md5/631b204574da7062802dac501a4b711f mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/sha512/daaed59d08fc49621479ed638dea0aac0cba123986e486571447e8e21e9a098776ce2e87fbd92ddea276782fc44621f23d40fa213296b28e1d4480553c7de4f7 -mmtk_julia.v0.30.2+0.x86_64-linux-gnu.tar.gz/md5/0b8d667927b7ae181245dd1aba467dcb -mmtk_julia.v0.30.2+0.x86_64-linux-gnu.tar.gz/sha512/df45a7b148efcbf8fe24b51899bdc78f052cd96bcf43ff8e96f4ca02851589f4d634f879e2ae23e98917ba235a533a13cf0cb092bfa4fc2a48e260ae1efc007d +mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/md5/73a8fbea71edce30a39a30f31969dd8e +mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/sha512/374848b7696b565dea66daa208830581f92c1fcb0138e7a7ab88564402e94bc79c54b6ed370ec68473e31e2bd411bf82c97793796c31d39aafbbfffea9c05588 +mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/md5/8cdeb14fd69945f64308be49f6912f9c +mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/sha512/3692502f65dec8c0971b56b9bf8178641892b390d520cbcd69880d75b7500e6341534d87882246e68998f590f824ec54c18f4b8fb4aa09b8f313de065c48450e \ No newline at end of file diff --git a/deps/mmtk/mmtk.h b/deps/mmtk/mmtk.h deleted file mode 100644 index c88204a769c41..0000000000000 --- a/deps/mmtk/mmtk.h +++ /dev/null @@ -1,113 +0,0 @@ -#ifndef MMTK_H -#define MMTK_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* MMTk_Mutator; -typedef void* MMTk_TraceLocal; -typedef void (*ProcessSlotFn)(void* closure, void* slot); -typedef void (*ProcessOffsetSlotFn)(void* closure, void* slot, int offset); - -typedef struct { - void** ptr; - size_t cap; -} RootsWorkBuffer; - -typedef struct { - RootsWorkBuffer (*report_slots_func)(void** buf, size_t size, size_t cap, void* data, bool renew); - RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); - RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); - void* data; -} RootsWorkClosure; - -/** - * Allocation - */ -extern MMTk_Mutator mmtk_bind_mutator(void *tls, int tid); -extern void mmtk_post_bind_mutator(MMTk_Mutator mutator, MMTk_Mutator original_mutator); -extern void mmtk_destroy_mutator(MMTk_Mutator mutator); - -extern void* mmtk_alloc(MMTk_Mutator mutator, size_t size, - size_t align, size_t offset, int allocator); - -extern void* mmtk_alloc_large(MMTk_Mutator mutator, size_t size, - size_t align, size_t offset, int allocator); - -extern void mmtk_post_alloc(MMTk_Mutator mutator, void* refer, - size_t bytes, int allocator); - -extern bool mmtk_is_live_object(void* ref); -extern bool mmtk_is_mapped_object(void* ref); -extern bool mmtk_is_mapped_address(void* addr); -extern int mmtk_object_is_managed_by_mmtk(void* addr); -extern void mmtk_runtime_panic(void); -extern void mmtk_unreachable(void); -extern unsigned char mmtk_pin_object(void* obj); -extern bool mmtk_is_pinned(void* obj); -extern const char* get_mmtk_version(void); - -extern void mmtk_set_vm_space(void* addr, size_t size); -extern void mmtk_immortal_region_post_alloc(void* addr, size_t size); - -// Write barriers -extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* src_addr, void* dst_obj, void* dst_addr, size_t size); -extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target); -extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target); -extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; - -extern _Atomic(uintptr_t) JULIA_MALLOC_BYTES; - -/** - * Misc - */ -extern void mmtk_gc_init(uintptr_t min_heap_size, uintptr_t max_heap_size, uintptr_t n_gcthreads, uintptr_t header_size, uintptr_t tag); -extern bool mmtk_will_never_move(void* object); -extern bool mmtk_process(char* name, char* value); -extern void mmtk_scan_region(void); -extern void mmtk_handle_user_collection_request(void *tls, uint8_t collection); -extern void mmtk_initialize_collection(void* tls); -extern void mmtk_start_control_collector(void *tls); -extern void mmtk_start_worker(void *tls, void* worker, void* mmtk); -extern void mmtk_process_julia_obj(void* addr); -extern void mmtk_register_finalizer(void* obj, void* function, bool is_ptr); -extern void mmtk_run_finalizers_for_obj(void* obj); -extern void mmtk_run_finalizers(bool at_exit); -extern void mmtk_gc_poll(void *tls); -extern void mmtk_julia_copy_stack_check(int copy_stack); -extern void* mmtk_get_possibly_forwarded(void* object); -extern void mmtk_block_thread_for_gc(void); -extern void* mmtk_new_mutator_iterator(void); -extern void* mmtk_get_next_mutator_tls(void*); -extern void* mmtk_close_mutator_iterator(void*); - - -/** - * VM Accounting - */ -extern size_t mmtk_free_bytes(void); -extern size_t mmtk_total_bytes(void); -extern size_t mmtk_used_bytes(void); -extern void* mmtk_starting_heap_address(void); -extern void* mmtk_last_heap_address(void); - -/** - * Reference Processing - */ -extern void mmtk_add_weak_candidate(void* ref); -extern void mmtk_add_soft_candidate(void* ref); -extern void mmtk_add_phantom_candidate(void* ref); - -extern void mmtk_harness_begin(void *tls); -extern void mmtk_harness_end(void); - -#ifdef __cplusplus -} -#endif - -#endif // MMTK_H diff --git a/deps/mmtk/mmtkMutator.h b/deps/mmtk/mmtkMutator.h deleted file mode 100644 index 2f513406353f2..0000000000000 --- a/deps/mmtk/mmtkMutator.h +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef MMTK_JULIA_MMTK_MUTATOR_H -#define MMTK_JULIA_MMTK_MUTATOR_H - -// mmtk_julia_types.h refers to the types in this file. -// So if this file is updated, make sure you regenerate Rust types for mmtk_julia_types.h. - -enum Allocator { - AllocatorDefault = 0, - AllocatorImmortal = 1, - AllocatorLos = 2, - AllocatorCode = 3, - AllocatorReadOnly = 4, -}; - -typedef struct { - void* data; - void* vtable; -} RustDynPtr; - -// These constants should match the constants defind in mmtk::util::alloc::allocators -#define MAX_BUMP_ALLOCATORS 6 -#define MAX_LARGE_OBJECT_ALLOCATORS 2 -#define MAX_MALLOC_ALLOCATORS 1 -#define MAX_IMMIX_ALLOCATORS 1 -#define MAX_FREE_LIST_ALLOCATORS 2 -#define MAX_MARK_COMPACT_ALLOCATORS 1 - -// The following types should have the same layout as the types with the same name in MMTk core (Rust) - -typedef struct { - void* tls; - void* cursor; - void* limit; - RustDynPtr space; - void* context; -} BumpAllocator; - -typedef struct { - void* tls; - void* space; - void* context; -} LargeObjectAllocator; - -typedef struct { - void* tls; - void* cursor; - void* limit; - void* immix_space; - void* context; - uint8_t hot; - uint8_t copy; - void* large_cursor; - void* large_limit; - uint8_t request_for_large; - uint8_t _align[7]; - uint8_t line_opt_tag; - uintptr_t line_opt; -} ImmixAllocator; - -typedef struct { - void* Address; -} FLBlock; - -typedef struct { - FLBlock first; - FLBlock last; - size_t size; - char lock; -} FLBlockList; - -typedef struct { - void* tls; - void* space; - void* context; - FLBlockList* available_blocks; - FLBlockList* available_blocks_stress; - FLBlockList* unswept_blocks; - FLBlockList* consumed_blocks; -} FreeListAllocator; - -typedef struct { - void* tls; - void* space; - void* context; -} MMTkMallocAllocator; // Prefix with MMTk to avoid name clash - -typedef struct { - BumpAllocator bump_allocator; -} MarkCompactAllocator; - -typedef struct { - BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS]; - LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS]; - MMTkMallocAllocator malloc[MAX_MALLOC_ALLOCATORS]; - ImmixAllocator immix[MAX_IMMIX_ALLOCATORS]; - FreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS]; - MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS]; -} Allocators; - -typedef struct { - void* allocator_mapping; - void* space_mapping; - RustDynPtr prepare_func; - RustDynPtr release_func; -} MutatorConfig; - -typedef struct { - Allocators allocators; - RustDynPtr barrier; - void* mutator_tls; - RustDynPtr plan; - MutatorConfig config; -} MMTkMutatorContext; -#endif // MMTK_MUTATOR_HPP diff --git a/deps/mmtk_julia.version b/deps/mmtk_julia.version index cb1e8064f9825..af5de9b374de8 100644 --- a/deps/mmtk_julia.version +++ b/deps/mmtk_julia.version @@ -1,6 +1,6 @@ MMTK_JULIA_BRANCH = master -MMTK_JULIA_SHA1 = f07d66aafc86af84ea988b35335acc9bbc770fa1 +MMTK_JULIA_SHA1 = c9e046baf3a0d52fe75d6c8b28f6afd69b045d95 MMTK_JULIA_GIT_URL := https://github.com/mmtk/mmtk-julia.git MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.3.tar.gz -MMTK_JULIA_JLL_VER := 0.30.3+1 +MMTK_JULIA_JLL_VER := 0.30.4+0 MMTK_JULIA_JLL_NAME := mmtk_julia From a78ea78872128d3167d269e78140e99330c827ba Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Fri, 17 Jan 2025 19:30:23 +0000 Subject: [PATCH 3/6] Updating BB version --- deps/checksums/mmtk_julia | 2 +- deps/mmtk_julia.version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/checksums/mmtk_julia b/deps/checksums/mmtk_julia index 3297dda790cb4..d0af87d5c831c 100644 --- a/deps/checksums/mmtk_julia +++ b/deps/checksums/mmtk_julia @@ -4,4 +4,4 @@ mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/sha512/daaed59d08fc49621479ed638dea mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/md5/73a8fbea71edce30a39a30f31969dd8e mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/sha512/374848b7696b565dea66daa208830581f92c1fcb0138e7a7ab88564402e94bc79c54b6ed370ec68473e31e2bd411bf82c97793796c31d39aafbbfffea9c05588 mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/md5/8cdeb14fd69945f64308be49f6912f9c -mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/sha512/3692502f65dec8c0971b56b9bf8178641892b390d520cbcd69880d75b7500e6341534d87882246e68998f590f824ec54c18f4b8fb4aa09b8f313de065c48450e \ No newline at end of file +mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/sha512/3692502f65dec8c0971b56b9bf8178641892b390d520cbcd69880d75b7500e6341534d87882246e68998f590f824ec54c18f4b8fb4aa09b8f313de065c48450e diff --git a/deps/mmtk_julia.version b/deps/mmtk_julia.version index af5de9b374de8..684197bbe3e4e 100644 --- a/deps/mmtk_julia.version +++ b/deps/mmtk_julia.version @@ -1,6 +1,6 @@ MMTK_JULIA_BRANCH = master MMTK_JULIA_SHA1 = c9e046baf3a0d52fe75d6c8b28f6afd69b045d95 MMTK_JULIA_GIT_URL := https://github.com/mmtk/mmtk-julia.git -MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.3.tar.gz +MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.4.tar.gz MMTK_JULIA_JLL_VER := 0.30.4+0 MMTK_JULIA_JLL_NAME := mmtk_julia From ec9a652930f4dcf3c135a7e98c75ed48cfa87d73 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 8 Jan 2025 10:26:57 +0000 Subject: [PATCH 4/6] Adding support for building using BB version of mmtk-julia --- Make.inc | 7 +++ deps/mmtk/mmtk.h | 113 +++++++++++++++++++++++++++++++++++++++ deps/mmtk/mmtkMutator.h | 114 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 deps/mmtk/mmtk.h create mode 100644 deps/mmtk/mmtkMutator.h diff --git a/Make.inc b/Make.inc index 16e238c6f0683..7af7c59fdf7c8 100644 --- a/Make.inc +++ b/Make.inc @@ -1426,6 +1426,13 @@ $(error "Attempting to build OpenBLAS or SuiteSparse without a functioning fortr endif endif +ifneq (${MMTK_PLAN},None) +# We still need some .h files from mmtk when +# building Julia using the BinaryBuilder +ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1) +MMTK_API_INC=$(JULIAHOME)/deps/mmtk/ +endif +endif # OS specific stuff diff --git a/deps/mmtk/mmtk.h b/deps/mmtk/mmtk.h new file mode 100644 index 0000000000000..c88204a769c41 --- /dev/null +++ b/deps/mmtk/mmtk.h @@ -0,0 +1,113 @@ +#ifndef MMTK_H +#define MMTK_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* MMTk_Mutator; +typedef void* MMTk_TraceLocal; +typedef void (*ProcessSlotFn)(void* closure, void* slot); +typedef void (*ProcessOffsetSlotFn)(void* closure, void* slot, int offset); + +typedef struct { + void** ptr; + size_t cap; +} RootsWorkBuffer; + +typedef struct { + RootsWorkBuffer (*report_slots_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); + void* data; +} RootsWorkClosure; + +/** + * Allocation + */ +extern MMTk_Mutator mmtk_bind_mutator(void *tls, int tid); +extern void mmtk_post_bind_mutator(MMTk_Mutator mutator, MMTk_Mutator original_mutator); +extern void mmtk_destroy_mutator(MMTk_Mutator mutator); + +extern void* mmtk_alloc(MMTk_Mutator mutator, size_t size, + size_t align, size_t offset, int allocator); + +extern void* mmtk_alloc_large(MMTk_Mutator mutator, size_t size, + size_t align, size_t offset, int allocator); + +extern void mmtk_post_alloc(MMTk_Mutator mutator, void* refer, + size_t bytes, int allocator); + +extern bool mmtk_is_live_object(void* ref); +extern bool mmtk_is_mapped_object(void* ref); +extern bool mmtk_is_mapped_address(void* addr); +extern int mmtk_object_is_managed_by_mmtk(void* addr); +extern void mmtk_runtime_panic(void); +extern void mmtk_unreachable(void); +extern unsigned char mmtk_pin_object(void* obj); +extern bool mmtk_is_pinned(void* obj); +extern const char* get_mmtk_version(void); + +extern void mmtk_set_vm_space(void* addr, size_t size); +extern void mmtk_immortal_region_post_alloc(void* addr, size_t size); + +// Write barriers +extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* src_addr, void* dst_obj, void* dst_addr, size_t size); +extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target); +extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target); +extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; + +extern _Atomic(uintptr_t) JULIA_MALLOC_BYTES; + +/** + * Misc + */ +extern void mmtk_gc_init(uintptr_t min_heap_size, uintptr_t max_heap_size, uintptr_t n_gcthreads, uintptr_t header_size, uintptr_t tag); +extern bool mmtk_will_never_move(void* object); +extern bool mmtk_process(char* name, char* value); +extern void mmtk_scan_region(void); +extern void mmtk_handle_user_collection_request(void *tls, uint8_t collection); +extern void mmtk_initialize_collection(void* tls); +extern void mmtk_start_control_collector(void *tls); +extern void mmtk_start_worker(void *tls, void* worker, void* mmtk); +extern void mmtk_process_julia_obj(void* addr); +extern void mmtk_register_finalizer(void* obj, void* function, bool is_ptr); +extern void mmtk_run_finalizers_for_obj(void* obj); +extern void mmtk_run_finalizers(bool at_exit); +extern void mmtk_gc_poll(void *tls); +extern void mmtk_julia_copy_stack_check(int copy_stack); +extern void* mmtk_get_possibly_forwarded(void* object); +extern void mmtk_block_thread_for_gc(void); +extern void* mmtk_new_mutator_iterator(void); +extern void* mmtk_get_next_mutator_tls(void*); +extern void* mmtk_close_mutator_iterator(void*); + + +/** + * VM Accounting + */ +extern size_t mmtk_free_bytes(void); +extern size_t mmtk_total_bytes(void); +extern size_t mmtk_used_bytes(void); +extern void* mmtk_starting_heap_address(void); +extern void* mmtk_last_heap_address(void); + +/** + * Reference Processing + */ +extern void mmtk_add_weak_candidate(void* ref); +extern void mmtk_add_soft_candidate(void* ref); +extern void mmtk_add_phantom_candidate(void* ref); + +extern void mmtk_harness_begin(void *tls); +extern void mmtk_harness_end(void); + +#ifdef __cplusplus +} +#endif + +#endif // MMTK_H diff --git a/deps/mmtk/mmtkMutator.h b/deps/mmtk/mmtkMutator.h new file mode 100644 index 0000000000000..2f513406353f2 --- /dev/null +++ b/deps/mmtk/mmtkMutator.h @@ -0,0 +1,114 @@ +#ifndef MMTK_JULIA_MMTK_MUTATOR_H +#define MMTK_JULIA_MMTK_MUTATOR_H + +// mmtk_julia_types.h refers to the types in this file. +// So if this file is updated, make sure you regenerate Rust types for mmtk_julia_types.h. + +enum Allocator { + AllocatorDefault = 0, + AllocatorImmortal = 1, + AllocatorLos = 2, + AllocatorCode = 3, + AllocatorReadOnly = 4, +}; + +typedef struct { + void* data; + void* vtable; +} RustDynPtr; + +// These constants should match the constants defind in mmtk::util::alloc::allocators +#define MAX_BUMP_ALLOCATORS 6 +#define MAX_LARGE_OBJECT_ALLOCATORS 2 +#define MAX_MALLOC_ALLOCATORS 1 +#define MAX_IMMIX_ALLOCATORS 1 +#define MAX_FREE_LIST_ALLOCATORS 2 +#define MAX_MARK_COMPACT_ALLOCATORS 1 + +// The following types should have the same layout as the types with the same name in MMTk core (Rust) + +typedef struct { + void* tls; + void* cursor; + void* limit; + RustDynPtr space; + void* context; +} BumpAllocator; + +typedef struct { + void* tls; + void* space; + void* context; +} LargeObjectAllocator; + +typedef struct { + void* tls; + void* cursor; + void* limit; + void* immix_space; + void* context; + uint8_t hot; + uint8_t copy; + void* large_cursor; + void* large_limit; + uint8_t request_for_large; + uint8_t _align[7]; + uint8_t line_opt_tag; + uintptr_t line_opt; +} ImmixAllocator; + +typedef struct { + void* Address; +} FLBlock; + +typedef struct { + FLBlock first; + FLBlock last; + size_t size; + char lock; +} FLBlockList; + +typedef struct { + void* tls; + void* space; + void* context; + FLBlockList* available_blocks; + FLBlockList* available_blocks_stress; + FLBlockList* unswept_blocks; + FLBlockList* consumed_blocks; +} FreeListAllocator; + +typedef struct { + void* tls; + void* space; + void* context; +} MMTkMallocAllocator; // Prefix with MMTk to avoid name clash + +typedef struct { + BumpAllocator bump_allocator; +} MarkCompactAllocator; + +typedef struct { + BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS]; + LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS]; + MMTkMallocAllocator malloc[MAX_MALLOC_ALLOCATORS]; + ImmixAllocator immix[MAX_IMMIX_ALLOCATORS]; + FreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS]; + MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS]; +} Allocators; + +typedef struct { + void* allocator_mapping; + void* space_mapping; + RustDynPtr prepare_func; + RustDynPtr release_func; +} MutatorConfig; + +typedef struct { + Allocators allocators; + RustDynPtr barrier; + void* mutator_tls; + RustDynPtr plan; + MutatorConfig config; +} MMTkMutatorContext; +#endif // MMTK_MUTATOR_HPP From 16779bbc11df1a4048ca90b1ebd83a2d1a4dcb35 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Fri, 17 Jan 2025 19:56:06 +0000 Subject: [PATCH 5/6] Use Immix by default --- Make.inc | 9 +--- deps/mmtk/mmtk.h | 113 --------------------------------------- deps/mmtk/mmtkMutator.h | 114 ---------------------------------------- 3 files changed, 1 insertion(+), 235 deletions(-) delete mode 100644 deps/mmtk/mmtk.h delete mode 100644 deps/mmtk/mmtkMutator.h diff --git a/Make.inc b/Make.inc index 7af7c59fdf7c8..97a8a3f47a62a 100644 --- a/Make.inc +++ b/Make.inc @@ -81,7 +81,7 @@ WITH_GC_VERIFY := 0 WITH_GC_DEBUG_ENV := 0 # Use stock if MMTK_PLAN hasn't been defined -MMTK_PLAN ?= None +MMTK_PLAN ?= Immix # Enable DTrace support WITH_DTRACE := 0 @@ -1426,13 +1426,6 @@ $(error "Attempting to build OpenBLAS or SuiteSparse without a functioning fortr endif endif -ifneq (${MMTK_PLAN},None) -# We still need some .h files from mmtk when -# building Julia using the BinaryBuilder -ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1) -MMTK_API_INC=$(JULIAHOME)/deps/mmtk/ -endif -endif # OS specific stuff diff --git a/deps/mmtk/mmtk.h b/deps/mmtk/mmtk.h deleted file mode 100644 index c88204a769c41..0000000000000 --- a/deps/mmtk/mmtk.h +++ /dev/null @@ -1,113 +0,0 @@ -#ifndef MMTK_H -#define MMTK_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* MMTk_Mutator; -typedef void* MMTk_TraceLocal; -typedef void (*ProcessSlotFn)(void* closure, void* slot); -typedef void (*ProcessOffsetSlotFn)(void* closure, void* slot, int offset); - -typedef struct { - void** ptr; - size_t cap; -} RootsWorkBuffer; - -typedef struct { - RootsWorkBuffer (*report_slots_func)(void** buf, size_t size, size_t cap, void* data, bool renew); - RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); - RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew); - void* data; -} RootsWorkClosure; - -/** - * Allocation - */ -extern MMTk_Mutator mmtk_bind_mutator(void *tls, int tid); -extern void mmtk_post_bind_mutator(MMTk_Mutator mutator, MMTk_Mutator original_mutator); -extern void mmtk_destroy_mutator(MMTk_Mutator mutator); - -extern void* mmtk_alloc(MMTk_Mutator mutator, size_t size, - size_t align, size_t offset, int allocator); - -extern void* mmtk_alloc_large(MMTk_Mutator mutator, size_t size, - size_t align, size_t offset, int allocator); - -extern void mmtk_post_alloc(MMTk_Mutator mutator, void* refer, - size_t bytes, int allocator); - -extern bool mmtk_is_live_object(void* ref); -extern bool mmtk_is_mapped_object(void* ref); -extern bool mmtk_is_mapped_address(void* addr); -extern int mmtk_object_is_managed_by_mmtk(void* addr); -extern void mmtk_runtime_panic(void); -extern void mmtk_unreachable(void); -extern unsigned char mmtk_pin_object(void* obj); -extern bool mmtk_is_pinned(void* obj); -extern const char* get_mmtk_version(void); - -extern void mmtk_set_vm_space(void* addr, size_t size); -extern void mmtk_immortal_region_post_alloc(void* addr, size_t size); - -// Write barriers -extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* src_addr, void* dst_obj, void* dst_addr, size_t size); -extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target); -extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target); -extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; - -extern _Atomic(uintptr_t) JULIA_MALLOC_BYTES; - -/** - * Misc - */ -extern void mmtk_gc_init(uintptr_t min_heap_size, uintptr_t max_heap_size, uintptr_t n_gcthreads, uintptr_t header_size, uintptr_t tag); -extern bool mmtk_will_never_move(void* object); -extern bool mmtk_process(char* name, char* value); -extern void mmtk_scan_region(void); -extern void mmtk_handle_user_collection_request(void *tls, uint8_t collection); -extern void mmtk_initialize_collection(void* tls); -extern void mmtk_start_control_collector(void *tls); -extern void mmtk_start_worker(void *tls, void* worker, void* mmtk); -extern void mmtk_process_julia_obj(void* addr); -extern void mmtk_register_finalizer(void* obj, void* function, bool is_ptr); -extern void mmtk_run_finalizers_for_obj(void* obj); -extern void mmtk_run_finalizers(bool at_exit); -extern void mmtk_gc_poll(void *tls); -extern void mmtk_julia_copy_stack_check(int copy_stack); -extern void* mmtk_get_possibly_forwarded(void* object); -extern void mmtk_block_thread_for_gc(void); -extern void* mmtk_new_mutator_iterator(void); -extern void* mmtk_get_next_mutator_tls(void*); -extern void* mmtk_close_mutator_iterator(void*); - - -/** - * VM Accounting - */ -extern size_t mmtk_free_bytes(void); -extern size_t mmtk_total_bytes(void); -extern size_t mmtk_used_bytes(void); -extern void* mmtk_starting_heap_address(void); -extern void* mmtk_last_heap_address(void); - -/** - * Reference Processing - */ -extern void mmtk_add_weak_candidate(void* ref); -extern void mmtk_add_soft_candidate(void* ref); -extern void mmtk_add_phantom_candidate(void* ref); - -extern void mmtk_harness_begin(void *tls); -extern void mmtk_harness_end(void); - -#ifdef __cplusplus -} -#endif - -#endif // MMTK_H diff --git a/deps/mmtk/mmtkMutator.h b/deps/mmtk/mmtkMutator.h deleted file mode 100644 index 2f513406353f2..0000000000000 --- a/deps/mmtk/mmtkMutator.h +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef MMTK_JULIA_MMTK_MUTATOR_H -#define MMTK_JULIA_MMTK_MUTATOR_H - -// mmtk_julia_types.h refers to the types in this file. -// So if this file is updated, make sure you regenerate Rust types for mmtk_julia_types.h. - -enum Allocator { - AllocatorDefault = 0, - AllocatorImmortal = 1, - AllocatorLos = 2, - AllocatorCode = 3, - AllocatorReadOnly = 4, -}; - -typedef struct { - void* data; - void* vtable; -} RustDynPtr; - -// These constants should match the constants defind in mmtk::util::alloc::allocators -#define MAX_BUMP_ALLOCATORS 6 -#define MAX_LARGE_OBJECT_ALLOCATORS 2 -#define MAX_MALLOC_ALLOCATORS 1 -#define MAX_IMMIX_ALLOCATORS 1 -#define MAX_FREE_LIST_ALLOCATORS 2 -#define MAX_MARK_COMPACT_ALLOCATORS 1 - -// The following types should have the same layout as the types with the same name in MMTk core (Rust) - -typedef struct { - void* tls; - void* cursor; - void* limit; - RustDynPtr space; - void* context; -} BumpAllocator; - -typedef struct { - void* tls; - void* space; - void* context; -} LargeObjectAllocator; - -typedef struct { - void* tls; - void* cursor; - void* limit; - void* immix_space; - void* context; - uint8_t hot; - uint8_t copy; - void* large_cursor; - void* large_limit; - uint8_t request_for_large; - uint8_t _align[7]; - uint8_t line_opt_tag; - uintptr_t line_opt; -} ImmixAllocator; - -typedef struct { - void* Address; -} FLBlock; - -typedef struct { - FLBlock first; - FLBlock last; - size_t size; - char lock; -} FLBlockList; - -typedef struct { - void* tls; - void* space; - void* context; - FLBlockList* available_blocks; - FLBlockList* available_blocks_stress; - FLBlockList* unswept_blocks; - FLBlockList* consumed_blocks; -} FreeListAllocator; - -typedef struct { - void* tls; - void* space; - void* context; -} MMTkMallocAllocator; // Prefix with MMTk to avoid name clash - -typedef struct { - BumpAllocator bump_allocator; -} MarkCompactAllocator; - -typedef struct { - BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS]; - LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS]; - MMTkMallocAllocator malloc[MAX_MALLOC_ALLOCATORS]; - ImmixAllocator immix[MAX_IMMIX_ALLOCATORS]; - FreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS]; - MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS]; -} Allocators; - -typedef struct { - void* allocator_mapping; - void* space_mapping; - RustDynPtr prepare_func; - RustDynPtr release_func; -} MutatorConfig; - -typedef struct { - Allocators allocators; - RustDynPtr barrier; - void* mutator_tls; - RustDynPtr plan; - MutatorConfig config; -} MMTkMutatorContext; -#endif // MMTK_MUTATOR_HPP From 0772f807af6ea9d79fb89c78eb5dddd1f8f87d87 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 23 Jan 2025 19:34:46 +0000 Subject: [PATCH 6/6] Skipping some tests that are currently incompatible with MMTk --- base/Base.jl | 2 ++ base/gcutils.jl | 11 +++++++ base/timing.jl | 6 ++-- contrib/refresh_checksums.mk | 2 +- src/gc-mmtk.c | 54 +++++++++++++++++++++++++-------- stdlib/Profile/test/allocs.jl | 13 ++++++-- stdlib/Profile/test/runtests.jl | 3 ++ test/checked.jl | 8 +++-- test/cmdlineargs.jl | 49 ++++++++++++++++++------------ test/gc.jl | 12 +++++--- test/misc.jl | 4 ++- 11 files changed, 118 insertions(+), 46 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index 04f732a4309c9..379347f027f69 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -128,6 +128,8 @@ include("sysinfo.jl") include("libc.jl") using .Libc: getpid, gethostname, time, memcpy, memset, memmove, memcmp +const USING_STOCK_GC = occursin("stock", GC.gc_active_impl()) + # These used to be in build_h.jl and are retained for backwards compatibility. # NOTE: keep in sync with `libblastrampoline_jll.libblastrampoline`. const libblas_name = "libblastrampoline" * (Sys.iswindows() ? "-5" : "") diff --git a/base/gcutils.jl b/base/gcutils.jl index 60b8ecdd17d65..d5e6f4597739f 100644 --- a/base/gcutils.jl +++ b/base/gcutils.jl @@ -281,4 +281,15 @@ function logging_enabled() ccall(:jl_is_gc_logging_enabled, Cint, ()) != 0 end +""" + GC.gc_active_impl() + +Return a string stating which GC implementation is being used and possibly +its version according to the list of supported GCs +""" +function gc_active_impl() + unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ())) +end + + end # module GC diff --git a/base/timing.jl b/base/timing.jl index 65c2a643d6a52..61fa73f2eff62 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -109,10 +109,8 @@ function gc_page_utilization_data() return Base.unsafe_wrap(Array, page_utilization_raw, JL_GC_N_MAX_POOLS, own=false) end - -const USING_STOCK_GC = occursin("stock", unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ()))) # Full sweep reasons are currently only available for the stock GC -@static if USING_STOCK_GC +@static if Base.USING_STOCK_GC # must be kept in sync with `src/gc-stock.h`` const FULL_SWEEP_REASONS = [:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL, :FULL_SWEEP_REASON_FORCED_FULL_SWEEP, :FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE] @@ -135,7 +133,7 @@ function full_sweep_reasons() d = Dict{Symbol, Int64}() # populate the dictionary according to the reasons above for the stock GC # otherwise return an empty dictionary for now - @static if USING_STOCK_GC + @static if Base.USING_STOCK_GC reason = cglobal(:jl_full_sweep_reasons, UInt64) reasons_as_array = Base.unsafe_wrap(Vector{UInt64}, reason, length(FULL_SWEEP_REASONS), own=false) for (i, r) in enumerate(FULL_SWEEP_REASONS) diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index 5a787b0b67cb1..fa7cddc705958 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient +BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 5ec1e34cc1acd..2f261a2e8e2fd 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -64,11 +64,37 @@ void jl_gc_init(void) { arraylist_new(&to_finalize, 0); arraylist_new(&finalizer_list_marked, 0); - + gc_num.interval = default_collect_interval; gc_num.allocd = 0; gc_num.max_pause = 0; gc_num.max_memory = 0; + // Necessary if we want to use Julia heap resizing heuristics + uint64_t mem_reserve = 250*1024*1024; // LLVM + other libraries need some amount of memory + uint64_t min_heap_size_hint = mem_reserve + 1*1024*1024; + uint64_t hint = jl_options.heap_size_hint; + + // check if heap size specified on command line + if (jl_options.heap_size_hint == 0) { + char *cp = getenv(HEAP_SIZE_HINT); + if (cp) + hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); + } +#ifdef _P64 + if (hint == 0) { + uint64_t constrained_mem = uv_get_constrained_memory(); + if (constrained_mem > 0 && constrained_mem < uv_get_total_memory()) + hint = constrained_mem; + } +#endif + if (hint) { + if (hint < min_heap_size_hint) + hint = min_heap_size_hint; + jl_gc_set_max_memory(hint - mem_reserve); + } + + // MMTK supports setting the heap size using the + // MMTK_MIN_HSIZE and MMTK_MAX_HSIZE environment variables long long min_heap_size; long long max_heap_size; char* min_size_def = getenv("MMTK_MIN_HSIZE"); @@ -77,7 +103,8 @@ void jl_gc_init(void) { char* max_size_def = getenv("MMTK_MAX_HSIZE"); char* max_size_gb = getenv("MMTK_MAX_HSIZE_G"); - // default min heap currently set as Julia's default_collect_interval + // If min and max values are not specified, set them to 0 here + // and use stock heuristics as defined in the binding if (min_size_def != NULL) { char *p; double min_size = strtod(min_size_def, &p); @@ -87,10 +114,9 @@ void jl_gc_init(void) { double min_size = strtod(min_size_gb, &p); min_heap_size = (long) 1024 * 1024 * 1024 * min_size; } else { - min_heap_size = default_collect_interval; + min_heap_size = 0; } - // default max heap currently set as 70% the free memory in the system if (max_size_def != NULL) { char *p; double max_size = strtod(max_size_def, &p); @@ -100,7 +126,7 @@ void jl_gc_init(void) { double max_size = strtod(max_size_gb, &p); max_heap_size = (long) 1024 * 1024 * 1024 * max_size; } else { - max_heap_size = uv_get_free_memory() * 70 / 100; + max_heap_size = 0; } // Assert that the number of stock GC threads is 0; MMTK uses the number of threads in jl_options.ngcthreads @@ -159,7 +185,17 @@ void jl_free_thread_gc_state(struct _jl_tls_states_t *ptls) { } JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem) { - // MMTk currently does not allow setting the heap size at runtime +#ifdef _P32 + max_mem = max_mem < MAX32HEAP ? max_mem : MAX32HEAP; +#endif + max_total_memory = max_mem; +} + +JL_DLLEXPORT uint64_t jl_gc_get_max_memory(void) +{ + // FIXME: We should return the max heap size set in MMTk + // when not using Julia's heap resizing heuristics + return max_total_memory; } STATIC_INLINE void maybe_collect(jl_ptls_t ptls) @@ -415,12 +451,6 @@ JL_DLLEXPORT void jl_gc_get_total_bytes(int64_t *bytes) JL_NOTSAFEPOINT *bytes = (num.total_allocd + num.deferred_alloc + num.allocd); } -JL_DLLEXPORT uint64_t jl_gc_get_max_memory(void) -{ - // FIXME: should probably return MMTk's heap size - return max_total_memory; -} - // These are needed to collect MMTk statistics from a Julia program using ccall JL_DLLEXPORT void (jl_mmtk_harness_begin)(void) { diff --git a/stdlib/Profile/test/allocs.jl b/stdlib/Profile/test/allocs.jl index d4930a2b7f5ed..5607783c782f9 100644 --- a/stdlib/Profile/test/allocs.jl +++ b/stdlib/Profile/test/allocs.jl @@ -73,8 +73,14 @@ end @test length(first_alloc.stacktrace) > 0 @test length(string(first_alloc.type)) > 0 - @testset for type in (Task, Vector{Float64},) - @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS + # Issue #57103: This test does not work with MMTk because of fastpath + # allocation which never calls the allocation profiler. + # TODO: We should port these observability tools (e.g. allocation + # profiler and heap snapshot) to MMTk + @static if Base.USING_STOCK_GC + @testset for type in (Task, Vector{Float64},) + @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS + end end # TODO: it would be nice to assert that these tasks @@ -143,6 +149,8 @@ end @test length([a for a in prof.allocs if a.type == String]) >= 1 end +# FIXME: Issue #57103 disabling test for MMTk. +@static if Base.USING_STOCK_GC @testset "alloc profiler catches allocs from codegen" begin @eval begin struct MyType x::Int; y::Int end @@ -162,6 +170,7 @@ end @test length(prof.allocs) >= 1 @test length([a for a in prof.allocs if a.type == MyType]) >= 1 end +end @testset "alloc profiler catches allocs from buffer resize" begin f(a) = for _ in 1:100; push!(a, 1); end diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index b73a2a618011b..a5a604d7923d0 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -344,6 +344,8 @@ end @test only(node.down).first == lidict[8] end +# FIXME: Issue #57103: heap snapshots is not currently supported in MMTk +@static if Base.USING_STOCK_GC @testset "HeapSnapshot" begin tmpdir = mktempdir() @@ -374,6 +376,7 @@ end rm(fname) rm(tmpdir, force = true, recursive = true) end +end @testset "PageProfile" begin fname = "$(getpid())_$(time_ns())" diff --git a/test/checked.jl b/test/checked.jl index 4031918a38730..b93c8796162c5 100644 --- a/test/checked.jl +++ b/test/checked.jl @@ -331,8 +331,12 @@ end @test checked_pow(BigInt(2), 2) == BigInt(4) @test checked_pow(BigInt(2), 100) == BigInt(1267650600228229401496703205376) - # Perf test: Make sure BigInts allocs don't scale with the power: - @test @allocations(checked_pow(BigInt(2), 2)) ≈ @allocations(checked_pow(BigInt(2), 10000)) rtol=0.9 + # FIXME: Issue #57103: the following test may fail because + # allocation may not be logged via MMTk's fastpath allocation + @static if Base.USING_STOCK_GC + # Perf test: Make sure BigInts allocs don't scale with the power: + @test @allocations(checked_pow(BigInt(2), 2)) ≈ @allocations(checked_pow(BigInt(2), 10000)) rtol=0.9 + end end @testset "Additional tests" begin diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 3ff7836223b84..ec7a51e804d3f 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -383,29 +383,33 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test p.exitcode == 1 && p.termsignal == 0 end - # --gcthreads - code = "print(Threads.ngcthreads())" - cpu_threads = ccall(:jl_effective_threads, Int32, ()) - @test string(cpu_threads) == - read(`$exename --threads auto -e $code`, String) == - read(`$exename --threads=auto -e $code`, String) == - read(`$exename -tauto -e $code`, String) == - read(`$exename -t auto -e $code`, String) - for nt in (nothing, "1") - withenv("JULIA_NUM_GC_THREADS" => nt) do - @test read(`$exename --gcthreads=2 -e $code`, String) == "2" - end - withenv("JULIA_NUM_GC_THREADS" => nt) do - @test read(`$exename --gcthreads=2,1 -e $code`, String) == "3" + # FIXME: Issue #57103 --gcthreads does not have the same semantics + # for Stock GC and MMTk, so the tests below are specific to the Stock GC + @static if Base.USING_STOCK_GC + # --gcthreads + code = "print(Threads.ngcthreads())" + cpu_threads = ccall(:jl_effective_threads, Int32, ()) + @test string(cpu_threads) == + read(`$exename --threads auto -e $code`, String) == + read(`$exename --threads=auto -e $code`, String) == + read(`$exename -tauto -e $code`, String) == + read(`$exename -t auto -e $code`, String) + for nt in (nothing, "1") + withenv("JULIA_NUM_GC_THREADS" => nt) do + @test read(`$exename --gcthreads=2 -e $code`, String) == "2" + end + withenv("JULIA_NUM_GC_THREADS" => nt) do + @test read(`$exename --gcthreads=2,1 -e $code`, String) == "3" + end end - end - withenv("JULIA_NUM_GC_THREADS" => 2) do - @test read(`$exename -e $code`, String) == "2" - end + withenv("JULIA_NUM_GC_THREADS" => 2) do + @test read(`$exename -e $code`, String) == "2" + end - withenv("JULIA_NUM_GC_THREADS" => "2,1") do - @test read(`$exename -e $code`, String) == "3" + withenv("JULIA_NUM_GC_THREADS" => "2,1") do + @test read(`$exename -e $code`, String) == "3" + end end # --machine-file @@ -1182,6 +1186,10 @@ end end end +# FIXME: Issue #57103: MMTK currently does not use --heap-size-hint since it only +# supports setting up a hard limit unlike the Stock GC +# which takes it as a soft limit. For now, we skip the tests below for MMTk +@static if Base.USING_STOCK_GC @testset "heap size hint" begin #heap-size-hint, we reserve 250 MB for non GC memory (llvm, etc.) @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$((500-250)*1024*1024)" @@ -1201,6 +1209,7 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)" end +end ## `Main.main` entrypoint diff --git a/test/gc.jl b/test/gc.jl index c532f17f04eb5..3e9f03ef40d92 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -67,6 +67,9 @@ end run_gctest("gc/chunks.jl") end +#FIXME: Issue #57103 disabling tests for MMTk, since +# they rely on information that is specific to the stock GC. +@static if Base.USING_STOCK_GC @testset "GC page metrics" begin run_nonzero_page_utilization_test() run_pg_size_test() @@ -76,13 +79,14 @@ end issue_54275_test() end -@testset "Base.GC docstrings" begin - @test isempty(Docs.undocumented_names(GC)) -end - @testset "Full GC reasons" begin full_sweep_reasons_test() end +end + +@testset "Base.GC docstrings" begin + @test isempty(Docs.undocumented_names(GC)) +end #testset doesn't work here because this needs to run in top level #Check that we ensure objects in toplevel exprs are rooted diff --git a/test/misc.jl b/test/misc.jl index 7070fd49f5f36..070952db89032 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1453,7 +1453,8 @@ end @test_throws ErrorException finalizer(x->nothing, 1) @test_throws ErrorException finalizer(C_NULL, 1) - +# FIXME: Issue #57103 Test is specific to Stock GC +@static if Base.USING_STOCK_GC @testset "GC utilities" begin GC.gc() GC.gc(true); GC.gc(false) @@ -1473,6 +1474,7 @@ end @test occursin("GC: pause", read(tmppath, String)) end end +end @testset "fieldtypes Module" begin @test fieldtypes(Module) === ()