DO NOT MERGE: Using mmtk-immix as a default GC

Make.inc

@@ -81,7 +81,7 @@ WITH_GC_VERIFY := 0
 WITH_GC_DEBUG_ENV := 0
 
 # Use stock if MMTK_PLAN hasn't been defined
-MMTK_PLAN ?= None
+MMTK_PLAN ?= Immix
 
 # Enable DTrace support
 WITH_DTRACE := 0

base/Base.jl

@@ -128,6 +128,8 @@ include("sysinfo.jl")
 include("libc.jl")
 using .Libc: getpid, gethostname, time, memcpy, memset, memmove, memcmp
 
+const USING_STOCK_GC = occursin("stock", GC.gc_active_impl())
+
 # These used to be in build_h.jl and are retained for backwards compatibility.
 # NOTE: keep in sync with `libblastrampoline_jll.libblastrampoline`.
 const libblas_name = "libblastrampoline" * (Sys.iswindows() ? "-5" : "")

base/gcutils.jl

@@ -281,4 +281,15 @@ function logging_enabled()
     ccall(:jl_is_gc_logging_enabled, Cint, ()) != 0
 end
 
+"""
+    GC.gc_active_impl()
+
+Return a string stating which GC implementation is being used and possibly
+its version according to the list of supported GCs
+"""
+function gc_active_impl()
+    unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ()))
+end
+
+
 end # module GC

base/timing.jl

@@ -109,10 +109,8 @@ function gc_page_utilization_data()
     return Base.unsafe_wrap(Array, page_utilization_raw, JL_GC_N_MAX_POOLS, own=false)
 end
 
-
-const USING_STOCK_GC = occursin("stock", unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ())))
 # Full sweep reasons are currently only available for the stock GC
-@static if USING_STOCK_GC
+@static if Base.USING_STOCK_GC
 # must be kept in sync with `src/gc-stock.h``
 const FULL_SWEEP_REASONS = [:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL, :FULL_SWEEP_REASON_FORCED_FULL_SWEEP,
                             :FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE]
@@ -135,7 +133,7 @@ function full_sweep_reasons()
     d = Dict{Symbol, Int64}()
     # populate the dictionary according to the reasons above for the stock GC
     # otherwise return an empty dictionary for now
-    @static if USING_STOCK_GC
+    @static if Base.USING_STOCK_GC
         reason = cglobal(:jl_full_sweep_reasons, UInt64)
         reasons_as_array = Base.unsafe_wrap(Vector{UInt64}, reason, length(FULL_SWEEP_REASONS), own=false)
         for (i, r) in enumerate(FULL_SWEEP_REASONS)

contrib/refresh_checksums.mk

@@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS))
 NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS))
 
 # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
-BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient
+BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia
 BB_GCC_EXPANDED_PROJECTS=openblas csl
 BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld
 # These are non-BB source-only deps

deps/checksums/mmtk_julia

@@ -1,6 +1,7 @@
-mmtk_julia-b69acf5af7a7dd97c1cc6fd99f7c2d51b477f214.tar.gz/md5/1911cf084d26c48e2ed58af3d268b4b6
-mmtk_julia-b69acf5af7a7dd97c1cc6fd99f7c2d51b477f214.tar.gz/sha512/75beab54398989c46b62e714b242cf6705d88d220f40c21e494e0f29161437f5fbe9ba05b543d2353a1ad76f4239ac4025b476be0be864649f310f14935289fe
-mmtk_julia-f07d66aafc86af84ea988b35335acc9bbc770fa1.tar.gz/md5/38afb5db6d8c55413a4ec96aefa2ebb4
 mmtk_julia-f07d66aafc86af84ea988b35335acc9bbc770fa1.tar.gz/sha512/78525582a46a6baf8d33df7b622e55cf244439afcd7192ba55489c1bc18393d1237d2903d517c610484bf9e2a7338ad31435a9cbf70889d6bcf87c40cec829e5
 mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/md5/631b204574da7062802dac501a4b711f
 mmtk_julia.v0.30.3+1.x86_64-linux-gnu.tar.gz/sha512/daaed59d08fc49621479ed638dea0aac0cba123986e486571447e8e21e9a098776ce2e87fbd92ddea276782fc44621f23d40fa213296b28e1d4480553c7de4f7
+mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/md5/73a8fbea71edce30a39a30f31969dd8e
+mmtk_julia-c9e046baf3a0d52fe75d6c8b28f6afd69b045d95.tar.gz/sha512/374848b7696b565dea66daa208830581f92c1fcb0138e7a7ab88564402e94bc79c54b6ed370ec68473e31e2bd411bf82c97793796c31d39aafbbfffea9c05588
+mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/md5/8cdeb14fd69945f64308be49f6912f9c
+mmtk_julia.v0.30.4+0.x86_64-linux-gnu.tar.gz/sha512/3692502f65dec8c0971b56b9bf8178641892b390d520cbcd69880d75b7500e6341534d87882246e68998f590f824ec54c18f4b8fb4aa09b8f313de065c48450e

deps/mmtk_julia.version

@@ -1,6 +1,6 @@
 MMTK_JULIA_BRANCH = master
-MMTK_JULIA_SHA1 = f07d66aafc86af84ea988b35335acc9bbc770fa1
+MMTK_JULIA_SHA1 = c9e046baf3a0d52fe75d6c8b28f6afd69b045d95
 MMTK_JULIA_GIT_URL := https://github.com/mmtk/mmtk-julia.git
-MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.3.tar.gz
-MMTK_JULIA_JLL_VER := 0.30.3+1
+MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.4.tar.gz
+MMTK_JULIA_JLL_VER := 0.30.4+0
 MMTK_JULIA_JLL_NAME := mmtk_julia

src/gc-mmtk.c

@@ -64,11 +64,37 @@ void jl_gc_init(void) {
 
     arraylist_new(&to_finalize, 0);
     arraylist_new(&finalizer_list_marked, 0);
-
+    gc_num.interval = default_collect_interval;
     gc_num.allocd = 0;
     gc_num.max_pause = 0;
     gc_num.max_memory = 0;
 
+    // Necessary if we want to use Julia heap resizing heuristics
+    uint64_t mem_reserve = 250*1024*1024; // LLVM + other libraries need some amount of memory
+    uint64_t min_heap_size_hint = mem_reserve + 1*1024*1024;
+    uint64_t hint = jl_options.heap_size_hint;
+
+    // check if heap size specified on command line
+    if (jl_options.heap_size_hint == 0) {
+        char *cp = getenv(HEAP_SIZE_HINT);
+        if (cp)
+            hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"<size>[<unit>]\"");
+    }
+#ifdef _P64
+    if (hint == 0) {
+        uint64_t constrained_mem = uv_get_constrained_memory();
+        if (constrained_mem > 0 && constrained_mem < uv_get_total_memory())
+            hint = constrained_mem;
+    }
+#endif
+    if (hint) {
+        if (hint < min_heap_size_hint)
+            hint = min_heap_size_hint;
+        jl_gc_set_max_memory(hint - mem_reserve);
+    }
+
+    // MMTK supports setting the heap size using the
+    // MMTK_MIN_HSIZE and MMTK_MAX_HSIZE environment variables
     long long min_heap_size;
     long long max_heap_size;
     char* min_size_def = getenv("MMTK_MIN_HSIZE");
@@ -77,7 +103,8 @@ void jl_gc_init(void) {
     char* max_size_def = getenv("MMTK_MAX_HSIZE");
     char* max_size_gb = getenv("MMTK_MAX_HSIZE_G");
 
-    // default min heap currently set as Julia's default_collect_interval
+    // If min and max values are not specified, set them to 0 here
+    // and use stock heuristics as defined in the binding
     if (min_size_def != NULL) {
         char *p;
         double min_size = strtod(min_size_def, &p);
@@ -87,10 +114,9 @@ void jl_gc_init(void) {
         double min_size = strtod(min_size_gb, &p);
         min_heap_size = (long) 1024 * 1024 * 1024 * min_size;
     } else {
-        min_heap_size = default_collect_interval;
+        min_heap_size = 0;
     }
 
-    // default max heap currently set as 70% the free memory in the system
     if (max_size_def != NULL) {
         char *p;
         double max_size = strtod(max_size_def, &p);
@@ -100,7 +126,7 @@ void jl_gc_init(void) {
         double max_size = strtod(max_size_gb, &p);
         max_heap_size = (long) 1024 * 1024 * 1024 * max_size;
     } else {
-        max_heap_size = uv_get_free_memory() * 70 / 100;
+        max_heap_size = 0;
     }
 
     // Assert that the number of stock GC threads is 0; MMTK uses the number of threads in jl_options.ngcthreads
@@ -159,7 +185,17 @@ void jl_free_thread_gc_state(struct _jl_tls_states_t *ptls) {
 }
 
 JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem) {
-    // MMTk currently does not allow setting the heap size at runtime
+#ifdef _P32
+    max_mem = max_mem < MAX32HEAP ? max_mem : MAX32HEAP;
+#endif
+    max_total_memory = max_mem;
+}
+
+JL_DLLEXPORT uint64_t jl_gc_get_max_memory(void)
+{
+    // FIXME: We should return the max heap size set in MMTk
+    // when not using Julia's heap resizing heuristics
+    return max_total_memory;
 }
 
 STATIC_INLINE void maybe_collect(jl_ptls_t ptls)
@@ -415,12 +451,6 @@ JL_DLLEXPORT void jl_gc_get_total_bytes(int64_t *bytes) JL_NOTSAFEPOINT
     *bytes = (num.total_allocd + num.deferred_alloc + num.allocd);
 }
 
-JL_DLLEXPORT uint64_t jl_gc_get_max_memory(void)
-{
-    // FIXME: should probably return MMTk's heap size
-    return max_total_memory;
-}
-
 // These are needed to collect MMTk statistics from a Julia program using ccall
 JL_DLLEXPORT void (jl_mmtk_harness_begin)(void)
 {

stdlib/Profile/test/allocs.jl

@@ -73,8 +73,14 @@ end
     @test length(first_alloc.stacktrace) > 0
     @test length(string(first_alloc.type)) > 0
 
-    @testset for type in (Task, Vector{Float64},)
-        @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS
+    # Issue #57103: This test does not work with MMTk because of fastpath
+    # allocation which never calls the allocation profiler.
+    # TODO: We should port these observability tools (e.g. allocation
+    # profiler and heap snapshot) to MMTk
+    @static if Base.USING_STOCK_GC
+        @testset for type in (Task, Vector{Float64},)
+            @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS
+        end
     end
 
     # TODO: it would be nice to assert that these tasks
@@ -143,6 +149,8 @@ end
     @test length([a for a in prof.allocs if a.type == String]) >= 1
 end
 
+# FIXME: Issue #57103 disabling test for MMTk.
+@static if Base.USING_STOCK_GC
 @testset "alloc profiler catches allocs from codegen" begin
     @eval begin
         struct MyType x::Int; y::Int end
@@ -162,6 +170,7 @@ end
     @test length(prof.allocs) >= 1
     @test length([a for a in prof.allocs if a.type == MyType]) >= 1
 end
+end
 
 @testset "alloc profiler catches allocs from buffer resize" begin
     f(a) = for _ in 1:100; push!(a, 1); end

stdlib/Profile/test/runtests.jl

@@ -344,6 +344,8 @@ end
     @test only(node.down).first == lidict[8]
 end
 
+# FIXME: Issue #57103: heap snapshots is not currently supported in MMTk
+@static if Base.USING_STOCK_GC
 @testset "HeapSnapshot" begin
     tmpdir = mktempdir()
 
@@ -374,6 +376,7 @@ end
     rm(fname)
     rm(tmpdir, force = true, recursive = true)
 end
+end
 
 @testset "PageProfile" begin
     fname = "$(getpid())_$(time_ns())"

test/checked.jl

@@ -331,8 +331,12 @@ end
     @test checked_pow(BigInt(2), 2) == BigInt(4)
     @test checked_pow(BigInt(2), 100) == BigInt(1267650600228229401496703205376)
 
-    # Perf test: Make sure BigInts allocs don't scale with the power:
-    @test @allocations(checked_pow(BigInt(2), 2)) ≈ @allocations(checked_pow(BigInt(2), 10000)) rtol=0.9
+    # FIXME: Issue #57103: the following test may fail because
+    # allocation may not be logged via MMTk's fastpath allocation
+    @static if Base.USING_STOCK_GC
+        # Perf test: Make sure BigInts allocs don't scale with the power:
+        @test @allocations(checked_pow(BigInt(2), 2)) ≈ @allocations(checked_pow(BigInt(2), 10000)) rtol=0.9
+    end
 end
 
 @testset "Additional tests" begin

test/cmdlineargs.jl

@@ -383,29 +383,33 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
         @test p.exitcode == 1 && p.termsignal == 0
     end
 
-    # --gcthreads
-    code = "print(Threads.ngcthreads())"
-    cpu_threads = ccall(:jl_effective_threads, Int32, ())
-    @test string(cpu_threads) ==
-          read(`$exename --threads auto -e $code`, String) ==
-          read(`$exename --threads=auto -e $code`, String) ==
-          read(`$exename -tauto -e $code`, String) ==
-          read(`$exename -t auto -e $code`, String)
-    for nt in (nothing, "1")
-        withenv("JULIA_NUM_GC_THREADS" => nt) do
-            @test read(`$exename --gcthreads=2 -e $code`, String) == "2"
-        end
-        withenv("JULIA_NUM_GC_THREADS" => nt) do
-            @test read(`$exename --gcthreads=2,1 -e $code`, String) == "3"
+    # FIXME: Issue #57103 --gcthreads does not have the same semantics
+    # for Stock GC and MMTk, so the tests below are specific to the Stock GC
+    @static if Base.USING_STOCK_GC
+        # --gcthreads
+        code = "print(Threads.ngcthreads())"
+        cpu_threads = ccall(:jl_effective_threads, Int32, ())
+        @test string(cpu_threads) ==
+            read(`$exename --threads auto -e $code`, String) ==
+            read(`$exename --threads=auto -e $code`, String) ==
+            read(`$exename -tauto -e $code`, String) ==
+            read(`$exename -t auto -e $code`, String)
+        for nt in (nothing, "1")
+            withenv("JULIA_NUM_GC_THREADS" => nt) do
+                @test read(`$exename --gcthreads=2 -e $code`, String) == "2"
+            end
+            withenv("JULIA_NUM_GC_THREADS" => nt) do
+                @test read(`$exename --gcthreads=2,1 -e $code`, String) == "3"
+            end
         end
-    end
 
-    withenv("JULIA_NUM_GC_THREADS" => 2) do
-        @test read(`$exename -e $code`, String) == "2"
-    end
+        withenv("JULIA_NUM_GC_THREADS" => 2) do
+            @test read(`$exename -e $code`, String) == "2"
+        end
 
-    withenv("JULIA_NUM_GC_THREADS" => "2,1") do
-        @test read(`$exename -e $code`, String) == "3"
+        withenv("JULIA_NUM_GC_THREADS" => "2,1") do
+            @test read(`$exename -e $code`, String) == "3"
+        end
     end
 
     # --machine-file
@@ -1182,6 +1186,10 @@ end
     end
 end
 
+# FIXME: Issue #57103: MMTK currently does not use --heap-size-hint since it only
+# supports setting up a hard limit unlike the Stock GC
+# which takes it as a soft limit. For now, we skip the tests below for MMTk
+@static if Base.USING_STOCK_GC
 @testset "heap size hint" begin
     #heap-size-hint, we reserve 250 MB for non GC memory (llvm, etc.)
     @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$((500-250)*1024*1024)"
@@ -1201,6 +1209,7 @@ end
 
     @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)"
 end
+end
 
 ## `Main.main` entrypoint
 

test/gc.jl

@@ -67,6 +67,9 @@ end
     run_gctest("gc/chunks.jl")
 end
 
+#FIXME: Issue #57103 disabling tests for MMTk, since
+# they rely on information that is specific to the stock GC.
+@static if Base.USING_STOCK_GC
 @testset "GC page metrics" begin
     run_nonzero_page_utilization_test()
     run_pg_size_test()
@@ -76,13 +79,14 @@ end
     issue_54275_test()
 end
 
-@testset "Base.GC docstrings" begin
-    @test isempty(Docs.undocumented_names(GC))
-end
-
 @testset "Full GC reasons" begin
     full_sweep_reasons_test()
 end
+end
+
+@testset "Base.GC docstrings" begin
+    @test isempty(Docs.undocumented_names(GC))
+end
 
 #testset doesn't work here because this needs to run in top level
 #Check that we ensure objects in toplevel exprs are rooted

test/misc.jl

@@ -1453,7 +1453,8 @@ end
 @test_throws ErrorException finalizer(x->nothing, 1)
 @test_throws ErrorException finalizer(C_NULL, 1)
 
-
+# FIXME: Issue #57103 Test is specific to Stock GC
+@static if Base.USING_STOCK_GC
 @testset "GC utilities" begin
     GC.gc()
     GC.gc(true); GC.gc(false)
@@ -1473,6 +1474,7 @@ end
         @test occursin("GC: pause", read(tmppath, String))
     end
 end
+end
 
 @testset "fieldtypes Module" begin
     @test fieldtypes(Module) === ()