diff --git a/crates/asm-macros/src/lib.rs b/crates/asm-macros/src/lib.rs index efd970d6bca0..694654a7eadb 100644 --- a/crates/asm-macros/src/lib.rs +++ b/crates/asm-macros/src/lib.rs @@ -66,12 +66,12 @@ cfg_if::cfg_if! { std::arch::global_asm!( concat!( ".p2align 4\n", - ".hidden ", $name, "\n", + //".hidden ", $name, "\n", ".global ", $name, "\n", - $crate::elf_func_type_header!($name), + //$crate::elf_func_type_header!($name), $name, ":\n", $body, - ".size ", $name, ",.-", $name, + //".size ", $name, ",.-", $name, ) $(, $($args)*)? ); diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 93ba8bde6693..ee6f30112bdd 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -46,3 +46,4 @@ ittapi = { version = "0.4.0", optional = true } [features] profiling = ['dep:wasmtime-jit-debug', 'dep:ittapi'] demangle = ['dep:rustc-demangle', 'dep:cpp_demangle'] +debug-builtins = [] diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 517b5d1b9768..2a08b5af2c20 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -1,7 +1,6 @@ //! Memory management for executable code. use crate::subslice_range; -use crate::unwind::UnwindRegistration; use anyhow::{anyhow, bail, Context, Result}; use object::read::{File, Object, ObjectSection}; use object::ObjectSymbol; @@ -9,8 +8,7 @@ use std::mem::ManuallyDrop; use std::ops::Range; use wasmtime_environ::obj; use wasmtime_jit_icache_coherence as icache_coherence; -use wasmtime_runtime::libcalls; -use wasmtime_runtime::MmapVec; +use wasmtime_runtime::{libcalls, MmapVec, UnwindRegistration}; /// Management of executable memory within a `MmapVec` /// diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index a991c9a4cbb8..d68b51cb3454 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -4,7 +4,6 @@ //! steps. use crate::code_memory::CodeMemory; -use crate::debug::create_gdbjit_image; use crate::profiling::ProfilingAgent; use anyhow::{bail, Context, Error, Result}; use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; @@ -19,9 +18,7 @@ use wasmtime_environ::{ DefinedFuncIndex, FuncIndex, FunctionLoc, MemoryInitialization, Module, ModuleTranslation, PrimaryMap, SignatureIndex, StackMapInformation, Tunables, WasmFunctionInfo, }; -use wasmtime_runtime::{ - CompiledModuleId, CompiledModuleIdAllocator, GdbJitImageRegistration, MmapVec, -}; +use wasmtime_runtime::{CompiledModuleId, CompiledModuleIdAllocator, MmapVec}; /// Secondary in-memory results of function compilation. #[derive(Serialize, Deserialize)] @@ -426,7 +423,8 @@ pub struct CompiledModule { wasm_to_native_trampolines: Vec<(SignatureIndex, FunctionLoc)>, meta: Metadata, code_memory: Arc, - dbg_jit_registration: Option, + #[cfg(feature = "debug-builtins")] + dbg_jit_registration: Option, /// A unique ID used to register this module with the engine. unique_id: CompiledModuleId, func_names: Vec, @@ -459,6 +457,7 @@ impl CompiledModule { module: Arc::new(info.module), funcs: info.funcs, wasm_to_native_trampolines: info.wasm_to_native_trampolines, + #[cfg(feature = "debug-builtins")] dbg_jit_registration: None, code_memory, meta: info.meta, @@ -471,11 +470,15 @@ impl CompiledModule { } fn register_debug_and_profiling(&mut self, profiler: &dyn ProfilingAgent) -> Result<()> { + #[cfg(feature = "debug-builtins")] if self.meta.native_debug_info_present { let text = self.text(); - let bytes = create_gdbjit_image(self.mmap().to_vec(), (text.as_ptr(), text.len())) - .context("failed to create jit image for gdb")?; - let reg = GdbJitImageRegistration::register(bytes); + let bytes = crate::debug::create_gdbjit_image( + self.mmap().to_vec(), + (text.as_ptr(), text.len()), + ) + .context("failed to create jit image for gdb")?; + let reg = wasmtime_runtime::GdbJitImageRegistration::register(bytes); self.dbg_jit_registration = Some(reg); } profiler.register_module(&self.code_memory, &|addr| { diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 438e44aacc24..7aac110b8416 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -3,11 +3,11 @@ #![deny(missing_docs)] mod code_memory; +#[cfg(feature = "debug-builtins")] mod debug; mod demangling; mod instantiate; pub mod profiling; -mod unwind; pub use crate::code_memory::CodeMemory; #[cfg(feature = "addr2line")] diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index fa0074e74886..70dfea1f922e 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -18,21 +18,22 @@ wasmtime-wmemcheck = { workspace = true } wasmtime-asm-macros = { workspace = true } wasmtime-environ = { workspace = true } wasmtime-fiber = { workspace = true, optional = true } -wasmtime-jit-debug = { workspace = true, features = ["gdb_jit_int"] } +wasmtime-jit-debug = { workspace = true, features = ["gdb_jit_int"], optional = true } wasmtime-versioned-export-macros = { workspace = true } libc = { version = "0.2.112", default-features = false } log = { workspace = true } memoffset = "0.9.0" indexmap = { workspace = true } cfg-if = { workspace = true } -rand = { version = "0.8.3", features = ['small_rng'] } anyhow = { workspace = true } -memfd = "0.6.2" paste = "1.0.3" encoding_rs = { version = "0.8.31", optional = true } sptr = "0.3.2" wasm-encoder = { workspace = true } +[target.'cfg(target_os = "linux")'.dependencies] +memfd = "0.6.2" + [target.'cfg(target_os = "macos")'.dependencies] mach = "0.3.2" @@ -54,6 +55,7 @@ features = [ [dev-dependencies] once_cell = { workspace = true } proptest = "1.0.0" +rand = { version = "0.8.3", features = ['small_rng'] } [build-dependencies] cc = "1.0" @@ -64,4 +66,4 @@ async = ["wasmtime-fiber"] pooling-allocator = [] component-model = ["wasmtime-environ/component-model", "dep:encoding_rs"] wmemcheck = [] -debug-builtins = [] +debug-builtins = ['wasmtime-jit-debug'] diff --git a/crates/runtime/build.rs b/crates/runtime/build.rs index 571f0c4b2267..d8ba14f6ccbd 100644 --- a/crates/runtime/build.rs +++ b/crates/runtime/build.rs @@ -2,6 +2,15 @@ use std::env; use wasmtime_versioned_export_macros::versioned_suffix; fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + // If this platform is neither unix nor windows then there's no default need + // for a C helper library since `helpers.c` is tailored for just these + // platforms currently. + if env::var("CARGO_CFG_UNIX").is_err() && env::var("CARGO_CFG_WINDOWS").is_err() { + return; + } + let mut build = cc::Build::new(); build.warnings(true); let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); @@ -11,7 +20,7 @@ fn main() { build.define("VERSIONED_SUFFIX", Some(versioned_suffix!())); if arch == "s390x" { println!("cargo:rerun-if-changed=src/trampolines/s390x.S"); - build.file("src/trampolines/s390x.S"); + build.file("src/arch/s390x.S"); } println!("cargo:rerun-if-changed=src/helpers.c"); build.file("src/helpers.c"); diff --git a/crates/runtime/src/traphandlers/backtrace/aarch64.rs b/crates/runtime/src/arch/aarch64.rs similarity index 62% rename from crates/runtime/src/traphandlers/backtrace/aarch64.rs rename to crates/runtime/src/arch/aarch64.rs index 8c36291e37af..b44547c0f45d 100644 --- a/crates/runtime/src/traphandlers/backtrace/aarch64.rs +++ b/crates/runtime/src/arch/aarch64.rs @@ -53,3 +53,47 @@ pub fn assert_fp_is_aligned(_fp: usize) { // // [0]: https://github.com/ARM-software/abi-aa/blob/2022Q1/aapcs64/aapcs64.rst#the-frame-pointer } + +#[rustfmt::skip] +macro_rules! wasm_to_libcall_trampoline { + ($libcall:ident ; $libcall_impl:ident) => { + wasmtime_asm_macros::asm_func!( + wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall), + " + .cfi_startproc + bti c + + // Load the pointer to `VMRuntimeLimits` in `x9`. + ldur x9, [x0, #8] + + // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits. + stur fp, [x9, #24] + + // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits. + stur lr, [x9, #32] + + // Tail call to the actual implementation of this libcall. + b {} + + .cfi_endproc + ", + sym $libcall_impl + ); + }; +} +pub(crate) use wasm_to_libcall_trampoline; + +#[cfg(test)] +mod wasm_to_libcall_trampoline_offsets_tests { + use wasmtime_environ::{Module, PtrSize, VMOffsets}; + + #[test] + fn test() { + let module = Module::new(); + let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module); + + assert_eq!(8, offsets.vmctx_runtime_limits()); + assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp()); + assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc()); + } +} diff --git a/crates/runtime/src/trampolines.rs b/crates/runtime/src/arch/mod.rs similarity index 64% rename from crates/runtime/src/trampolines.rs rename to crates/runtime/src/arch/mod.rs index 16741075cef5..28f2dd54be69 100644 --- a/crates/runtime/src/trampolines.rs +++ b/crates/runtime/src/arch/mod.rs @@ -1,18 +1,16 @@ -//! Wasm-to-libcall trampolines. - cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { - #[macro_use] mod x86_64; + pub use x86_64::*; } else if #[cfg(target_arch = "aarch64")] { - #[macro_use] mod aarch64; + pub use aarch64::*; } else if #[cfg(target_arch = "s390x")] { - #[macro_use] mod s390x; - }else if #[cfg(target_arch = "riscv64")] { - #[macro_use] + pub use s390x::*; + } else if #[cfg(target_arch = "riscv64")] { mod riscv64; + pub use riscv64::*; } else { compile_error!("unsupported architecture"); } diff --git a/crates/runtime/src/trampolines/riscv64.rs b/crates/runtime/src/arch/riscv64.rs similarity index 71% rename from crates/runtime/src/trampolines/riscv64.rs rename to crates/runtime/src/arch/riscv64.rs index b43afcbf4518..4bb291b2f888 100644 --- a/crates/runtime/src/trampolines/riscv64.rs +++ b/crates/runtime/src/arch/riscv64.rs @@ -1,3 +1,22 @@ +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + *(fp as *mut usize).offset(1) +} + +// And the current frame pointer points to the next older frame pointer. +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { + fp >= entry_sp +} + +pub fn assert_entry_sp_is_aligned(sp: usize) { + assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); +} + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); +} + #[rustfmt::skip] macro_rules! wasm_to_libcall_trampoline { ($libcall:ident ; $libcall_impl:ident) => { @@ -28,6 +47,7 @@ macro_rules! wasm_to_libcall_trampoline { ); }; } +pub(crate) use wasm_to_libcall_trampoline; #[cfg(test)] mod wasm_to_libcall_trampoline_offsets_tests { diff --git a/crates/runtime/src/trampolines/s390x.S b/crates/runtime/src/arch/s390x.S similarity index 100% rename from crates/runtime/src/trampolines/s390x.S rename to crates/runtime/src/arch/s390x.S diff --git a/crates/runtime/src/trampolines/s390x.rs b/crates/runtime/src/arch/s390x.rs similarity index 55% rename from crates/runtime/src/trampolines/s390x.rs rename to crates/runtime/src/arch/s390x.rs index f8ca65ca4fc9..66794c93cdab 100644 --- a/crates/runtime/src/trampolines/s390x.rs +++ b/crates/runtime/src/arch/s390x.rs @@ -1,3 +1,26 @@ +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + // The next older PC can be found in register %r14 at function entry, which + // was saved into slot 14 of the register save area pointed to by "FP" (the + // backchain pointer). + *(fp as *mut usize).offset(14) +} + +// The next older "FP" (backchain pointer) was saved in the slot pointed to +// by the current "FP". +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { + fp > entry_sp +} + +pub fn assert_entry_sp_is_aligned(sp: usize) { + assert_eq!(sp % 8, 0, "stack should always be aligned to 8"); +} + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); +} + // The implementation for libcall trampolines is in the s390x.S // file. We provide this dummy definition of wasm_to_libcall_trampoline // here to make libcalls.rs compile on s390x. Note that this means we @@ -6,6 +29,7 @@ macro_rules! wasm_to_libcall_trampoline { ($libcall:ident ; $libcall_impl:ident) => {}; } +pub(crate) use wasm_to_libcall_trampoline; // The wasm_to_host_trampoline implementation is in the s390x.S // file, but we still want to have this unit test here. diff --git a/crates/runtime/src/trampolines/x86_64.rs b/crates/runtime/src/arch/x86_64.rs similarity index 57% rename from crates/runtime/src/trampolines/x86_64.rs rename to crates/runtime/src/arch/x86_64.rs index d22bcaf1a0e3..2f81bb82cf95 100644 --- a/crates/runtime/src/trampolines/x86_64.rs +++ b/crates/runtime/src/arch/x86_64.rs @@ -1,21 +1,45 @@ +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + // The calling convention always pushes the return pointer (aka the PC of + // the next older frame) just before this frame. + *(fp as *mut usize).offset(1) +} + +// And the current frame pointer points to the next older frame pointer. +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { + fp >= entry_sp +} + +pub fn assert_entry_sp_is_aligned(sp: usize) { + assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); +} + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); +} + // Helper macros for getting the first and second arguments according to the // system calling convention, as well as some callee-saved scratch registers we // can safely use in the trampolines. cfg_if::cfg_if! { if #[cfg(windows)] { macro_rules! callee_vmctx { () => ("rcx") } - #[allow(unused)] macro_rules! caller_vmctx { () => ("rdx") } macro_rules! scratch0 { () => ("r10") } macro_rules! scratch1 { () => ("r11") } - } else if #[cfg(unix)] { + } else if #[cfg(any(unix, wasmtime_custom_platform))] { macro_rules! callee_vmctx { () => ("rdi") } - #[allow(unused)] macro_rules! caller_vmctx { () => ("rsi") } macro_rules! scratch0 { () => ("r10") } macro_rules! scratch1 { () => ("r11") } } else { - compile_error!("platform not supported"); + compile_error!("default calling convention for this platform is not known"); + + macro_rules! callee_vmctx { () => ("") } + macro_rules! scratch0 { () => ("") } + macro_rules! scratch1 { () => ("") } } } +pub(crate) use {callee_vmctx, scratch0, scratch1}; #[rustfmt::skip] macro_rules! wasm_to_libcall_trampoline { @@ -27,15 +51,15 @@ macro_rules! wasm_to_libcall_trampoline { .cfi_startproc simple .cfi_def_cfa_offset 0 - // Load the pointer to `VMRuntimeLimits` in `", scratch0!(), "`. - mov ", scratch0!(), ", 8[", callee_vmctx!(), "] + // Load the pointer to `VMRuntimeLimits` in `scratch0!()`. + mov ", crate::arch::scratch0!(), ", 8[", crate::arch::callee_vmctx!(), "] // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits. - mov 24[", scratch0!(), "], rbp + mov 24[", crate::arch::scratch0!(), "], rbp // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits. - mov ", scratch1!(), ", [rsp] - mov 32[", scratch0!(), "], ", scratch1!(), " + mov ", crate::arch::scratch1!(), ", [rsp] + mov 32[", crate::arch::scratch0!(), "], ", crate::arch::scratch1!(), " // Tail call to the actual implementation of this libcall. jmp {} @@ -47,6 +71,7 @@ macro_rules! wasm_to_libcall_trampoline { ); }; } +pub(crate) use wasm_to_libcall_trampoline; #[cfg(test)] mod wasm_to_libcall_trampoline_offsets_tests { diff --git a/crates/runtime/src/cow.rs b/crates/runtime/src/cow.rs index ccd34b828910..5277f85ef4b2 100644 --- a/crates/runtime/src/cow.rs +++ b/crates/runtime/src/cow.rs @@ -5,7 +5,7 @@ use crate::{MmapVec, SendSyncPtr}; use anyhow::Result; -use libc::c_void; +use std::ffi::c_void; use std::fs::File; use std::ptr::NonNull; use std::sync::Arc; @@ -124,7 +124,7 @@ impl MemoryImage { // files, but for now this is still a Linux-specific region of Wasmtime. // Some work will be needed to get this file compiling for macOS and // Windows. - #[cfg(not(any(windows, miri)))] + #[cfg(all(unix, not(miri)))] if let Some(mmap) = mmap { let start = mmap.as_ptr() as usize; let end = start + mmap.len(); @@ -726,30 +726,10 @@ impl MemoryImageSlot { unsafe { let start = self.base.as_ptr().add(range.start); - cfg_if::cfg_if! { - if #[cfg(miri)] { - if readwrite { - std::ptr::write_bytes(start, 0u8, range.len()); - } - } else if #[cfg(unix)] { - let flags = if readwrite { - rustix::mm::MprotectFlags::READ | rustix::mm::MprotectFlags::WRITE - } else { - rustix::mm::MprotectFlags::empty() - }; - rustix::mm::mprotect(start.cast(), range.len(), flags)?; - } else { - use windows_sys::Win32::System::Memory::*; - - let failure = if readwrite { - VirtualAlloc(start.cast(), range.len(), MEM_COMMIT, PAGE_READWRITE).is_null() - } else { - VirtualFree(start.cast(), range.len(), MEM_DECOMMIT) == 0 - }; - if failure { - return Err(std::io::Error::last_os_error().into()); - } - } + if readwrite { + crate::sys::vm::expose_exisiting_mapping(start, range.len())?; + } else { + crate::sys::vm::hide_existing_mapping(start, range.len())?; } } @@ -775,24 +755,7 @@ impl MemoryImageSlot { } unsafe { - cfg_if::cfg_if! { - if #[cfg(miri)] { - std::ptr::write_bytes(self.base.as_ptr(), 0, self.static_size); - } else if #[cfg(unix)] { - let ptr = rustix::mm::mmap_anonymous( - self.base.as_ptr().cast(), - self.static_size, - rustix::mm::ProtFlags::empty(), - rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED, - )?; - assert_eq!(ptr, self.base.as_ptr().cast()); - } else { - use windows_sys::Win32::System::Memory::*; - if VirtualFree(self.base.as_ptr().cast(), self.static_size, MEM_DECOMMIT) == 0 { - return Err(std::io::Error::last_os_error().into()); - } - } - } + crate::sys::vm::erase_existing_mapping(self.base.as_ptr(), self.static_size)?; } self.image = None; diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index e0584726fa67..9fa5d83b301f 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -65,16 +65,6 @@ mod table_pool; #[cfg(all(feature = "async", unix, not(miri)))] mod stack_pool; -cfg_if::cfg_if! { - if #[cfg(windows)] { - mod windows; - use windows as imp; - } else { - mod unix; - use unix as imp; - } -} - use super::{ InstanceAllocationRequest, InstanceAllocatorImpl, MemoryAllocationIndex, TableAllocationIndex, }; diff --git a/crates/runtime/src/instance/allocator/pooling/memory_pool.rs b/crates/runtime/src/instance/allocator/pooling/memory_pool.rs index 25c000101dc1..319db985874f 100644 --- a/crates/runtime/src/instance/allocator/pooling/memory_pool.rs +++ b/crates/runtime/src/instance/allocator/pooling/memory_pool.rs @@ -68,7 +68,7 @@ use crate::{ MpkEnabled, PoolingInstanceAllocatorConfig, }; use anyhow::{anyhow, bail, Context, Result}; -use libc::c_void; +use std::ffi::c_void; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; use wasmtime_environ::{ diff --git a/crates/runtime/src/instance/allocator/pooling/stack_pool.rs b/crates/runtime/src/instance/allocator/pooling/stack_pool.rs index edb169f348da..eb1581d5f669 100644 --- a/crates/runtime/src/instance/allocator/pooling/stack_pool.rs +++ b/crates/runtime/src/instance/allocator/pooling/stack_pool.rs @@ -1,8 +1,8 @@ use super::{ - imp::{commit_stack_pages, reset_stack_pages_to_zero}, index_allocator::{SimpleIndexAllocator, SlotId}, round_up_to_pow2, }; +use crate::sys::vm::{commit_stack_pages, reset_stack_pages_to_zero}; use crate::{Mmap, PoolingInstanceAllocatorConfig}; use anyhow::{anyhow, bail, Context, Result}; @@ -166,10 +166,10 @@ impl StackPool { 0, size_to_memset, ); - } - // Use the system to reset remaining stack pages to zero. - reset_stack_pages_to_zero(bottom as _, size - size_to_memset).unwrap(); + // Use the system to reset remaining stack pages to zero. + reset_stack_pages_to_zero(bottom as _, size - size_to_memset).unwrap(); + } } } diff --git a/crates/runtime/src/instance/allocator/pooling/table_pool.rs b/crates/runtime/src/instance/allocator/pooling/table_pool.rs index 558cb548e966..45ce3c3b965f 100644 --- a/crates/runtime/src/instance/allocator/pooling/table_pool.rs +++ b/crates/runtime/src/instance/allocator/pooling/table_pool.rs @@ -1,8 +1,8 @@ use super::{ - imp::{commit_table_pages, decommit_table_pages}, index_allocator::{SimpleIndexAllocator, SlotId}, round_up_to_pow2, TableAllocationIndex, }; +use crate::sys::vm::{commit_table_pages, decommit_table_pages}; use crate::{InstanceAllocationRequest, Mmap, PoolingInstanceAllocatorConfig, SendSyncPtr, Table}; use anyhow::{anyhow, bail, Context, Result}; use std::mem; @@ -129,10 +129,12 @@ impl TablePool { match (|| { let base = self.get(allocation_index); - commit_table_pages( - base as *mut u8, - self.table_elements * mem::size_of::<*mut u8>(), - )?; + unsafe { + commit_table_pages( + base as *mut u8, + self.table_elements * mem::size_of::<*mut u8>(), + )?; + } let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut( base.cast(), diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs deleted file mode 100644 index 00259f4b0afd..000000000000 --- a/crates/runtime/src/instance/allocator/pooling/unix.rs +++ /dev/null @@ -1,56 +0,0 @@ -use anyhow::Result; - -fn decommit(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - unsafe { - cfg_if::cfg_if! { - if #[cfg(miri)] { - std::ptr::write_bytes(addr, 0, len); - } else if #[cfg(target_os = "linux")] { - use rustix::mm::{madvise, Advice}; - - // On Linux, this is enough to cause the kernel to initialize - // the pages to 0 on next access - madvise(addr as _, len, Advice::LinuxDontNeed)?; - } else { - use rustix::mm::{mmap_anonymous, ProtFlags, MapFlags}; - - // By creating a new mapping at the same location, this will - // discard the mapping for the pages in the given range. - // The new mapping will be to the CoW zero page, so this - // effectively zeroes the pages. - mmap_anonymous( - addr as _, - len, - ProtFlags::READ | ProtFlags::WRITE, - MapFlags::PRIVATE | MapFlags::FIXED, - )?; - } - } - } - - Ok(()) -} - -pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { - // A no-op as table pages remain READ|WRITE - Ok(()) -} - -pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len) -} - -#[cfg(all(feature = "async", not(miri)))] -pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { - // A no-op as stack pages remain READ|WRITE - Ok(()) -} - -#[cfg(all(feature = "async", not(miri)))] -pub fn reset_stack_pages_to_zero(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len) -} diff --git a/crates/runtime/src/instance/allocator/pooling/windows.rs b/crates/runtime/src/instance/allocator/pooling/windows.rs deleted file mode 100644 index 5e9d0c51e414..000000000000 --- a/crates/runtime/src/instance/allocator/pooling/windows.rs +++ /dev/null @@ -1,38 +0,0 @@ -use anyhow::{bail, Result}; -use windows_sys::Win32::System::Memory::*; - -pub fn commit(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - // Memory needs to be committed, so don't use the `region` crate - if unsafe { VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() } { - bail!("failed to commit memory as read/write"); - } - - Ok(()) -} - -pub fn decommit(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - if unsafe { VirtualFree(addr as _, len, MEM_DECOMMIT) } == 0 { - bail!( - "failed to decommit memory pages: {}", - std::io::Error::last_os_error() - ); - } - - Ok(()) -} - -pub fn commit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - commit(addr, len) -} - -pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len) -} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index d3cf7565abbf..2d4680266239 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -10,9 +10,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets}; -#[macro_use] -mod trampolines; - +mod arch; #[cfg(feature = "component-model")] pub mod component; mod export; @@ -25,6 +23,7 @@ mod mmap_vec; mod parking_spot; mod send_sync_ptr; mod store_box; +mod sys; mod table; mod traphandlers; mod vmcontext; @@ -34,6 +33,7 @@ pub mod debug_builtins; pub mod libcalls; pub mod mpk; +#[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; pub use crate::export::*; @@ -54,6 +54,7 @@ pub use crate::mmap::Mmap; pub use crate::mmap_vec::MmapVec; pub use crate::mpk::MpkEnabled; pub use crate::store_box::*; +pub use crate::sys::unwind::UnwindRegistration; pub use crate::table::{Table, TableElement}; pub use crate::traphandlers::*; pub use crate::vmcontext::{ @@ -219,30 +220,13 @@ pub fn page_size() -> usize { return match PAGE_SIZE.load(Ordering::Relaxed) { 0 => { - let size = get_page_size(); + let size = sys::vm::get_page_size(); assert!(size != 0); PAGE_SIZE.store(size, Ordering::Relaxed); size } n => n, }; - - #[cfg(windows)] - fn get_page_size() -> usize { - use std::mem::MaybeUninit; - use windows_sys::Win32::System::SystemInformation::*; - - unsafe { - let mut info = MaybeUninit::uninit(); - GetSystemInfo(info.as_mut_ptr()); - info.assume_init_ref().dwPageSize as usize - } - } - - #[cfg(unix)] - fn get_page_size() -> usize { - unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() } - } } /// Result of [`Memory::atomic_wait32`] and [`Memory::atomic_wait64`] diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index b43c11b33cee..539a1b3f5f31 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -80,6 +80,7 @@ use wasmtime_wmemcheck::AccessError::{ /// now to ensure that the fp/sp on exit are recorded for backtraces to work /// properly. pub mod trampolines { + use crate::arch::wasm_to_libcall_trampoline; use crate::{Instance, TrapReason, VMContext}; macro_rules! libcall { diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index 341cd4deea36..4275cd68f833 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -1,30 +1,18 @@ //! Low-level abstraction for allocating and managing zero-filled pages //! of memory. +use crate::sys::mmap; use anyhow::{Context, Result}; use std::fs::File; use std::ops::Range; use std::path::Path; use std::sync::Arc; -cfg_if::cfg_if! { - if #[cfg(windows)] { - mod windows; - use windows as sys; - } else if #[cfg(miri)] { - mod miri; - use miri as sys; - } else { - mod unix; - use unix as sys; - } -} - /// A simple struct consisting of a page-aligned pointer to page-aligned /// and initially-zeroed memory and a length. #[derive(Debug)] pub struct Mmap { - sys: sys::Mmap, + sys: mmap::Mmap, file: Option>, } @@ -47,7 +35,7 @@ impl Mmap { /// The memory mapping and the length of the file within the mapping are /// returned. pub fn from_file(path: &Path) -> Result { - let (sys, file) = sys::Mmap::from_file(path)?; + let (sys, file) = mmap::Mmap::from_file(path)?; Ok(Mmap { sys, file: Some(Arc::new(file)), @@ -70,18 +58,18 @@ impl Mmap { if mapping_size == 0 { Ok(Mmap { - sys: sys::Mmap::new_empty(), + sys: mmap::Mmap::new_empty(), file: None, }) } else if accessible_size == mapping_size { Ok(Mmap { - sys: sys::Mmap::new(mapping_size) + sys: mmap::Mmap::new(mapping_size) .context(format!("mmap failed to allocate {mapping_size:#x} bytes"))?, file: None, }) } else { let mut result = Mmap { - sys: sys::Mmap::reserve(mapping_size) + sys: mmap::Mmap::reserve(mapping_size) .context(format!("mmap failed to reserve {mapping_size:#x} bytes"))?, file: None, }; diff --git a/crates/runtime/src/sys/custom/capi.rs b/crates/runtime/src/sys/custom/capi.rs new file mode 100644 index 000000000000..7e8edac04720 --- /dev/null +++ b/crates/runtime/src/sys/custom/capi.rs @@ -0,0 +1,127 @@ +#![allow(non_camel_case_types)] + +// Flags to either `wasmtime_mmap_anonymous` or `wasmtime_mprotect`. + +/// Indicates that the memory region should be readable. +pub const WASMTIME_PROT_READ: u32 = 1 << 0; +/// Indicates that the memory region should be writable. +pub const WASMTIME_PROT_WRITE: u32 = 1 << 1; +/// Indicates that the memory region should be executable. +pub const WASMTIME_PROT_EXEC: u32 = 1 << 2; + +pub use WASMTIME_PROT_EXEC as PROT_EXEC; +pub use WASMTIME_PROT_READ as PROT_READ; +pub use WASMTIME_PROT_WRITE as PROT_WRITE; + +/// Handler function for traps in Wasmtime passed to `wasmtime_init_traps`. +/// +/// This function is invoked whenever a trap is caught by the system. For +/// example this would be invoked during a signal handler on Linux. This +/// function is passed a number of parameters indicating information about the +/// trap: +/// +/// * `ip` - the instruction pointer at the time of the trap. +/// * `fp` - the frame pointer register's value at the time of the trap. +/// * `has_faulting_addr` - whether this trap is associated with an access +/// violation (e.g. a segfault) meaning memory was accessed when it shouldn't +/// be. If this is `true` then the next parameter is filled in. +/// * `faulting_addr` - if `has_faulting_addr` is true then this is the address +/// that was attempted to be accessed. Otherwise this value is not used. +/// +/// If this function returns then the trap was not handled. This probably means +/// that a fatal exception happened and the process should be aborted. +/// +/// This function may not return as it may invoke `wasmtime_longjmp` if a wasm +/// trap is detected. +pub type wasmtime_trap_handler_t = + extern "C" fn(ip: usize, fp: usize, has_faulting_addr: bool, faulting_addr: usize); + +extern "C" { + /// Creates a new virtual memory mapping of the `size` specified with + /// protection bits specified in `prot_flags`. + /// + /// Memory can be lazily committed. + /// + /// Returns the base pointer of the new mapping. Aborts the process on + /// failure. + /// + /// Similar to `mmap(0, size, prot_flags, MAP_PRIVATE, 0, -1)` on Linux. + pub fn wasmtime_mmap_new(size: usize, prot_flags: u32) -> *mut u8; + + /// Remaps the virtual memory starting at `addr` going for `size` bytes to + /// the protections specified with a new blank mapping. + /// + /// This will unmap any prior mappings and decommit them. New mappings for + /// anonymous memory are used to replace these mappings and the new area + /// should have the protection specified by `prot_flags`. + /// + /// Aborts the process on failure. + /// + /// Similar to `mmap(addr, size, prot_flags, MAP_PRIVATE | MAP_FIXED, 0, -1)` on Linux. + pub fn wasmtime_mmap_remap(addr: *mut u8, size: usize, prot_flags: u32); + + /// Unmaps memory at the specified `ptr` for `size` bytes. + /// + /// The memory should be discarded and decommitted and should generate a + /// segfault if accessed after this function call. + /// + /// Aborts the process on failure. + /// + /// Similar to `munmap` on Linux. + pub fn wasmtime_munmap(ptr: *mut u8, size: usize); + + /// Configures the protections associated with a region of virtual memory + /// starting at `ptr` and going to `size`. + /// + /// Aborts the process on failure. + /// + /// Similar to `mprotect` on Linux. + pub fn wasmtime_mprotect(ptr: *mut u8, size: usize, prot_flags: u32); + + /// Returns the page size, in bytes, of the current system. + pub fn wasmtime_page_size() -> usize; + + /// Used to setup a frame on the stack to longjmp back to in the future. + /// + /// This function is used for handling traps in WebAssembly and is paried + /// with `wasmtime_longjmp`. + /// + /// * `jmp_buf` - this argument is filled in with a pointer which if used + /// will be passed to `wasmtime_longjmp` later on by the runtime. + /// * `callback` - this callback should be invoked after `jmp_buf` is + /// configured. + /// * `payload` and `callee` - the two arguments to pass to `callback`. + /// + /// Returns 0 if `wasmtime_longjmp` was used to return to this function. + /// Returns 1 if `wasmtime_longjmp` was not called an `callback` returned. + pub fn wasmtime_setjmp( + jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut u8), + payload: *mut u8, + callee: *mut u8, + ) -> i32; + + /// Paired with `wasmtime_setjmp` this is used to jump back to the `setjmp` + /// point. + /// + /// The argument here was originally passed to `wasmtime_setjmp` through its + /// out-param. + /// + /// This function cannot return. + /// + /// This function may be invoked from the `wasmtime_trap_handler_t` + /// configured by `wasmtime_init_traps`. + pub fn wasmtime_longjmp(jmp_buf: *const u8) -> !; + + /// Initializes trap-handling logic for this platform. + /// + /// Wasmtime's implementation of WebAssembly relies on the ability to catch + /// signals/traps/etc. For example divide-by-zero may raise a machine + /// exception. Out-of-bounds memory accesses may also raise a machine + /// exception. This function is used to initialize trap handling. + /// + /// The `handler` provided is a function pointer to invoke whenever a trap + /// is encountered. The `handler` is invoked whenever a trap is caught by + /// the system. + pub fn wasmtime_init_traps(handler: wasmtime_trap_handler_t); +} diff --git a/crates/runtime/src/sys/custom/mmap.rs b/crates/runtime/src/sys/custom/mmap.rs new file mode 100644 index 000000000000..348350152176 --- /dev/null +++ b/crates/runtime/src/sys/custom/mmap.rs @@ -0,0 +1,102 @@ +use crate::sys::capi; +use crate::SendSyncPtr; +use anyhow::{bail, Result}; +use std::fs::File; +use std::ops::Range; +use std::path::Path; +use std::ptr::NonNull; + +#[derive(Debug)] +pub struct Mmap { + memory: SendSyncPtr<[u8]>, +} + +impl Mmap { + pub fn new_empty() -> Mmap { + Mmap { + memory: SendSyncPtr::from(&mut [][..]), + } + } + + pub fn new(size: usize) -> Result { + let ptr = unsafe { capi::wasmtime_mmap_new(size, capi::PROT_READ | capi::PROT_WRITE) }; + let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), size); + let memory = SendSyncPtr::new(NonNull::new(memory).unwrap()); + Ok(Mmap { memory }) + } + + pub fn reserve(size: usize) -> Result { + let ptr = unsafe { capi::wasmtime_mmap_new(size, 0) }; + let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), size); + let memory = SendSyncPtr::new(NonNull::new(memory).unwrap()); + Ok(Mmap { memory }) + } + + pub fn from_file(_path: &Path) -> Result<(Self, File)> { + bail!("not supported on this platform"); + } + + pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<()> { + let ptr = self.memory.as_ptr().cast::(); + unsafe { + capi::wasmtime_mprotect( + ptr.add(start).cast(), + len, + capi::PROT_READ | capi::PROT_WRITE, + ) + } + + Ok(()) + } + + #[inline] + pub fn as_ptr(&self) -> *const u8 { + self.memory.as_ptr() as *const u8 + } + + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut u8 { + self.memory.as_ptr().cast() + } + + #[inline] + pub fn len(&self) -> usize { + unsafe { (*self.memory.as_ptr()).len() } + } + + pub unsafe fn make_executable( + &self, + range: Range, + enable_branch_protection: bool, + ) -> Result<()> { + let base = self.memory.as_ptr().cast::().add(range.start).cast(); + let len = range.end - range.start; + + // not mapped into the C API at this time. + let _ = enable_branch_protection; + + capi::wasmtime_mprotect(base, len, capi::PROT_READ | capi::PROT_EXEC); + Ok(()) + } + + pub unsafe fn make_readonly(&self, range: Range) -> Result<()> { + let base = self.memory.as_ptr().cast::().add(range.start).cast(); + let len = range.end - range.start; + + capi::wasmtime_mprotect(base, len, capi::PROT_READ); + Ok(()) + } +} + +impl Drop for Mmap { + fn drop(&mut self) { + unsafe { + let ptr = self.memory.as_ptr().cast(); + let len = (*self.memory.as_ptr()).len(); + if len == 0 { + return; + } + capi::wasmtime_munmap(ptr, len); + } + } +} diff --git a/crates/runtime/src/sys/custom/mod.rs b/crates/runtime/src/sys/custom/mod.rs new file mode 100644 index 000000000000..c25b60ddab99 --- /dev/null +++ b/crates/runtime/src/sys/custom/mod.rs @@ -0,0 +1,6 @@ +/// TODO: dox +pub mod capi; +pub mod mmap; +pub mod traphandlers; +pub mod unwind; +pub mod vm; diff --git a/crates/runtime/src/sys/custom/traphandlers.rs b/crates/runtime/src/sys/custom/traphandlers.rs new file mode 100644 index 000000000000..d5cb0782de7b --- /dev/null +++ b/crates/runtime/src/sys/custom/traphandlers.rs @@ -0,0 +1,57 @@ +// With MIRI set up just enough of a setjmp/longjmp with catching panics +// to get a few tests working that use this. +// +// Note that no actual JIT code runs in MIRI so this is purely here for +// host-to-host calls. + +use crate::traphandlers::tls; +use crate::VMContext; +use std::mem; + +pub use crate::sys::capi::{self, wasmtime_longjmp}; + +#[allow(missing_docs)] +pub type SignalHandler<'a> = dyn Fn() + Send + Sync + 'a; + +pub unsafe fn wasmtime_setjmp( + jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, +) -> i32 { + let callback = mem::transmute::< + extern "C" fn(*mut u8, *mut VMContext), + extern "C" fn(*mut u8, *mut u8), + >(callback); + capi::wasmtime_setjmp(jmp_buf, callback, payload, callee.cast()) +} + +pub fn platform_init() { + unsafe { + capi::wasmtime_init_traps(handle_trap); + } +} + +extern "C" fn handle_trap(ip: usize, fp: usize, has_faulting_addr: bool, faulting_addr: usize) { + tls::with(|info| { + let info = match info { + Some(info) => info, + None => return, + }; + let faulting_addr = if has_faulting_addr { + Some(faulting_addr) + } else { + None + }; + let ip = ip as *const u8; + let jmp_buf = info.take_jmp_buf_if_trap(ip, |_handler| { + panic!("custom signal handlers are not supported on this platform"); + }); + if !jmp_buf.is_null() { + info.set_jit_trap(ip, fp, faulting_addr); + unsafe { wasmtime_longjmp(jmp_buf) } + } + }) +} + +pub fn lazy_per_thread_init() {} diff --git a/crates/jit/src/unwind/miri.rs b/crates/runtime/src/sys/custom/unwind.rs similarity index 93% rename from crates/jit/src/unwind/miri.rs rename to crates/runtime/src/sys/custom/unwind.rs index 4b4527e66834..d6181940cd0f 100644 --- a/crates/jit/src/unwind/miri.rs +++ b/crates/runtime/src/sys/custom/unwind.rs @@ -1,3 +1,5 @@ +#![allow(missing_docs)] + use anyhow::Result; pub struct UnwindRegistration {} diff --git a/crates/runtime/src/sys/custom/vm.rs b/crates/runtime/src/sys/custom/vm.rs new file mode 100644 index 000000000000..54a8899b201e --- /dev/null +++ b/crates/runtime/src/sys/custom/vm.rs @@ -0,0 +1,44 @@ +use crate::sys::capi; +use std::io; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + capi::wasmtime_mprotect(ptr.cast(), len, capi::PROT_READ | capi::PROT_WRITE); + Ok(()) +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + capi::wasmtime_mprotect(ptr.cast(), len, 0); + Ok(()) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + capi::wasmtime_mmap_remap(ptr.cast(), len, 0); + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn commit_table_pages(_addr: *mut u8, _len: usize) -> io::Result<()> { + // Table pages are always READ | WRITE so there's nothing that needs to be + // done here. + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn decommit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + if len == 0 { + return Ok(()); + } + + capi::wasmtime_mmap_anonymous( + addr, + len, + capi::PROT_READ | capi::PROT_WRITE, + capi::MAP_ANONYMOUS | capi::MAP_FIXED, + ); + + Ok(()) +} + +pub fn get_page_size() -> usize { + unsafe { capi::wasmtime_page_size() } +} diff --git a/crates/runtime/src/mmap/miri.rs b/crates/runtime/src/sys/miri/mmap.rs similarity index 100% rename from crates/runtime/src/mmap/miri.rs rename to crates/runtime/src/sys/miri/mmap.rs diff --git a/crates/runtime/src/sys/miri/mod.rs b/crates/runtime/src/sys/miri/mod.rs new file mode 100644 index 000000000000..c8a35ad10645 --- /dev/null +++ b/crates/runtime/src/sys/miri/mod.rs @@ -0,0 +1,4 @@ +pub mod mmap; +pub mod traphandlers; +pub mod unwind; +pub mod vm; diff --git a/crates/runtime/src/sys/miri/traphandlers.rs b/crates/runtime/src/sys/miri/traphandlers.rs new file mode 100644 index 000000000000..ed984fdb8f4d --- /dev/null +++ b/crates/runtime/src/sys/miri/traphandlers.rs @@ -0,0 +1,44 @@ +// With MIRI set up just enough of a setjmp/longjmp with catching panics +// to get a few tests working that use this. +// +// Note that no actual JIT code runs in MIRI so this is purely here for +// host-to-host calls. + +use crate::VMContext; + +struct WasmtimeLongjmp; + +#[wasmtime_versioned_export_macros::versioned_export] +pub unsafe extern "C" fn wasmtime_setjmp( + _jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, +) -> i32 { + use std::panic::{self, AssertUnwindSafe}; + let result = panic::catch_unwind(AssertUnwindSafe(|| { + callback(payload, callee); + })); + match result { + Ok(()) => 1, + Err(e) => { + if e.is::() { + 0 + } else { + panic::resume_unwind(e) + } + } + } +} + +#[wasmtime_versioned_export_macros::versioned_export] +pub unsafe extern "C" fn wasmtime_longjmp(_jmp_buf: *const u8) -> ! { + std::panic::panic_any(WasmtimeLongjmp) +} + +#[allow(missing_docs)] +pub type SignalHandler<'a> = dyn Fn() + Send + Sync + 'a; + +pub fn platform_init() {} + +pub fn lazy_per_thread_init() {} diff --git a/crates/runtime/src/sys/miri/unwind.rs b/crates/runtime/src/sys/miri/unwind.rs new file mode 100644 index 000000000000..d6181940cd0f --- /dev/null +++ b/crates/runtime/src/sys/miri/unwind.rs @@ -0,0 +1,17 @@ +#![allow(missing_docs)] + +use anyhow::Result; + +pub struct UnwindRegistration {} + +impl UnwindRegistration { + pub const SECTION_NAME: &'static str = ".eh_frame"; + + pub unsafe fn new( + _base_address: *const u8, + _unwind_info: *const u8, + _unwind_len: usize, + ) -> Result { + Ok(UnwindRegistration {}) + } +} diff --git a/crates/runtime/src/sys/miri/vm.rs b/crates/runtime/src/sys/miri/vm.rs new file mode 100644 index 000000000000..9fa106a84773 --- /dev/null +++ b/crates/runtime/src/sys/miri/vm.rs @@ -0,0 +1,30 @@ +use std::io; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0u8, len); + Ok(()) +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +pub unsafe fn commit_table_pages(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +pub unsafe fn decommit_table_pages(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +pub fn get_page_size() -> usize { + 4096 +} diff --git a/crates/runtime/src/sys/mod.rs b/crates/runtime/src/sys/mod.rs new file mode 100644 index 000000000000..3d7b3b60420d --- /dev/null +++ b/crates/runtime/src/sys/mod.rs @@ -0,0 +1,19 @@ +#![allow(clippy::cast_sign_loss)] // platforms too fiddly to worry about this + +cfg_if::cfg_if! { + if #[cfg(miri)] { + mod miri; + pub use miri::*; + } else if #[cfg(windows)] { + mod windows; + pub use windows::*; + } else if #[cfg(unix)] { + mod unix; + pub use unix::*; + } else if #[cfg(wasmtime_custom_platform)] { + mod custom; + pub use custom::*; + } else { + compile_error!("unsupported platform"); + } +} diff --git a/crates/runtime/src/mmap/unix.rs b/crates/runtime/src/sys/unix/mmap.rs similarity index 100% rename from crates/runtime/src/mmap/unix.rs rename to crates/runtime/src/sys/unix/mmap.rs diff --git a/crates/runtime/src/sys/unix/mod.rs b/crates/runtime/src/sys/unix/mod.rs new file mode 100644 index 000000000000..c8a35ad10645 --- /dev/null +++ b/crates/runtime/src/sys/unix/mod.rs @@ -0,0 +1,4 @@ +pub mod mmap; +pub mod traphandlers; +pub mod unwind; +pub mod vm; diff --git a/crates/runtime/src/traphandlers/unix.rs b/crates/runtime/src/sys/unix/traphandlers.rs similarity index 97% rename from crates/runtime/src/traphandlers/unix.rs rename to crates/runtime/src/sys/unix/traphandlers.rs index c4a993f15068..1cb7b820b9e5 100644 --- a/crates/runtime/src/traphandlers/unix.rs +++ b/crates/runtime/src/sys/unix/traphandlers.rs @@ -1,6 +1,21 @@ -#![allow(clippy::cast_sign_loss)] // platforms too fiddly to worry about this +use crate::VMContext; -use crate::traphandlers::{tls, wasmtime_longjmp}; +#[link(name = "wasmtime-helpers")] +extern "C" { + #[wasmtime_versioned_export_macros::versioned_link] + #[allow(improper_ctypes)] + pub fn wasmtime_setjmp( + jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, + ) -> i32; + + #[wasmtime_versioned_export_macros::versioned_link] + pub fn wasmtime_longjmp(jmp_buf: *const u8) -> !; +} + +use crate::traphandlers::tls; use std::cell::RefCell; use std::io; use std::mem::{self, MaybeUninit}; @@ -16,9 +31,6 @@ static mut PREV_SIGILL: MaybeUninit = MaybeUninit::uninit(); static mut PREV_SIGFPE: MaybeUninit = MaybeUninit::uninit(); pub unsafe fn platform_init() { - if cfg!(miri) { - return; - } let register = |slot: &mut MaybeUninit, signal: i32| { let mut handler: libc::sigaction = mem::zeroed(); // The flags here are relatively careful, and they are... @@ -317,10 +329,6 @@ pub fn lazy_per_thread_init() { }); unsafe fn allocate_sigaltstack() -> Option { - if cfg!(miri) { - return None; - } - // Check to see if the existing sigaltstack, if it exists, is big // enough. If so we don't need to allocate our own. let mut old_stack = mem::zeroed(); diff --git a/crates/jit/src/unwind/systemv.rs b/crates/runtime/src/sys/unix/unwind.rs similarity index 97% rename from crates/jit/src/unwind/systemv.rs rename to crates/runtime/src/sys/unix/unwind.rs index 2233e4dcb350..87d0a28d421a 100644 --- a/crates/jit/src/unwind/systemv.rs +++ b/crates/runtime/src/sys/unix/unwind.rs @@ -14,6 +14,7 @@ extern "C" { } impl UnwindRegistration { + #[allow(missing_docs)] pub const SECTION_NAME: &'static str = ".eh_frame"; /// Registers precompiled unwinding information with the system. @@ -28,7 +29,7 @@ impl UnwindRegistration { unwind_len: usize, ) -> Result { debug_assert_eq!( - unwind_info as usize % wasmtime_runtime::page_size(), + unwind_info as usize % crate::page_size(), 0, "The unwind info must always be aligned to a page" ); diff --git a/crates/runtime/src/sys/unix/vm.rs b/crates/runtime/src/sys/unix/vm.rs new file mode 100644 index 000000000000..b08cee8d47a0 --- /dev/null +++ b/crates/runtime/src/sys/unix/vm.rs @@ -0,0 +1,82 @@ +use rustix::mm::{mmap_anonymous, mprotect, MapFlags, MprotectFlags, ProtFlags}; +use std::io; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE)?; + Ok(()) +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + mprotect(ptr.cast(), len, MprotectFlags::empty())?; + Ok(()) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + let ret = mmap_anonymous( + ptr.cast(), + len, + ProtFlags::empty(), + MapFlags::PRIVATE | MapFlags::FIXED, + )?; + assert_eq!(ptr, ret.cast()); + Ok(()) +} + +unsafe fn decommit(addr: *mut u8, len: usize) -> io::Result<()> { + if len == 0 { + return Ok(()); + } + + unsafe { + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + use rustix::mm::{madvise, Advice}; + + // On Linux, this is enough to cause the kernel to initialize + // the pages to 0 on next access + madvise(addr as _, len, Advice::LinuxDontNeed)?; + } else { + // By creating a new mapping at the same location, this will + // discard the mapping for the pages in the given range. + // The new mapping will be to the CoW zero page, so this + // effectively zeroes the pages. + mmap_anonymous( + addr as _, + len, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE | MapFlags::FIXED, + )?; + } + } + } + + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn commit_table_pages(_addr: *mut u8, _len: usize) -> io::Result<()> { + // Table pages are always READ | WRITE so there's nothing that needs to be + // done here. + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn decommit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + decommit(addr, len) +} + +#[cfg(feature = "async")] +pub unsafe fn commit_stack_pages(_addr: *mut u8, _len: usize) -> io::Result<()> { + // Like table pages stack pages are always READ | WRITE so nothing extra + // needs to be done to ensure they can be committed. + Ok(()) +} + +#[cfg(feature = "async")] +pub unsafe fn reset_stack_pages_to_zero(addr: *mut u8, len: usize) -> io::Result<()> { + decommit(addr, len) +} + +pub fn get_page_size() -> usize { + unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() } +} diff --git a/crates/runtime/src/mmap/windows.rs b/crates/runtime/src/sys/windows/mmap.rs similarity index 100% rename from crates/runtime/src/mmap/windows.rs rename to crates/runtime/src/sys/windows/mmap.rs diff --git a/crates/runtime/src/sys/windows/mod.rs b/crates/runtime/src/sys/windows/mod.rs new file mode 100644 index 000000000000..c8a35ad10645 --- /dev/null +++ b/crates/runtime/src/sys/windows/mod.rs @@ -0,0 +1,4 @@ +pub mod mmap; +pub mod traphandlers; +pub mod unwind; +pub mod vm; diff --git a/crates/runtime/src/traphandlers/windows.rs b/crates/runtime/src/sys/windows/traphandlers.rs similarity index 88% rename from crates/runtime/src/traphandlers/windows.rs rename to crates/runtime/src/sys/windows/traphandlers.rs index 5ad7295f5e3c..3e87c0e7d380 100644 --- a/crates/runtime/src/traphandlers/windows.rs +++ b/crates/runtime/src/sys/windows/traphandlers.rs @@ -1,9 +1,25 @@ -use crate::traphandlers::{tls, wasmtime_longjmp}; +use crate::traphandlers::tls; +use crate::VMContext; use std::io; use windows_sys::Win32::Foundation::*; use windows_sys::Win32::System::Diagnostics::Debug::*; use windows_sys::Win32::System::Kernel::*; +#[link(name = "wasmtime-helpers")] +extern "C" { + #[wasmtime_versioned_export_macros::versioned_link] + #[allow(improper_ctypes)] + pub fn wasmtime_setjmp( + jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, + ) -> i32; + + #[wasmtime_versioned_export_macros::versioned_link] + pub fn wasmtime_longjmp(jmp_buf: *const u8) -> !; +} + /// Function which may handle custom signals while processing traps. pub type SignalHandler<'a> = dyn Fn(*mut EXCEPTION_POINTERS) -> bool + Send + Sync + 'a; diff --git a/crates/jit/src/unwind/winx64.rs b/crates/runtime/src/sys/windows/unwind.rs similarity index 95% rename from crates/jit/src/unwind/winx64.rs rename to crates/runtime/src/sys/windows/unwind.rs index 6468b87c3ec4..30f136180c15 100644 --- a/crates/jit/src/unwind/winx64.rs +++ b/crates/runtime/src/sys/windows/unwind.rs @@ -10,8 +10,10 @@ pub struct UnwindRegistration { } impl UnwindRegistration { + #[allow(missing_docs)] pub const SECTION_NAME: &'static str = ".pdata"; + #[allow(missing_docs)] pub unsafe fn new( base_address: *const u8, unwind_info: *const u8, diff --git a/crates/runtime/src/sys/windows/vm.rs b/crates/runtime/src/sys/windows/vm.rs new file mode 100644 index 000000000000..7d68ebd8e377 --- /dev/null +++ b/crates/runtime/src/sys/windows/vm.rs @@ -0,0 +1,40 @@ +use std::io; +use std::mem::MaybeUninit; +use windows_sys::Win32::System::Memory::*; +use windows_sys::Win32::System::SystemInformation::*; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + if VirtualAlloc(ptr.cast(), len, MEM_COMMIT, PAGE_READWRITE).is_null() { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + erase_existing_mapping(ptr, len) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + if VirtualFree(ptr.cast(), len, MEM_DECOMMIT) == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +pub unsafe fn commit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + expose_exisiting_mapping(addr, len) +} + +pub unsafe fn decommit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + erase_existing_mapping(addr, len) +} + +pub fn get_page_size() -> usize { + unsafe { + let mut info = MaybeUninit::uninit(); + GetSystemInfo(info.as_mut_ptr()); + info.assume_init_ref().dwPageSize as usize + } +} diff --git a/crates/runtime/src/trampolines/aarch64.rs b/crates/runtime/src/trampolines/aarch64.rs deleted file mode 100644 index 29852bad1ae1..000000000000 --- a/crates/runtime/src/trampolines/aarch64.rs +++ /dev/null @@ -1,42 +0,0 @@ -#[rustfmt::skip] -macro_rules! wasm_to_libcall_trampoline { - ($libcall:ident ; $libcall_impl:ident) => { - wasmtime_asm_macros::asm_func!( - wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall), - " - .cfi_startproc - bti c - - // Load the pointer to `VMRuntimeLimits` in `x9`. - ldur x9, [x0, #8] - - // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits. - stur fp, [x9, #24] - - // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits. - stur lr, [x9, #32] - - // Tail call to the actual implementation of this libcall. - b {} - - .cfi_endproc - ", - sym $libcall_impl - ); - }; -} - -#[cfg(test)] -mod wasm_to_libcall_trampoline_offsets_tests { - use wasmtime_environ::{Module, PtrSize, VMOffsets}; - - #[test] - fn test() { - let module = Module::new(); - let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module); - - assert_eq!(8, offsets.vmctx_runtime_limits()); - assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp()); - assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc()); - } -} diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 21a7a519a1d1..8296d953c263 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -4,6 +4,7 @@ mod backtrace; mod coredump; +use crate::sys::traphandlers; use crate::{Instance, VMContext, VMRuntimeLimits}; use anyhow::Error; use std::any::Any; @@ -16,74 +17,10 @@ pub use self::backtrace::{Backtrace, Frame}; pub use self::coredump::CoreDumpStack; pub use self::tls::{tls_eager_initialize, AsyncWasmCallState, PreviousAsyncWasmCallState}; -cfg_if::cfg_if! { - if #[cfg(miri)] { - // With MIRI set up just enough of a setjmp/longjmp with catching panics - // to get a few tests working that use this. - // - // Note that no actual JIT code runs in MIRI so this is purely here for - // host-to-host calls. - - struct WasmtimeLongjmp; - - #[wasmtime_versioned_export_macros::versioned_export] - unsafe extern "C" fn wasmtime_setjmp( - _jmp_buf: *mut *const u8, - callback: extern "C" fn(*mut u8, *mut VMContext), - payload: *mut u8, - callee: *mut VMContext, - ) -> i32 { - use std::panic::{self, AssertUnwindSafe}; - let result = panic::catch_unwind(AssertUnwindSafe(|| { - callback(payload, callee); - })); - match result { - Ok(()) => 1, - Err(e) => { - if e.is::() { - 0 - } else { - panic::resume_unwind(e) - } - } - } - } - - #[wasmtime_versioned_export_macros::versioned_export] - unsafe extern "C" fn wasmtime_longjmp(_jmp_buf: *const u8) -> ! { - std::panic::panic_any(WasmtimeLongjmp) - } - } else { - #[link(name = "wasmtime-helpers")] - extern "C" { - #[wasmtime_versioned_export_macros::versioned_link] - #[allow(improper_ctypes)] - fn wasmtime_setjmp( - jmp_buf: *mut *const u8, - callback: extern "C" fn(*mut u8, *mut VMContext), - payload: *mut u8, - callee: *mut VMContext, - ) -> i32; - #[wasmtime_versioned_export_macros::versioned_link] - fn wasmtime_longjmp(jmp_buf: *const u8) -> !; - } - } -} - -cfg_if::cfg_if! { - if #[cfg(unix)] { - mod unix; - use unix as sys; - } else if #[cfg(target_os = "windows")] { - mod windows; - use windows as sys; - } -} - #[cfg(target_os = "macos")] mod macos; -pub use sys::SignalHandler; +pub use traphandlers::SignalHandler; /// Globally-set callback to determine whether a program counter is actually a /// wasm trap. @@ -123,7 +60,7 @@ pub fn init_traps(is_wasm_pc: fn(usize) -> bool, macos_use_mach_ports: bool) { MACOS_USE_MACH_PORTS = macos_use_mach_ports; return macos::platform_init(); } - sys::platform_init(); + traphandlers::platform_init(); }); #[cfg(target_os = "macos")] @@ -143,7 +80,7 @@ fn lazy_per_thread_init() { } } - sys::lazy_per_thread_init(); + traphandlers::lazy_per_thread_init(); } /// Raises a trap immediately. @@ -303,7 +240,7 @@ where let result = CallThreadState::new(signal_handler, capture_backtrace, capture_coredump, *limits) .with(|cx| { - wasmtime_setjmp( + traphandlers::wasmtime_setjmp( cx.jmp_buf.as_ptr(), call_closure::, &mut closure as *mut F as *mut u8, @@ -470,7 +407,7 @@ impl CallThreadState { (*self.unwind.get()) .as_mut_ptr() .write((reason, backtrace, coredump)); - wasmtime_longjmp(self.jmp_buf.get()); + traphandlers::wasmtime_longjmp(self.jmp_buf.get()); } } @@ -490,7 +427,7 @@ impl CallThreadState { /// * a different pointer - a jmp_buf buffer to longjmp to, meaning that /// the wasm trap was succesfully handled. #[cfg_attr(target_os = "macos", allow(dead_code))] // macOS is more raw and doesn't use this - fn take_jmp_buf_if_trap( + pub(crate) fn take_jmp_buf_if_trap( &self, pc: *const u8, call_handler: impl Fn(&SignalHandler) -> bool, @@ -519,7 +456,7 @@ impl CallThreadState { self.jmp_buf.replace(ptr::null()) } - fn set_jit_trap(&self, pc: *const u8, fp: usize, faulting_addr: Option) { + pub(crate) fn set_jit_trap(&self, pc: *const u8, fp: usize, faulting_addr: Option) { let backtrace = self.capture_backtrace(self.limits, Some((pc as usize, fp))); let coredump = self.capture_coredump(self.limits, Some((pc as usize, fp))); unsafe { @@ -581,7 +518,7 @@ impl Drop for ResetCell<'_, T> { // happen which requires us to read some contextual state to figure out what to // do with the trap. This `tls` module is used to persist that information from // the caller to the trap site. -mod tls { +pub(crate) mod tls { use super::CallThreadState; use std::mem; use std::ops::Range; diff --git a/crates/runtime/src/traphandlers/backtrace.rs b/crates/runtime/src/traphandlers/backtrace.rs index 0b61fb425f5d..4cab189dcfc4 100644 --- a/crates/runtime/src/traphandlers/backtrace.rs +++ b/crates/runtime/src/traphandlers/backtrace.rs @@ -20,39 +20,13 @@ //! exit FP and stopping once we reach the entry SP (meaning that the next older //! frame is a host frame). +use crate::arch; use crate::{ traphandlers::{tls, CallThreadState}, VMRuntimeLimits, }; -use cfg_if::cfg_if; use std::ops::ControlFlow; -// Architecture-specific bits for stack walking. Each of these modules should -// define and export the following functions: -// -// * `unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize` -// * `unsafe fn get_next_older_fp_from_fp(fp: usize) -> usize` -// * `fn reached_entry_sp(fp: usize, first_wasm_sp: usize) -> bool` -// * `fn assert_entry_sp_is_aligned(sp: usize)` -// * `fn assert_fp_is_aligned(fp: usize)` -cfg_if! { - if #[cfg(target_arch = "x86_64")] { - mod x86_64; - use x86_64 as arch; - } else if #[cfg(target_arch = "aarch64")] { - mod aarch64; - use aarch64 as arch; - } else if #[cfg(target_arch = "s390x")] { - mod s390x; - use s390x as arch; - } else if #[cfg(target_arch = "riscv64")] { - mod riscv64; - use riscv64 as arch; - } else { - compile_error!("unsupported architecture"); - } -} - /// A WebAssembly stack trace. #[derive(Debug)] pub struct Backtrace(Vec); diff --git a/crates/runtime/src/traphandlers/backtrace/riscv64.rs b/crates/runtime/src/traphandlers/backtrace/riscv64.rs deleted file mode 100644 index bc3f5b0840d3..000000000000 --- a/crates/runtime/src/traphandlers/backtrace/riscv64.rs +++ /dev/null @@ -1,18 +0,0 @@ -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - *(fp as *mut usize).offset(1) -} - -// And the current frame pointer points to the next older frame pointer. -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { - fp >= entry_sp -} - -pub fn assert_entry_sp_is_aligned(sp: usize) { - assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); -} - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); -} diff --git a/crates/runtime/src/traphandlers/backtrace/s390x.rs b/crates/runtime/src/traphandlers/backtrace/s390x.rs deleted file mode 100644 index 7290c4850c9a..000000000000 --- a/crates/runtime/src/traphandlers/backtrace/s390x.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - // The next older PC can be found in register %r14 at function entry, which - // was saved into slot 14 of the register save area pointed to by "FP" (the - // backchain pointer). - *(fp as *mut usize).offset(14) -} - -// The next older "FP" (backchain pointer) was saved in the slot pointed to -// by the current "FP". -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { - fp > entry_sp -} - -pub fn assert_entry_sp_is_aligned(sp: usize) { - assert_eq!(sp % 8, 0, "stack should always be aligned to 8"); -} - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); -} diff --git a/crates/runtime/src/traphandlers/backtrace/x86_64.rs b/crates/runtime/src/traphandlers/backtrace/x86_64.rs deleted file mode 100644 index ed1a3eb4b14b..000000000000 --- a/crates/runtime/src/traphandlers/backtrace/x86_64.rs +++ /dev/null @@ -1,20 +0,0 @@ -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - // The calling convention always pushes the return pointer (aka the PC of - // the next older frame) just before this frame. - *(fp as *mut usize).offset(1) -} - -// And the current frame pointer points to the next older frame pointer. -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { - fp >= entry_sp -} - -pub fn assert_entry_sp_is_aligned(sp: usize) { - assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); -} - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); -} diff --git a/crates/runtime/src/traphandlers/macos.rs b/crates/runtime/src/traphandlers/macos.rs index 34d707498e7b..f7906036a00a 100644 --- a/crates/runtime/src/traphandlers/macos.rs +++ b/crates/runtime/src/traphandlers/macos.rs @@ -33,7 +33,8 @@ #![allow(non_snake_case, clippy::cast_sign_loss)] -use crate::traphandlers::{tls, wasmtime_longjmp}; +use crate::sys::traphandlers::wasmtime_longjmp; +use crate::traphandlers::tls; use mach::exception_types::*; use mach::kern_return::*; use mach::mach_init::*;