From 175fb32fedabf3c835aaebd157131ce1e0c6033a Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 10 Aug 2024 15:59:00 +0800 Subject: [PATCH 1/4] update --- vm/llvm-jit/include/compat_llvm.hpp | 10 +- vm/llvm-jit/src/compat/compat_llvm.cpp | 49 +++++- vm/llvm-jit/src/llvm/compiler.cpp | 190 ++++++++------------- vm/llvm-jit/src/llvm/llvm_jit_context.cpp | 198 ++++++++++++---------- vm/llvm-jit/src/llvm/llvm_jit_context.hpp | 5 +- 5 files changed, 229 insertions(+), 223 deletions(-) diff --git a/vm/llvm-jit/include/compat_llvm.hpp b/vm/llvm-jit/include/compat_llvm.hpp index 2e900e96..e115066f 100644 --- a/vm/llvm-jit/include/compat_llvm.hpp +++ b/vm/llvm-jit/include/compat_llvm.hpp @@ -41,11 +41,11 @@ class bpftime_llvm_jit_vm : public bpftime::vm::compat::bpftime_vm_impl { } private: - uint64_t (*map_by_fd)(uint32_t) = nullptr; - uint64_t (*map_by_idx)(uint32_t) = nullptr; - uint64_t (*map_val)(uint64_t) = nullptr; - uint64_t (*var_addr)(uint32_t) = nullptr; - uint64_t (*code_addr)(uint32_t) = nullptr; + uint64_t (*map_by_fd)(uint32_t); + uint64_t (*map_by_idx)(uint32_t); + uint64_t (*map_val)(uint64_t); + uint64_t (*var_addr)(uint32_t); + uint64_t (*code_addr)(uint32_t); std::vector instructions; std::vector > ext_funcs; diff --git a/vm/llvm-jit/src/compat/compat_llvm.cpp b/vm/llvm-jit/src/compat/compat_llvm.cpp index bbb7108e..2a484e31 100644 --- a/vm/llvm-jit/src/compat/compat_llvm.cpp +++ b/vm/llvm-jit/src/compat/compat_llvm.cpp @@ -1,10 +1,39 @@ #include "spdlog/spdlog.h" #include #include +#include #include #include #include "compat_llvm.hpp" #include "../llvm/llvm_jit_context.hpp" + +static uint64_t default_map_by_fd(uint32_t) +{ + SPDLOG_WARN("map_by_fd not defined, but called"); + return 0; +} + +static uint64_t default_map_by_idx(uint32_t) +{ + SPDLOG_WARN("map_by_idx not defined, but called"); + return 0; +} + +static uint64_t default_map_val(uint64_t) +{ + SPDLOG_WARN("map_val not defined, but called"); + return 0; +} +static uint64_t default_var_addr(uint32_t) +{ + SPDLOG_WARN("var_addr not defined, but called"); + return 0; +} +static uint64_t default_code_addr(uint32_t) +{ + SPDLOG_WARN("code_addr not defined, but called"); + return 0; +} namespace bpftime::vm::compat { @@ -21,6 +50,11 @@ bpftime_llvm_jit_vm::bpftime_llvm_jit_vm() : ext_funcs(MAX_EXT_FUNCS) { this->jit_ctx = std::make_unique(this); + map_by_fd = &default_map_by_fd; + map_by_idx = &default_map_by_idx; + map_val = &default_map_val; + code_addr = &default_code_addr; + var_addr = &default_var_addr; } std::string bpftime_llvm_jit_vm::get_error_message() @@ -98,11 +132,16 @@ void bpftime_llvm_jit_vm::set_lddw_helpers(uint64_t (*map_by_fd)(uint32_t), uint64_t (*var_addr)(uint32_t), uint64_t (*code_addr)(uint32_t)) { - this->map_by_fd = map_by_fd; - this->map_by_idx = map_by_idx; - this->map_val = map_val; - this->var_addr = var_addr; - this->code_addr = code_addr; + if (map_by_fd) + this->map_by_fd = map_by_fd; + if (map_by_idx) + this->map_by_idx = map_by_idx; + if (map_val) + this->map_val = map_val; + if (var_addr) + this->var_addr = var_addr; + if (code_addr) + this->code_addr = code_addr; } std::vector bpftime_llvm_jit_vm::do_aot_compile(bool print_ir) diff --git a/vm/llvm-jit/src/llvm/compiler.cpp b/vm/llvm-jit/src/llvm/compiler.cpp index 138caeff..7b95e617 100644 --- a/vm/llvm-jit/src/llvm/compiler.cpp +++ b/vm/llvm-jit/src/llvm/compiler.cpp @@ -86,8 +86,11 @@ const size_t MAX_LOCAL_FUNC_DEPTH = 32; */ Expected llvm_bpf_jit_context::generateModule( const std::vector &extFuncNames, - const std::vector &lddwHelpers) + const std::vector &lddwHelpers, + bool patch_map_val_at_compile_time) { + SPDLOG_DEBUG("Generating module: patch_map_val_at_compile_time={}", + patch_map_val_at_compile_time); auto context = std::make_unique(); auto jitModule = std::make_unique("bpf-jit", *context); const auto &insts = vm->instructions; @@ -155,17 +158,15 @@ Expected llvm_bpf_jit_context::generateModule( // The main function Function *bpf_func = Function::Create( FunctionType::get(Type::getInt64Ty(*context), - { llvm::PointerType::getUnqual(llvm::Type::getInt8Ty(*context)), + { llvm::PointerType::getUnqual( + llvm::Type::getInt8Ty(*context)), Type::getInt64Ty(*context) }, false), Function::ExternalLinkage, "bpf_main", jitModule.get()); - - // Get args of uint64_t bpf_main(uint64_t, uint64_t) - llvm::Argument* mem = bpf_func->getArg(0); - llvm::Argument* mem_len = bpf_func->getArg(1); - - + // Get args of uint64_t bpf_main(uint64_t, uint64_t) + llvm::Argument *mem = bpf_func->getArg(0); + llvm::Argument *mem_len = bpf_func->getArg(1); std::vector regs; std::vector allBlocks; @@ -639,43 +640,38 @@ Expected llvm_bpf_jit_context::generateModule( builder.CreateStore(builder.getInt64(val), regs[inst.dst_reg]); } else if (inst.src_reg == 1) { - if (auto itr = lddwHelper.find( - LDDW_HELPER_MAP_BY_FD); - itr != lddwHelper.end()) - - { + SPDLOG_DEBUG( + "Emit lddw helper 1 (map_by_fd) at pc {}, imm={}, patched at compile time", + pc, inst.imm); + builder.CreateStore( + builder.getInt32( + vm->map_by_fd(inst.imm)), + regs[inst.dst_reg]); + } else if (inst.src_reg == 2) { + SPDLOG_DEBUG( + "Emit lddw helper 2 (map_by_fd + map_val) at pc {}, imm1={}, imm2={}", + pc, inst.imm, nextInst.imm); + auto mapPtr = vm->map_by_fd(inst.imm); + if (patch_map_val_at_compile_time) { SPDLOG_DEBUG( - "Emit lddw helper 1 (map_by_fd) at pc {}, imm={}", - pc, inst.imm); + "map_val is required to be evaluated at compile time"); builder.CreateStore( - builder.CreateCall( - lddwHelperWithUint32, - itr->second, - { builder.getInt32( - inst.imm) }), + builder.getInt64( + vm->map_val(mapPtr) + + nextInst.imm), regs[inst.dst_reg]); } else { - return llvm::make_error< - llvm::StringError>( - "Using lddw helper 1, which requires map_by_fd", - llvm::inconvertibleErrorCode()); - } - } else if (inst.src_reg == 2) { - if (auto itrMapByFd = lddwHelper.find( - LDDW_HELPER_MAP_BY_FD); - itrMapByFd != lddwHelper.end()) { + SPDLOG_DEBUG( + "map_val is requires to be evaluated at runtime, emitting calling instructions"); + if (auto itrMapVal = lddwHelper.find( LDDW_HELPER_MAP_VAL); itrMapVal != lddwHelper.end()) { - auto retMapByFd = builder.CreateCall( - lddwHelperWithUint32, - itrMapByFd->second, - { builder.getInt32( - inst.imm) }); auto retMapVal = builder.CreateCall( lddwHelperWithUint64, itrMapVal->second, - { retMapByFd }); + { builder.getInt64( + mapPtr) }); auto finalRet = builder.CreateAdd( retMapVal, builder.getInt64( @@ -683,99 +679,63 @@ Expected llvm_bpf_jit_context::generateModule( builder.CreateStore( finalRet, regs[inst.dst_reg]); - SPDLOG_DEBUG( - "Emit lddw helper 2 (map_by_fd + map_val) at pc {}, imm1={}, imm2={}", - pc, inst.imm, - nextInst.imm); + } else { return llvm::make_error< llvm::StringError>( - "Using lddw helper 2, which requires map_val", + "Using lddw helper 2, which requires map_val to be defined. This should not happen", llvm::inconvertibleErrorCode()); } - - } else { - return llvm::make_error< - llvm::StringError>( - "Using lddw helper 2, which requires map_by_fd", - llvm::inconvertibleErrorCode()); } + } else if (inst.src_reg == 3) { - if (auto itr = lddwHelper.find( - LDDW_HELPER_VAR_ADDR); - itr != lddwHelper.end()) { - builder.CreateStore( - builder.CreateCall( - lddwHelperWithUint32, - itr->second, - { builder.getInt32( - inst.imm) }), - regs[inst.dst_reg]); - SPDLOG_DEBUG( - "Emit lddw helper 3 (var_addr) at pc {}, imm1={}", - pc, inst.imm); - } else { - return llvm::make_error< - llvm::StringError>( - "Using lddw helper 3, which requires var_addr", - llvm::inconvertibleErrorCode()); - } + SPDLOG_DEBUG( + "Emit lddw helper 3 (var_addr) at pc {}, imm1={}", + pc, inst.imm); + builder.CreateStore( + builder.getInt64( + vm->var_addr(inst.imm)), + regs[inst.dst_reg]); } else if (inst.src_reg == 4) { - if (auto itr = lddwHelper.find( - LDDW_HELPER_CODE_ADDR); - itr != lddwHelper.end()) { - builder.CreateStore( - builder.CreateCall( - lddwHelperWithUint32, - itr->second, - { builder.getInt32( - inst.imm) }), - regs[inst.dst_reg]); - SPDLOG_DEBUG( - "Emit lddw helper 4 (code_addr) at pc {}, imm1={}", - pc, inst.imm); - } else { - return llvm::make_error< - llvm::StringError>( - "Using lddw helper 4, which requires code_addr", - llvm::inconvertibleErrorCode()); - } + SPDLOG_DEBUG( + "Emit lddw helper 4 (code_addr) at pc {}, imm1={}", + pc, inst.imm); + builder.CreateStore( + builder.getInt64( + vm->code_addr(inst.imm)), + regs[inst.dst_reg]); } else if (inst.src_reg == 5) { - if (auto itr = lddwHelper.find( - LDDW_HELPER_MAP_BY_IDX); - itr != lddwHelper.end()) { + SPDLOG_DEBUG( + "Emit lddw helper 4 (map_by_idx) at pc {}, imm1={}", + pc, inst.imm); + builder.CreateStore( + builder.getInt64( + vm->map_by_idx(inst.imm)), + regs[inst.dst_reg]); + } else if (inst.src_reg == 6) { + auto mapPtr = vm->map_by_idx(inst.imm); + SPDLOG_DEBUG( + "Emit lddw helper 6 (map_by_idx + map_val) at pc {}, imm1={}, imm2={}", + pc, inst.imm, nextInst.imm); + if (patch_map_val_at_compile_time) { + SPDLOG_DEBUG( + "Required to evaluate map_val at compile time"); builder.CreateStore( - builder.CreateCall( - lddwHelperWithUint32, - itr->second, - { builder.getInt32( - inst.imm) }), + builder.getInt64( + vm->map_val(mapPtr) + + nextInst.imm), regs[inst.dst_reg]); - SPDLOG_DEBUG( - "Emit lddw helper 4 (map_by_idx) at pc {}, imm1={}", - pc, inst.imm); } else { - return llvm::make_error< - llvm::StringError>( - "Using lddw helper 5, which requires map_by_idx", - llvm::inconvertibleErrorCode()); - } - } else if (inst.src_reg == 6) { - if (auto itrMapByIdx = lddwHelper.find( - LDDW_HELPER_MAP_BY_IDX); - itrMapByIdx != lddwHelper.end()) { + SPDLOG_DEBUG( + "Required to evaluate map_val at runtime time"); if (auto itrMapVal = lddwHelper.find( LDDW_HELPER_MAP_VAL); itrMapVal != lddwHelper.end()) { - auto retMapByIdx = builder.CreateCall( - lddwHelperWithUint32, - itrMapByIdx->second, - { builder.getInt32( - inst.imm) }); auto retMapVal = builder.CreateCall( lddwHelperWithUint64, itrMapVal->second, - { retMapByIdx }); + { builder.getInt64( + mapPtr) }); auto finalRet = builder.CreateAdd( retMapVal, builder.getInt64( @@ -783,21 +743,13 @@ Expected llvm_bpf_jit_context::generateModule( builder.CreateStore( finalRet, regs[inst.dst_reg]); - SPDLOG_DEBUG( - "Emit lddw helper 6 (map_by_idx + map_val) at pc {}, imm1={}, imm2={}", - pc, inst.imm, - nextInst.imm); + } else { return llvm::make_error< llvm::StringError>( "Using lddw helper 6, which requires map_val", llvm::inconvertibleErrorCode()); } - } else { - return llvm::make_error< - llvm::StringError>( - "Using lddw helper 6, which requires map_by_idx", - llvm::inconvertibleErrorCode()); } } break; diff --git a/vm/llvm-jit/src/llvm/llvm_jit_context.cpp b/vm/llvm-jit/src/llvm/llvm_jit_context.cpp index 7534acd3..c3781389 100644 --- a/vm/llvm-jit/src/llvm/llvm_jit_context.cpp +++ b/vm/llvm-jit/src/llvm/llvm_jit_context.cpp @@ -13,7 +13,6 @@ #include #include - #include #include #include @@ -177,10 +176,6 @@ using namespace llvm::orc; using namespace bpftime; using namespace std; - - - - struct spin_lock_guard { pthread_spinlock_t *spin; spin_lock_guard(pthread_spinlock_t *spin) : spin(spin) @@ -197,46 +192,48 @@ static ExitOnError ExitOnErr; static void optimizeModule(llvm::Module &M) { - // std::cout << "LLVM_VERSION_MAJOR: " << LLVM_VERSION_MAJOR << std::endl; - #if LLVM_VERSION_MAJOR >= 17 - // ===================== - // Create the analysis managers. - // These must be declared in this order so that they are destroyed in the - // correct order due to inter-analysis-manager references. - LoopAnalysisManager LAM; - FunctionAnalysisManager FAM; - CGSCCAnalysisManager CGAM; - ModuleAnalysisManager MAM; - - // Create the new pass manager builder. - // Take a look at the PassBuilder constructor parameters for more - // customization, e.g. specifying a TargetMachine or various debugging - // options. - PassBuilder PB; - - // Register all the basic analyses with the managers. - PB.registerModuleAnalyses(MAM); - PB.registerCGSCCAnalyses(CGAM); - PB.registerFunctionAnalyses(FAM); - PB.registerLoopAnalyses(LAM); - PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - - // Create the pass manager. - // This one corresponds to a typical -O2 optimization pipeline. - ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O3); - - // Optimize the IR! - MPM.run(M, MAM); - // ===================================== - #else - llvm::legacy::PassManager PM; - - llvm::PassManagerBuilder PMB; - PMB.OptLevel = 3; - PMB.populateModulePassManager(PM); - - PM.run(M); - #endif + // std::cout << "LLVM_VERSION_MAJOR: " << LLVM_VERSION_MAJOR << + // std::endl; +#if LLVM_VERSION_MAJOR >= 17 + // ===================== + // Create the analysis managers. + // These must be declared in this order so that they are destroyed in + // the correct order due to inter-analysis-manager references. + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + // Create the new pass manager builder. + // Take a look at the PassBuilder constructor parameters for more + // customization, e.g. specifying a TargetMachine or various debugging + // options. + PassBuilder PB; + + // Register all the basic analyses with the managers. + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // Create the pass manager. + // This one corresponds to a typical -O2 optimization pipeline. + ModulePassManager MPM = + PB.buildPerModuleDefaultPipeline(OptimizationLevel::O3); + + // Optimize the IR! + MPM.run(M, MAM); + // ===================================== +#else + llvm::legacy::PassManager PM; + + llvm::PassManagerBuilder PMB; + PMB.OptLevel = 3; + PMB.populateModulePassManager(PM); + + PM.run(M); +#endif } #if defined(__arm__) || defined(_M_ARM) @@ -264,8 +261,8 @@ void llvm_bpf_jit_context::do_jit_compile() { auto [jit, extFuncNames, definedLddwHelpers] = create_and_initialize_lljit_instance(); - auto bpfModule = - ExitOnErr(generateModule(extFuncNames, definedLddwHelpers)); + auto bpfModule = ExitOnErr( + generateModule(extFuncNames, definedLddwHelpers, true)); bpfModule.withModuleDo([](auto &M) { optimizeModule(M); }); ExitOnErr(jit->addIRModule(std::move(bpfModule))); this->jit = std::move(jit); @@ -293,7 +290,8 @@ std::vector llvm_bpf_jit_context::do_aot_compile( const std::vector &lddwHelpers, bool print_ir) { SPDLOG_DEBUG("AOT: start"); - if (auto module = generateModule(extFuncNames, lddwHelpers); module) { + if (auto module = generateModule(extFuncNames, lddwHelpers, false); + module) { auto defaultTargetTriple = llvm::sys::getDefaultTargetTriple(); SPDLOG_DEBUG("AOT: target triple: {}", defaultTargetTriple); return module->withModuleDo([&](auto &module) @@ -327,20 +325,29 @@ std::vector llvm_bpf_jit_context::do_aot_compile( std::make_unique( objStream); -legacy::PassManager pass; + legacy::PassManager pass; // auto FileType = CGFT_ObjectFile; #if LLVM_VERSION_MAJOR >= 18 -if (targetMachine->addPassesToEmitFile(pass, *BOS, nullptr, CodeGenFileType::ObjectFile)) { + if (targetMachine->addPassesToEmitFile( + pass, *BOS, nullptr, + CodeGenFileType::ObjectFile)) { #elif LLVM_VERSION_MAJOR >= 10 -if (targetMachine->addPassesToEmitFile(pass, *BOS, nullptr, CGFT_ObjectFile)) { + if (targetMachine->addPassesToEmitFile( + pass, *BOS, nullptr, CGFT_ObjectFile)) { #elif LLVM_VERSION_MAJOR >= 8 -if (targetMachine->addPassesToEmitFile(pass, *BOS, nullptr, TargetMachine::CGFT_ObjectFile)) { + if (targetMachine->addPassesToEmitFile( + pass, *BOS, nullptr, + TargetMachine::CGFT_ObjectFile)) { #else -if (targetMachine->addPassesToEmitFile(pass, *BOS, TargetMachine::CGFT_ObjectFile, true)) { + if (targetMachine->addPassesToEmitFile( + pass, *BOS, TargetMachine::CGFT_ObjectFile, + true)) { #endif - SPDLOG_ERROR("Unable to emit module for target machine"); - throw std::runtime_error("Unable to emit module for target machine"); -} + SPDLOG_ERROR( + "Unable to emit module for target machine"); + throw std::runtime_error( + "Unable to emit module for target machine"); + } pass.run(module); SPDLOG_INFO("AOT: done, received {} bytes", @@ -364,28 +371,30 @@ std::vector llvm_bpf_jit_context::do_aot_compile(bool print_ir) std::vector extNames, lddwNames; for (uint32_t i = 0; i < std::size(vm->ext_funcs); i++) { if (vm->ext_funcs[i].has_value()) { - #if LLVM_VERSION_MAJOR >= 16 - extNames.emplace_back(ext_func_sym(i)); - #else - extNames.push_back(ext_func_sym(i)); - #endif +#if LLVM_VERSION_MAJOR >= 16 + extNames.emplace_back(ext_func_sym(i)); +#else + extNames.push_back(ext_func_sym(i)); +#endif } } const auto tryDefineLddwHelper = [&](const char *name, void *func) { if (func) { - #if LLVM_VERSION_MAJOR >= 16 - lddwNames.emplace_back(name); - #else - lddwNames.push_back(name); - #endif +#if LLVM_VERSION_MAJOR >= 16 + lddwNames.emplace_back(name); +#else + lddwNames.push_back(name); +#endif } }; - tryDefineLddwHelper(LDDW_HELPER_MAP_BY_FD, (void *)vm->map_by_fd); - tryDefineLddwHelper(LDDW_HELPER_MAP_BY_IDX, (void *)vm->map_by_idx); + // Only map_val will have a chance to be called at runtime tryDefineLddwHelper(LDDW_HELPER_MAP_VAL, (void *)vm->map_val); - tryDefineLddwHelper(LDDW_HELPER_CODE_ADDR, (void *)vm->code_addr); - tryDefineLddwHelper(LDDW_HELPER_VAR_ADDR, (void *)vm->var_addr); + // These symbols won't be used at runtime + // tryDefineLddwHelper(LDDW_HELPER_MAP_BY_FD, (void *)vm->map_by_fd); + // tryDefineLddwHelper(LDDW_HELPER_MAP_BY_IDX, (void *)vm->map_by_idx); + // tryDefineLddwHelper(LDDW_HELPER_CODE_ADDR, (void *)vm->code_addr); + // tryDefineLddwHelper(LDDW_HELPER_VAR_ADDR, (void *)vm->var_addr); return this->do_aot_compile(extNames, lddwNames, print_ir); } @@ -433,17 +442,17 @@ llvm_bpf_jit_context::create_and_initialize_lljit_instance() ext_func_sym(i)); sym.setFlags(JITSymbolFlags::Callable | JITSymbolFlags::Exported); - - #if LLVM_VERSION_MAJOR < 17 +#if LLVM_VERSION_MAJOR < 17 extSymbols.try_emplace(symName, sym); extFuncNames.push_back(ext_func_sym(i)); - #else - auto symbol = ::llvm::orc::ExecutorSymbolDef (::llvm::orc::ExecutorAddr (sym.getAddress()), sym.getFlags()); +#else + auto symbol = ::llvm::orc::ExecutorSymbolDef( + ::llvm::orc::ExecutorAddr(sym.getAddress()), + sym.getFlags()); extSymbols.try_emplace(symName, symbol); extFuncNames.emplace_back(ext_func_sym(i)); - #endif - +#endif } } #if defined(__arm__) || defined(_M_ARM) @@ -464,31 +473,36 @@ llvm_bpf_jit_context::create_and_initialize_lljit_instance() // printf("The type of sym %s\n", typeid(sym).name()); sym.setFlags(JITSymbolFlags::Callable | JITSymbolFlags::Exported); - - - #if LLVM_VERSION_MAJOR < 17 - lddwSyms.try_emplace(jit->getExecutionSession().intern(name), sym); +#if LLVM_VERSION_MAJOR < 17 + lddwSyms.try_emplace( + jit->getExecutionSession().intern(name), sym); definedLddwHelpers.push_back(name); - #else - auto symbol = ::llvm::orc::ExecutorSymbolDef (::llvm::orc::ExecutorAddr (sym.getAddress()), sym.getFlags()); - lddwSyms.try_emplace(jit->getExecutionSession().intern(name), symbol); +#else + auto symbol = ::llvm::orc::ExecutorSymbolDef( + ::llvm::orc::ExecutorAddr(sym.getAddress()), + sym.getFlags()); + lddwSyms.try_emplace( + jit->getExecutionSession().intern(name), + symbol); definedLddwHelpers.emplace_back(name); - #endif - - +#endif } }; - tryDefineLddwHelper(LDDW_HELPER_MAP_BY_FD, (void *)vm->map_by_fd); - tryDefineLddwHelper(LDDW_HELPER_MAP_BY_IDX, (void *)vm->map_by_idx); + // Only map_val will have a chance to be called at runtime, so it's the + // only symbol to be defined tryDefineLddwHelper(LDDW_HELPER_MAP_VAL, (void *)vm->map_val); - tryDefineLddwHelper(LDDW_HELPER_CODE_ADDR, (void *)vm->code_addr); - tryDefineLddwHelper(LDDW_HELPER_VAR_ADDR, (void *)vm->var_addr); + // These symbols won't be used at runtime + // tryDefineLddwHelper(LDDW_HELPER_MAP_BY_FD, (void *)vm->map_by_fd); + // tryDefineLddwHelper(LDDW_HELPER_MAP_BY_IDX, (void *)vm->map_by_idx); + // tryDefineLddwHelper(LDDW_HELPER_CODE_ADDR, (void *)vm->code_addr); + // tryDefineLddwHelper(LDDW_HELPER_VAR_ADDR, (void *)vm->var_addr); ExitOnErr(mainDylib.define(absoluteSymbols(lddwSyms))); return { std::move(jit), extFuncNames, definedLddwHelpers }; } -bpftime::vm::compat::precompiled_ebpf_function llvm_bpf_jit_context::get_entry_address() +bpftime::vm::compat::precompiled_ebpf_function +llvm_bpf_jit_context::get_entry_address() { if (!this->jit.has_value()) { SPDLOG_CRITICAL( @@ -506,4 +520,4 @@ bpftime::vm::compat::precompiled_ebpf_function llvm_bpf_jit_context::get_entry_a SPDLOG_DEBUG("LLVM-JIT: Entry func is {:x}", (uintptr_t)addr); return addr; } -} \ No newline at end of file +} diff --git a/vm/llvm-jit/src/llvm/llvm_jit_context.hpp b/vm/llvm-jit/src/llvm/llvm_jit_context.hpp index fc188738..61452fa8 100644 --- a/vm/llvm-jit/src/llvm/llvm_jit_context.hpp +++ b/vm/llvm-jit/src/llvm/llvm_jit_context.hpp @@ -33,12 +33,13 @@ class llvm_bpf_jit_context { std::unique_ptr compiling; llvm::Expected generateModule(const std::vector &extFuncNames, - const std::vector &lddwHelpers); + const std::vector &lddwHelpers, + bool patch_map_val_at_compile_time); std::vector do_aot_compile(const std::vector &extFuncNames, const std::vector &lddwHelpers, bool print_ir); - // (JIT, extFuncs, lddwHelpers) + // (JIT, extFuncs, definedLddwSymbols) std::tuple, std::vector, std::vector > create_and_initialize_lljit_instance(); From 88ca79f9ddfed4f6f758479774fcb0d6098b530e Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 10 Aug 2024 16:13:00 +0800 Subject: [PATCH 2/4] update --- vm/llvm-jit/src/llvm/compiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/llvm-jit/src/llvm/compiler.cpp b/vm/llvm-jit/src/llvm/compiler.cpp index 7b95e617..fc2bcb6f 100644 --- a/vm/llvm-jit/src/llvm/compiler.cpp +++ b/vm/llvm-jit/src/llvm/compiler.cpp @@ -662,7 +662,7 @@ Expected llvm_bpf_jit_context::generateModule( regs[inst.dst_reg]); } else { SPDLOG_DEBUG( - "map_val is requires to be evaluated at runtime, emitting calling instructions"); + "map_val is required to be evaluated at runtime, emitting calling instructions"); if (auto itrMapVal = lddwHelper.find( LDDW_HELPER_MAP_VAL); From 25d0c24f06998764db293b0a75a872c369e22d16 Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 10 Aug 2024 16:23:00 +0800 Subject: [PATCH 3/4] update --- vm/llvm-jit/src/llvm/compiler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/llvm-jit/src/llvm/compiler.cpp b/vm/llvm-jit/src/llvm/compiler.cpp index fc2bcb6f..d7304f4f 100644 --- a/vm/llvm-jit/src/llvm/compiler.cpp +++ b/vm/llvm-jit/src/llvm/compiler.cpp @@ -632,7 +632,7 @@ Expected llvm_bpf_jit_context::generateModule( (((uint64_t)((uint32_t)nextInst.imm)) << 32); pc++; - SPDLOG_TRACE("Load LDDW val= {} part1={:x} part2={:x}", + SPDLOG_DEBUG("Load LDDW val= {} part1={:x} part2={:x}", val, (uint64_t)inst.imm, (uint64_t)nextInst.imm); if (inst.src_reg == 0) { @@ -644,7 +644,7 @@ Expected llvm_bpf_jit_context::generateModule( "Emit lddw helper 1 (map_by_fd) at pc {}, imm={}, patched at compile time", pc, inst.imm); builder.CreateStore( - builder.getInt32( + builder.getInt64( vm->map_by_fd(inst.imm)), regs[inst.dst_reg]); } else if (inst.src_reg == 2) { From 30950923b63f9796452923f357665fc667d3137f Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 10 Aug 2024 17:16:38 +0800 Subject: [PATCH 4/4] update --- vm/llvm-jit/include/compat_llvm.hpp | 10 +-- vm/llvm-jit/src/compat/compat_llvm.cpp | 47 ++--------- vm/llvm-jit/src/llvm/compiler.cpp | 98 +++++++++++++++++++---- vm/llvm-jit/src/llvm/llvm_jit_context.cpp | 6 -- 4 files changed, 92 insertions(+), 69 deletions(-) diff --git a/vm/llvm-jit/include/compat_llvm.hpp b/vm/llvm-jit/include/compat_llvm.hpp index e115066f..2e900e96 100644 --- a/vm/llvm-jit/include/compat_llvm.hpp +++ b/vm/llvm-jit/include/compat_llvm.hpp @@ -41,11 +41,11 @@ class bpftime_llvm_jit_vm : public bpftime::vm::compat::bpftime_vm_impl { } private: - uint64_t (*map_by_fd)(uint32_t); - uint64_t (*map_by_idx)(uint32_t); - uint64_t (*map_val)(uint64_t); - uint64_t (*var_addr)(uint32_t); - uint64_t (*code_addr)(uint32_t); + uint64_t (*map_by_fd)(uint32_t) = nullptr; + uint64_t (*map_by_idx)(uint32_t) = nullptr; + uint64_t (*map_val)(uint64_t) = nullptr; + uint64_t (*var_addr)(uint32_t) = nullptr; + uint64_t (*code_addr)(uint32_t) = nullptr; std::vector instructions; std::vector > ext_funcs; diff --git a/vm/llvm-jit/src/compat/compat_llvm.cpp b/vm/llvm-jit/src/compat/compat_llvm.cpp index 2a484e31..6ca96d19 100644 --- a/vm/llvm-jit/src/compat/compat_llvm.cpp +++ b/vm/llvm-jit/src/compat/compat_llvm.cpp @@ -7,33 +7,6 @@ #include "compat_llvm.hpp" #include "../llvm/llvm_jit_context.hpp" -static uint64_t default_map_by_fd(uint32_t) -{ - SPDLOG_WARN("map_by_fd not defined, but called"); - return 0; -} - -static uint64_t default_map_by_idx(uint32_t) -{ - SPDLOG_WARN("map_by_idx not defined, but called"); - return 0; -} - -static uint64_t default_map_val(uint64_t) -{ - SPDLOG_WARN("map_val not defined, but called"); - return 0; -} -static uint64_t default_var_addr(uint32_t) -{ - SPDLOG_WARN("var_addr not defined, but called"); - return 0; -} -static uint64_t default_code_addr(uint32_t) -{ - SPDLOG_WARN("code_addr not defined, but called"); - return 0; -} namespace bpftime::vm::compat { @@ -50,11 +23,6 @@ bpftime_llvm_jit_vm::bpftime_llvm_jit_vm() : ext_funcs(MAX_EXT_FUNCS) { this->jit_ctx = std::make_unique(this); - map_by_fd = &default_map_by_fd; - map_by_idx = &default_map_by_idx; - map_val = &default_map_val; - code_addr = &default_code_addr; - var_addr = &default_var_addr; } std::string bpftime_llvm_jit_vm::get_error_message() @@ -132,16 +100,11 @@ void bpftime_llvm_jit_vm::set_lddw_helpers(uint64_t (*map_by_fd)(uint32_t), uint64_t (*var_addr)(uint32_t), uint64_t (*code_addr)(uint32_t)) { - if (map_by_fd) - this->map_by_fd = map_by_fd; - if (map_by_idx) - this->map_by_idx = map_by_idx; - if (map_val) - this->map_val = map_val; - if (var_addr) - this->var_addr = var_addr; - if (code_addr) - this->code_addr = code_addr; + this->map_by_fd = map_by_fd; + this->map_by_idx = map_by_idx; + this->map_val = map_val; + this->var_addr = var_addr; + this->code_addr = code_addr; } std::vector bpftime_llvm_jit_vm::do_aot_compile(bool print_ir) diff --git a/vm/llvm-jit/src/llvm/compiler.cpp b/vm/llvm-jit/src/llvm/compiler.cpp index d7304f4f..0cc910cd 100644 --- a/vm/llvm-jit/src/llvm/compiler.cpp +++ b/vm/llvm-jit/src/llvm/compiler.cpp @@ -643,18 +643,43 @@ Expected llvm_bpf_jit_context::generateModule( SPDLOG_DEBUG( "Emit lddw helper 1 (map_by_fd) at pc {}, imm={}, patched at compile time", pc, inst.imm); - builder.CreateStore( - builder.getInt64( - vm->map_by_fd(inst.imm)), - regs[inst.dst_reg]); + if (vm->map_by_fd) { + builder.CreateStore( + builder.getInt64(vm->map_by_fd( + inst.imm)), + regs[inst.dst_reg]); + } else { + SPDLOG_INFO( + "map_by_fd is called in eBPF code, but is not provided, will use the default behavior"); + // Default: input value + builder.CreateStore( + builder.getInt64( + (int64_t)inst.imm), + regs[inst.dst_reg]); + } + } else if (inst.src_reg == 2) { SPDLOG_DEBUG( "Emit lddw helper 2 (map_by_fd + map_val) at pc {}, imm1={}, imm2={}", pc, inst.imm, nextInst.imm); - auto mapPtr = vm->map_by_fd(inst.imm); + uint64_t mapPtr; + if (vm->map_by_fd) { + mapPtr = vm->map_by_fd(inst.imm); + } else { + SPDLOG_INFO( + "map_by_fd is called in eBPF code, but is not provided, will use the default behavior"); + // Default: returns the input value + mapPtr = (uint64_t)inst.imm; + } if (patch_map_val_at_compile_time) { SPDLOG_DEBUG( "map_val is required to be evaluated at compile time"); + if (!vm->map_val) { + return llvm::make_error< + llvm::StringError>( + "map_val is not provided, unable to compile", + llvm::inconvertibleErrorCode()); + } builder.CreateStore( builder.getInt64( vm->map_val(mapPtr) + @@ -683,7 +708,7 @@ Expected llvm_bpf_jit_context::generateModule( } else { return llvm::make_error< llvm::StringError>( - "Using lddw helper 2, which requires map_val to be defined. This should not happen", + "Using lddw helper 2, which requires map_val to be defined.", llvm::inconvertibleErrorCode()); } } @@ -692,6 +717,12 @@ Expected llvm_bpf_jit_context::generateModule( SPDLOG_DEBUG( "Emit lddw helper 3 (var_addr) at pc {}, imm1={}", pc, inst.imm); + if (!vm->var_addr) { + return llvm::make_error< + llvm::StringError>( + "var_addr is not provided, unable to compile", + llvm::inconvertibleErrorCode()); + } builder.CreateStore( builder.getInt64( vm->var_addr(inst.imm)), @@ -700,6 +731,12 @@ Expected llvm_bpf_jit_context::generateModule( SPDLOG_DEBUG( "Emit lddw helper 4 (code_addr) at pc {}, imm1={}", pc, inst.imm); + if (!vm->code_addr) { + return llvm::make_error< + llvm::StringError>( + "code_addr is not provided, unable to compile", + llvm::inconvertibleErrorCode()); + } builder.CreateStore( builder.getInt64( vm->code_addr(inst.imm)), @@ -708,23 +745,52 @@ Expected llvm_bpf_jit_context::generateModule( SPDLOG_DEBUG( "Emit lddw helper 4 (map_by_idx) at pc {}, imm1={}", pc, inst.imm); - builder.CreateStore( - builder.getInt64( - vm->map_by_idx(inst.imm)), - regs[inst.dst_reg]); + if (vm->map_by_idx) { + builder.CreateStore( + builder.getInt64(vm->map_by_idx( + inst.imm)), + regs[inst.dst_reg]); + } else { + SPDLOG_INFO( + "map_by_idx is called in eBPF code, but it's not provided, will use the default behavior"); + // Default: returns the input value + builder.CreateStore( + builder.getInt64( + (int64_t)inst.imm), + regs[inst.dst_reg]); + } + } else if (inst.src_reg == 6) { - auto mapPtr = vm->map_by_idx(inst.imm); SPDLOG_DEBUG( "Emit lddw helper 6 (map_by_idx + map_val) at pc {}, imm1={}, imm2={}", pc, inst.imm, nextInst.imm); + + uint64_t mapPtr; + if (vm->map_by_idx) { + mapPtr = vm->map_by_idx(inst.imm); + } else { + SPDLOG_DEBUG( + "map_by_idx is called in eBPF code, but it's not provided, will use the default behavior"); + // Default: returns the input value + mapPtr = (int64_t)inst.imm; + } if (patch_map_val_at_compile_time) { SPDLOG_DEBUG( "Required to evaluate map_val at compile time"); - builder.CreateStore( - builder.getInt64( - vm->map_val(mapPtr) + - nextInst.imm), - regs[inst.dst_reg]); + if (vm->map_val) { + builder.CreateStore( + builder.getInt64( + vm->map_val( + mapPtr) + + nextInst.imm), + regs[inst.dst_reg]); + } else { + return llvm::make_error< + llvm::StringError>( + "map_val is not provided, unable to compile", + llvm::inconvertibleErrorCode()); + } + } else { SPDLOG_DEBUG( "Required to evaluate map_val at runtime time"); diff --git a/vm/llvm-jit/src/llvm/llvm_jit_context.cpp b/vm/llvm-jit/src/llvm/llvm_jit_context.cpp index c3781389..9d1fc75f 100644 --- a/vm/llvm-jit/src/llvm/llvm_jit_context.cpp +++ b/vm/llvm-jit/src/llvm/llvm_jit_context.cpp @@ -141,9 +141,6 @@ #include "bpftime_vm_compat.hpp" #include "compiler_utils.hpp" #include "spdlog/spdlog.h" -#include -#include -#include #include #include "llvm/IR/Module.h" @@ -159,12 +156,9 @@ #include #include #include -#include #include #include -#include #include -#include #include #include #include