From c7f370bc06bb382dd476af8d679ea48a79bb5424 Mon Sep 17 00:00:00 2001 From: officeyutong Date: Thu, 2 Nov 2023 00:13:53 +0800 Subject: [PATCH 01/16] Stage commits --- daemon/kernel/bpf_tracer.bpf.c | 4 +- daemon/user/bpftime_driver.cpp | 2 +- daemon/user/handle_bpf_event.cpp | 8 ++- runtime/CMakeLists.txt | 2 +- runtime/include/bpftime_shm.hpp | 3 + runtime/src/bpf_map/array_map_kernel_user.cpp | 2 +- .../bpf_map/percpu_array_map_kernel_user.cpp | 69 +++++++++++++++++++ .../bpf_map/percpu_array_map_kernel_user.hpp | 31 +++++++++ runtime/src/handler/map_handler.cpp | 37 +++++++++- 9 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 runtime/src/bpf_map/percpu_array_map_kernel_user.cpp create mode 100644 runtime/src/bpf_map/percpu_array_map_kernel_user.hpp diff --git a/daemon/kernel/bpf_tracer.bpf.c b/daemon/kernel/bpf_tracer.bpf.c index f0a4d86a..f7f9bed8 100644 --- a/daemon/kernel/bpf_tracer.bpf.c +++ b/daemon/kernel/bpf_tracer.bpf.c @@ -356,7 +356,7 @@ process_perf_event_open_enter(struct trace_event_raw_sys_enter *ctx) // found uprobe if (enable_replace_uprobe) { new_attr.probe_offset = 0; - int size = bpf_probe_read_user_str( + long size = bpf_probe_read_user_str( old_uprobe_path, sizeof(old_uprobe_path), (void *)new_attr.uprobe_path); if (size <= 0) { @@ -367,7 +367,7 @@ process_perf_event_open_enter(struct trace_event_raw_sys_enter *ctx) size = PATH_LENTH; } bpf_probe_write_user((void *)new_attr.uprobe_path, - new_uprobe_path, size); + &new_uprobe_path, (size_t)size); bpf_probe_write_user(attr, &new_attr, sizeof(new_attr)); } return 0; diff --git a/daemon/user/bpftime_driver.cpp b/daemon/user/bpftime_driver.cpp index 2c09a770..568e129c 100644 --- a/daemon/user/bpftime_driver.cpp +++ b/daemon/user/bpftime_driver.cpp @@ -57,7 +57,7 @@ static int get_bpf_map_id_from_pid_fd(bpf_tracer_bpf *obj, int pid, int fd) } if (data.type != BPF_FD_TYPE_MAP) { spdlog::error("Invalid bpf fd type {} for pid {} fd {}", - data.type, pid, fd); + (int)data.type, pid, fd); } map_id = data.kernel_id; return map_id; diff --git a/daemon/user/handle_bpf_event.cpp b/daemon/user/handle_bpf_event.cpp index cc70f840..e2354249 100644 --- a/daemon/user/handle_bpf_event.cpp +++ b/daemon/user/handle_bpf_event.cpp @@ -194,7 +194,7 @@ int bpf_event_handler::handle_bpf_event(const struct event *e) /* prepare fields */ const char *cmd_str; if (e->bpf_data.bpf_cmd >= - (sizeof(bpf_cmd_strings) / sizeof(bpf_cmd_strings[0]))) { + (sizeof(bpf_cmd_strings) / sizeof(bpf_cmd_strings[0]))) { cmd_str = "UNKNOWN COMMAND"; } else { cmd_str = bpf_cmd_strings[e->bpf_data.bpf_cmd]; @@ -322,8 +322,9 @@ int bpf_event_handler::handle_ioctl(const struct event *e) fd); } } else if (req == PERF_EVENT_IOC_SET_BPF) { - spdlog::info("Setting bpf for perf event {} and bpf {} (id: {})", fd, - data, e->ioctl_data.bpf_prog_id); + spdlog::info( + "Setting bpf for perf event {} and bpf {} (id: {})", fd, + data, e->ioctl_data.bpf_prog_id); if (config.is_driving_bpftime) { return driver.bpftime_attach_perf_to_bpf_server( e->pid, fd, e->ioctl_data.bpf_prog_id); @@ -339,6 +340,7 @@ int bpf_event_handler::handle_event(const struct event *e) if (e->pid == current_pid) { return 0; } + spdlog::debug("Received event with type {}", (int)e->type); switch (e->type) { case SYS_OPEN: return handle_open_events(e); diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 02649221..eeb42c28 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -74,6 +74,7 @@ set(sources src/bpf_map/per_cpu_hash_map.cpp src/bpf_map/array_map_kernel_user.cpp src/bpf_map/hash_map_kernel_user.cpp + src/bpf_map/percpu_array_map_kernel_user.cpp ) # list(APPEND sources @@ -122,7 +123,6 @@ target_link_libraries(${PROJECT_NAME} ) add_dependencies(${PROJECT_NAME} vm-bpf FridaGum syscall_id_table spdlog::spdlog libbpf) - if(${ENABLE_EBPF_VERIFIER}) target_include_directories(${PROJECT_NAME} PRIVATE ${BPFTIME_VERIFIER_INCLUDE}) target_link_libraries(${PROJECT_NAME} PRIVATE bpftime-verifier) diff --git a/runtime/include/bpftime_shm.hpp b/runtime/include/bpftime_shm.hpp index 9544d03e..3ca4f933 100644 --- a/runtime/include/bpftime_shm.hpp +++ b/runtime/include/bpftime_shm.hpp @@ -94,6 +94,9 @@ enum class bpf_map_type { BPF_MAP_TYPE_KERNEL_USER_HASH = KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_HASH, BPF_MAP_TYPE_KERNEL_USER_ARRAY = KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_ARRAY, + BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY = KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_PERCPU_ARRAY, + + }; enum class shm_open_type { diff --git a/runtime/src/bpf_map/array_map_kernel_user.cpp b/runtime/src/bpf_map/array_map_kernel_user.cpp index 359a9b94..653d3dfd 100644 --- a/runtime/src/bpf_map/array_map_kernel_user.cpp +++ b/runtime/src/bpf_map/array_map_kernel_user.cpp @@ -6,7 +6,7 @@ #ifndef roundup #define roundup(x, y) \ ({ \ - const typeof(y) __y = y; \ + const decltype(y) __y = y; \ (((x) + (__y - 1)) / __y) * __y; \ }) #endif // roundup diff --git a/runtime/src/bpf_map/percpu_array_map_kernel_user.cpp b/runtime/src/bpf_map/percpu_array_map_kernel_user.cpp new file mode 100644 index 00000000..0e3f04e8 --- /dev/null +++ b/runtime/src/bpf_map/percpu_array_map_kernel_user.cpp @@ -0,0 +1,69 @@ +#include "bpf/bpf.h" +#include "spdlog/spdlog.h" +#include +#include +namespace bpftime +{ +void percpu_array_map_kernel_user_impl::init_map_fd() +{ + kernel_map_fd = bpf_map_get_fd_by_id(kernel_map_id); + if (kernel_map_fd < 0) { + spdlog::error( + "Failed to retrive kernel map fd by kernel map id: {}", + kernel_map_fd); + return; + } + bpf_map_info map_info; + uint32_t sz = sizeof(map_info); + if (int err = bpf_obj_get_info_by_fd(kernel_map_fd, &map_info, &sz); + err < 0) { + spdlog::error( + "Failed to get map detail of kernel map fd {}, err={}", + kernel_map_fd, err); + return; + } + _value_size = map_info.value_size; + _max_entries = map_info.max_entries; + value_data.resize(_value_size * ncpu); +} +void *percpu_array_map_kernel_user_impl::elem_lookup(const void *key) +{ + int err = bpf_map_lookup_elem(kernel_map_fd, key, + (void *)value_data.data()); + if (err < 0) { + spdlog::error( + "Failed to perform elem lookup of percpu array {}", + kernel_map_fd); + return nullptr; + } + return value_data.data(); +} + +long percpu_array_map_kernel_user_impl::elem_update(const void *key, + const void *value, + uint64_t flags) +{ + return bpf_map_update_elem(kernel_map_fd, key, value, flags); +} + +long percpu_array_map_kernel_user_impl::elem_delete(const void *key) +{ + return bpf_map_delete_elem(kernel_map_fd, key); +} + +int percpu_array_map_kernel_user_impl::map_get_next_key(const void *key, + void *next_key) +{ + return bpf_map_get_next_key(kernel_map_fd, key, next_key); +} +percpu_array_map_kernel_user_impl::percpu_array_map_kernel_user_impl( + boost::interprocess::managed_shared_memory &memory, int kernel_map_id) + : kernel_map_id(kernel_map_id), + value_data(1, memory.get_segment_manager()) +{ + ncpu = sysconf(_SC_NPROCESSORS_ONLN); +} +percpu_array_map_kernel_user_impl::~percpu_array_map_kernel_user_impl() +{ +} +} // namespace bpftime diff --git a/runtime/src/bpf_map/percpu_array_map_kernel_user.hpp b/runtime/src/bpf_map/percpu_array_map_kernel_user.hpp new file mode 100644 index 00000000..96783f10 --- /dev/null +++ b/runtime/src/bpf_map/percpu_array_map_kernel_user.hpp @@ -0,0 +1,31 @@ +#ifndef _BPFTIME_PERCPU_ARRAY_MAP_KERNEL_USER_HPP +#define _BPFTIME_PERCPU_ARRAY_MAP_KERNEL_USER_HPP +#include +#include +namespace bpftime +{ +class percpu_array_map_kernel_user_impl { + uint32_t _value_size; + uint32_t _max_entries; + int kernel_map_fd = -1; + int kernel_map_id; + int ncpu; + void init_map_fd(); + bytes_vec value_data; + + public: + const static bool should_lock = true; + void *elem_lookup(const void *key); + + long elem_update(const void *key, const void *value, uint64_t flags); + + long elem_delete(const void *key); + + int map_get_next_key(const void *key, void *next_key); + percpu_array_map_kernel_user_impl( + boost::interprocess::managed_shared_memory &memory, + int kernel_map_id); + ~percpu_array_map_kernel_user_impl(); +}; +} // namespace bpftime +#endif diff --git a/runtime/src/handler/map_handler.cpp b/runtime/src/handler/map_handler.cpp index 31d78215..aeb6eaa0 100644 --- a/runtime/src/handler/map_handler.cpp +++ b/runtime/src/handler/map_handler.cpp @@ -8,6 +8,7 @@ #include #include #include +#include using boost::interprocess::interprocess_sharable_mutex; using boost::interprocess::scoped_lock; @@ -104,6 +105,11 @@ const void *bpf_map_handler::map_lookup_elem(const void *key, map_impl_ptr.get()); return do_lookup(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_lookup(impl); + } default: assert(false && "Unsupported map type"); } @@ -172,6 +178,11 @@ long bpf_map_handler::map_update_elem(const void *key, const void *value, map_impl_ptr.get()); return do_update(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_update(impl); + } default: assert(false && "Unsupported map type"); } @@ -228,6 +239,11 @@ int bpf_map_handler::bpf_map_get_next_key(const void *key, void *next_key, map_impl_ptr.get()); return do_get_next_key(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_get_next_key(impl); + } default: assert(false && "Unsupported map type"); } @@ -296,6 +312,11 @@ long bpf_map_handler::map_delete_elem(const void *key, map_impl_ptr.get()); return do_delete(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_delete(impl); + } default: assert(false && "Unsupported map type"); } @@ -358,12 +379,20 @@ int bpf_map_handler::map_init(managed_shared_memory &memory) } case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_HASH: { map_impl_ptr = memory.construct( - container_name.c_str())(memory, attr.kernel_bpf_map_id); + container_name.c_str())(memory, attr.kernel_bpf_map_id); + return 0; + } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY: { + map_impl_ptr = + memory.construct( + container_name.c_str())(memory, + attr.kernel_bpf_map_id); return 0; } default: spdlog::error("Unsupported map type: {}", (int)type); - assert(false && "Unsupported map type"); + // assert(false && "Unsupported map type"); + return -1; } return 0; } @@ -399,6 +428,10 @@ void bpf_map_handler::map_free(managed_shared_memory &memory) memory.destroy( container_name.c_str()); break; + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY: + memory.destroy( + container_name.c_str()); + break; default: assert(false && "Unsupported map type"); } From f4712a1b452421492e283e26475731cffe2d443d Mon Sep 17 00:00:00 2001 From: officeyutong Date: Thu, 2 Nov 2023 21:48:34 +0800 Subject: [PATCH 02/16] Ignore enabling of non-uprobe attachings --- daemon/user/bpftime_driver.cpp | 5 +-- daemon/user/handle_bpf_event.cpp | 57 +++++++++++++++++++++----------- daemon/user/handle_bpf_event.hpp | 2 +- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/daemon/user/bpftime_driver.cpp b/daemon/user/bpftime_driver.cpp index 568e129c..6e407529 100644 --- a/daemon/user/bpftime_driver.cpp +++ b/daemon/user/bpftime_driver.cpp @@ -297,8 +297,9 @@ int bpftime_driver::bpftime_perf_event_enable_server(int server_pid, int fd) { int fd_id = check_and_get_pid_fd(server_pid, fd); if (fd_id < 0) { - spdlog::error("fd {} for pid {} not exists", fd, server_pid); - return -1; + spdlog::warn("Unrecorded uprobe fd: {} for pid {}", fd, + server_pid); + return 0; } int res = bpftime_perf_event_enable(fd_id); if (res < 0) { diff --git a/daemon/user/handle_bpf_event.cpp b/daemon/user/handle_bpf_event.cpp index e2354249..3e9e7be3 100644 --- a/daemon/user/handle_bpf_event.cpp +++ b/daemon/user/handle_bpf_event.cpp @@ -244,7 +244,7 @@ static const char *perf_type_id_strings[PERF_TYPE_MAX_ID] = { "PERF_TYPE_HW_CACHE", "PERF_TYPE_RAW", "PERF_TYPE_BREAKPOINT", }; -int bpf_event_handler::handle_perf_event(const struct event *e) +int bpf_event_handler::handle_perf_event_open(const struct event *e) { spdlog::debug("handle_perf_event"); const char *type_id_str = "UNKNOWN TYPE"; @@ -257,24 +257,43 @@ int bpf_event_handler::handle_perf_event(const struct event *e) /* print output */ spdlog::info("PERF {:<6} {:<16} type:{:<16} ret:{}\n", e->pid, e->comm, type_id_str, e->perf_event_data.ret); - if (config.is_driving_bpftime && e->perf_event_data.ret > 0) { - if (perf_type == (unsigned int)uprobe_type) { - auto attr = &e->perf_event_data.attr; - // NO legacy bpf types - bool retprobe = attr->config & + + if (config.is_driving_bpftime) { + if (e->perf_event_data.ret >= 0) { + spdlog::debug( + "Handling perf event creating with perf type {}", + perf_type); + if (perf_type == (unsigned int)uprobe_type) { + auto attr = &e->perf_event_data.attr; + // NO legacy bpf types + bool retprobe = + attr->config & (1 << determine_uprobe_retprobe_bit()); - spdlog::debug("retprobe {}", retprobe); - size_t ref_ctr_off = attr->config >> - PERF_UPROBE_REF_CTR_OFFSET_SHIFT; - const char *name = e->perf_event_data.name_or_path; - uint64_t offset = e->perf_event_data.attr.probe_offset; - spdlog::debug("Creating uprobe name {} offset {} " - "ref_ctr_off {} attr->config={:x}", - name, offset, ref_ctr_off, attr->config); - driver.bpftime_uprobe_create_server( - e->pid, e->perf_event_data.ret, - e->perf_event_data.pid, name, offset, retprobe, - ref_ctr_off); + spdlog::debug("retprobe {}", retprobe); + size_t ref_ctr_off = + attr->config >> + PERF_UPROBE_REF_CTR_OFFSET_SHIFT; + const char *name = + e->perf_event_data.name_or_path; + uint64_t offset = + e->perf_event_data.attr.probe_offset; + spdlog::debug( + "Creating uprobe name {} offset {} " + "ref_ctr_off {} attr->config={:x}", + name, offset, ref_ctr_off, + attr->config); + driver.bpftime_uprobe_create_server( + e->pid, e->perf_event_data.ret, + e->perf_event_data.pid, name, offset, + retprobe, ref_ctr_off); + } else { + spdlog::warn("Unsupported perf event type: {}", + perf_type); + } + } else { + spdlog::debug( + "Ignore failed perf event creation, err={}", + e->perf_event_data.ret); } } return 0; @@ -346,7 +365,7 @@ int bpf_event_handler::handle_event(const struct event *e) return handle_open_events(e); break; case SYS_PERF_EVENT_OPEN: - return handle_perf_event(e); + return handle_perf_event_open(e); break; case SYS_BPF: return handle_bpf_event(e); diff --git a/daemon/user/handle_bpf_event.hpp b/daemon/user/handle_bpf_event.hpp index b3bea89d..cc67e6b5 100644 --- a/daemon/user/handle_bpf_event.hpp +++ b/daemon/user/handle_bpf_event.hpp @@ -20,7 +20,7 @@ class bpf_event_handler { int handle_close_event(const struct event *e); int handle_bpf_event(const struct event *e); int handle_open_events(const struct event *e); - int handle_perf_event(const struct event *e); + int handle_perf_event_open(const struct event *e); int handle_load_bpf_prog_event(const struct event *e); int handle_ioctl(const struct event *e); public: From 6ad3813ff0b98475939cb680ca9a3bda82ed05bf Mon Sep 17 00:00:00 2001 From: officeyutong Date: Thu, 2 Nov 2023 22:13:53 +0800 Subject: [PATCH 03/16] Re-organize map impls --- runtime/CMakeLists.txt | 18 +++++++++--------- .../{ => shared}/array_map_kernel_user.cpp | 2 +- .../{ => shared}/array_map_kernel_user.hpp | 0 .../{ => shared}/hash_map_kernel_user.cpp | 4 +--- .../{ => shared}/hash_map_kernel_user.hpp | 0 .../percpu_array_map_kernel_user.cpp | 2 +- .../percpu_array_map_kernel_user.hpp | 4 ++++ .../src/bpf_map/{ => userspace}/array_map.cpp | 2 +- .../src/bpf_map/{ => userspace}/array_map.hpp | 0 .../src/bpf_map/{ => userspace}/hash_map.cpp | 2 +- .../src/bpf_map/{ => userspace}/hash_map.hpp | 0 .../{ => userspace}/per_cpu_array_map.cpp | 2 +- .../{ => userspace}/per_cpu_array_map.hpp | 0 .../{ => userspace}/per_cpu_hash_map.cpp | 2 +- .../{ => userspace}/per_cpu_hash_map.hpp | 0 .../{ => userspace}/perf_event_array_map.cpp | 2 +- .../{ => userspace}/perf_event_array_map.hpp | 0 .../bpf_map/{ => userspace}/ringbuf_map.cpp | 2 +- .../bpf_map/{ => userspace}/ringbuf_map.hpp | 0 runtime/src/bpftime_shm_internal.hpp | 4 ++-- runtime/src/handler/epoll_handler.hpp | 2 +- runtime/src/handler/map_handler.cpp | 18 +++++++++--------- runtime/src/handler/map_handler.hpp | 4 ++-- runtime/unit-test/maps/test_per_cpu_array.cpp | 2 +- runtime/unit-test/maps/test_per_cpu_hash.cpp | 2 +- runtime/unit-test/test_bpftime_shm_json.cpp | 2 +- 26 files changed, 39 insertions(+), 37 deletions(-) rename runtime/src/bpf_map/{ => shared}/array_map_kernel_user.cpp (98%) rename runtime/src/bpf_map/{ => shared}/array_map_kernel_user.hpp (100%) rename runtime/src/bpf_map/{ => shared}/hash_map_kernel_user.cpp (96%) rename runtime/src/bpf_map/{ => shared}/hash_map_kernel_user.hpp (100%) rename runtime/src/bpf_map/{ => shared}/percpu_array_map_kernel_user.cpp (96%) rename runtime/src/bpf_map/{ => shared}/percpu_array_map_kernel_user.hpp (79%) rename runtime/src/bpf_map/{ => userspace}/array_map.cpp (97%) rename runtime/src/bpf_map/{ => userspace}/array_map.hpp (100%) rename runtime/src/bpf_map/{ => userspace}/hash_map.cpp (98%) rename runtime/src/bpf_map/{ => userspace}/hash_map.hpp (100%) rename runtime/src/bpf_map/{ => userspace}/per_cpu_array_map.cpp (98%) rename runtime/src/bpf_map/{ => userspace}/per_cpu_array_map.hpp (100%) rename runtime/src/bpf_map/{ => userspace}/per_cpu_hash_map.cpp (99%) rename runtime/src/bpf_map/{ => userspace}/per_cpu_hash_map.hpp (100%) rename runtime/src/bpf_map/{ => userspace}/perf_event_array_map.cpp (96%) rename runtime/src/bpf_map/{ => userspace}/perf_event_array_map.hpp (100%) rename runtime/src/bpf_map/{ => userspace}/ringbuf_map.cpp (99%) rename runtime/src/bpf_map/{ => userspace}/ringbuf_map.hpp (100%) diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index eeb42c28..38538430 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -66,15 +66,15 @@ set(sources src/ffi.cpp src/bpf_helper.cpp - src/bpf_map/array_map.cpp - src/bpf_map/hash_map.cpp - src/bpf_map/ringbuf_map.cpp - src/bpf_map/perf_event_array_map.cpp - src/bpf_map/per_cpu_array_map.cpp - src/bpf_map/per_cpu_hash_map.cpp - src/bpf_map/array_map_kernel_user.cpp - src/bpf_map/hash_map_kernel_user.cpp - src/bpf_map/percpu_array_map_kernel_user.cpp + src/bpf_map/userspace/array_map.cpp + src/bpf_map/userspace/hash_map.cpp + src/bpf_map/userspace/ringbuf_map.cpp + src/bpf_map/userspace/perf_event_array_map.cpp + src/bpf_map/userspace/per_cpu_array_map.cpp + src/bpf_map/userspace/per_cpu_hash_map.cpp + src/bpf_map/shared/array_map_kernel_user.cpp + src/bpf_map/shared/hash_map_kernel_user.cpp + src/bpf_map/shared/percpu_array_map_kernel_user.cpp ) # list(APPEND sources diff --git a/runtime/src/bpf_map/array_map_kernel_user.cpp b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp similarity index 98% rename from runtime/src/bpf_map/array_map_kernel_user.cpp rename to runtime/src/bpf_map/shared/array_map_kernel_user.cpp index 653d3dfd..1ebd7aab 100644 --- a/runtime/src/bpf_map/array_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/runtime/src/bpf_map/array_map_kernel_user.hpp b/runtime/src/bpf_map/shared/array_map_kernel_user.hpp similarity index 100% rename from runtime/src/bpf_map/array_map_kernel_user.hpp rename to runtime/src/bpf_map/shared/array_map_kernel_user.hpp diff --git a/runtime/src/bpf_map/hash_map_kernel_user.cpp b/runtime/src/bpf_map/shared/hash_map_kernel_user.cpp similarity index 96% rename from runtime/src/bpf_map/hash_map_kernel_user.cpp rename to runtime/src/bpf_map/shared/hash_map_kernel_user.cpp index 9ec4c4df..bab0f824 100644 --- a/runtime/src/bpf_map/hash_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/hash_map_kernel_user.cpp @@ -1,7 +1,5 @@ #include "spdlog/spdlog.h" -#include -#include -#include +#include #include #include #include diff --git a/runtime/src/bpf_map/hash_map_kernel_user.hpp b/runtime/src/bpf_map/shared/hash_map_kernel_user.hpp similarity index 100% rename from runtime/src/bpf_map/hash_map_kernel_user.hpp rename to runtime/src/bpf_map/shared/hash_map_kernel_user.hpp diff --git a/runtime/src/bpf_map/percpu_array_map_kernel_user.cpp b/runtime/src/bpf_map/shared/percpu_array_map_kernel_user.cpp similarity index 96% rename from runtime/src/bpf_map/percpu_array_map_kernel_user.cpp rename to runtime/src/bpf_map/shared/percpu_array_map_kernel_user.cpp index 0e3f04e8..edbcb15f 100644 --- a/runtime/src/bpf_map/percpu_array_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/percpu_array_map_kernel_user.cpp @@ -1,6 +1,6 @@ #include "bpf/bpf.h" #include "spdlog/spdlog.h" -#include +#include #include namespace bpftime { diff --git a/runtime/src/bpf_map/percpu_array_map_kernel_user.hpp b/runtime/src/bpf_map/shared/percpu_array_map_kernel_user.hpp similarity index 79% rename from runtime/src/bpf_map/percpu_array_map_kernel_user.hpp rename to runtime/src/bpf_map/shared/percpu_array_map_kernel_user.hpp index 96783f10..7d54a25e 100644 --- a/runtime/src/bpf_map/percpu_array_map_kernel_user.hpp +++ b/runtime/src/bpf_map/shared/percpu_array_map_kernel_user.hpp @@ -4,6 +4,10 @@ #include namespace bpftime { + +// This is a simple percpu array map implementation, which could be accessed from both userspace ebpf programs and kernel ebpf programs +// It just uses syscalls to operate the corresponding kernel map :( + class percpu_array_map_kernel_user_impl { uint32_t _value_size; uint32_t _max_entries; diff --git a/runtime/src/bpf_map/array_map.cpp b/runtime/src/bpf_map/userspace/array_map.cpp similarity index 97% rename from runtime/src/bpf_map/array_map.cpp rename to runtime/src/bpf_map/userspace/array_map.cpp index 67081645..8129567a 100644 --- a/runtime/src/bpf_map/array_map.cpp +++ b/runtime/src/bpf_map/userspace/array_map.cpp @@ -1,4 +1,4 @@ -#include +#include #include namespace bpftime { diff --git a/runtime/src/bpf_map/array_map.hpp b/runtime/src/bpf_map/userspace/array_map.hpp similarity index 100% rename from runtime/src/bpf_map/array_map.hpp rename to runtime/src/bpf_map/userspace/array_map.hpp diff --git a/runtime/src/bpf_map/hash_map.cpp b/runtime/src/bpf_map/userspace/hash_map.cpp similarity index 98% rename from runtime/src/bpf_map/hash_map.cpp rename to runtime/src/bpf_map/userspace/hash_map.cpp index c9e6b0f1..2c1bcc3f 100644 --- a/runtime/src/bpf_map/hash_map.cpp +++ b/runtime/src/bpf_map/userspace/hash_map.cpp @@ -1,5 +1,5 @@ #include "spdlog/spdlog.h" -#include +#include #include #include #include diff --git a/runtime/src/bpf_map/hash_map.hpp b/runtime/src/bpf_map/userspace/hash_map.hpp similarity index 100% rename from runtime/src/bpf_map/hash_map.hpp rename to runtime/src/bpf_map/userspace/hash_map.hpp diff --git a/runtime/src/bpf_map/per_cpu_array_map.cpp b/runtime/src/bpf_map/userspace/per_cpu_array_map.cpp similarity index 98% rename from runtime/src/bpf_map/per_cpu_array_map.cpp rename to runtime/src/bpf_map/userspace/per_cpu_array_map.cpp index 24b38cfa..82bc264f 100644 --- a/runtime/src/bpf_map/per_cpu_array_map.cpp +++ b/runtime/src/bpf_map/userspace/per_cpu_array_map.cpp @@ -1,7 +1,7 @@ #include "bpf_map/map_common_def.hpp" #include "spdlog/spdlog.h" #include -#include +#include #include #include namespace bpftime diff --git a/runtime/src/bpf_map/per_cpu_array_map.hpp b/runtime/src/bpf_map/userspace/per_cpu_array_map.hpp similarity index 100% rename from runtime/src/bpf_map/per_cpu_array_map.hpp rename to runtime/src/bpf_map/userspace/per_cpu_array_map.hpp diff --git a/runtime/src/bpf_map/per_cpu_hash_map.cpp b/runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp similarity index 99% rename from runtime/src/bpf_map/per_cpu_hash_map.cpp rename to runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp index 1589250b..db215c84 100644 --- a/runtime/src/bpf_map/per_cpu_hash_map.cpp +++ b/runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp @@ -2,7 +2,7 @@ #include "spdlog/fmt/bin_to_hex.h" #include "spdlog/spdlog.h" #include -#include +#include #include namespace bpftime { diff --git a/runtime/src/bpf_map/per_cpu_hash_map.hpp b/runtime/src/bpf_map/userspace/per_cpu_hash_map.hpp similarity index 100% rename from runtime/src/bpf_map/per_cpu_hash_map.hpp rename to runtime/src/bpf_map/userspace/per_cpu_hash_map.hpp diff --git a/runtime/src/bpf_map/perf_event_array_map.cpp b/runtime/src/bpf_map/userspace/perf_event_array_map.cpp similarity index 96% rename from runtime/src/bpf_map/perf_event_array_map.cpp rename to runtime/src/bpf_map/userspace/perf_event_array_map.cpp index 7afd34fa..3176464b 100644 --- a/runtime/src/bpf_map/perf_event_array_map.cpp +++ b/runtime/src/bpf_map/userspace/perf_event_array_map.cpp @@ -1,5 +1,5 @@ #include "spdlog/spdlog.h" -#include +#include #include #include #include diff --git a/runtime/src/bpf_map/perf_event_array_map.hpp b/runtime/src/bpf_map/userspace/perf_event_array_map.hpp similarity index 100% rename from runtime/src/bpf_map/perf_event_array_map.hpp rename to runtime/src/bpf_map/userspace/perf_event_array_map.hpp diff --git a/runtime/src/bpf_map/ringbuf_map.cpp b/runtime/src/bpf_map/userspace/ringbuf_map.cpp similarity index 99% rename from runtime/src/bpf_map/ringbuf_map.cpp rename to runtime/src/bpf_map/userspace/ringbuf_map.cpp index bbbe97d6..58f63995 100644 --- a/runtime/src/bpf_map/ringbuf_map.cpp +++ b/runtime/src/bpf_map/userspace/ringbuf_map.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include enum { diff --git a/runtime/src/bpf_map/ringbuf_map.hpp b/runtime/src/bpf_map/userspace/ringbuf_map.hpp similarity index 100% rename from runtime/src/bpf_map/ringbuf_map.hpp rename to runtime/src/bpf_map/userspace/ringbuf_map.hpp diff --git a/runtime/src/bpftime_shm_internal.hpp b/runtime/src/bpftime_shm_internal.hpp index eb5c7c6f..9e48590d 100644 --- a/runtime/src/bpftime_shm_internal.hpp +++ b/runtime/src/bpftime_shm_internal.hpp @@ -1,7 +1,7 @@ #ifndef _BPFTIME_SHM_INTERNAL #define _BPFTIME_SHM_INTERNAL -#include "bpf_map/array_map.hpp" -#include "bpf_map/ringbuf_map.hpp" +#include "bpf_map/userspace/array_map.hpp" +#include "bpf_map/userspace/ringbuf_map.hpp" #include #include #include diff --git a/runtime/src/handler/epoll_handler.hpp b/runtime/src/handler/epoll_handler.hpp index a11dbcb3..aedbb303 100644 --- a/runtime/src/handler/epoll_handler.hpp +++ b/runtime/src/handler/epoll_handler.hpp @@ -3,7 +3,7 @@ #include "handler/perf_event_handler.hpp" #include #include -#include +#include #include #include namespace bpftime diff --git a/runtime/src/handler/map_handler.cpp b/runtime/src/handler/map_handler.cpp index aeb6eaa0..c9c71bf4 100644 --- a/runtime/src/handler/map_handler.cpp +++ b/runtime/src/handler/map_handler.cpp @@ -1,14 +1,14 @@ -#include "bpf_map/per_cpu_array_map.hpp" -#include "bpf_map/per_cpu_hash_map.hpp" -#include +#include "bpf_map/userspace/per_cpu_array_map.hpp" +#include "bpf_map/userspace/per_cpu_hash_map.hpp" +#include #include "spdlog/spdlog.h" #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include using boost::interprocess::interprocess_sharable_mutex; using boost::interprocess::scoped_lock; diff --git a/runtime/src/handler/map_handler.hpp b/runtime/src/handler/map_handler.hpp index ccd6b036..b6f7f7af 100644 --- a/runtime/src/handler/map_handler.hpp +++ b/runtime/src/handler/map_handler.hpp @@ -1,7 +1,7 @@ #ifndef _MAP_HANDLER #define _MAP_HANDLER -#include "bpf_map/array_map.hpp" -#include "bpf_map/ringbuf_map.hpp" +#include "bpf_map/userspace/array_map.hpp" +#include "bpf_map/userspace/ringbuf_map.hpp" #include "bpftime_shm.hpp" #include "spdlog/spdlog.h" #include diff --git a/runtime/unit-test/maps/test_per_cpu_array.cpp b/runtime/unit-test/maps/test_per_cpu_array.cpp index 7c04e3f6..20868b82 100644 --- a/runtime/unit-test/maps/test_per_cpu_array.cpp +++ b/runtime/unit-test/maps/test_per_cpu_array.cpp @@ -1,7 +1,7 @@ #include "catch2/catch_message.hpp" #include #include -#include +#include #include #include #include diff --git a/runtime/unit-test/maps/test_per_cpu_hash.cpp b/runtime/unit-test/maps/test_per_cpu_hash.cpp index 9d91269d..e709a042 100644 --- a/runtime/unit-test/maps/test_per_cpu_hash.cpp +++ b/runtime/unit-test/maps/test_per_cpu_hash.cpp @@ -2,7 +2,7 @@ #include "../common_def.hpp" #include #include -#include +#include #include #include #include diff --git a/runtime/unit-test/test_bpftime_shm_json.cpp b/runtime/unit-test/test_bpftime_shm_json.cpp index 22280321..34bce948 100644 --- a/runtime/unit-test/test_bpftime_shm_json.cpp +++ b/runtime/unit-test/test_bpftime_shm_json.cpp @@ -1,7 +1,7 @@ #include "catch2/catch_message.hpp" #include #include -#include +#include #include #include #include From 3d775d1ff9bdf77893fff5a6ffd1bc2985ab52ee Mon Sep 17 00:00:00 2001 From: officeyutong Date: Fri, 3 Nov 2023 00:07:38 +0800 Subject: [PATCH 04/16] Update map implementation of kernel-user perf array --- runtime/CMakeLists.txt | 2 + .../shared/perf_event_array_kernel_user.cpp | 137 ++++++++++++++++++ .../shared/perf_event_array_kernel_user.hpp | 53 +++++++ 3 files changed, 192 insertions(+) create mode 100644 runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp create mode 100644 runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 38538430..11fbee0e 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -72,9 +72,11 @@ set(sources src/bpf_map/userspace/perf_event_array_map.cpp src/bpf_map/userspace/per_cpu_array_map.cpp src/bpf_map/userspace/per_cpu_hash_map.cpp + src/bpf_map/shared/array_map_kernel_user.cpp src/bpf_map/shared/hash_map_kernel_user.cpp src/bpf_map/shared/percpu_array_map_kernel_user.cpp + src/bpf_map/shared/perf_event_array_kernel_user.cpp ) # list(APPEND sources diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp new file mode 100644 index 00000000..cf86b627 --- /dev/null +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -0,0 +1,137 @@ +#include "bpf/bpf.h" +#include "bpf/libbpf_common.h" +#include "bpftool/libbpf/src/libbpf.h" +#include +#include +#include +#include +#include +#include +namespace bpftime +{ + +int perf_event_array_kernel_user_impl::output_data(const void *buf, size_t size) +{ + ensure_init_user_ringbuf(); + void *mem = nullptr; + { + std::scoped_lock guard( + *reserve_mutex); + // Reserving is *NOT* thread safe + mem = user_ring_buffer__reserve(user_rb, size); + } + if (!mem) { + spdlog::error("Failed to reserve for user ringbuf: {}", errno); + return errno; + } + memcpy(mem, buf, size); + user_ring_buffer__submit(user_rb, mem); + return 0; +} +void perf_event_array_kernel_user_impl::init_user_ringbuf() +{ + user_rb_fd = bpf_map_get_fd_by_id(user_rb_id); + if (user_rb_fd < 0) { + spdlog::error( + "Failed to get user ringbuf fd from id {}, err={}", + user_rb_id, user_rb_fd); + return; + } + LIBBPF_OPTS(user_ring_buffer_opts, opts); + + user_rb = user_ring_buffer__new(user_rb_fd, &opts); + if (!user_rb) { + spdlog::error("Failed to create user ringbuf for fd {}, err={}", + user_rb_fd, errno); + return; + } +} +perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( + boost::interprocess::managed_shared_memory &memory, uint32_t key_size, + uint32_t value_size, uint32_t max_entries, int user_rb_id) + : max_ent(max_entries), user_rb_id(user_rb_id), + reserve_mutex(boost::interprocess::make_managed_unique_ptr( + memory.construct( + boost::interprocess::anonymous_instance)(), + memory)) +{ + if (key_size != 4 || value_size != 4) { + spdlog::error( + "Key size and value size of perf_event_array must be 4"); + assert(false); + } + ensure_init_user_ringbuf(); +} +perf_event_array_kernel_user_impl::~perf_event_array_kernel_user_impl() +{ + user_ring_buffer__free(user_rb); +} + +void *perf_event_array_kernel_user_impl::elem_lookup(const void *key) +{ + ensure_init_user_ringbuf(); + int32_t k = *(int32_t *)key; + if (k < 0 || (size_t)k >= max_ent) { + errno = EINVAL; + return nullptr; + } + spdlog::info( + "Looking up key {} from perf event array kernel user, which is useless", + k); + return &dummy; +} + +long perf_event_array_kernel_user_impl::elem_update(const void *key, + const void *value, + uint64_t flags) +{ + ensure_init_user_ringbuf(); + int32_t k = *(int32_t *)key; + if (k < 0 || (size_t)k >= max_ent) { + errno = EINVAL; + return -1; + } + int32_t v = *(int32_t *)value; + spdlog::info( + "Updating key {}, value {} from perf event array kernel user, which is useless", + k, v); + return 0; +} + +long perf_event_array_kernel_user_impl::elem_delete(const void *key) +{ + ensure_init_user_ringbuf(); + int32_t k = *(int32_t *)key; + if (k < 0 || (size_t)k >= max_ent) { + errno = EINVAL; + return -1; + } + spdlog::info( + "Deleting key {} from perf event array kernel user, which is useless", + k); + return 0; +} + +int perf_event_array_kernel_user_impl::map_get_next_key(const void *key, + void *next_key) +{ + ensure_init_user_ringbuf(); + int32_t *out = (int32_t *)next_key; + if (key == nullptr) { + *out = 0; + return 0; + } + int32_t k = *(int32_t *)key; + // The last key + if ((size_t)(k + 1) == max_ent) { + errno = ENOENT; + return -1; + } + if (k < 0 || (size_t)k >= max_ent) { + errno = EINVAL; + return -1; + } + *out = k + 1; + return 0; +} +} // namespace bpftime diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp new file mode 100644 index 00000000..528fb7db --- /dev/null +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp @@ -0,0 +1,53 @@ +#include "bpftool/libbpf/src/libbpf.h" +#include +#include +#include +#include +namespace bpftime +{ + +using mutex_ptr = boost::interprocess::managed_unique_ptr< + boost::interprocess::interprocess_mutex, + boost::interprocess::managed_shared_memory>::type; + +// Here is an implementation of a perf event array that can output data from +// both userspace and kernel space It's corresponded with a normal perf event +// array in kernel, and it will be used by kernel ebpf programs But from the +// userspace, it will hold a user ringbuf (provided by libbpf). Once data was +// written, it will commit the data to the ringbuf. A kernel program will be +// attached to an intervally triggered event (e.g a timer perf event). This +// program will examine if data was available in the user ringbuf, and writes +// the data into the corresponding kernel perf event array. +class perf_event_array_kernel_user_impl { + user_ring_buffer *user_rb = nullptr; + uint32_t dummy = 0xffffffff; + void init_user_ringbuf(); + uint32_t max_ent; + int user_rb_id; + int user_rb_fd; + mutex_ptr reserve_mutex; + + public: + const static bool should_lock = false; + perf_event_array_kernel_user_impl( + boost::interprocess::managed_shared_memory &memory, + uint32_t key_size, uint32_t value_size, uint32_t max_entries, + int user_rb_id); + virtual ~perf_event_array_kernel_user_impl(); + + void *elem_lookup(const void *key); + + long elem_update(const void *key, const void *value, uint64_t flags); + + long elem_delete(const void *key); + + int map_get_next_key(const void *key, void *next_key); + + void ensure_init_user_ringbuf() + { + if (!user_rb) + init_user_ringbuf(); + } + int output_data(const void *buf, size_t size); +}; +} // namespace bpftime From 3dd32112f49cf5e4d6a04a9f4ba74c15b6a758d4 Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 4 Nov 2023 01:10:47 +0800 Subject: [PATCH 05/16] Update --- daemon/user/bpf_tracer.cpp | 3 + daemon/user/handle_bpf_event.cpp | 3 + runtime/include/bpftime_shm.hpp | 12 +- .../shared/perf_event_array_kernel_user.cpp | 294 +++++++++++++++--- .../shared/perf_event_array_kernel_user.hpp | 54 +++- runtime/src/handler/map_handler.cpp | 35 +++ 6 files changed, 339 insertions(+), 62 deletions(-) diff --git a/daemon/user/bpf_tracer.cpp b/daemon/user/bpf_tracer.cpp index 54bca1ac..090b8dd4 100644 --- a/daemon/user/bpf_tracer.cpp +++ b/daemon/user/bpf_tracer.cpp @@ -31,7 +31,10 @@ static int libbpf_print_fn(enum libbpf_print_level level, const char *format, { if (level == LIBBPF_DEBUG && !verbose) return 0; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" return vfprintf(stderr, format, args); +#pragma GCC diagnostic pop } static void sig_int(int signo) diff --git a/daemon/user/handle_bpf_event.cpp b/daemon/user/handle_bpf_event.cpp index 3e9e7be3..ff9e1ea8 100644 --- a/daemon/user/handle_bpf_event.cpp +++ b/daemon/user/handle_bpf_event.cpp @@ -31,7 +31,10 @@ static int parse_uint_from_file(const char *file, const char *fmt) fprintf(stderr, "failed to open '%s\n", file); exit(1); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" err = fscanf(f, fmt, &ret); +#pragma GCC diagnostic pop if (err != 1) { err = err == EOF ? -EIO : -errno; fprintf(stderr, "failed to parse '%s'\n", file); diff --git a/runtime/include/bpftime_shm.hpp b/runtime/include/bpftime_shm.hpp index 3ca4f933..76594296 100644 --- a/runtime/include/bpftime_shm.hpp +++ b/runtime/include/bpftime_shm.hpp @@ -92,11 +92,15 @@ enum class bpf_map_type { BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, - BPF_MAP_TYPE_KERNEL_USER_HASH = KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_HASH, - BPF_MAP_TYPE_KERNEL_USER_ARRAY = KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_ARRAY, - BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY = KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_PERCPU_ARRAY, + BPF_MAP_TYPE_KERNEL_USER_HASH = + KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_HASH, + BPF_MAP_TYPE_KERNEL_USER_ARRAY = + KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_ARRAY, + BPF_MAP_TYPE_KERNEL_USER_PERCPU_ARRAY = + KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_PERCPU_ARRAY, + BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY = + KERNEL_USER_MAP_OFFSET + BPF_MAP_TYPE_PERF_EVENT_ARRAY, - }; enum class shm_open_type { diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index cf86b627..cb4c14f3 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -1,75 +1,75 @@ #include "bpf/bpf.h" #include "bpf/libbpf_common.h" #include "bpftool/libbpf/src/libbpf.h" +#include "linux/bpf_common.h" #include #include +#include #include #include -#include +#include +#include +#include +#include +#include #include +#include +#include +#include +// kernel perf event array map id -> user_ring_buffer* for the current process +using user_ringbuf_map = + std::map >; +user_ringbuf_map *user_rb_map = nullptr; +static inline void ensure_user_rb_map_initialized() +{ + if (!user_rb_map) + user_rb_map = new user_ringbuf_map; +} namespace bpftime { -int perf_event_array_kernel_user_impl::output_data(const void *buf, size_t size) -{ - ensure_init_user_ringbuf(); - void *mem = nullptr; - { - std::scoped_lock guard( - *reserve_mutex); - // Reserving is *NOT* thread safe - mem = user_ring_buffer__reserve(user_rb, size); - } +int perf_event_array_kernel_user_impl::output_data_into_kernel(const void *buf, + size_t size) +{ + auto user_rb = ensure_current_map_user_ringbuf(); + void *mem = user_rb->reserve(size + 8); if (!mem) { spdlog::error("Failed to reserve for user ringbuf: {}", errno); return errno; } - memcpy(mem, buf, size); - user_ring_buffer__submit(user_rb, mem); + *(uint64_t *)(mem) = (uint64_t)size; + memcpy((char *)mem + 8, buf, size); + user_rb->submit(mem); + spdlog::trace("Commited {} bytes of data into kernel", size); return 0; } -void perf_event_array_kernel_user_impl::init_user_ringbuf() -{ - user_rb_fd = bpf_map_get_fd_by_id(user_rb_id); - if (user_rb_fd < 0) { - spdlog::error( - "Failed to get user ringbuf fd from id {}, err={}", - user_rb_id, user_rb_fd); - return; - } - LIBBPF_OPTS(user_ring_buffer_opts, opts); - user_rb = user_ring_buffer__new(user_rb_fd, &opts); - if (!user_rb) { - spdlog::error("Failed to create user ringbuf for fd {}, err={}", - user_rb_fd, errno); - return; - } -} perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( boost::interprocess::managed_shared_memory &memory, uint32_t key_size, - uint32_t value_size, uint32_t max_entries, int user_rb_id) - : max_ent(max_entries), user_rb_id(user_rb_id), - reserve_mutex(boost::interprocess::make_managed_unique_ptr( - memory.construct( - boost::interprocess::anonymous_instance)(), - memory)) + uint32_t value_size, uint32_t max_entries, int kernel_perf_id) + : max_ent(max_entries), kernel_perf_id(kernel_perf_id) { if (key_size != 4 || value_size != 4) { spdlog::error( "Key size and value size of perf_event_array must be 4"); assert(false); } - ensure_init_user_ringbuf(); + kernel_perf_fd = bpf_map_get_fd_by_id(kernel_perf_id); + if (kernel_perf_fd < 0) { + spdlog::error("Unable to get fd of kernel perf id {}", + kernel_perf_id); + assert(false); + } + spdlog::debug( + "Initialized perf_event_array_kernel_user_impl, kernel perf fd {}, kernel perf id {}", + kernel_perf_fd, kernel_perf_id); } perf_event_array_kernel_user_impl::~perf_event_array_kernel_user_impl() { - user_ring_buffer__free(user_rb); } void *perf_event_array_kernel_user_impl::elem_lookup(const void *key) { - ensure_init_user_ringbuf(); int32_t k = *(int32_t *)key; if (k < 0 || (size_t)k >= max_ent) { errno = EINVAL; @@ -85,7 +85,6 @@ long perf_event_array_kernel_user_impl::elem_update(const void *key, const void *value, uint64_t flags) { - ensure_init_user_ringbuf(); int32_t k = *(int32_t *)key; if (k < 0 || (size_t)k >= max_ent) { errno = EINVAL; @@ -100,7 +99,6 @@ long perf_event_array_kernel_user_impl::elem_update(const void *key, long perf_event_array_kernel_user_impl::elem_delete(const void *key) { - ensure_init_user_ringbuf(); int32_t k = *(int32_t *)key; if (k < 0 || (size_t)k >= max_ent) { errno = EINVAL; @@ -115,7 +113,6 @@ long perf_event_array_kernel_user_impl::elem_delete(const void *key) int perf_event_array_kernel_user_impl::map_get_next_key(const void *key, void *next_key) { - ensure_init_user_ringbuf(); int32_t *out = (int32_t *)next_key; if (key == nullptr) { *out = 0; @@ -134,4 +131,217 @@ int perf_event_array_kernel_user_impl::map_get_next_key(const void *key, *out = k + 1; return 0; } +user_ringbuffer_wrapper * +perf_event_array_kernel_user_impl::ensure_current_map_user_ringbuf() +{ + ensure_user_rb_map_initialized(); + user_ringbuffer_wrapper *result; + if (auto itr = user_rb_map->find(kernel_perf_id); + itr == user_rb_map->end()) { + result = itr->second.get(); + } else { + auto ptr = std::make_unique( + kernel_perf_id); + auto raw_ptr = ptr.get(); + user_rb_map->emplace(kernel_perf_id, std::move(ptr)); + result = raw_ptr; + + int pfd = create_intervally_triggered_perf_event(10); + auto prog = create_transporting_kernel_ebpf_program( + result->user_rb_id, kernel_perf_id); + LIBBPF_OPTS(bpf_prog_load_opts, opts); + char log_buffer[2048]; + opts.log_buf = log_buffer; + opts.log_size = sizeof(log_buffer); + opts.log_level = 1; + int bpf_fd = + bpf_prog_load(BPF_PROG_TYPE_PERF_EVENT, "transporter", + "GPL", (bpf_insn *)prog.data(), + prog.size(), &opts); + if (bpf_fd < 0) { + spdlog::error( + "Failed to load bpf prog: err={}, message={}", + errno, log_buffer); + } + assert(bpf_fd >= 0); + int err; + err = ioctl(pfd, PERF_EVENT_IOC_SET_BPF, bpf_fd); + if (err < 0) { + spdlog::error( + "Failed to run PERF_EVENT_IOC_SET_BPF: {}", + err); + assert(false); + } + err = ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0); + if (err < 0) { + spdlog::error("Failed to run PERF_EVENT_IOC_ENABLE: {}", + err); + assert(false); + } + } + return result; +} +user_ringbuffer_wrapper::user_ringbuffer_wrapper(int kernel_perf_id) +{ + pthread_spin_init(&reserve_lock, PTHREAD_PROCESS_PRIVATE); + LIBBPF_OPTS(bpf_map_create_opts, user_rb_opts); + std::string name = "ku_perf_id_" + std::to_string(kernel_perf_id); + user_rb_fd = bpf_map_create(BPF_MAP_TYPE_USER_RINGBUF, name.c_str(), 0, + 0, 1024 * 1000, &user_rb_opts); + if (user_rb_fd < 0) { + spdlog::error( + "Failed to create user ringbuffer for shared perf event array id {}", + kernel_perf_id); + return; + } + bpf_map_info map_info; + uint32_t map_info_size = sizeof(map_info); + if (int err = bpf_obj_get_info_by_fd(user_rb_fd, &map_info, + &map_info_size); + err < 0) { + spdlog::error("Failed to get map info for user rb fd {}", + user_rb_fd); + return; + } + user_rb_id = map_info.id; + rb = user_ring_buffer__new(user_rb_fd, nullptr); + assert(rb && + "Failed to initialize user_rb_fd! This SHOULD NOT Happen."); +} + +user_ringbuffer_wrapper::~user_ringbuffer_wrapper() +{ + pthread_spin_destroy(&reserve_lock); + user_ring_buffer__free(rb); +} +void *user_ringbuffer_wrapper::reserve(uint32_t size) +{ + return user_ring_buffer__reserve(rb, (uint32_t)size); +} +void user_ringbuffer_wrapper::submit(void *mem) +{ + user_ring_buffer__submit(rb, mem); +} + +// The original program was like: +/* +#define NULL 0 +static long (*bpf_dynptr_read)(void *dst, int len, const struct bpf_dynptr *src, +int offset, long flags) = (void *) 201; static long +(*bpf_perf_event_output)(void *ctx, void *map, long flags, void *data, long +size) = (void *) 25; static long (*bpf_user_ringbuf_drain)(void *map, void +*callback_fn, void *ctx, long flags) = (void *) 209; + +static int cb(void *dynptr, void *ctx); + +int func(void* ctx) +{ + bpf_user_ringbuf_drain((void*)0x234, &cb, ctx, 0); + return 0; +} +static int cb(void *dynptr, void *ctx) +{ + long val; + char buf[496]; + bpf_dynptr_read(&val, 8, dynptr, 0, 0); + if(val>400) return 1; + bpf_dynptr_read(&buf, val, dynptr, 8, 0); + bpf_perf_event_output(ctx, (void*)0x123, 0, buf, val); + return 1; +} +*/ + +// Compiled using gcc.godbolt.org + +std::vector +create_transporting_kernel_ebpf_program(int user_ringbuf_id, + int perf_event_array_id) +{ + static_assert( + sizeof(bpf_insn) == sizeof(uint64_t), + "bpf_insn is expected to be in the same size of uint64_t"); + bpf_insn insns[] = { + // r3 = r1 + BPF_MOV64_REG(3, 1), + // r1 = map_by_fd(user_ringbuf_fd) + BPF_LD_IMM64_RAW_FULL(1, 5, 0, 0, user_ringbuf_id, 0), + // r2 = callback fn + BPF_MOV64_IMM(2, 64), + // r4 = 0 + BPF_MOV64_IMM(4, 0), + // call bpf_user_ringbuf_drain(void *map, void *callback_fn, + // void *ctx, long flags) = 0x209 + BPF_EMIT_CALL(0x209), + // r0 = 0 + BPF_MOV64_IMM(0, 0), BPF_EXIT_INSN(), + // static int cb(void *dynptr, void *ctx) + // r6 = r2 + BPF_MOV64_REG(6, 2), + // r7 = r1 + BPF_MOV64_REG(7, 1), + // r1 = r10 + BPF_MOV64_REG(1, 10), + // r1 += -0x8 + BPF_ALU64_IMM(BPF_ADD, 1, -8), + // r2=8, + BPF_MOV64_IMM(2, 8), + // r3 = r7 + BPF_MOV64_IMM(3, 7), + // r4=0 + BPF_MOV64_IMM(4, 0), + // r5 = 0 + BPF_MOV64_IMM(5, 0), + // call 0xc9 bpf_dynptr_read(void *dst, int len, const struct + // bpf_dynptr *src, int offset, long flags) = (void *) 201 + BPF_EMIT_CALL(0xc9), + // r2 = *(u64 *)(r10 - 0x8) + BPF_LDX_MEM(BPF_DW, 2, 10, -8), + // if r2 s> 0x1f0 goto +0xd + BPF_RAW_INSN(0x65, 0x2, 0x0, 0x0d, 0x1f0), + // r8 = r10 + BPF_MOV64_REG(8, 10), + // r8 += -0x1f8 + BPF_ALU64_IMM(BPF_ADD, 8, -0x1f8), + // r1 = r8 + BPF_MOV64_REG(1, 8), + // r3 = r7 + BPF_MOV64_REG(3, 7), + // r4 = 8 + BPF_MOV64_IMM(4, 8), + // r5 = 0 + BPF_MOV64_IMM(5, 0), + // call 0xc9 bpf_dynptr_read(void *dst, int len, const struct + // bpf_dynptr *src, int offset, long flags) = (void *) 201 + BPF_EMIT_CALL(0xc9), + // r5 = *(u64 *)(r10 - 0x8) + BPF_LDX_MEM(BPF_DW, 5, 10, -8), + // r1 = r6 + BPF_MOV64_REG(1, 6), + // r2 = map_by_fd(perf_event_array_fd) + BPF_LD_IMM64_RAW_FULL(2, 5, 0, 0, perf_event_array_id, 0), + // r3 = 0 + BPF_MOV64_IMM(3, 0), + // r4 = r8 + BPF_MOV64_REG(4, 8), + // call 0x19 bpf_perf_event_output(void *ctx, void *map, long + // flags, void *data, long size) = 25 + BPF_EMIT_CALL(25), + // r0 = 1 + BPF_MOV64_IMM(0, 1), BPF_EXIT_INSN() + }; + return std::vector((uint64_t *)&insns, + (uint64_t *)&insns + std::size(insns)); +} + +int create_intervally_triggered_perf_event(int freq) +{ + perf_event_attr pe_attr = { + .type = PERF_TYPE_SOFTWARE, + .size = sizeof(struct perf_event_attr), + .config = PERF_COUNT_SW_CPU_CLOCK, + .sample_freq = (__u64)freq, + .freq = 1, + }; + return syscall(__NR_perf_event_open, &pe_attr, 0, -1, -1, 0); +} } // namespace bpftime diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp index 528fb7db..9e3fde9d 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp @@ -1,14 +1,23 @@ #include "bpftool/libbpf/src/libbpf.h" #include #include -#include -#include +#include +#include +#include namespace bpftime { -using mutex_ptr = boost::interprocess::managed_unique_ptr< - boost::interprocess::interprocess_mutex, - boost::interprocess::managed_shared_memory>::type; +// Wraps a user_ringbuffer and it's spinlock that locks the reservation +struct user_ringbuffer_wrapper { + pthread_spinlock_t reserve_lock; + user_ring_buffer *rb; + int user_rb_id; + int user_rb_fd; + user_ringbuffer_wrapper(int kernel_perf_id); + ~user_ringbuffer_wrapper(); + void *reserve(uint32_t size); + void submit(void *mem); +}; // Here is an implementation of a perf event array that can output data from // both userspace and kernel space It's corresponded with a normal perf event @@ -18,21 +27,20 @@ using mutex_ptr = boost::interprocess::managed_unique_ptr< // attached to an intervally triggered event (e.g a timer perf event). This // program will examine if data was available in the user ringbuf, and writes // the data into the corresponding kernel perf event array. + +// Data definition of things from userspace to kernel through user rb +// [8 bytes of data length][data] class perf_event_array_kernel_user_impl { - user_ring_buffer *user_rb = nullptr; uint32_t dummy = 0xffffffff; - void init_user_ringbuf(); uint32_t max_ent; - int user_rb_id; - int user_rb_fd; - mutex_ptr reserve_mutex; - + int kernel_perf_id; + int kernel_perf_fd = -1; public: const static bool should_lock = false; perf_event_array_kernel_user_impl( boost::interprocess::managed_shared_memory &memory, uint32_t key_size, uint32_t value_size, uint32_t max_entries, - int user_rb_id); + int kernel_perf_id); virtual ~perf_event_array_kernel_user_impl(); void *elem_lookup(const void *key); @@ -43,11 +51,25 @@ class perf_event_array_kernel_user_impl { int map_get_next_key(const void *key, void *next_key); - void ensure_init_user_ringbuf() + user_ringbuffer_wrapper *ensure_current_map_user_ringbuf(); + int output_data_into_kernel(const void *buf, size_t size); + int get_kernel_perf_fd() + { + return kernel_perf_fd; + } + int get_user_ringbuf_fd() { - if (!user_rb) - init_user_ringbuf(); + return ensure_current_map_user_ringbuf()->user_rb_fd; } - int output_data(const void *buf, size_t size); }; + +// Create a bpf program that will check if the user_ringbuf has data and copy +// the data into the perf event +std::vector +create_transporting_kernel_ebpf_program(int user_ringbuf_id, + int perf_event_array_id); + +// Create an intervally triggered perf event +int create_intervally_triggered_perf_event(int freq); + } // namespace bpftime diff --git a/runtime/src/handler/map_handler.cpp b/runtime/src/handler/map_handler.cpp index c9c71bf4..d0dab49a 100644 --- a/runtime/src/handler/map_handler.cpp +++ b/runtime/src/handler/map_handler.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include using boost::interprocess::interprocess_sharable_mutex; using boost::interprocess::scoped_lock; @@ -110,6 +112,11 @@ const void *bpf_map_handler::map_lookup_elem(const void *key, map_impl_ptr.get()); return do_lookup(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_lookup(impl); + } default: assert(false && "Unsupported map type"); } @@ -183,6 +190,11 @@ long bpf_map_handler::map_update_elem(const void *key, const void *value, map_impl_ptr.get()); return do_update(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_update(impl); + } default: assert(false && "Unsupported map type"); } @@ -244,6 +256,11 @@ int bpf_map_handler::bpf_map_get_next_key(const void *key, void *next_key, map_impl_ptr.get()); return do_get_next_key(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_get_next_key(impl); + } default: assert(false && "Unsupported map type"); } @@ -317,6 +334,11 @@ long bpf_map_handler::map_delete_elem(const void *key, map_impl_ptr.get()); return do_delete(impl); } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_delete(impl); + } default: assert(false && "Unsupported map type"); } @@ -389,6 +411,15 @@ int bpf_map_handler::map_init(managed_shared_memory &memory) attr.kernel_bpf_map_id); return 0; } + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY: { + map_impl_ptr = + memory.construct( + container_name.c_str())( + memory, 4, 4, sysconf(_SC_NPROCESSORS_ONLN), + attr.kernel_bpf_map_id); + return 0; + } + default: spdlog::error("Unsupported map type: {}", (int)type); // assert(false && "Unsupported map type"); @@ -432,6 +463,10 @@ void bpf_map_handler::map_free(managed_shared_memory &memory) memory.destroy( container_name.c_str()); break; + case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY: + memory.destroy( + container_name.c_str()); + break; default: assert(false && "Unsupported map type"); } From be3a27643d499b8875856758667345510dd5b970 Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 4 Nov 2023 20:53:18 +0800 Subject: [PATCH 06/16] Make malloc work --- .gitignore | 1 + daemon/kernel/bpf_tracer.bpf.c | 62 ++++++++++++++----- daemon/user/bpf_tracer.cpp | 6 +- daemon/user/bpftime_driver.cpp | 23 +++++-- daemon/user/bpftime_driver.hpp | 5 +- daemon/user/daemon_config.hpp | 2 +- daemon/user/handle_bpf_event.cpp | 18 +++++- runtime/src/attach/bpf_attach_ctx.cpp | 16 +++++ .../bpf_map/shared/hash_map_kernel_user.cpp | 1 + runtime/src/bpftime_shm_internal.cpp | 8 +-- runtime/src/handler/prog_handler.hpp | 3 + 11 files changed, 114 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index a757ac01..948572bc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ test.txt test.gdb index.html* dump* +test* ### VisualStudioCode ### .vscode/settings.json .vscode/runtime.json diff --git a/daemon/kernel/bpf_tracer.bpf.c b/daemon/kernel/bpf_tracer.bpf.c index f7f9bed8..4065b849 100644 --- a/daemon/kernel/bpf_tracer.bpf.c +++ b/daemon/kernel/bpf_tracer.bpf.c @@ -185,7 +185,7 @@ static int process_bpf_prog_load_events(union bpf_attr *attr) // check whether write is success bpf_probe_read_user(&new_attr, sizeof(new_attr), attr); if (new_attr.insn_cnt != 2) { - bpf_printk("write failed\n"); + bpf_printk("bpftime: write failed\n"); } } return 0; @@ -210,6 +210,7 @@ SEC("tracepoint/syscalls/sys_enter_bpf") int tracepoint__syscalls__sys_enter_bpf(struct trace_event_raw_sys_enter *ctx) { if (!filter_target()) { + bpf_printk("bpftime: Skipping sys enter bpf"); return 0; } @@ -226,9 +227,8 @@ int tracepoint__syscalls__sys_enter_bpf(struct trace_event_raw_sys_enter *ctx) process_bpf_syscall_enter(ctx); return 0; } - -inline struct event *get_ringbuf_sys_exit_bpf_event(struct bpf_args_t *ap, - int ret) +static __always_inline struct event * +get_ringbuf_sys_exit_bpf_event(struct bpf_args_t *ap, int ret) { struct event *event = fill_basic_event_info(); if (!event) { @@ -257,47 +257,68 @@ static int process_bpf_syscall_exit(struct bpf_args_t *ap, int ret) &bpf_progs_new_fd_args_map, &pid_tgid); if (!id_ptr) return 0; /* missed entry */ - bpf_map_delete_elem(&bpf_progs_new_fd_args_map, - &pid_tgid); + // bpf_map_delete_elem(&bpf_progs_new_fd_args_map, + // &pid_tgid); struct bpf_fd_data data = { .type = BPF_FD_TYPE_PROG, .kernel_id = *id_ptr }; set_bpf_fd_data(ret, &data); - bpf_printk("bpf_prog_load exit id: %d fd: %d\n", - *id_ptr, ret); + bpf_printk( + "bpftime: bpf_prog_load exit id: %d fd: %d\n", + *id_ptr, ret); break; } - + case BPF_MAP_CREATE: { u64 pid_tgid = bpf_get_current_pid_tgid(); u32 *id_ptr = (u32 *)bpf_map_lookup_elem( &bpf_map_new_fd_args_map, &pid_tgid); if (!id_ptr) return 0; /* missed entry */ - bpf_map_delete_elem(&bpf_map_new_fd_args_map, - &pid_tgid); + u32 id = *id_ptr; + // bpf_map_delete_elem(&bpf_map_new_fd_args_map, + // &pid_tgid); struct bpf_fd_data data = { .type = BPF_FD_TYPE_MAP, - .kernel_id = *id_ptr }; + .kernel_id = id }; // record map fd and id set_bpf_fd_data(ret, &data); - bpf_printk("bpf_map_create exit id: %d fd: %d\n", - *id_ptr, ret); + bpf_printk( + "bpftime: bpf_map_create exit id: %d fd: %d\n", + id, ret); if (submit_bpf_events) { // submit event to user for record struct event *event = get_ringbuf_sys_exit_bpf_event(ap, ret); - event->bpf_data.map_id = *id_ptr; + // Make verifier happy + if (!event) + return 0; + event->bpf_data.map_id = (int)id; bpf_ringbuf_submit(event, 0); } break; } case BPF_LINK_CREATE: { + u64 pid_tgid = bpf_get_current_pid_tgid(); + u32 *id_ptr = (u32 *)bpf_map_lookup_elem( + &bpf_progs_new_fd_args_map, &pid_tgid); + if (!id_ptr) { + bpf_printk( + "bpftime: Unable to get prog id when creating link"); + return 0; /* missed entry */ + } + u32 id = *id_ptr; + // bpf_map_delete_elem(&bpf_map_new_fd_args_map, + // &pid_tgid); struct bpf_fd_data data = { .type = BPF_FD_TYPE_OTHERS, - .kernel_id = 0 }; + .kernel_id = id }; set_bpf_fd_data(ret, &data); + bpf_printk("bpftime: bpf_link_create"); if (submit_bpf_events) { + bpf_printk("bpftime: Submitting link creation"); struct event *event = get_ringbuf_sys_exit_bpf_event(ap, ret); + if (!event) + return 0; bpf_ringbuf_submit(event, 0); } break; @@ -309,6 +330,8 @@ static int process_bpf_syscall_exit(struct bpf_args_t *ap, int ret) if (submit_bpf_events) { struct event *event = get_ringbuf_sys_exit_bpf_event(ap, ret); + if (!event) + return 0; bpf_ringbuf_submit(event, 0); } break; @@ -369,6 +392,11 @@ process_perf_event_open_enter(struct trace_event_raw_sys_enter *ctx) bpf_probe_write_user((void *)new_attr.uprobe_path, &new_uprobe_path, (size_t)size); bpf_probe_write_user(attr, &new_attr, sizeof(new_attr)); + + char buf[64]; + bpf_probe_read_user_str(buf, sizeof(buf), + (void *)new_attr.uprobe_path); + bpf_printk("Writting new uprobe path: %s", buf); } return 0; } @@ -408,7 +436,7 @@ int tracepoint__syscalls__sys_enter_perf_event_open( args.cpu = (int)ctx->args[2]; bpf_probe_read_user_str(args.name_or_path, PATH_LENTH, (const void *)args.attr.uprobe_path); - + bpf_printk("Received path: %s", args.name_or_path); u64 pid_tgid = bpf_get_current_pid_tgid(); bpf_map_update_elem(&perf_event_open_param_start, &pid_tgid, &args, 0); diff --git a/daemon/user/bpf_tracer.cpp b/daemon/user/bpf_tracer.cpp index 090b8dd4..7197f269 100644 --- a/daemon/user/bpf_tracer.cpp +++ b/daemon/user/bpf_tracer.cpp @@ -78,7 +78,11 @@ int bpftime::start_daemon(struct daemon_config env) /* initialize global data (filtering options) */ obj->rodata->target_pid = env.pid; obj->rodata->enable_replace_prog = env.enable_replace_prog; - strncpy(obj->rodata->new_uprobe_path, env.new_uprobe_path, PATH_LENTH); +// strncpy(obj->rodata->new_uprobe_path, env.new_uprobe_path, +// PATH_LENTH); +#warning FIXME: currently using `/a` as the replacing executable path to uprobe perf event in the kernel, since long strings (such as bpftime_daemon it self) may break userspace memory. Find a better way to solve this in the future + strncpy(obj->rodata->new_uprobe_path, "/a", PATH_LENTH); + obj->rodata->enable_replace_uprobe = env.enable_replace_uprobe; obj->rodata->uprobe_perf_type = determine_uprobe_perf_type(); obj->rodata->kprobe_perf_type = determine_kprobe_perf_type(); diff --git a/daemon/user/bpftime_driver.cpp b/daemon/user/bpftime_driver.cpp index 6e407529..7b06fbbb 100644 --- a/daemon/user/bpftime_driver.cpp +++ b/daemon/user/bpftime_driver.cpp @@ -154,7 +154,8 @@ int bpftime_driver::bpftime_progs_create_server(int kernel_id, int server_pid) { int fd = bpf_prog_get_fd_by_id(kernel_id); if (fd < 0) { - spdlog::error("Failed to get prog fd for id {}", kernel_id); + spdlog::error("Failed to get prog fd by prog id {}, err={}", + kernel_id, errno); return -1; } spdlog::debug("get prog fd {} for id {}", fd, kernel_id); @@ -237,7 +238,20 @@ int bpftime_driver::bpftime_maps_create_server(int kernel_id) spdlog::info("create map in kernel id {}", kernel_id); return kernel_id; } - +int bpftime_driver::bpftime_attach_perf_to_bpf_fd_server(int server_pid, + int perf_fd, + int bpf_prog_fd) +{ + int prog_id = + get_bpf_map_id_from_pid_fd(object, server_pid, bpf_prog_fd); + if (prog_id < 0) { + spdlog::error( + "Failed to lookup bpf prog id from bpf prog fd {}", + bpf_prog_fd); + return -1; + } + return bpftime_attach_perf_to_bpf_server(server_pid, perf_fd, prog_id); +} int bpftime_driver::bpftime_attach_perf_to_bpf_server(int server_pid, int perf_fd, int kernel_bpf_id) @@ -254,8 +268,9 @@ int bpftime_driver::bpftime_attach_perf_to_bpf_server(int server_pid, int res = bpftime_progs_create_server(kernel_bpf_id, server_pid); if (res < 0) { - spdlog::error("Failed to create bpf for id {}", - kernel_bpf_id); + spdlog::error( + "Failed to create bpf program (userspace side) for kernel program id {}", + kernel_bpf_id); return -1; } } diff --git a/daemon/user/bpftime_driver.hpp b/daemon/user/bpftime_driver.hpp index 587b80c7..7bf528ab 100644 --- a/daemon/user/bpftime_driver.hpp +++ b/daemon/user/bpftime_driver.hpp @@ -34,7 +34,8 @@ class bpftime_driver { daemon_config config; struct bpf_tracer_bpf *object = NULL; - int check_and_create_prog_related_maps(int fd, const bpf_prog_info* info); + int check_and_create_prog_related_maps(int fd, + const bpf_prog_info *info); public: // create a bpf prog in the global shared memory @@ -51,6 +52,8 @@ class bpftime_driver { int bpftime_attach_perf_to_bpf_server(int server_pid, int perf_fd, int kernel_bpf_id); + int bpftime_attach_perf_to_bpf_fd_server(int server_pid, int perf_fd, + int bpf_prog_fd); // create uprobe in the global shared memory // diff --git a/daemon/user/daemon_config.hpp b/daemon/user/daemon_config.hpp index 5ff1cd50..d72dd2b4 100644 --- a/daemon/user/daemon_config.hpp +++ b/daemon/user/daemon_config.hpp @@ -29,7 +29,7 @@ struct daemon_config { // should bpftime be involve bool is_driving_bpftime = true; - bool submit_bpf_events = false; + bool submit_bpf_events = true; }; #endif // BPFTIME_DAEMON_CONFIG_HPP diff --git a/daemon/user/handle_bpf_event.cpp b/daemon/user/handle_bpf_event.cpp index ff9e1ea8..e2817782 100644 --- a/daemon/user/handle_bpf_event.cpp +++ b/daemon/user/handle_bpf_event.cpp @@ -215,12 +215,24 @@ int bpf_event_handler::handle_bpf_event(const struct event *e) (enum bpf_map_type)e->bpf_data.attr.map_type), e->bpf_data.attr.map_name, e->bpf_data.map_id); break; - case BPF_LINK_CREATE: + case BPF_LINK_CREATE: { + int prog_fd = e->bpf_data.attr.link_create.prog_fd; + int perf = e->bpf_data.attr.link_create.target_fd; /* code */ spdlog::info(" BPF_LINK_CREATE prog_fd:{} target_fd:{}", - e->bpf_data.attr.link_create.prog_fd, - e->bpf_data.attr.link_create.target_fd); + prog_fd, perf); + if (config.is_driving_bpftime) { + if (int err = + driver.bpftime_attach_perf_to_bpf_fd_server( + e->pid, perf, prog_fd); + err < 0) { + spdlog::warn( + "Unable to attach perf {} to bpf prog {}, err={}", + perf, prog_fd, err); + } + } break; + } case BPF_PROG_LOAD: /* code */ spdlog::info( diff --git a/runtime/src/attach/bpf_attach_ctx.cpp b/runtime/src/attach/bpf_attach_ctx.cpp index e7fe4448..b5aa5bed 100644 --- a/runtime/src/attach/bpf_attach_ctx.cpp +++ b/runtime/src/attach/bpf_attach_ctx.cpp @@ -248,6 +248,9 @@ int bpf_attach_ctx::init_attach_ctx_from_handlers( // attach base on events switch (event_handler.type) { case bpf_event_type::BPF_TYPE_FILTER: { + spdlog::debug( + "Creating filter for perf event fd {}", + i); auto progs = handler_prog_fds[i]; if (progs.size() > 1) { spdlog::error( @@ -273,6 +276,9 @@ int bpf_attach_ctx::init_attach_ctx_from_handlers( break; } case bpf_event_type::BPF_TYPE_REPLACE: { + spdlog::debug( + "Creating replace for perf event fd {}", + i); auto progs = handler_prog_fds[i]; if (progs.size() > 1) { spdlog::error( @@ -307,8 +313,12 @@ int bpf_attach_ctx::init_attach_ctx_from_handlers( progs.size(), (uintptr_t)func_addr); err = attach_manager->attach_uprobe_at( func_addr, [=](const pt_regs ®s) { + spdlog::trace( + "Uprobe triggered"); uint64_t ret; for (auto &[k, prog] : progs) { + spdlog::trace( + "Calling ebpf programs in uprobe callback"); prog->bpftime_prog_exec( (void *)®s, sizeof(regs), @@ -346,6 +356,9 @@ int bpf_attach_ctx::init_attach_ctx_from_handlers( break; } case bpf_event_type::PERF_TYPE_TRACEPOINT: { + spdlog::debug( + "Creating tracepoint for perf event fd {}", + i); err = create_tracepoint( event_handler.tracepoint_id, i, manager); @@ -361,8 +374,11 @@ int bpf_attach_ctx::init_attach_ctx_from_handlers( spdlog::debug( "Attaching software perf event, nothing need to do"); err = i; + break; } default: + spdlog::warn("Unexpected bpf_event_type: {}", + (int)event_handler.type); break; } spdlog::debug("Create attach event {} {} {} for {}", i, diff --git a/runtime/src/bpf_map/shared/hash_map_kernel_user.cpp b/runtime/src/bpf_map/shared/hash_map_kernel_user.cpp index bab0f824..23e0668b 100644 --- a/runtime/src/bpf_map/shared/hash_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/hash_map_kernel_user.cpp @@ -59,6 +59,7 @@ long hash_map_kernel_user_impl::elem_update(const void *key, const void *value, if (map_fd < 0) { init_map_fd(); } + spdlog::debug("Update shared hash map"); // Allocate as a local variable to make // it thread safe, since we use sharable lock return bpf_map_update_elem(map_fd, key, value, flags); diff --git a/runtime/src/bpftime_shm_internal.cpp b/runtime/src/bpftime_shm_internal.cpp index e8c9e696..7e21188e 100644 --- a/runtime/src/bpftime_shm_internal.cpp +++ b/runtime/src/bpftime_shm_internal.cpp @@ -176,6 +176,8 @@ int bpftime_shm::attach_perf_to_bpf(int perf_fd, int bpf_fd) int bpftime_shm::add_bpf_prog_attach_target(int perf_fd, int bpf_fd) { + spdlog::debug("Try attaching prog fd {} to perf fd {}", bpf_fd, + perf_fd); if (!is_prog_fd(bpf_fd)) { spdlog::error("Fd {} not prog fd", bpf_fd); errno = ENOENT; @@ -228,8 +230,7 @@ int bpftime_shm::add_software_perf_event_to_epoll(int swpe_fd, int epoll_fd, } auto &perf_handler = std::get(manager->get_handler(swpe_fd)); - if (perf_handler.type != - bpf_event_type::PERF_TYPE_SOFTWARE) { + if (perf_handler.type != bpf_event_type::PERF_TYPE_SOFTWARE) { spdlog::error( "Expected perf fd {} to be a software perf event instance", swpe_fd); @@ -554,8 +555,7 @@ bool bpftime_shm::is_software_perf_event_handler_fd(int fd) const if (!is_perf_event_handler_fd(fd)) return false; const auto &hd = std::get(get_handler(fd)); - return hd.type == - bpf_event_type::PERF_TYPE_SOFTWARE; + return hd.type == bpf_event_type::PERF_TYPE_SOFTWARE; } bpftime::agent_config &bpftime_get_agent_config() diff --git a/runtime/src/handler/prog_handler.hpp b/runtime/src/handler/prog_handler.hpp index d369f2a7..f57f3ad0 100644 --- a/runtime/src/handler/prog_handler.hpp +++ b/runtime/src/handler/prog_handler.hpp @@ -1,5 +1,6 @@ #ifndef _PROG_HANDLER_HPP #define _PROG_HANDLER_HPP +#include "spdlog/spdlog.h" #include #include #include @@ -56,6 +57,8 @@ class bpf_prog_handler { void add_attach_fd(int fd) const { + spdlog::debug("Add attach fd {} for bpf_prog {}", fd, + name.c_str()); attach_fds.push_back(fd); } From ddf36ea0b196ba921e0e8586f6888eafba13e3df Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sat, 4 Nov 2023 23:10:33 +0800 Subject: [PATCH 07/16] Update --- daemon/kernel/bpf_kernel_config.h | 16 +- daemon/user/bpf_tracer.cpp | 2 +- example/.gitignore | 1 + runtime/include/bpftime_shm.hpp | 2 +- runtime/src/bpf_helper.cpp | 53 ++++-- .../shared/perf_event_array_kernel_user.cpp | 168 +++++++++++------- .../shared/perf_event_array_kernel_user.hpp | 35 +--- runtime/src/bpftime_shm.cpp | 32 +++- runtime/src/bpftime_shm_internal.cpp | 8 + runtime/src/bpftime_shm_internal.hpp | 1 + runtime/src/handler/map_handler.cpp | 9 +- runtime/src/handler/map_handler.hpp | 39 ++-- 12 files changed, 226 insertions(+), 140 deletions(-) diff --git a/daemon/kernel/bpf_kernel_config.h b/daemon/kernel/bpf_kernel_config.h index 4c0d3e7b..91a77d74 100644 --- a/daemon/kernel/bpf_kernel_config.h +++ b/daemon/kernel/bpf_kernel_config.h @@ -22,16 +22,16 @@ const volatile bool submit_bpf_events = 0; static __always_inline bool filter_target(void) { - u64 pid = bpf_get_current_pid_tgid() >> 32; + u64 pid = bpf_get_current_pid_tgid() & 0xffffffff; if (target_pid && pid != target_pid) { - // filter target pid + // filter target pid return false; } - if (current_pid && pid == current_pid) { - // avoid breaking current process - return false; - } - return true; + if (current_pid && pid == current_pid) { + // avoid breaking current process + return false; + } + return true; } -#endif // BPFTIME_KERNEL_CONFIG_H \ No newline at end of file +#endif // BPFTIME_KERNEL_CONFIG_H diff --git a/daemon/user/bpf_tracer.cpp b/daemon/user/bpf_tracer.cpp index 7197f269..30a75575 100644 --- a/daemon/user/bpf_tracer.cpp +++ b/daemon/user/bpf_tracer.cpp @@ -87,7 +87,7 @@ int bpftime::start_daemon(struct daemon_config env) obj->rodata->uprobe_perf_type = determine_uprobe_perf_type(); obj->rodata->kprobe_perf_type = determine_kprobe_perf_type(); obj->rodata->submit_bpf_events = env.submit_bpf_events; - + obj->rodata->current_pid = getpid(); if (!env.show_open) { bpf_program__set_autoload( obj->progs.tracepoint__syscalls__sys_exit_open, false); diff --git a/example/.gitignore b/example/.gitignore index 65e669fb..671cbc6a 100644 --- a/example/.gitignore +++ b/example/.gitignore @@ -1 +1,2 @@ simple_uretprobe_test* +malloc-perf* diff --git a/runtime/include/bpftime_shm.hpp b/runtime/include/bpftime_shm.hpp index 76594296..55bfc7c1 100644 --- a/runtime/include/bpftime_shm.hpp +++ b/runtime/include/bpftime_shm.hpp @@ -152,7 +152,6 @@ bpftime::agent_config &bpftime_get_agent_config(); } // namespace bpftime extern "C" { - // initialize the global shared memory for store bpf progs and maps void bpftime_initialize_global_shm(bpftime::shm_open_type type); // destroy the global shared memory data structure @@ -267,6 +266,7 @@ int bpftime_add_software_perf_event(int cpu, int32_t sample_type, int bpftime_is_software_perf_event(int fd); void *bpftime_get_software_perf_event_raw_buffer(int fd, size_t expected_size); int bpftime_perf_event_output(int fd, const void *buf, size_t sz); +int bpftime_shared_perf_event_output(int map_fd, const void *buf, size_t sz); } #endif // BPFTIME_SHM_CPP_H diff --git a/runtime/src/bpf_helper.cpp b/runtime/src/bpf_helper.cpp index 506e4485..8059801c 100644 --- a/runtime/src/bpf_helper.cpp +++ b/runtime/src/bpf_helper.cpp @@ -115,23 +115,21 @@ uint64_t bpftime_get_current_comm(uint64_t buf, uint64_t size, uint64_t, uint64_t bpftime_map_lookup_elem_helper(uint64_t map, uint64_t key, uint64_t, uint64_t, uint64_t) { - return (uint64_t)bpftime_helper_map_lookup_elem(map >> 32, - (void *)key); + return (uint64_t)bpftime_helper_map_lookup_elem(map >> 32, (void *)key); } uint64_t bpftime_map_update_elem_helper(uint64_t map, uint64_t key, uint64_t value, uint64_t flags, uint64_t) { - return (uint64_t)bpftime_helper_map_update_elem( - map >> 32, (void *)key, (void *)value, flags); + return (uint64_t)bpftime_helper_map_update_elem(map >> 32, (void *)key, + (void *)value, flags); } uint64_t bpftime_map_delete_elem_helper(uint64_t map, uint64_t key, uint64_t, uint64_t, uint64_t) { - return (uint64_t)bpftime_helper_map_delete_elem(map >> 32, - (void *)key); + return (uint64_t)bpftime_helper_map_delete_elem(map >> 32, (void *)key); } uint64_t bpf_probe_read_str(uint64_t buf, uint64_t bufsz, uint64_t ptr, @@ -228,17 +226,40 @@ uint64_t bpf_perf_event_output(uint64_t ctx, uint64_t map, uint64_t flags, return (uint64_t)(-1); } int fd = map >> 32; - const int32_t *val_ptr = - (int32_t *)(uintptr_t)bpftime_helper_map_lookup_elem( - fd, ¤t_cpu); - if (val_ptr == nullptr) { - spdlog::error("Invalid map fd for perf event output: {}", fd); - errno = EINVAL; - return (uint64_t)(-1); + // Check map type. userspace perf event array, or shared perf event + // array? + bpftime::bpf_map_type map_ty; + if (int err = bpftime_map_get_info(fd, nullptr, nullptr, &map_ty); + err < 0) { + spdlog::error("Unable to query map type of fd {}", fd); + return -1; + } + int ret; + if (map_ty == bpftime::bpf_map_type::BPF_MAP_TYPE_PERF_EVENT_ARRAY) { + const int32_t *val_ptr = + (int32_t *)(uintptr_t)bpftime_helper_map_lookup_elem( + fd, ¤t_cpu); + if (val_ptr == nullptr) { + spdlog::error( + "Invalid map fd for perf event output: {}", fd); + errno = EINVAL; + return (uint64_t)(-1); + } + int32_t perf_handler_fd = *val_ptr; + ret = bpftime_perf_event_output(perf_handler_fd, + (const void *)(uintptr_t)data, + (size_t)size); + } else if (map_ty == + bpftime::bpf_map_type:: + BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY) { + ret = bpftime_shared_perf_event_output( + fd, (const void *)(uintptr_t)data, (size_t)size); + } else { + spdlog::error( + "Attempting to run perf_output on a non-perf array map"); + ret = -1; } - int32_t perf_handler_fd = *val_ptr; - int ret = bpftime_perf_event_output( - perf_handler_fd, (const void *)(uintptr_t)data, (size_t)size); + sched_setaffinity(0, sizeof(orig), &orig); return (uint64_t)ret; } diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index cb4c14f3..01ce64f5 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -13,9 +13,12 @@ #include #include #include +#include #include #include #include +#include + // kernel perf event array map id -> user_ring_buffer* for the current process using user_ringbuf_map = std::map >; @@ -27,11 +30,29 @@ static inline void ensure_user_rb_map_initialized() } namespace bpftime { +// Wraps a user_ringbuffer and it's spinlock that locks the reservation +struct user_ringbuffer_wrapper { + pthread_spinlock_t reserve_lock; + user_ring_buffer *rb; + int user_rb_id; + int user_rb_fd; + user_ringbuffer_wrapper(int user_rb_id); + ~user_ringbuffer_wrapper(); + void *reserve(uint32_t size); + void submit(void *mem); +}; +int perf_event_array_kernel_user_impl::get_user_ringbuf_fd() +{ + return ensure_current_map_user_ringbuf()->user_rb_fd; +} int perf_event_array_kernel_user_impl::output_data_into_kernel(const void *buf, size_t size) { + spdlog::debug("Received data output for kernel perf event array {}", + kernel_perf_id); auto user_rb = ensure_current_map_user_ringbuf(); + spdlog::debug("User ringbuf ensured: {:x}", (uintptr_t)user_rb); void *mem = user_rb->reserve(size + 8); if (!mem) { spdlog::error("Failed to reserve for user ringbuf: {}", errno); @@ -43,7 +64,9 @@ int perf_event_array_kernel_user_impl::output_data_into_kernel(const void *buf, spdlog::trace("Commited {} bytes of data into kernel", size); return 0; } - +// Put the creation of user ringbuffer & transporter ebpf program in the +// constructor of perf_event_array_kernel_user_impl Only one instance of map and +// ebpf program is required, so just put them in the daemon perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( boost::interprocess::managed_shared_memory &memory, uint32_t key_size, uint32_t value_size, uint32_t max_entries, int kernel_perf_id) @@ -54,15 +77,66 @@ perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( "Key size and value size of perf_event_array must be 4"); assert(false); } - kernel_perf_fd = bpf_map_get_fd_by_id(kernel_perf_id); - if (kernel_perf_fd < 0) { - spdlog::error("Unable to get fd of kernel perf id {}", - kernel_perf_id); - assert(false); + // Create corresponding user ringbuffer + LIBBPF_OPTS(bpf_map_create_opts, user_rb_opts); + std::string name = "ku_perf_id_" + std::to_string(kernel_perf_id); + int user_rb_fd = bpf_map_create(BPF_MAP_TYPE_USER_RINGBUF, name.c_str(), + 0, 0, 1024 * 1024, &user_rb_opts); + if (user_rb_fd < 0) { + spdlog::error( + "Failed to create user ringbuffer for shared perf event array id {}, err={}", + kernel_perf_id, errno); + return; + } + bpf_map_info map_info; + uint32_t map_info_size = sizeof(map_info); + if (int err = bpf_obj_get_info_by_fd(user_rb_fd, &map_info, + &map_info_size); + err < 0) { + spdlog::error("Failed to get map info for user rb fd {}", + user_rb_fd); + return; } + user_rb_id = map_info.id; spdlog::debug( - "Initialized perf_event_array_kernel_user_impl, kernel perf fd {}, kernel perf id {}", - kernel_perf_fd, kernel_perf_id); + "Initialized perf_event_array_kernel_user_impl, kernel perf id {}, user ringbuffer id {}, user ringbuffer map type {}", + kernel_perf_id, user_rb_id, (int)map_info.type); + int pfd = create_intervally_triggered_perf_event(10); + int kernel_perf_fd = bpf_map_get_fd_by_id(kernel_perf_id); + auto prog = create_transporting_kernel_ebpf_program(user_rb_fd, + kernel_perf_fd); + std::vector fds; + fds.push_back(user_rb_fd); + fds.push_back(kernel_perf_fd); + + LIBBPF_OPTS(bpf_prog_load_opts, opts); + char log_buffer[2048]; + opts.log_buf = log_buffer; + opts.log_size = sizeof(log_buffer); + opts.log_level = 5; + opts.fd_array = fds.data(); + + spdlog::debug("Loading transporter program with {} insns", prog.size()); + int bpf_fd = + bpf_prog_load(BPF_PROG_TYPE_PERF_EVENT, "transporter", "GPL", + (bpf_insn *)prog.data(), prog.size(), &opts); + if (bpf_fd < 0) { + spdlog::error("Failed to load bpf prog: err={}, message={}", + errno, log_buffer); + } + assert(bpf_fd >= 0); + int err; + err = ioctl(pfd, PERF_EVENT_IOC_SET_BPF, bpf_fd); + if (err < 0) { + spdlog::error("Failed to run PERF_EVENT_IOC_SET_BPF: {}", err); + assert(false); + } + err = ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0); + if (err < 0) { + spdlog::error("Failed to run PERF_EVENT_IOC_ENABLE: {}", err); + assert(false); + } + spdlog::debug("Attached transporter ebpf program"); } perf_event_array_kernel_user_impl::~perf_event_array_kernel_user_impl() { @@ -137,7 +211,7 @@ perf_event_array_kernel_user_impl::ensure_current_map_user_ringbuf() ensure_user_rb_map_initialized(); user_ringbuffer_wrapper *result; if (auto itr = user_rb_map->find(kernel_perf_id); - itr == user_rb_map->end()) { + itr != user_rb_map->end()) { result = itr->second.get(); } else { auto ptr = std::make_unique( @@ -145,68 +219,30 @@ perf_event_array_kernel_user_impl::ensure_current_map_user_ringbuf() auto raw_ptr = ptr.get(); user_rb_map->emplace(kernel_perf_id, std::move(ptr)); result = raw_ptr; - - int pfd = create_intervally_triggered_perf_event(10); - auto prog = create_transporting_kernel_ebpf_program( - result->user_rb_id, kernel_perf_id); - LIBBPF_OPTS(bpf_prog_load_opts, opts); - char log_buffer[2048]; - opts.log_buf = log_buffer; - opts.log_size = sizeof(log_buffer); - opts.log_level = 1; - int bpf_fd = - bpf_prog_load(BPF_PROG_TYPE_PERF_EVENT, "transporter", - "GPL", (bpf_insn *)prog.data(), - prog.size(), &opts); - if (bpf_fd < 0) { - spdlog::error( - "Failed to load bpf prog: err={}, message={}", - errno, log_buffer); - } - assert(bpf_fd >= 0); - int err; - err = ioctl(pfd, PERF_EVENT_IOC_SET_BPF, bpf_fd); - if (err < 0) { - spdlog::error( - "Failed to run PERF_EVENT_IOC_SET_BPF: {}", - err); - assert(false); - } - err = ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0); - if (err < 0) { - spdlog::error("Failed to run PERF_EVENT_IOC_ENABLE: {}", - err); - assert(false); - } } return result; } -user_ringbuffer_wrapper::user_ringbuffer_wrapper(int kernel_perf_id) +user_ringbuffer_wrapper::user_ringbuffer_wrapper(int user_rb_id) + : user_rb_id(user_rb_id) { pthread_spin_init(&reserve_lock, PTHREAD_PROCESS_PRIVATE); - LIBBPF_OPTS(bpf_map_create_opts, user_rb_opts); - std::string name = "ku_perf_id_" + std::to_string(kernel_perf_id); - user_rb_fd = bpf_map_create(BPF_MAP_TYPE_USER_RINGBUF, name.c_str(), 0, - 0, 1024 * 1000, &user_rb_opts); + + LIBBPF_OPTS(user_ring_buffer_opts, opts); + user_rb_fd = bpf_map_get_fd_by_id(user_rb_id); + spdlog::debug("map id {} -> fd {}, user ring buffer", user_rb_id, + user_rb_fd); if (user_rb_fd < 0) { spdlog::error( - "Failed to create user ringbuffer for shared perf event array id {}", - kernel_perf_id); - return; + "Failed to get user_rb_fd from user_rb_id {}, err={}", + user_rb_id, errno); + throw std::runtime_error( + "Failed to get user_rb_fd from user_rb_id"); } - bpf_map_info map_info; - uint32_t map_info_size = sizeof(map_info); - if (int err = bpf_obj_get_info_by_fd(user_rb_fd, &map_info, - &map_info_size); - err < 0) { - spdlog::error("Failed to get map info for user rb fd {}", - user_rb_fd); - return; - } - user_rb_id = map_info.id; - rb = user_ring_buffer__new(user_rb_fd, nullptr); + rb = user_ring_buffer__new(user_rb_fd, &opts); assert(rb && - "Failed to initialize user_rb_fd! This SHOULD NOT Happen."); + "Failed to initialize user ringbuffer! This SHOULD NOT Happen."); + spdlog::debug("User ringbuffer wrapper created, fd={}, id={}", + user_rb_fd, user_rb_id); } user_ringbuffer_wrapper::~user_ringbuffer_wrapper() @@ -254,8 +290,8 @@ static int cb(void *dynptr, void *ctx) // Compiled using gcc.godbolt.org std::vector -create_transporting_kernel_ebpf_program(int user_ringbuf_id, - int perf_event_array_id) +create_transporting_kernel_ebpf_program(int user_ringbuf_fd, + int perf_event_array_fd) { static_assert( sizeof(bpf_insn) == sizeof(uint64_t), @@ -264,7 +300,7 @@ create_transporting_kernel_ebpf_program(int user_ringbuf_id, // r3 = r1 BPF_MOV64_REG(3, 1), // r1 = map_by_fd(user_ringbuf_fd) - BPF_LD_IMM64_RAW_FULL(1, 5, 0, 0, user_ringbuf_id, 0), + BPF_LD_IMM64_RAW_FULL(1, 1, 0, 0, user_ringbuf_fd, 0), // r2 = callback fn BPF_MOV64_IMM(2, 64), // r4 = 0 @@ -318,7 +354,7 @@ create_transporting_kernel_ebpf_program(int user_ringbuf_id, // r1 = r6 BPF_MOV64_REG(1, 6), // r2 = map_by_fd(perf_event_array_fd) - BPF_LD_IMM64_RAW_FULL(2, 5, 0, 0, perf_event_array_id, 0), + BPF_LD_IMM64_RAW_FULL(2, 1, 0, 0, perf_event_array_fd, 0), // r3 = 0 BPF_MOV64_IMM(3, 0), // r4 = r8 diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp index 9e3fde9d..53f47d03 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp @@ -1,5 +1,5 @@ -#include "bpftool/libbpf/src/libbpf.h" -#include +#ifndef _BPFTIME_PERF_EVENT_ARRAY_KERNEL_USER +#define _BPFTIME_PERF_EVENT_ARRAY_KERNEL_USER #include #include #include @@ -7,18 +7,6 @@ namespace bpftime { -// Wraps a user_ringbuffer and it's spinlock that locks the reservation -struct user_ringbuffer_wrapper { - pthread_spinlock_t reserve_lock; - user_ring_buffer *rb; - int user_rb_id; - int user_rb_fd; - user_ringbuffer_wrapper(int kernel_perf_id); - ~user_ringbuffer_wrapper(); - void *reserve(uint32_t size); - void submit(void *mem); -}; - // Here is an implementation of a perf event array that can output data from // both userspace and kernel space It's corresponded with a normal perf event // array in kernel, and it will be used by kernel ebpf programs But from the @@ -34,7 +22,8 @@ class perf_event_array_kernel_user_impl { uint32_t dummy = 0xffffffff; uint32_t max_ent; int kernel_perf_id; - int kernel_perf_fd = -1; + int user_rb_id; + public: const static bool should_lock = false; perf_event_array_kernel_user_impl( @@ -51,25 +40,19 @@ class perf_event_array_kernel_user_impl { int map_get_next_key(const void *key, void *next_key); - user_ringbuffer_wrapper *ensure_current_map_user_ringbuf(); + struct user_ringbuffer_wrapper *ensure_current_map_user_ringbuf(); int output_data_into_kernel(const void *buf, size_t size); - int get_kernel_perf_fd() - { - return kernel_perf_fd; - } - int get_user_ringbuf_fd() - { - return ensure_current_map_user_ringbuf()->user_rb_fd; - } + int get_user_ringbuf_fd(); }; // Create a bpf program that will check if the user_ringbuf has data and copy // the data into the perf event std::vector -create_transporting_kernel_ebpf_program(int user_ringbuf_id, - int perf_event_array_id); +create_transporting_kernel_ebpf_program(int user_ringbuf_fd, + int perf_event_array_fd); // Create an intervally triggered perf event int create_intervally_triggered_perf_event(int freq); } // namespace bpftime +#endif diff --git a/runtime/src/bpftime_shm.cpp b/runtime/src/bpftime_shm.cpp index 590b3de4..92f1b388 100644 --- a/runtime/src/bpftime_shm.cpp +++ b/runtime/src/bpftime_shm.cpp @@ -130,7 +130,7 @@ void bpftime_close(int fd) } int bpftime_map_get_info(int fd, bpftime::bpf_map_attr *out_attr, - const char **out_name, bpf_map_type *type) + const char **out_name, bpftime::bpf_map_type *type) { if (!shm_holder.global_shared_memory.is_map_fd(fd)) { errno = ENOENT; @@ -351,7 +351,35 @@ int bpftime_perf_event_output(int fd, const void *buf, size_t sz) } } -#define INVALID_MAP_PTR ((uint64_t)0 - 1) +int bpftime_shared_perf_event_output(int map_fd, const void *buf, size_t sz) +{ + spdlog::debug("Output data into shared perf event array fd {}", map_fd); + auto &shm = shm_holder.global_shared_memory; + if (!shm.is_shared_perf_event_array_map_fd(map_fd)) { + spdlog::error("Expected fd {} to be a shared perf event array", + map_fd); + errno = EINVAL; + return -1; + } + auto &map_handler = std::get(shm.get_handler(map_fd)); + if (auto p = map_handler.try_get_shared_perf_event_array_map_impl(); + p.has_value()) { + int err = p.value()->output_data_into_kernel(buf, sz); + if (err < 0) { + errno = -err; + return -1; + } + return 0; + } else { + spdlog::error( + "Expected map {} to be a shared perf event array map", + map_fd); + errno = EINVAL; + return -1; + } +} + +const uint64_t INVALID_MAP_PTR = ((uint64_t)0 - 1); extern "C" uint64_t map_ptr_by_fd(uint32_t fd) { diff --git a/runtime/src/bpftime_shm_internal.cpp b/runtime/src/bpftime_shm_internal.cpp index 7e21188e..ef1ebc83 100644 --- a/runtime/src/bpftime_shm_internal.cpp +++ b/runtime/src/bpftime_shm_internal.cpp @@ -327,6 +327,14 @@ bool bpftime_shm::is_ringbuf_map_fd(int fd) const auto &map_impl = std::get(manager->get_handler(fd)); return map_impl.type == bpf_map_type::BPF_MAP_TYPE_RINGBUF; } +bool bpftime_shm::is_shared_perf_event_array_map_fd(int fd) const +{ + if (!is_map_fd(fd)) + return false; + auto &map_impl = std::get(manager->get_handler(fd)); + return map_impl.type == + bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY; +} bool bpftime_shm::is_array_map_fd(int fd) const { if (!is_map_fd(fd)) diff --git a/runtime/src/bpftime_shm_internal.hpp b/runtime/src/bpftime_shm_internal.hpp index 9e48590d..9f3b4d2e 100644 --- a/runtime/src/bpftime_shm_internal.hpp +++ b/runtime/src/bpftime_shm_internal.hpp @@ -50,6 +50,7 @@ class bpftime_shm { bool is_map_fd(int fd) const; bool is_ringbuf_map_fd(int fd) const; bool is_array_map_fd(int fd) const; + bool is_shared_perf_event_array_map_fd(int fd) const; bool is_perf_event_handler_fd(int fd) const; bool is_software_perf_event_handler_fd(int fd) const; diff --git a/runtime/src/handler/map_handler.cpp b/runtime/src/handler/map_handler.cpp index d0dab49a..3161aaa1 100644 --- a/runtime/src/handler/map_handler.cpp +++ b/runtime/src/handler/map_handler.cpp @@ -473,5 +473,12 @@ void bpf_map_handler::map_free(managed_shared_memory &memory) map_impl_ptr = nullptr; return; } - +std::optional +bpf_map_handler::try_get_shared_perf_event_array_map_impl() const +{ + if (type != bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_PERF_EVENT_ARRAY) + return {}; + return static_cast( + map_impl_ptr.get()); +} } // namespace bpftime diff --git a/runtime/src/handler/map_handler.hpp b/runtime/src/handler/map_handler.hpp index b6f7f7af..9865b801 100644 --- a/runtime/src/handler/map_handler.hpp +++ b/runtime/src/handler/map_handler.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace bpftime { using char_allocator = boost::interprocess::allocator< @@ -90,36 +91,34 @@ class bpf_map_handler { bool from_userspace = false) const; // * BPF_MAP_UPDATE_ELEM // * Description - // * Create or update an element (key/value pair) in a specified map. + // * Create or update an element (key/value pair) in a + // specified map. // * // * The *flags* argument should be specified as one of the // * following: // * // * **BPF_ANY** - // * Create a new element or update an existing element. - // * **BPF_NOEXIST** - // * Create a new element only if it did not exist. - // * **BPF_EXIST** - // * Update an existing element. - // * **BPF_F_LOCK** - // * Update a spin_lock-ed map element. + // * Create a new element or update an existing + // element. * **BPF_NOEXIST** * Create a + // new element only if it did not exist. * **BPF_EXIST** * + // Update an existing element. * **BPF_F_LOCK** * + // Update a spin_lock-ed map element. // * // * Return - // * Returns zero on success. On error, -1 is returned and *errno* - // * is set appropriately. + // * Returns zero on success. On error, -1 is returned and + // *errno* * is set appropriately. // * // * May set *errno* to **EINVAL**, **EPERM**, **ENOMEM**, // * **E2BIG**, **EEXIST**, or **ENOENT**. // * // * **E2BIG** // * The number of elements in the map reached the - // * *max_entries* limit specified at map creation time. - // * **EEXIST** - // * If *flags* specifies **BPF_NOEXIST** and the element - // * with *key* already exists in the map. - // * **ENOENT** - // * If *flags* specifies **BPF_EXIST** and the element with - // * *key* does not exist in the map. + // * *max_entries* limit specified at map creation + // time. * **EEXIST** * If *flags* + // specifies **BPF_NOEXIST** and the element * with + // *key* already exists in the map. * **ENOENT** * + // If *flags* specifies **BPF_EXIST** and the element with * + // *key* does not exist in the map. // * long map_update_elem(const void *key, const void *value, uint64_t flags, bool from_userspace = false) const; @@ -128,8 +127,8 @@ class bpf_map_handler { // * Look up and delete an element by key in a specified map. // * // * Return - // * Returns zero on success. On error, -1 is returned and *errno* - // * is set appropriately. + // * Returns zero on success. On error, -1 is returned and + // *errno* * is set appropriately. long map_delete_elem(const void *key, bool from_userspace = false) const; // * BPF_MAP_GET_NEXT_KEY @@ -162,6 +161,8 @@ class bpf_map_handler { uint32_t get_value_size() const; std::optional try_get_ringbuf_map_impl() const; std::optional try_get_array_map_impl() const; + std::optional + try_get_shared_perf_event_array_map_impl() const; private: std::string get_container_name(); From 81e577f74046ba9317b09c73ee470bc2007fcf0b Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 02:51:32 +0800 Subject: [PATCH 08/16] Stage changes --- daemon/user/bpftime_driver.cpp | 24 +- .../shared/perf_event_array_kernel_user.cpp | 205 ++++++++++-------- .../shared/perf_event_array_kernel_user.hpp | 1 - runtime/src/bpftime_shm_internal.cpp | 9 +- 4 files changed, 133 insertions(+), 106 deletions(-) diff --git a/daemon/user/bpftime_driver.cpp b/daemon/user/bpftime_driver.cpp index 7b06fbbb..c8236464 100644 --- a/daemon/user/bpftime_driver.cpp +++ b/daemon/user/bpftime_driver.cpp @@ -42,7 +42,7 @@ static int get_kernel_bpf_prog_insns(int fd, const bpf_prog_info *info, struct bpf_insn_data insn_data; -static int get_bpf_map_id_from_pid_fd(bpf_tracer_bpf *obj, int pid, int fd) +static int get_bpf_obj_id_from_pid_fd(bpf_tracer_bpf *obj, int pid, int fd) { unsigned long long key = MAKE_PFD(pid, fd); int map_id = -1; @@ -55,10 +55,10 @@ static int get_bpf_map_id_from_pid_fd(bpf_tracer_bpf *obj, int pid, int fd) pid, fd); return -1; } - if (data.type != BPF_FD_TYPE_MAP) { - spdlog::error("Invalid bpf fd type {} for pid {} fd {}", - (int)data.type, pid, fd); - } + // if (data.type != BPF_FD_TYPE_MAP) { + // spdlog::error("Invalid bpf fd type {} for pid {} fd {}", + // (int)data.type, pid, fd); + // } map_id = data.kernel_id; return map_id; } @@ -90,7 +90,7 @@ static int relocate_bpf_prog_insns(std::vector &insns, switch (inst.code) { case EBPF_OP_LDDW: if (inst.src_reg == 1 || inst.src_reg == 2) { - int map_id = get_bpf_map_id_from_pid_fd( + int map_id = get_bpf_obj_id_from_pid_fd( obj, pid, inst.imm); if (map_id < 0) { return -1; @@ -178,13 +178,14 @@ int bpftime_driver::bpftime_progs_create_server(int kernel_id, int server_pid) kernel_id); return -1; } - res = bpftime_progs_create(kernel_id, buffer.data(), buffer.size(), - info.name, info.type); + res = bpftime_progs_create(fd, + buffer.data(), buffer.size(), info.name, + info.type); if (res < 0) { spdlog::error("Failed to create prog for id {}", kernel_id); return -1; } - spdlog::info("create prog {} in shm success", kernel_id); + spdlog::info("create prog {}, fd {} in shm success", kernel_id, fd); // check and created related maps res = check_and_create_prog_related_maps(fd, &info); if (res < 0) { @@ -243,7 +244,7 @@ int bpftime_driver::bpftime_attach_perf_to_bpf_fd_server(int server_pid, int bpf_prog_fd) { int prog_id = - get_bpf_map_id_from_pid_fd(object, server_pid, bpf_prog_fd); + get_bpf_obj_id_from_pid_fd(object, server_pid, bpf_prog_fd); if (prog_id < 0) { spdlog::error( "Failed to lookup bpf prog id from bpf prog fd {}", @@ -275,7 +276,8 @@ int bpftime_driver::bpftime_attach_perf_to_bpf_server(int server_pid, } } - int res = bpftime_attach_perf_to_bpf(perf_id, kernel_bpf_id); + int res = bpftime_attach_perf_to_bpf( + perf_id, bpf_prog_get_fd_by_id(kernel_bpf_id)); if (res < 0) { spdlog::error("Failed to attach perf to bpf"); return -1; diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index 01ce64f5..8569c7e1 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -1,7 +1,9 @@ #include "bpf/bpf.h" #include "bpf/libbpf_common.h" +#include "bpftool/libbpf/src/btf.h" #include "bpftool/libbpf/src/libbpf.h" #include "linux/bpf_common.h" +#include "linux/btf.h" #include #include #include @@ -19,6 +21,8 @@ #include #include +static int create_transporter_prog(int user_ringbuf_fd, int kernel_perf_fd); + // kernel perf event array map id -> user_ring_buffer* for the current process using user_ringbuf_map = std::map >; @@ -102,28 +106,10 @@ perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( "Initialized perf_event_array_kernel_user_impl, kernel perf id {}, user ringbuffer id {}, user ringbuffer map type {}", kernel_perf_id, user_rb_id, (int)map_info.type); int pfd = create_intervally_triggered_perf_event(10); - int kernel_perf_fd = bpf_map_get_fd_by_id(kernel_perf_id); - auto prog = create_transporting_kernel_ebpf_program(user_rb_fd, - kernel_perf_fd); - std::vector fds; - fds.push_back(user_rb_fd); - fds.push_back(kernel_perf_fd); + int kernel_perf_fd = bpf_map_get_fd_by_id(kernel_perf_id); - LIBBPF_OPTS(bpf_prog_load_opts, opts); - char log_buffer[2048]; - opts.log_buf = log_buffer; - opts.log_size = sizeof(log_buffer); - opts.log_level = 5; - opts.fd_array = fds.data(); + int bpf_fd = create_transporter_prog(user_rb_fd, kernel_perf_fd); - spdlog::debug("Loading transporter program with {} insns", prog.size()); - int bpf_fd = - bpf_prog_load(BPF_PROG_TYPE_PERF_EVENT, "transporter", "GPL", - (bpf_insn *)prog.data(), prog.size(), &opts); - if (bpf_fd < 0) { - spdlog::error("Failed to load bpf prog: err={}, message={}", - errno, log_buffer); - } assert(bpf_fd >= 0); int err; err = ioctl(pfd, PERF_EVENT_IOC_SET_BPF, bpf_fd); @@ -268,23 +254,31 @@ int offset, long flags) = (void *) 201; static long size) = (void *) 25; static long (*bpf_user_ringbuf_drain)(void *map, void *callback_fn, void *ctx, long flags) = (void *) 209; -static int cb(void *dynptr, void *ctx); +struct context { + unsigned long size; + char buf[32]; +}; +static int cb(void *dynptr, struct context *ctx); int func(void* ctx) { - bpf_user_ringbuf_drain((void*)0x234, &cb, ctx, 0); + struct context ctx_buf; + ctx_buf.size = 0; + for(int i=0;i400) return 1; - bpf_dynptr_read(&buf, val, dynptr, 8, 0); - bpf_perf_event_output(ctx, (void*)0x123, 0, buf, val); + if(bpf_dynptr_read(&ctx->size, 8, dynptr, 0, 0)<0) return 1; + if(ctx->size>sizeof(ctx->buf)) return 1; + if(bpf_dynptr_read(ctx->buf, ctx->size, dynptr, 8, 0)<0) return 1; return 1; } + */ // Compiled using gcc.godbolt.org @@ -297,73 +291,37 @@ create_transporting_kernel_ebpf_program(int user_ringbuf_fd, sizeof(bpf_insn) == sizeof(uint64_t), "bpf_insn is expected to be in the same size of uint64_t"); bpf_insn insns[] = { - // r3 = r1 - BPF_MOV64_REG(3, 1), + BPF_MOV64_REG(6, 1), BPF_MOV64_IMM(1, 0), + BPF_STX_MEM(BPF_DW, 10, 1, -8), + BPF_STX_MEM(BPF_DW, 10, 1, -0x10), + BPF_STX_MEM(BPF_DW, 10, 1, -0x18), + BPF_STX_MEM(BPF_DW, 10, 1, -0x20), + BPF_STX_MEM(BPF_DW, 10, 1, -0x28), BPF_MOV64_REG(3, 10), + BPF_ALU64_IMM(BPF_ADD, 3, -0x28), // r1 = map_by_fd(user_ringbuf_fd) BPF_LD_IMM64_RAW_FULL(1, 1, 0, 0, user_ringbuf_fd, 0), - // r2 = callback fn - BPF_MOV64_IMM(2, 64), - // r4 = 0 - BPF_MOV64_IMM(4, 0), - // call bpf_user_ringbuf_drain(void *map, void *callback_fn, - // void *ctx, long flags) = 0x209 - BPF_EMIT_CALL(0x209), - // r0 = 0 - BPF_MOV64_IMM(0, 0), BPF_EXIT_INSN(), - // static int cb(void *dynptr, void *ctx) - // r6 = r2 - BPF_MOV64_REG(6, 2), - // r7 = r1 - BPF_MOV64_REG(7, 1), - // r1 = r10 - BPF_MOV64_REG(1, 10), - // r1 += -0x8 - BPF_ALU64_IMM(BPF_ADD, 1, -8), - // r2=8, - BPF_MOV64_IMM(2, 8), - // r3 = r7 - BPF_MOV64_IMM(3, 7), - // r4=0 - BPF_MOV64_IMM(4, 0), - // r5 = 0 - BPF_MOV64_IMM(5, 0), - // call 0xc9 bpf_dynptr_read(void *dst, int len, const struct - // bpf_dynptr *src, int offset, long flags) = (void *) 201 - BPF_EMIT_CALL(0xc9), - // r2 = *(u64 *)(r10 - 0x8) - BPF_LDX_MEM(BPF_DW, 2, 10, -8), - // if r2 s> 0x1f0 goto +0xd - BPF_RAW_INSN(0x65, 0x2, 0x0, 0x0d, 0x1f0), - // r8 = r10 - BPF_MOV64_REG(8, 10), - // r8 += -0x1f8 - BPF_ALU64_IMM(BPF_ADD, 8, -0x1f8), - // r1 = r8 - BPF_MOV64_REG(1, 8), - // r3 = r7 - BPF_MOV64_REG(3, 7), - // r4 = 8 - BPF_MOV64_IMM(4, 8), - // r5 = 0 - BPF_MOV64_IMM(5, 0), - // call 0xc9 bpf_dynptr_read(void *dst, int len, const struct - // bpf_dynptr *src, int offset, long flags) = (void *) 201 - BPF_EMIT_CALL(0xc9), - // r5 = *(u64 *)(r10 - 0x8) - BPF_LDX_MEM(BPF_DW, 5, 10, -8), - // r1 = r6 + // r2 = callback fn (+7) + BPF_LD_IMM64_RAW_FULL(2, 4, 0, 0, 16, 0), BPF_MOV64_IMM(4, 0), + BPF_EMIT_CALL(0xd1), BPF_RAW_INSN(0x55, 0, 0, 0xa, 1), + BPF_LDX_MEM(BPF_DW, 5, 10, -0x28), + BPF_ALU64_IMM(BPF_AND, 5, 0x1f), + BPF_STX_MEM(BPF_DW, 10, 5, -0x28), + + BPF_MOV64_REG(4, 10), BPF_ALU64_IMM(BPF_ADD, 4, -0x20), BPF_MOV64_REG(1, 6), - // r2 = map_by_fd(perf_event_array_fd) BPF_LD_IMM64_RAW_FULL(2, 1, 0, 0, perf_event_array_fd, 0), - // r3 = 0 - BPF_MOV64_IMM(3, 0), - // r4 = r8 - BPF_MOV64_REG(4, 8), - // call 0x19 bpf_perf_event_output(void *ctx, void *map, long - // flags, void *data, long size) = 25 - BPF_EMIT_CALL(25), - // r0 = 1 - BPF_MOV64_IMM(0, 1), BPF_EXIT_INSN() + BPF_MOV64_IMM(3, 0), BPF_EMIT_CALL(0x19), BPF_MOV64_IMM(0, 0), + BPF_EXIT_INSN(), + // callback func + BPF_MOV64_REG(7, 2), BPF_MOV64_REG(6, 1), BPF_MOV64_IMM(8, 0), + BPF_MOV64_REG(1, 7), BPF_MOV64_IMM(2, 0x8), BPF_MOV64_REG(3, 6), + BPF_MOV64_IMM(4, 0), BPF_MOV64_IMM(5, 0), BPF_EMIT_CALL(0xc9), + BPF_RAW_INSN(0x6d, 0x8, 0, 8, 0), BPF_LDX_MEM(BPF_DW, 2, 7, 0), + BPF_RAW_INSN(0x25, 0x2, 0x0, 0x6, 0x20), + BPF_ALU64_IMM(BPF_ADD, 7, 8), BPF_MOV64_REG(1, 7), + BPF_MOV64_REG(3, 6), BPF_MOV64_IMM(4, 8), BPF_MOV64_IMM(5, 0), + BPF_EMIT_CALL(0xc9), BPF_MOV64_IMM(0, 1), BPF_EXIT_INSN() + }; return std::vector((uint64_t *)&insns, (uint64_t *)&insns + std::size(insns)); @@ -381,3 +339,66 @@ int create_intervally_triggered_perf_event(int freq) return syscall(__NR_perf_event_open, &pe_attr, 0, -1, -1, 0); } } // namespace bpftime + +static int create_transporter_prog(int user_ringbuf_fd, int kernel_perf_fd) +{ + // Maniuplate a corresponding btf + btf *btf = btf__new_empty(); + int bpf_dynptr_st = btf__add_struct(btf, "bpf_dynptr", 0); + int bpf_dynptr_ptr = btf__add_ptr(btf, bpf_dynptr_st); + int void_ptr = btf__add_ptr(btf, 0); + int long_ty = btf__add_int(btf, "long", 8, 1); + int int_ty = btf__add_int(btf, "int", 4, 1); + + int cb_func_proto = btf__add_func_proto(btf, long_ty); + btf__add_func_param(btf, "dynptr", bpf_dynptr_ptr); + btf__add_func_param(btf, "context", void_ptr); + int cb_func = btf__add_func(btf, "transporter_cb", BTF_FUNC_STATIC, + cb_func_proto); + // int main_func_proto = btf__add_func(btf, + // "transporter",BTF_FUNC_GLOBAL , int proto_type_id) + int main_func_proto = btf__add_func_proto(btf, int_ty); + btf__add_func_param(btf, "ctx", void_ptr); + int main_func = btf__add_func(btf, "transporter", BTF_FUNC_GLOBAL, + main_func_proto); + uint32_t size; + + auto btf_raw_data = btf__raw_data(btf, &size); + LIBBPF_OPTS(bpf_btf_load_opts, btf_load_opts); + int btf_fd = bpf_btf_load(btf_raw_data, size, &btf_load_opts); + if (btf_fd < 0) { + spdlog::error("Failed to load btf into kernel: {}", errno); + return -1; + } + auto prog = bpftime::create_transporting_kernel_ebpf_program( + user_ringbuf_fd, kernel_perf_fd); + std::vector fds; + fds.push_back(user_ringbuf_fd); + fds.push_back(kernel_perf_fd); + std::vector func_info; + func_info.push_back( + bpf_func_info{ .insn_off = 0, .type_id = (uint32_t)main_func }); + func_info.push_back( + bpf_func_info{ .insn_off = 28, .type_id = (uint32_t)cb_func }); + + LIBBPF_OPTS(bpf_prog_load_opts, prog_load_opts); + char log_buffer[4096]; + prog_load_opts.log_buf = log_buffer; + prog_load_opts.log_size = sizeof(log_buffer); + prog_load_opts.log_level = 5; + prog_load_opts.fd_array = fds.data(); + prog_load_opts.prog_btf_fd = btf_fd; + prog_load_opts.func_info = func_info.data(); + prog_load_opts.func_info_cnt = func_info.size(); + prog_load_opts.func_info_rec_size = sizeof(bpf_func_info); + spdlog::debug("Loading transporter program with {} insns", prog.size()); + int bpf_fd = bpf_prog_load(BPF_PROG_TYPE_PERF_EVENT, "transporter", + "GPL", (bpf_insn *)prog.data(), prog.size(), + &prog_load_opts); + + if (bpf_fd < 0) { + spdlog::error("Failed to load bpf prog: err={}, message=\n{}", + errno, log_buffer); + } + return bpf_fd; +} diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp index 53f47d03..070c8559 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp @@ -50,7 +50,6 @@ class perf_event_array_kernel_user_impl { std::vector create_transporting_kernel_ebpf_program(int user_ringbuf_fd, int perf_event_array_fd); - // Create an intervally triggered perf event int create_intervally_triggered_perf_event(int freq); diff --git a/runtime/src/bpftime_shm_internal.cpp b/runtime/src/bpftime_shm_internal.cpp index ef1ebc83..560fb913 100644 --- a/runtime/src/bpftime_shm_internal.cpp +++ b/runtime/src/bpftime_shm_internal.cpp @@ -134,6 +134,8 @@ int bpftime_shm::add_uprobe(int fd, int pid, const char *name, uint64_t offset, // if fd is negative, we need to create a new fd for allocating fd = open_fake_fd(); } + spdlog::debug("Set fd {} to uprobe, pid={}, name={}, offset={}", fd, + pid, name, offset); return manager->set_handler( fd, bpftime::bpf_perf_event_handler{ retprobe, offset, pid, name, @@ -167,7 +169,7 @@ int bpftime_shm::add_software_perf_event(int cpu, int32_t sample_type, int bpftime_shm::attach_perf_to_bpf(int perf_fd, int bpf_fd) { if (!is_perf_fd(perf_fd)) { - spdlog::error("Fd {} not a perf fd", perf_fd); + spdlog::error("Fd {} is not a perf fd", perf_fd); errno = ENOENT; return -1; } @@ -179,7 +181,7 @@ int bpftime_shm::add_bpf_prog_attach_target(int perf_fd, int bpf_fd) spdlog::debug("Try attaching prog fd {} to perf fd {}", bpf_fd, perf_fd); if (!is_prog_fd(bpf_fd)) { - spdlog::error("Fd {} not prog fd", bpf_fd); + spdlog::error("Fd {} is not a prog fd", bpf_fd); errno = ENOENT; return -1; } @@ -396,6 +398,9 @@ int bpftime_shm::add_bpf_prog(int fd, const ebpf_inst *insn, size_t insn_cnt, // if fd is negative, we need to create a new fd for allocating fd = open_fake_fd(); } + spdlog::debug( + "Set handler fd {} to bpf_prog_handler, name {}, prog_type {}, insn_cnt {}", + fd, prog_name, prog_type, insn_cnt); return manager->set_handler( fd, bpftime::bpf_prog_handler(segment, insn, insn_cnt, prog_name, From ab188cdb36a84a4795b2a6aa74eb506b1d1477d5 Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 03:03:48 +0800 Subject: [PATCH 09/16] update --- .../shared/perf_event_array_kernel_user.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index 8569c7e1..152787f2 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -200,8 +200,8 @@ perf_event_array_kernel_user_impl::ensure_current_map_user_ringbuf() itr != user_rb_map->end()) { result = itr->second.get(); } else { - auto ptr = std::make_unique( - kernel_perf_id); + auto ptr = + std::make_unique(user_rb_id); auto raw_ptr = ptr.get(); user_rb_map->emplace(kernel_perf_id, std::move(ptr)); result = raw_ptr; @@ -224,6 +224,7 @@ user_ringbuffer_wrapper::user_ringbuffer_wrapper(int user_rb_id) throw std::runtime_error( "Failed to get user_rb_fd from user_rb_id"); } + rb = user_ring_buffer__new(user_rb_fd, &opts); assert(rb && "Failed to initialize user ringbuffer! This SHOULD NOT Happen."); @@ -291,18 +292,18 @@ create_transporting_kernel_ebpf_program(int user_ringbuf_fd, sizeof(bpf_insn) == sizeof(uint64_t), "bpf_insn is expected to be in the same size of uint64_t"); bpf_insn insns[] = { - BPF_MOV64_REG(6, 1), BPF_MOV64_IMM(1, 0), - BPF_STX_MEM(BPF_DW, 10, 1, -8), - BPF_STX_MEM(BPF_DW, 10, 1, -0x10), - BPF_STX_MEM(BPF_DW, 10, 1, -0x18), - BPF_STX_MEM(BPF_DW, 10, 1, -0x20), - BPF_STX_MEM(BPF_DW, 10, 1, -0x28), BPF_MOV64_REG(3, 10), + BPF_MOV64_REG(6, 1), BPF_MOV64_IMM(7, 0), + BPF_STX_MEM(BPF_DW, 10, 7, -8), + BPF_STX_MEM(BPF_DW, 10, 7, -0x10), + BPF_STX_MEM(BPF_DW, 10, 7, -0x18), + BPF_STX_MEM(BPF_DW, 10, 7, -0x20), + BPF_STX_MEM(BPF_DW, 10, 7, -0x28), BPF_MOV64_REG(3, 10), BPF_ALU64_IMM(BPF_ADD, 3, -0x28), // r1 = map_by_fd(user_ringbuf_fd) BPF_LD_IMM64_RAW_FULL(1, 1, 0, 0, user_ringbuf_fd, 0), // r2 = callback fn (+7) BPF_LD_IMM64_RAW_FULL(2, 4, 0, 0, 16, 0), BPF_MOV64_IMM(4, 0), - BPF_EMIT_CALL(0xd1), BPF_RAW_INSN(0x55, 0, 0, 0xa, 1), + BPF_EMIT_CALL(0xd1), BPF_RAW_INSN(0x6d, 7, 0, 0xa, 0), BPF_LDX_MEM(BPF_DW, 5, 10, -0x28), BPF_ALU64_IMM(BPF_AND, 5, 0x1f), BPF_STX_MEM(BPF_DW, 10, 5, -0x28), From 314ca05f2c816a98b0ed56e8eb293b59c9ecee38 Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 20:05:50 +0800 Subject: [PATCH 10/16] Initial working shared perf event array --- .../bpf_map/shared/array_map_kernel_user.cpp | 38 +++--- .../shared/perf_event_array_kernel_user.cpp | 118 +++++++++++++----- .../shared/perf_event_array_kernel_user.hpp | 5 +- runtime/src/handler/handler_manager.hpp | 2 +- 4 files changed, 112 insertions(+), 51 deletions(-) diff --git a/runtime/src/bpf_map/shared/array_map_kernel_user.cpp b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp index 1ebd7aab..1a7dbbca 100644 --- a/runtime/src/bpf_map/shared/array_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp @@ -1,3 +1,4 @@ +#include "spdlog/spdlog.h" #include #include #include @@ -6,7 +7,7 @@ #ifndef roundup #define roundup(x, y) \ ({ \ - const decltype(y) __y = y; \ + const decltype(y) __y = y; \ (((x) + (__y - 1)) / __y) * __y; \ }) #endif // roundup @@ -51,26 +52,29 @@ void array_map_kernel_user_impl::init_map_fd() // return; // } size_t mmap_sz = bpf_map_mmap_sz(_value_size, _max_entries); + spdlog::debug("mmap shared array map, fd={}, mmap_sz={}", map_fd, + mmap_sz); mmap_ptr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (mmap_ptr == MAP_FAILED) { - spdlog::error("Failed to mmap for kernel map id {}", - kernel_map_id); - return; - } - int prot; - if (info.map_flags & BPF_F_RDONLY_PROG) - prot = PROT_READ; - else - prot = PROT_READ | PROT_WRITE; - void *mmaped = mmap(mmap_ptr, mmap_sz, prot, MAP_SHARED | MAP_FIXED, - map_fd, 0); - if (mmaped == MAP_FAILED) { - spdlog::error("Failed to mmap for kernel map id {}", - kernel_map_id); + spdlog::error("Failed to mmap for kernel map id {}, err={}", + kernel_map_id, errno); return; } - mmap_ptr = mmaped; + // What does this piece of code do? + // int prot; + // if (info.map_flags & BPF_F_RDONLY_PROG) + // prot = PROT_READ; + // else + // prot = PROT_READ | PROT_WRITE; + // void *mmaped = mmap(mmap_ptr, mmap_sz, prot, MAP_SHARED | MAP_FIXED, + // map_fd, 0); + // if (mmaped == MAP_FAILED) { + // spdlog::error("Failed to mmap for kernel map id {}, err={}", + // kernel_map_id, errno); + // return; + // } + // mmap_ptr = mmaped; } array_map_kernel_user_impl::array_map_kernel_user_impl( @@ -97,7 +101,7 @@ void *array_map_kernel_user_impl::elem_lookup(const void *key) if (res < 0) { return nullptr; } - return (void*)value_data.data(); + return (void *)value_data.data(); } long array_map_kernel_user_impl::elem_update(const void *key, const void *value, diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index 152787f2..d1782f20 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -4,6 +4,8 @@ #include "bpftool/libbpf/src/libbpf.h" #include "linux/bpf_common.h" #include "linux/btf.h" +#include "linux/perf_event.h" +#include "spdlog/fmt/bin_to_hex.h" #include #include #include @@ -20,6 +22,7 @@ #include #include #include +#include static int create_transporter_prog(int user_ringbuf_fd, int kernel_perf_fd); @@ -65,7 +68,8 @@ int perf_event_array_kernel_user_impl::output_data_into_kernel(const void *buf, *(uint64_t *)(mem) = (uint64_t)size; memcpy((char *)mem + 8, buf, size); user_rb->submit(mem); - spdlog::trace("Commited {} bytes of data into kernel", size); + spdlog::trace("Commited {} bytes of data into kernel: {:n}", size, + spdlog::to_hex((uint8_t *)buf, (uint8_t *)buf + size)); return 0; } // Put the creation of user ringbuffer & transporter ebpf program in the @@ -105,10 +109,12 @@ perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( spdlog::debug( "Initialized perf_event_array_kernel_user_impl, kernel perf id {}, user ringbuffer id {}, user ringbuffer map type {}", kernel_perf_id, user_rb_id, (int)map_info.type); - int pfd = create_intervally_triggered_perf_event(10); + int &pfd = this->pfd; + pfd = create_intervally_triggered_perf_event(10); int kernel_perf_fd = bpf_map_get_fd_by_id(kernel_perf_id); - int bpf_fd = create_transporter_prog(user_rb_fd, kernel_perf_fd); + int &bpf_fd = this->transporter_prog_fd; + bpf_fd = create_transporter_prog(user_rb_fd, kernel_perf_fd); assert(bpf_fd >= 0); int err; @@ -122,10 +128,13 @@ perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( spdlog::error("Failed to run PERF_EVENT_IOC_ENABLE: {}", err); assert(false); } + spdlog::debug("Attached transporter ebpf program"); } perf_event_array_kernel_user_impl::~perf_event_array_kernel_user_impl() { + close(pfd); + close(transporter_prog_fd); } void *perf_event_array_kernel_user_impl::elem_lookup(const void *key) @@ -292,51 +301,98 @@ create_transporting_kernel_ebpf_program(int user_ringbuf_fd, sizeof(bpf_insn) == sizeof(uint64_t), "bpf_insn is expected to be in the same size of uint64_t"); bpf_insn insns[] = { - BPF_MOV64_REG(6, 1), BPF_MOV64_IMM(7, 0), - BPF_STX_MEM(BPF_DW, 10, 7, -8), - BPF_STX_MEM(BPF_DW, 10, 7, -0x10), - BPF_STX_MEM(BPF_DW, 10, 7, -0x18), - BPF_STX_MEM(BPF_DW, 10, 7, -0x20), - BPF_STX_MEM(BPF_DW, 10, 7, -0x28), BPF_MOV64_REG(3, 10), - BPF_ALU64_IMM(BPF_ADD, 3, -0x28), + BPF_MOV64_REG(6, 1), + // bpf_printk("triggered") + BPF_MOV64_IMM(1, 0x64), BPF_STX_MEM(BPF_H, 10, 1, -8), + BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x67697274, 0x65726567), + BPF_STX_MEM(BPF_DW, 10, 1, -0x10), BPF_MOV64_REG(1, 10), + BPF_ALU64_IMM(BPF_ADD, 1, -0x10), BPF_MOV64_IMM(2, 0xa), + BPF_EMIT_CALL(0x6), + // Other + BPF_MOV64_IMM(8, 0), BPF_STX_MEM(BPF_DW, 10, 8, -0x18), + BPF_STX_MEM(BPF_DW, 10, 8, -0x20), + BPF_STX_MEM(BPF_DW, 10, 8, -0x28), + BPF_STX_MEM(BPF_DW, 10, 8, -0x30), + BPF_STX_MEM(BPF_DW, 10, 8, -0x38), BPF_MOV64_REG(3, 10), + BPF_ALU64_IMM(BPF_ADD, 3, -0x38), // r1 = map_by_fd(user_ringbuf_fd) BPF_LD_IMM64_RAW_FULL(1, 1, 0, 0, user_ringbuf_fd, 0), // r2 = callback fn (+7) - BPF_LD_IMM64_RAW_FULL(2, 4, 0, 0, 16, 0), BPF_MOV64_IMM(4, 0), - BPF_EMIT_CALL(0xd1), BPF_RAW_INSN(0x6d, 7, 0, 0xa, 0), - BPF_LDX_MEM(BPF_DW, 5, 10, -0x28), + BPF_LD_IMM64_RAW_FULL(2, 4, 0, 0, 39, 0), BPF_MOV64_IMM(4, 0), + BPF_EMIT_CALL(0xd1), BPF_MOV64_REG(7, 0), + BPF_MOV64_IMM(1, 0x6425), BPF_STX_MEM(BPF_H, 10, 1, -0x3c), + BPF_MOV64_IMM(1, 0x3d727265), BPF_STX_MEM(BPF_W, 10, 1, -0x40), + BPF_STX_MEM(BPF_B, 10, 8, -0x3a), BPF_MOV64_REG(1, 10), + BPF_ALU64_IMM(BPF_ADD, 1, -0x40), BPF_MOV64_IMM(2, 7), + BPF_MOV64_REG(3, 7), BPF_EMIT_CALL(6), + BPF_ALU64_IMM(BPF_LSH, 7, 0x20), + BPF_ALU64_IMM(BPF_ARSH, 7, 0x20), BPF_MOV64_IMM(1, 1), + BPF_RAW_INSN(0x6d, 0x1, 0x7, 0x13, 0), + BPF_LDX_MEM(BPF_DW, 5, 10, -0x38), BPF_ALU64_IMM(BPF_AND, 5, 0x1f), - BPF_STX_MEM(BPF_DW, 10, 5, -0x28), + BPF_STX_MEM(BPF_DW, 10, 5, -0x38), - BPF_MOV64_REG(4, 10), BPF_ALU64_IMM(BPF_ADD, 4, -0x20), + BPF_MOV64_REG(4, 10), BPF_ALU64_IMM(BPF_ADD, 4, -0x30), BPF_MOV64_REG(1, 6), BPF_LD_IMM64_RAW_FULL(2, 1, 0, 0, perf_event_array_fd, 0), - BPF_MOV64_IMM(3, 0), BPF_EMIT_CALL(0x19), BPF_MOV64_IMM(0, 0), + BPF_LD_IMM64_RAW_FULL(3, 0, 0, 0, (__s32)0xffffffff, 0), + BPF_EMIT_CALL(0x19), + + BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x32727265, 0x64253d), + BPF_STX_MEM(BPF_DW, 10, 1, -0x48), BPF_MOV64_REG(1, 10), + BPF_ALU64_IMM(BPF_ADD, 1, -0x48), BPF_MOV64_IMM(2, 8), + BPF_MOV64_REG(3, 0), BPF_EMIT_CALL(6), BPF_MOV64_IMM(0, 0), BPF_EXIT_INSN(), // callback func - BPF_MOV64_REG(7, 2), BPF_MOV64_REG(6, 1), BPF_MOV64_IMM(8, 0), - BPF_MOV64_REG(1, 7), BPF_MOV64_IMM(2, 0x8), BPF_MOV64_REG(3, 6), - BPF_MOV64_IMM(4, 0), BPF_MOV64_IMM(5, 0), BPF_EMIT_CALL(0xc9), - BPF_RAW_INSN(0x6d, 0x8, 0, 8, 0), BPF_LDX_MEM(BPF_DW, 2, 7, 0), + BPF_MOV64_REG(7, 2), BPF_MOV64_REG(6, 1), + + BPF_MOV64_IMM(1, 0x62632064), BPF_STX_MEM(BPF_W, 10, 1, -8), + BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x67697274, 0x65726567), + BPF_STX_MEM(BPF_DW, 10, 1, -0x10), BPF_MOV64_IMM(8, 0), + BPF_STX_MEM(BPF_B, 10, 8, -4), BPF_MOV64_REG(1, 10), + BPF_ALU64_IMM(BPF_ADD, 1, -0x10), BPF_MOV64_IMM(2, 0xd), + BPF_EMIT_CALL(0x6), BPF_MOV64_REG(1, 7), BPF_MOV64_IMM(2, 0x8), + BPF_MOV64_REG(3, 6), BPF_MOV64_IMM(4, 0), BPF_MOV64_IMM(5, 0), + BPF_EMIT_CALL(0xc9), BPF_RAW_INSN(0x6d, 0x8, 0, 0x16, 0), + + BPF_STX_MEM(BPF_B, 10, 8, -0x16), BPF_MOV64_IMM(1, 0x646c), + BPF_STX_MEM(BPF_H, 10, 1, -0x18), + BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x7a697320, 0x25203a65), + BPF_STX_MEM(BPF_DW, 10, 1, -0x20), + + BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x65636552, 0x64657669), + BPF_STX_MEM(BPF_DW, 10, 1, -0x28), BPF_LDX_MEM(BPF_DW, 3, 7, 0), + BPF_MOV64_REG(1, 10), BPF_ALU64_IMM(BPF_ADD, 1, -0x28), + BPF_MOV64_IMM(2, 0x13), BPF_EMIT_CALL(0x6), + + BPF_LDX_MEM(BPF_DW, 2, 7, 0), BPF_RAW_INSN(0x25, 0x2, 0x0, 0x6, 0x20), BPF_ALU64_IMM(BPF_ADD, 7, 8), BPF_MOV64_REG(1, 7), BPF_MOV64_REG(3, 6), BPF_MOV64_IMM(4, 8), BPF_MOV64_IMM(5, 0), - BPF_EMIT_CALL(0xc9), BPF_MOV64_IMM(0, 1), BPF_EXIT_INSN() + BPF_EMIT_CALL(0xc9), BPF_RAW_INSN(0x6d, 8, 0, 7, 0), + BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x64206263, 0x656e6f), + BPF_STX_MEM(BPF_DW, 10, 1, -0x30), BPF_MOV64_REG(1, 10), + BPF_ALU64_IMM(BPF_ADD, 1, -0x30), BPF_MOV64_IMM(2, 8), + BPF_EMIT_CALL(0x6), + + BPF_MOV64_IMM(0, 1), BPF_EXIT_INSN() }; return std::vector((uint64_t *)&insns, (uint64_t *)&insns + std::size(insns)); } -int create_intervally_triggered_perf_event(int freq) +int create_intervally_triggered_perf_event(int duration_ms) { - perf_event_attr pe_attr = { - .type = PERF_TYPE_SOFTWARE, - .size = sizeof(struct perf_event_attr), - .config = PERF_COUNT_SW_CPU_CLOCK, - .sample_freq = (__u64)freq, - .freq = 1, - }; + perf_event_attr pe_attr; + memset(&pe_attr, 0, sizeof(pe_attr)); + pe_attr.type = PERF_TYPE_SOFTWARE; + pe_attr.size = sizeof(struct perf_event_attr); + pe_attr.config = PERF_COUNT_SW_CPU_CLOCK; + pe_attr.sample_period = (__u64)duration_ms * 1000; + pe_attr.sample_type = PERF_SAMPLE_RAW; + pe_attr.freq = 0; + return syscall(__NR_perf_event_open, &pe_attr, 0, -1, -1, 0); } } // namespace bpftime @@ -380,10 +436,10 @@ static int create_transporter_prog(int user_ringbuf_fd, int kernel_perf_fd) func_info.push_back( bpf_func_info{ .insn_off = 0, .type_id = (uint32_t)main_func }); func_info.push_back( - bpf_func_info{ .insn_off = 28, .type_id = (uint32_t)cb_func }); + bpf_func_info{ .insn_off = 60, .type_id = (uint32_t)cb_func }); LIBBPF_OPTS(bpf_prog_load_opts, prog_load_opts); - char log_buffer[4096]; + char log_buffer[8192]; prog_load_opts.log_buf = log_buffer; prog_load_opts.log_size = sizeof(log_buffer); prog_load_opts.log_level = 5; diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp index 070c8559..332c6f89 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.hpp @@ -23,7 +23,8 @@ class perf_event_array_kernel_user_impl { uint32_t max_ent; int kernel_perf_id; int user_rb_id; - + int pfd = -1; + int transporter_prog_fd = -1; public: const static bool should_lock = false; perf_event_array_kernel_user_impl( @@ -51,7 +52,7 @@ std::vector create_transporting_kernel_ebpf_program(int user_ringbuf_fd, int perf_event_array_fd); // Create an intervally triggered perf event -int create_intervally_triggered_perf_event(int freq); +int create_intervally_triggered_perf_event(int duration_ms); } // namespace bpftime #endif diff --git a/runtime/src/handler/handler_manager.hpp b/runtime/src/handler/handler_manager.hpp index d4143cdb..b838acee 100644 --- a/runtime/src/handler/handler_manager.hpp +++ b/runtime/src/handler/handler_manager.hpp @@ -37,7 +37,7 @@ using boost_shm_string = boost::interprocess::basic_string, char_allocator>; -const size_t DEFAULT_MAX_FD = 1024 * 4; +const size_t DEFAULT_MAX_FD = 1024 * 8; struct unused_handler {}; From c1d253007dc651d879efdb434bd9f38fc61a3d0a Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 21:19:12 +0800 Subject: [PATCH 11/16] final --- daemon/user/bpftime_driver.cpp | 1 - daemon/user/bpftime_driver.hpp | 2 - daemon/user/handle_bpf_event.cpp | 1 + .../shared/perf_event_array_kernel_user.cpp | 186 +++++++++++------- 4 files changed, 116 insertions(+), 74 deletions(-) diff --git a/daemon/user/bpftime_driver.cpp b/daemon/user/bpftime_driver.cpp index c8236464..cbc5657a 100644 --- a/daemon/user/bpftime_driver.cpp +++ b/daemon/user/bpftime_driver.cpp @@ -1,7 +1,6 @@ #include "bpftime_driver.hpp" #include #include -#include "ebpf-vm.h" #include "ebpf_inst.h" #include #include "bpftime_shm.hpp" diff --git a/daemon/user/bpftime_driver.hpp b/daemon/user/bpftime_driver.hpp index 7bf528ab..fa56a299 100644 --- a/daemon/user/bpftime_driver.hpp +++ b/daemon/user/bpftime_driver.hpp @@ -2,8 +2,6 @@ #define BPFTIME_DRIVER_HPP #include "daemon_config.hpp" -#include "bpftime_shm.hpp" -#include #include #include diff --git a/daemon/user/handle_bpf_event.cpp b/daemon/user/handle_bpf_event.cpp index e2817782..a0ca2635 100644 --- a/daemon/user/handle_bpf_event.cpp +++ b/daemon/user/handle_bpf_event.cpp @@ -9,6 +9,7 @@ #include #include "handle_bpf_event.hpp" #include "../bpf_tracer_event.h" +#include #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index d1782f20..7e2c274b 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -1,5 +1,6 @@ #include "bpf/bpf.h" #include "bpf/libbpf_common.h" +#include "bpftool/libbpf/include/uapi/linux/bpf.h" #include "bpftool/libbpf/src/btf.h" #include "bpftool/libbpf/src/libbpf.h" #include "linux/bpf_common.h" @@ -56,6 +57,13 @@ int perf_event_array_kernel_user_impl::get_user_ringbuf_fd() int perf_event_array_kernel_user_impl::output_data_into_kernel(const void *buf, size_t size) { + if (size > 480 - 32) { + spdlog::error( + "Max data size for shared perf event array is {} bytes", + 480 - 32); + errno = E2BIG; + return -1; + } spdlog::debug("Received data output for kernel perf event array {}", kernel_perf_id); auto user_rb = ensure_current_map_user_ringbuf(); @@ -300,84 +308,117 @@ create_transporting_kernel_ebpf_program(int user_ringbuf_fd, static_assert( sizeof(bpf_insn) == sizeof(uint64_t), "bpf_insn is expected to be in the same size of uint64_t"); +// r1 = map_by_fd(user_ringbuf_fd) +// BPF_LD_IMM64_RAW_FULL(1, 1, 0, 0, user_ringbuf_fd, 0), +#define ZERO(off) BPF_STX_MEM(BPF_DW, 10, 1, -off) bpf_insn insns[] = { + // r6 = r1 BPF_MOV64_REG(6, 1), - // bpf_printk("triggered") - BPF_MOV64_IMM(1, 0x64), BPF_STX_MEM(BPF_H, 10, 1, -8), - BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x67697274, 0x65726567), - BPF_STX_MEM(BPF_DW, 10, 1, -0x10), BPF_MOV64_REG(1, 10), - BPF_ALU64_IMM(BPF_ADD, 1, -0x10), BPF_MOV64_IMM(2, 0xa), - BPF_EMIT_CALL(0x6), - // Other - BPF_MOV64_IMM(8, 0), BPF_STX_MEM(BPF_DW, 10, 8, -0x18), - BPF_STX_MEM(BPF_DW, 10, 8, -0x20), - BPF_STX_MEM(BPF_DW, 10, 8, -0x28), - BPF_STX_MEM(BPF_DW, 10, 8, -0x30), - BPF_STX_MEM(BPF_DW, 10, 8, -0x38), BPF_MOV64_REG(3, 10), - BPF_ALU64_IMM(BPF_ADD, 3, -0x38), + // r1 = 0 + BPF_MOV64_IMM(1, 0), + // r10-0x8...=r10-0x1c8 := 0 + ZERO(0x8), ZERO(0x10), ZERO(0x18), ZERO(0x20), ZERO(0x28), + ZERO(0x30), ZERO(0x38), ZERO(0x40), ZERO(0x48), ZERO(0x50), + ZERO(0x58), ZERO(0x60), ZERO(0x68), ZERO(0x70), ZERO(0x78), + ZERO(0x80), ZERO(0x88), ZERO(0x90), ZERO(0x98), ZERO(0xa0), + ZERO(0xa8), ZERO(0xb0), ZERO(0xb8), ZERO(0xc0), ZERO(0xc8), + ZERO(0xd0), ZERO(0xd8), ZERO(0xe0), ZERO(0xe8), ZERO(0xf0), + ZERO(0xf8), ZERO(0x100), ZERO(0x108), ZERO(0x110), ZERO(0x118), + ZERO(0x120), ZERO(0x128), ZERO(0x130), ZERO(0x138), ZERO(0x140), + ZERO(0x148), ZERO(0x150), ZERO(0x158), ZERO(0x160), ZERO(0x168), + ZERO(0x170), ZERO(0x178), ZERO(0x180), ZERO(0x188), ZERO(0x190), + ZERO(0x198), ZERO(0x1a0), ZERO(0x1a8), ZERO(0x1b0), ZERO(0x1b8), + ZERO(0x1c0), ZERO(0x1c8), + // r3 = r10 + BPF_MOV64_REG(3, 10), + // r3 += -0x1e8 + BPF_ALU64_IMM(BPF_ADD, 3, -0x1c8), // r1 = map_by_fd(user_ringbuf_fd) BPF_LD_IMM64_RAW_FULL(1, 1, 0, 0, user_ringbuf_fd, 0), - // r2 = callback fn (+7) - BPF_LD_IMM64_RAW_FULL(2, 4, 0, 0, 39, 0), BPF_MOV64_IMM(4, 0), - BPF_EMIT_CALL(0xd1), BPF_MOV64_REG(7, 0), - BPF_MOV64_IMM(1, 0x6425), BPF_STX_MEM(BPF_H, 10, 1, -0x3c), - BPF_MOV64_IMM(1, 0x3d727265), BPF_STX_MEM(BPF_W, 10, 1, -0x40), - BPF_STX_MEM(BPF_B, 10, 8, -0x3a), BPF_MOV64_REG(1, 10), - BPF_ALU64_IMM(BPF_ADD, 1, -0x40), BPF_MOV64_IMM(2, 7), - BPF_MOV64_REG(3, 7), BPF_EMIT_CALL(6), - BPF_ALU64_IMM(BPF_LSH, 7, 0x20), - BPF_ALU64_IMM(BPF_ARSH, 7, 0x20), BPF_MOV64_IMM(1, 1), - BPF_RAW_INSN(0x6d, 0x1, 0x7, 0x13, 0), - BPF_LDX_MEM(BPF_DW, 5, 10, -0x38), - BPF_ALU64_IMM(BPF_AND, 5, 0x1f), - BPF_STX_MEM(BPF_DW, 10, 5, -0x38), - - BPF_MOV64_REG(4, 10), BPF_ALU64_IMM(BPF_ADD, 4, -0x30), + // r2 = code(callback) + BPF_LD_IMM64_RAW_FULL(2, 4, 0, 0, 22, 0), + // r4 = 0 + BPF_MOV64_IMM(4, 0), + // call 0xd1 (bpf_user_ringbuf_drain) + BPF_EMIT_CALL(0xd1), + // r0 <<= 0x20 + BPF_ALU64_IMM(BPF_LSH, 0, 0x20), + // r0 s>>= 0x20 + BPF_ALU64_IMM(BPF_ARSH, 0, 0x20), + // r1 = 1 + BPF_MOV64_IMM(1, 1), + // if r1 s> r0 goto +0xd + BPF_RAW_INSN(0x6d, 1, 0, 0xd, 0), + // r4 = r10 + BPF_MOV64_REG(4, 10), + // r4 += -0x1e0 + BPF_ALU64_IMM(BPF_ADD, 4, -0x1c0), + // r5 = *(u64 *)(r10 - 0x1e8) + BPF_LDX_MEM(BPF_DW, 5, 10, -0x1c8), + // r1 = 0x1e1 + BPF_MOV64_IMM(1, 0x1c1), + // if r1 > r5 goto +0x2 + BPF_RAW_INSN(0x2d, 1, 5, 2, 0), + // r5 = 0x1e0 + BPF_MOV64_IMM(5, 0x1c0), + // *(u64 *)(r10 - 0x1e8) = r5 + BPF_STX_MEM(BPF_DW, 10, 5, -0x1c8), + // r1 = r6 BPF_MOV64_REG(1, 6), + // r2 = map_by_fd(perf_event_array_fd) BPF_LD_IMM64_RAW_FULL(2, 1, 0, 0, perf_event_array_fd, 0), + // r3 = 0xffffffff BPF_LD_IMM64_RAW_FULL(3, 0, 0, 0, (__s32)0xffffffff, 0), + // call 0x19 bpf_perf_event_output BPF_EMIT_CALL(0x19), - - BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x32727265, 0x64253d), - BPF_STX_MEM(BPF_DW, 10, 1, -0x48), BPF_MOV64_REG(1, 10), - BPF_ALU64_IMM(BPF_ADD, 1, -0x48), BPF_MOV64_IMM(2, 8), - BPF_MOV64_REG(3, 0), BPF_EMIT_CALL(6), BPF_MOV64_IMM(0, 0), + // r0 = 0 + BPF_MOV64_IMM(0, 0), + // exit BPF_EXIT_INSN(), - // callback func - BPF_MOV64_REG(7, 2), BPF_MOV64_REG(6, 1), - - BPF_MOV64_IMM(1, 0x62632064), BPF_STX_MEM(BPF_W, 10, 1, -8), - BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x67697274, 0x65726567), - BPF_STX_MEM(BPF_DW, 10, 1, -0x10), BPF_MOV64_IMM(8, 0), - BPF_STX_MEM(BPF_B, 10, 8, -4), BPF_MOV64_REG(1, 10), - BPF_ALU64_IMM(BPF_ADD, 1, -0x10), BPF_MOV64_IMM(2, 0xd), - BPF_EMIT_CALL(0x6), BPF_MOV64_REG(1, 7), BPF_MOV64_IMM(2, 0x8), - BPF_MOV64_REG(3, 6), BPF_MOV64_IMM(4, 0), BPF_MOV64_IMM(5, 0), - BPF_EMIT_CALL(0xc9), BPF_RAW_INSN(0x6d, 0x8, 0, 0x16, 0), - - BPF_STX_MEM(BPF_B, 10, 8, -0x16), BPF_MOV64_IMM(1, 0x646c), - BPF_STX_MEM(BPF_H, 10, 1, -0x18), - BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x7a697320, 0x25203a65), - BPF_STX_MEM(BPF_DW, 10, 1, -0x20), - - BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x65636552, 0x64657669), - BPF_STX_MEM(BPF_DW, 10, 1, -0x28), BPF_LDX_MEM(BPF_DW, 3, 7, 0), - BPF_MOV64_REG(1, 10), BPF_ALU64_IMM(BPF_ADD, 1, -0x28), - BPF_MOV64_IMM(2, 0x13), BPF_EMIT_CALL(0x6), - + // callback function + // r7 = r2 + BPF_MOV64_REG(7, 2), + // r6 = r1 + BPF_MOV64_REG(6, 1), + // r8 = 0 + BPF_MOV64_IMM(8, 0), + // r1 = r7 + BPF_MOV64_REG(1, 7), + // r2 = 8 + BPF_MOV64_IMM(2, 8), + // r3 = r6 + BPF_MOV64_REG(3, 6), + // r4 = 0 + BPF_MOV64_IMM(4, 0), + // r5 = 0 + BPF_MOV64_IMM(5, 0), + // call 0xc9 bpf_dynptr_read + BPF_EMIT_CALL(0xc9), + // if r8 s> r0 goto +0x8 + BPF_RAW_INSN(0x6d, 8, 0, 8, 0), + // r2 = *(u64 *)(r7 + 0x0) BPF_LDX_MEM(BPF_DW, 2, 7, 0), - BPF_RAW_INSN(0x25, 0x2, 0x0, 0x6, 0x20), - BPF_ALU64_IMM(BPF_ADD, 7, 8), BPF_MOV64_REG(1, 7), - BPF_MOV64_REG(3, 6), BPF_MOV64_IMM(4, 8), BPF_MOV64_IMM(5, 0), - BPF_EMIT_CALL(0xc9), BPF_RAW_INSN(0x6d, 8, 0, 7, 0), - BPF_LD_IMM64_RAW_FULL(1, 0, 0, 0, 0x64206263, 0x656e6f), - BPF_STX_MEM(BPF_DW, 10, 1, -0x30), BPF_MOV64_REG(1, 10), - BPF_ALU64_IMM(BPF_ADD, 1, -0x30), BPF_MOV64_IMM(2, 8), - BPF_EMIT_CALL(0x6), - - BPF_MOV64_IMM(0, 1), BPF_EXIT_INSN() + // if r2 > 0x1e0 goto +0x6 + BPF_RAW_INSN(0x25, 2, 0, 6, 0x01c0), + // r7 += 0x8 + BPF_ALU64_IMM(BPF_ADD, 7, 8), + // r1 = r7 + BPF_MOV64_REG(1, 7), + // r3 = r6 + BPF_MOV64_REG(3, 6), + // r4 = 8 + BPF_MOV64_IMM(4, 8), + // r5 = 0 + BPF_MOV64_IMM(5, 0), + // call 0xc9 + BPF_EMIT_CALL(0xc9), + // r0 = 1 + BPF_MOV64_IMM(0, 1), + // exit + BPF_EXIT_INSN() }; +#undef ZERO return std::vector((uint64_t *)&insns, (uint64_t *)&insns + std::size(insns)); } @@ -436,12 +477,14 @@ static int create_transporter_prog(int user_ringbuf_fd, int kernel_perf_fd) func_info.push_back( bpf_func_info{ .insn_off = 0, .type_id = (uint32_t)main_func }); func_info.push_back( - bpf_func_info{ .insn_off = 60, .type_id = (uint32_t)cb_func }); + bpf_func_info{ .insn_off = 86, .type_id = (uint32_t)cb_func }); LIBBPF_OPTS(bpf_prog_load_opts, prog_load_opts); - char log_buffer[8192]; + // char log_buffer[8192]; + const size_t log_buffer_size = 1 << 20; + char *log_buffer = new char[log_buffer_size]; prog_load_opts.log_buf = log_buffer; - prog_load_opts.log_size = sizeof(log_buffer); + prog_load_opts.log_size = log_buffer_size; prog_load_opts.log_level = 5; prog_load_opts.fd_array = fds.data(); prog_load_opts.prog_btf_fd = btf_fd; @@ -452,10 +495,11 @@ static int create_transporter_prog(int user_ringbuf_fd, int kernel_perf_fd) int bpf_fd = bpf_prog_load(BPF_PROG_TYPE_PERF_EVENT, "transporter", "GPL", (bpf_insn *)prog.data(), prog.size(), &prog_load_opts); - + std::string log_message(log_buffer); + delete[] log_buffer; if (bpf_fd < 0) { spdlog::error("Failed to load bpf prog: err={}, message=\n{}", - errno, log_buffer); + errno, log_message); } return bpf_fd; } From 7a39d8d96fa3dc39e200a59f8d617d5948a1bb7f Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 21:20:55 +0800 Subject: [PATCH 12/16] PERF_EVENT_IOC_DISABLE --- runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp index 7e2c274b..44ee40f3 100644 --- a/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/perf_event_array_kernel_user.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -141,6 +142,7 @@ perf_event_array_kernel_user_impl::perf_event_array_kernel_user_impl( } perf_event_array_kernel_user_impl::~perf_event_array_kernel_user_impl() { + ioctl(pfd, PERF_EVENT_IOC_DISABLE, 0); close(pfd); close(transporter_prog_fd); } From 55457cb38292ccafc9d42c996b891de0b2baa1bd Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 22:22:36 +0800 Subject: [PATCH 13/16] Fix --- daemon/user/bpftime_driver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/user/bpftime_driver.cpp b/daemon/user/bpftime_driver.cpp index cbc5657a..84f4e5fb 100644 --- a/daemon/user/bpftime_driver.cpp +++ b/daemon/user/bpftime_driver.cpp @@ -177,7 +177,7 @@ int bpftime_driver::bpftime_progs_create_server(int kernel_id, int server_pid) kernel_id); return -1; } - res = bpftime_progs_create(fd, + res = bpftime_progs_create(kernel_id, buffer.data(), buffer.size(), info.name, info.type); if (res < 0) { @@ -276,7 +276,7 @@ int bpftime_driver::bpftime_attach_perf_to_bpf_server(int server_pid, } int res = bpftime_attach_perf_to_bpf( - perf_id, bpf_prog_get_fd_by_id(kernel_bpf_id)); + perf_id, kernel_bpf_id); if (res < 0) { spdlog::error("Failed to attach perf to bpf"); return -1; From c90a70f079b5e93340561fd595de8f0e756022dd Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 23:01:21 +0800 Subject: [PATCH 14/16] update DEFAULT_MAX_FD --- runtime/src/handler/handler_manager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/handler/handler_manager.hpp b/runtime/src/handler/handler_manager.hpp index b838acee..bd0e53b2 100644 --- a/runtime/src/handler/handler_manager.hpp +++ b/runtime/src/handler/handler_manager.hpp @@ -37,7 +37,7 @@ using boost_shm_string = boost::interprocess::basic_string, char_allocator>; -const size_t DEFAULT_MAX_FD = 1024 * 8; +const size_t DEFAULT_MAX_FD = 1024 * 6; struct unused_handler {}; From 4be7fe39d51a446112c306dcb1e29b7d9c2d326d Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 23:17:16 +0800 Subject: [PATCH 15/16] Revert some changes --- .../bpf_map/shared/array_map_kernel_user.cpp | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/runtime/src/bpf_map/shared/array_map_kernel_user.cpp b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp index 1a7dbbca..936e8990 100644 --- a/runtime/src/bpf_map/shared/array_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp @@ -62,19 +62,19 @@ void array_map_kernel_user_impl::init_map_fd() return; } // What does this piece of code do? - // int prot; - // if (info.map_flags & BPF_F_RDONLY_PROG) - // prot = PROT_READ; - // else - // prot = PROT_READ | PROT_WRITE; - // void *mmaped = mmap(mmap_ptr, mmap_sz, prot, MAP_SHARED | MAP_FIXED, - // map_fd, 0); - // if (mmaped == MAP_FAILED) { - // spdlog::error("Failed to mmap for kernel map id {}, err={}", - // kernel_map_id, errno); - // return; - // } - // mmap_ptr = mmaped; + int prot; + if (info.map_flags & BPF_F_RDONLY_PROG) + prot = PROT_READ; + else + prot = PROT_READ | PROT_WRITE; + void *mmaped = mmap(mmap_ptr, mmap_sz, prot, MAP_SHARED | MAP_FIXED, + map_fd, 0); + if (mmaped == MAP_FAILED) { + spdlog::error("Failed to mmap for kernel map id {}, err={}", + kernel_map_id, errno); + return; + } + mmap_ptr = mmaped; } array_map_kernel_user_impl::array_map_kernel_user_impl( From 91df95d727035e0a96553b24484765b311eed84f Mon Sep 17 00:00:00 2001 From: officeyutong Date: Sun, 5 Nov 2023 23:32:21 +0800 Subject: [PATCH 16/16] Update changes --- runtime/src/bpf_map/shared/array_map_kernel_user.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/bpf_map/shared/array_map_kernel_user.cpp b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp index 936e8990..cc56e7cb 100644 --- a/runtime/src/bpf_map/shared/array_map_kernel_user.cpp +++ b/runtime/src/bpf_map/shared/array_map_kernel_user.cpp @@ -15,7 +15,7 @@ namespace bpftime { -size_t bpf_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) +static size_t bpf_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) { const long page_sz = sysconf(_SC_PAGE_SIZE); size_t map_sz;