diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index 6b992332..e4c851be 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -81,6 +81,10 @@ jobs: enable: false examples: path: tailcall_minimal + - privilege_options: + enable: false + examples: + path: malloc privilege_options: - options: "--privileged -v /sys/kernel/debug/:/sys/kernel/debug:rw -v /sys/kernel/tracing:/sys/kernel/tracing:rw" enable: true @@ -208,3 +212,9 @@ jobs: ROOT=$(pwd) cd example/${{matrix.examples.path}} python3 $ROOT/.github/script/run_example.py "${{matrix.examples.executable}}" "${{matrix.examples.victim}}" "${{matrix.examples.expected_str}}" "/github/home/.bpftime/bpftime -i /github/home/.bpftime" 0 + - name: Setup tmate session + # Setup SSH when manually triggered and failing, so we can debug CI more conveniently + if: "${{ failure() && github.event_name == 'workflow_dispatch' }}" + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/.github/workflows/test-runtime.yml b/.github/workflows/test-runtime.yml index 769d117c..dd50f95c 100644 --- a/.github/workflows/test-runtime.yml +++ b/.github/workflows/test-runtime.yml @@ -24,6 +24,11 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'recursive' + - name: Remount shm dev + # The size of /dev/shm defaults to be 64M, but boost won't detect this at all, leaving bus error to us.. + # So we remount it to make it larger + run: | + mount -o remount,size=1G /dev/shm - name: Install lcov if: "matrix.container == 'ubuntu-2204'" run: | @@ -39,6 +44,7 @@ jobs: cmake --build build --config Debug --target bpftime_runtime_tests -j$(nproc) - name: Test Runtime run: ./build/runtime/unit-test/bpftime_runtime_tests + - name: Generate runtime coverage (Ubuntu) if: "matrix.container == 'ubuntu-2204'" run: | diff --git a/attach/frida_uprobe_attach_impl/CMakeLists.txt b/attach/frida_uprobe_attach_impl/CMakeLists.txt index ccb44b4c..f211cce8 100644 --- a/attach/frida_uprobe_attach_impl/CMakeLists.txt +++ b/attach/frida_uprobe_attach_impl/CMakeLists.txt @@ -27,7 +27,7 @@ option(TEST_LCOV "option for lcov" OFF) add_executable(bpftime_frida_uprobe_attach_tests ${TEST_SOURCES}) if (${TEST_LCOV}) - target_compile_options(bpftime_frida_uprobe_attach_tests PRIVATE -fprofile-arcs -ftest-coverage) + target_compile_options(bpftime_frida_uprobe_attach_tests PRIVATE -fprofile-arcs -ftest-coverage -fprofile-update=atomic) endif() if(${ENABLE_EBPF_VERIFIER} AND NOT TARGET Catch2) diff --git a/attach/syscall_trace_attach_impl/CMakeLists.txt b/attach/syscall_trace_attach_impl/CMakeLists.txt index 2241cab2..05e53ea4 100644 --- a/attach/syscall_trace_attach_impl/CMakeLists.txt +++ b/attach/syscall_trace_attach_impl/CMakeLists.txt @@ -32,7 +32,7 @@ option(TEST_LCOV "option for lcov" OFF) add_executable(bpftime_syscall_trace_attach_tests ${TEST_SOURCES}) if (${TEST_LCOV}) - target_compile_options(bpftime_syscall_trace_attach_tests PRIVATE -fprofile-arcs -ftest-coverage) + target_compile_options(bpftime_syscall_trace_attach_tests PRIVATE -fprofile-arcs -ftest-coverage -fprofile-update=atomic) endif() if(${ENABLE_EBPF_VERIFIER} AND NOT TARGET Catch2) diff --git a/bpftime-verifier/CMakeLists.txt b/bpftime-verifier/CMakeLists.txt index 5ab5be9e..6d21b60a 100644 --- a/bpftime-verifier/CMakeLists.txt +++ b/bpftime-verifier/CMakeLists.txt @@ -38,7 +38,7 @@ option(TEST_LCOV "option for lcov" OFF) add_executable(bpftime_verifier_tests ${TEST_SOURCES}) if (${TEST_LCOV}) - target_compile_options(bpftime_verifier_tests PRIVATE -fprofile-arcs -ftest-coverage) + target_compile_options(bpftime_verifier_tests PRIVATE -fprofile-arcs -ftest-coverage -fprofile-update=atomic) endif() add_dependencies(bpftime_verifier_tests bpftime-verifier) diff --git a/runtime/src/bpf_map/map_common_def.hpp b/runtime/src/bpf_map/map_common_def.hpp index 5fbce558..2dad56ce 100644 --- a/runtime/src/bpf_map/map_common_def.hpp +++ b/runtime/src/bpf_map/map_common_def.hpp @@ -68,6 +68,16 @@ struct bytes_vec_hasher { return seed; } }; + +static inline bool check_update_flags(uint64_t flags) +{ + if (flags != 0 /*BPF_ANY*/ && flags != 1 /*BPF_NOEXIST*/ && + flags != 2 /*BPF_EXIST*/) { + errno = EINVAL; + return false; + } + return true; +} } // namespace bpftime #endif diff --git a/runtime/src/bpf_map/userspace/array_map.cpp b/runtime/src/bpf_map/userspace/array_map.cpp index abfd4896..30143990 100644 --- a/runtime/src/bpf_map/userspace/array_map.cpp +++ b/runtime/src/bpf_map/userspace/array_map.cpp @@ -3,6 +3,8 @@ * Copyright (c) 2022, eunomia-bpf org * All rights reserved. */ +#include "bpf_map/map_common_def.hpp" +#include "linux/bpf.h" #include #include @@ -34,9 +36,16 @@ void *array_map_impl::elem_lookup(const void *key) long array_map_impl::elem_update(const void *key, const void *value, uint64_t flags) { + if (!check_update_flags(flags)) + return -1; auto key_val = *(uint32_t *)key; + if (key_val < _max_entries && flags == BPF_NOEXIST) { + errno = EEXIST; + return -1; + } + if (key_val >= _max_entries) { - errno = ENOENT; + errno = E2BIG; return -1; } std::copy((uint8_t *)value, (uint8_t *)value + _value_size, @@ -47,13 +56,16 @@ long array_map_impl::elem_update(const void *key, const void *value, long array_map_impl::elem_delete(const void *key) { auto key_val = *(uint32_t *)key; - if (key_val >= _max_entries) { - errno = ENOENT; - return -1; - } - std::fill(&data[key_val * _value_size], - &data[key_val * _value_size] + _value_size, 0); - return 0; + // kernel tests says element in an array map can't be deleted... + errno = EINVAL; + return -1; + // if (key_val >= _max_entries) { + // errno = ENOENT; + // return -1; + // } + // std::fill(&data[key_val * _value_size], + // &data[key_val * _value_size] + _value_size, 0); + // return 0; } int array_map_impl::map_get_next_key(const void *key, void *next_key) diff --git a/runtime/src/bpf_map/userspace/per_cpu_array_map.cpp b/runtime/src/bpf_map/userspace/per_cpu_array_map.cpp index b4114f9a..55f21cbd 100644 --- a/runtime/src/bpf_map/userspace/per_cpu_array_map.cpp +++ b/runtime/src/bpf_map/userspace/per_cpu_array_map.cpp @@ -4,6 +4,7 @@ * All rights reserved. */ #include "bpf_map/map_common_def.hpp" +#include "linux/bpf.h" #include "spdlog/spdlog.h" #include #include @@ -39,7 +40,7 @@ void *per_cpu_array_map_impl::elem_lookup(const void *key) } uint32_t key_val = *(uint32_t *)key; if (key_val >= max_ent) { - errno = E2BIG; + errno = ENOENT; return nullptr; } return data_at(key_val, cpu); @@ -49,6 +50,8 @@ void *per_cpu_array_map_impl::elem_lookup(const void *key) long per_cpu_array_map_impl::elem_update(const void *key, const void *value, uint64_t flags) { + if (!check_update_flags(flags)) + return -1; return ensure_on_current_cpu([&](int cpu) -> long { // return impl[cpu].elem_update(key, value, flags); if (key == nullptr) { @@ -56,6 +59,10 @@ long per_cpu_array_map_impl::elem_update(const void *key, const void *value, return -1; } uint32_t key_val = *(uint32_t *)key; + if (key_val < max_ent && flags == BPF_NOEXIST) { + errno = EEXIST; + return -1; + } if (key_val >= max_ent) { errno = E2BIG; return -1; @@ -68,13 +75,12 @@ long per_cpu_array_map_impl::elem_update(const void *key, const void *value, long per_cpu_array_map_impl::elem_delete(const void *key) { - errno = ENOTSUP; + errno = EINVAL; SPDLOG_DEBUG("Deleting of per cpu array is not supported"); return -1; } -int per_cpu_array_map_impl::map_get_next_key(const void *key, - void *next_key) +int per_cpu_array_map_impl::map_get_next_key(const void *key, void *next_key) { // Not found if (key == nullptr || *(uint32_t *)key >= max_ent) { @@ -100,7 +106,7 @@ void *per_cpu_array_map_impl::elem_lookup_userspace(const void *key) } uint32_t key_val = *(uint32_t *)key; if (key_val >= max_ent) { - errno = E2BIG; + errno = ENOENT; return nullptr; } return data_at(key_val, 0); @@ -110,11 +116,17 @@ long per_cpu_array_map_impl::elem_update_userspace(const void *key, const void *value, uint64_t flags) { + if (!check_update_flags(flags)) + return -1; if (key == nullptr) { errno = ENOENT; return -1; } uint32_t key_val = *(uint32_t *)key; + if (key_val < max_ent && flags == BPF_NOEXIST) { + errno = EEXIST; + return -1; + } if (key_val >= max_ent) { errno = E2BIG; return -1; @@ -126,8 +138,8 @@ long per_cpu_array_map_impl::elem_update_userspace(const void *key, long per_cpu_array_map_impl::elem_delete_userspace(const void *key) { - errno = ENOTSUP; - SPDLOG_ERROR("Element delete is not supported by per cpu array"); + errno = EINVAL; + SPDLOG_WARN("Element delete is not supported by per cpu array"); return -1; } diff --git a/runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp b/runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp index 8b19bc1f..a08197b6 100644 --- a/runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp +++ b/runtime/src/bpf_map/userspace/per_cpu_hash_map.cpp @@ -4,28 +4,32 @@ * All rights reserved. */ #include "bpf_map/map_common_def.hpp" +#include "linux/bpf.h" #include "spdlog/fmt/bin_to_hex.h" #include "spdlog/spdlog.h" #include #include +#include +#include #include #include "platform_utils.hpp" + namespace bpftime { per_cpu_hash_map_impl::per_cpu_hash_map_impl( boost::interprocess::managed_shared_memory &memory, uint32_t key_size, - uint32_t value_size) - : per_cpu_hash_map_impl(memory, key_size, value_size, + uint32_t value_size, uint32_t max_entries) + : per_cpu_hash_map_impl(memory, key_size, value_size, max_entries, sysconf(_SC_NPROCESSORS_ONLN)) { } per_cpu_hash_map_impl::per_cpu_hash_map_impl( boost::interprocess::managed_shared_memory &memory, uint32_t key_size, - uint32_t value_size, int ncpu) + uint32_t value_size, uint32_t max_entries, int ncpu) : impl(memory.get_segment_manager()), key_size(key_size), - value_size(value_size), ncpu(ncpu), + value_size(value_size), ncpu(ncpu), _max_entries(max_entries), value_template(value_size * ncpu, memory.get_segment_manager()), key_templates(memory.get_segment_manager()), single_value_templates(memory.get_segment_manager()) @@ -64,6 +68,8 @@ void *per_cpu_hash_map_impl::elem_lookup(const void *key) long per_cpu_hash_map_impl::elem_update(const void *key, const void *value, uint64_t flags) { + if (!check_update_flags(flags)) + return -1; int cpu = my_sched_getcpu(); SPDLOG_DEBUG("Per cpu update, key {}, value {}", (const char *)key, *(long *)value); @@ -115,7 +121,7 @@ int per_cpu_hash_map_impl::map_get_next_key(const void *key, void *next_key) } // No need to be allocated at shm. Allocate as a local variable to make // it thread safe, since we use sharable lock - bytes_vec key_vec = this->key_templates[0]; + bytes_vec &key_vec = this->key_templates[0]; key_vec.assign((uint8_t *)key, (uint8_t *)key + key_size); auto itr = impl.find(key_vec); @@ -139,7 +145,7 @@ void *per_cpu_hash_map_impl::elem_lookup_userspace(const void *key) errno = ENOENT; return nullptr; } - bytes_vec key_vec = this->key_templates[0]; + bytes_vec &key_vec = this->key_templates[0]; key_vec.assign((uint8_t *)key, (uint8_t *)key + key_size); if (auto itr = impl.find(key_vec); itr != impl.end()) { SPDLOG_TRACE("Exit elem lookup of hash map: {}", @@ -157,24 +163,54 @@ long per_cpu_hash_map_impl::elem_update_userspace(const void *key, const void *value, uint64_t flags) { - bytes_vec key_vec = this->key_templates[0]; + if (!check_update_flags(flags)) + return -1; + bytes_vec &key_vec = this->key_templates[0]; bytes_vec value_vec = this->value_template; key_vec.assign((uint8_t *)key, (uint8_t *)key + key_size); value_vec.assign((uint8_t *)value, (uint8_t *)value + value_size * ncpu); - - if (auto itr = impl.find(key_vec); itr != impl.end()) { - itr->second = value_vec; - } else { - impl.insert(bi_map_value_ty(key_vec, value_vec)); + bool elem_exists = impl.find(key_vec) != impl.end(); + if (flags == BPF_NOEXIST && elem_exists) { + errno = EEXIST; + return -1; } + if (flags == BPF_EXIST && !elem_exists) { + errno = ENOENT; + return -1; + } + if (elem_exists == false && impl.size() == _max_entries) { + errno = E2BIG; + return -1; + } + impl.insert_or_assign(key_vec, value_vec); return 0; } long per_cpu_hash_map_impl::elem_delete_userspace(const void *key) { - bytes_vec key_vec = this->key_templates[0]; + bytes_vec &key_vec = this->key_templates[0]; + key_vec.assign((uint8_t *)key, (uint8_t *)key + key_size); + auto itr = impl.find(key_vec); + if (itr == impl.end()) { + errno = ENOENT; + return -1; + } + impl.erase(itr); + return 0; +} + +long per_cpu_hash_map_impl::lookup_and_delete_userspace(const void *key, + void *value) +{ + bytes_vec &key_vec = this->key_templates[0]; key_vec.assign((uint8_t *)key, (uint8_t *)key + key_size); - impl.erase(key_vec); + auto itr = this->impl.find(key_vec); + if (itr == impl.end()) { + errno = ENOENT; + return -1; + } + memcpy(value, itr->second.data(), ncpu * value_size); + impl.erase(itr); return 0; } } // namespace bpftime diff --git a/runtime/src/bpf_map/userspace/per_cpu_hash_map.hpp b/runtime/src/bpf_map/userspace/per_cpu_hash_map.hpp index 8cee8ae7..420ac515 100644 --- a/runtime/src/bpf_map/userspace/per_cpu_hash_map.hpp +++ b/runtime/src/bpf_map/userspace/per_cpu_hash_map.hpp @@ -27,27 +27,36 @@ class per_cpu_hash_map_impl { std::equal_to, bi_map_allocator>; using shm_hash_map_vec_allocator = boost::interprocess::allocator< - shm_hash_map, boost::interprocess::managed_shared_memory::segment_manager>; - using shm_hash_map_vec = boost::interprocess::vector; - + shm_hash_map, + boost::interprocess::managed_shared_memory::segment_manager>; + using shm_hash_map_vec = + boost::interprocess::vector; + using bytes_vec_vec_allocator = boost::interprocess::allocator< - bytes_vec, boost::interprocess::managed_shared_memory::segment_manager>; - using bytes_vec_vec = boost::interprocess::vector; + bytes_vec, + boost::interprocess::managed_shared_memory::segment_manager>; + using bytes_vec_vec = + boost::interprocess::vector; shm_hash_map impl; uint32_t key_size; uint32_t value_size; int ncpu; + uint32_t _max_entries; bytes_vec value_template; bytes_vec_vec key_templates, single_value_templates; + public: const static bool should_lock = false; per_cpu_hash_map_impl(boost::interprocess::managed_shared_memory &memory, - uint32_t key_size, uint32_t value_size); + uint32_t key_size, uint32_t value_size, + uint32_t max_entries); per_cpu_hash_map_impl(boost::interprocess::managed_shared_memory &memory, - uint32_t key_size, uint32_t value_size, int ncpu); + uint32_t key_size, uint32_t value_size, + uint32_t max_entries, int ncpu); void *elem_lookup(const void *key); long elem_update(const void *key, const void *value, uint64_t flags); @@ -62,6 +71,15 @@ class per_cpu_hash_map_impl { uint64_t flags); long elem_delete_userspace(const void *key); + long lookup_and_delete_userspace(const void *key, void *value); + uint32_t get_value_size() const + { + return value_size; + } + int getncpu() const + { + return ncpu; + } }; } // namespace bpftime diff --git a/runtime/src/bpf_map/userspace/var_hash_map.cpp b/runtime/src/bpf_map/userspace/var_hash_map.cpp index abfdcd61..c9e8e8a5 100644 --- a/runtime/src/bpf_map/userspace/var_hash_map.cpp +++ b/runtime/src/bpf_map/userspace/var_hash_map.cpp @@ -3,21 +3,32 @@ * Copyright (c) 2022, eunomia-bpf org * All rights reserved. */ +#include "linux/bpf.h" #include "spdlog/spdlog.h" #include #include +#include +#include #include #include +static bool is_good_update_flag(uint64_t flags) +{ + return flags == BPF_ANY || flags == BPF_EXIST || flags == BPF_NOEXIST; +} + namespace bpftime { var_size_hash_map_impl::var_size_hash_map_impl(managed_shared_memory &memory, uint32_t key_size, - uint32_t value_size) - : map_impl(10, bytes_vec_hasher(), std::equal_to(), + uint32_t value_size, + uint32_t max_entries, + uint32_t flags) + : map_impl(max_entries, bytes_vec_hasher(), std::equal_to(), bi_map_allocator(memory.get_segment_manager())), _key_size(key_size), _value_size(value_size), + _max_entries(max_entries), flags(flags), key_vec(key_size, memory.get_segment_manager()), value_vec(value_size, memory.get_segment_manager()) { @@ -25,6 +36,10 @@ var_size_hash_map_impl::var_size_hash_map_impl(managed_shared_memory &memory, void *var_size_hash_map_impl::elem_lookup(const void *key) { + if (flags & BPF_F_WRONLY) { + errno = EPERM; + return nullptr; + } SPDLOG_TRACE("Peform elem lookup of hash map"); // Since we use lock here, we don't need to allocate key_vec and // value_vec @@ -42,8 +57,32 @@ void *var_size_hash_map_impl::elem_lookup(const void *key) long var_size_hash_map_impl::elem_update(const void *key, const void *value, uint64_t flags) { + // Check flags + if (!is_good_update_flag(flags)) { + errno = EINVAL; + return -1; + } key_vec.assign((uint8_t *)key, (uint8_t *)key + _key_size); value_vec.assign((uint8_t *)value, (uint8_t *)value + _value_size); + bool element_exists = map_impl.find(key_vec) != map_impl.end(); + // Check if the element exists... + if (flags == BPF_NOEXIST && element_exists) { + errno = EEXIST; + return -1; + } + if (flags == BPF_EXIST && !element_exists) { + errno = ENOENT; + return -1; + } + // Ensure max_entries + if (element_exists == false && map_impl.size() == _max_entries) { + errno = E2BIG; + return -1; + } + if (element_exists == false && (this->flags & BPF_F_RDONLY)) { + errno = EPERM; + return -1; + } map_impl.insert_or_assign(key_vec, value_vec); return 0; } @@ -51,12 +90,32 @@ long var_size_hash_map_impl::elem_update(const void *key, const void *value, long var_size_hash_map_impl::elem_delete(const void *key) { key_vec.assign((uint8_t *)key, (uint8_t *)key + _key_size); - map_impl.erase(key_vec); + auto itr = map_impl.find(key_vec); + if (itr == map_impl.end()) { + errno = ENOENT; + return -1; + } + map_impl.erase(itr); + return 0; +} +int var_size_hash_map_impl::lookup_and_delete(const void *key, void *value_out) +{ + key_vec.assign((uint8_t *)key, (uint8_t *)key + _key_size); + auto itr = map_impl.find(key_vec); + if (itr == map_impl.end()) { + errno = ENOENT; + return -1; + } + memcpy(value_out, &itr->second[0], _value_size); + map_impl.erase(itr); return 0; } - int var_size_hash_map_impl::map_get_next_key(const void *key, void *next_key) { + if (flags & BPF_F_WRONLY) { + errno = EPERM; + return -1; + } if (key == nullptr) { // nullptr means the first key auto itr = map_impl.begin(); diff --git a/runtime/src/bpf_map/userspace/var_hash_map.hpp b/runtime/src/bpf_map/userspace/var_hash_map.hpp index 456cf266..af469ad5 100644 --- a/runtime/src/bpf_map/userspace/var_hash_map.hpp +++ b/runtime/src/bpf_map/userspace/var_hash_map.hpp @@ -30,7 +30,8 @@ class var_size_hash_map_impl { shm_hash_map map_impl; uint32_t _key_size; uint32_t _value_size; - + uint32_t _max_entries; + uint32_t flags; // buffers used to access the key and value in hash map bytes_vec key_vec; bytes_vec value_vec; @@ -38,7 +39,8 @@ class var_size_hash_map_impl { public: const static bool should_lock = true; var_size_hash_map_impl(managed_shared_memory &memory, uint32_t key_size, - uint32_t value_size); + uint32_t value_size, uint32_t max_entries, + uint32_t flags); void *elem_lookup(const void *key); @@ -47,6 +49,12 @@ class var_size_hash_map_impl { long elem_delete(const void *key); int map_get_next_key(const void *key, void *next_key); + + int lookup_and_delete(const void *key, void *value_out); + uint32_t get_value_size() const + { + return _value_size; + } }; } // namespace bpftime diff --git a/runtime/src/handler/map_handler.cpp b/runtime/src/handler/map_handler.cpp index 8224d2fe..f542e3c3 100644 --- a/runtime/src/handler/map_handler.cpp +++ b/runtime/src/handler/map_handler.cpp @@ -119,7 +119,7 @@ const void *bpf_map_handler::map_lookup_elem(const void *key, return from_syscall ? do_lookup_userspace(impl) : do_lookup(impl); } - #ifdef BPFTIME_BUILD_WITH_LIBBPF +#ifdef BPFTIME_BUILD_WITH_LIBBPF case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_ARRAY: { auto impl = static_cast( map_impl_ptr.get()); @@ -145,7 +145,7 @@ const void *bpf_map_handler::map_lookup_elem(const void *key, static_cast(map_impl_ptr.get()); return do_lookup(impl); } - #endif +#endif case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { auto impl = static_cast( map_impl_ptr.get()); @@ -213,7 +213,7 @@ long bpf_map_handler::map_update_elem(const void *key, const void *value, return from_syscall ? do_update_userspace(impl) : do_update(impl); } - #ifdef BPFTIME_BUILD_WITH_LIBBPF +#ifdef BPFTIME_BUILD_WITH_LIBBPF case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_ARRAY: { auto impl = static_cast( map_impl_ptr.get()); @@ -239,7 +239,7 @@ long bpf_map_handler::map_update_elem(const void *key, const void *value, static_cast(map_impl_ptr.get()); return do_update(impl); } - #endif +#endif case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { if (!from_syscall) { // Map in maps only support update from syscall @@ -300,7 +300,7 @@ int bpf_map_handler::bpf_map_get_next_key(const void *key, void *next_key, map_impl_ptr.get()); return do_get_next_key(impl); } - #if __linux__ && defined(BPFTIME_BUILD_WITH_LIBBPF) +#if __linux__ && defined(BPFTIME_BUILD_WITH_LIBBPF) case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_ARRAY: { auto impl = static_cast( map_impl_ptr.get()); @@ -326,7 +326,7 @@ int bpf_map_handler::bpf_map_get_next_key(const void *key, void *next_key, static_cast(map_impl_ptr.get()); return do_get_next_key(impl); } - #endif +#endif case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { auto impl = static_cast( map_impl_ptr.get()); @@ -394,7 +394,7 @@ long bpf_map_handler::map_delete_elem(const void *key, bool from_syscall) const return from_syscall ? do_delete_userspace(impl) : do_delete(impl); } - #ifdef BPFTIME_BUILD_WITH_LIBBPF +#ifdef BPFTIME_BUILD_WITH_LIBBPF case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_ARRAY: { auto impl = static_cast( map_impl_ptr.get()); @@ -420,7 +420,7 @@ long bpf_map_handler::map_delete_elem(const void *key, bool from_syscall) const static_cast(map_impl_ptr.get()); return do_delete(impl); } - #endif +#endif case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { if (!from_syscall) { // Map in maps only support update from syscall @@ -447,8 +447,9 @@ int bpf_map_handler::map_init(managed_shared_memory &memory) auto container_name = get_container_name(); switch (type) { case bpf_map_type::BPF_MAP_TYPE_HASH: { - map_impl_ptr = memory.construct( - container_name.c_str())(memory, max_entries, key_size, value_size); + map_impl_ptr = + memory.construct(container_name.c_str())( + memory, max_entries, key_size, value_size); return 0; } case bpf_map_type::BPF_MAP_TYPE_ARRAY: { @@ -488,10 +489,11 @@ int bpf_map_handler::map_init(managed_shared_memory &memory) } case bpf_map_type::BPF_MAP_TYPE_PERCPU_HASH: { map_impl_ptr = memory.construct( - container_name.c_str())(memory, key_size, value_size); + container_name.c_str())(memory, key_size, value_size, + max_entries); return 0; } - #ifdef BPFTIME_BUILD_WITH_LIBBPF +#ifdef BPFTIME_BUILD_WITH_LIBBPF case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_ARRAY: { map_impl_ptr = memory.construct( container_name.c_str())(memory, attr.kernel_bpf_map_id); @@ -523,7 +525,7 @@ int bpf_map_handler::map_init(managed_shared_memory &memory) max_entries); return 0; } - #endif +#endif case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { map_impl_ptr = memory.construct( container_name.c_str())(memory, max_entries); @@ -570,7 +572,7 @@ void bpf_map_handler::map_free(managed_shared_memory &memory) case bpf_map_type::BPF_MAP_TYPE_PERCPU_HASH: memory.destroy(container_name.c_str()); break; - #ifdef BPFTIME_BUILD_WITH_LIBBPF +#ifdef BPFTIME_BUILD_WITH_LIBBPF case bpf_map_type::BPF_MAP_TYPE_KERNEL_USER_ARRAY: memory.destroy( container_name.c_str()); @@ -590,7 +592,7 @@ void bpf_map_handler::map_free(managed_shared_memory &memory) case bpf_map_type::BPF_MAP_TYPE_PROG_ARRAY: memory.destroy(container_name.c_str()); break; - #endif +#endif default: auto func_ptr = global_map_ops_table[(int)type].map_free; if (func_ptr) { diff --git a/runtime/unit-test/CMakeLists.txt b/runtime/unit-test/CMakeLists.txt index 6f7bf94c..b84a3d9a 100644 --- a/runtime/unit-test/CMakeLists.txt +++ b/runtime/unit-test/CMakeLists.txt @@ -19,7 +19,8 @@ set(TEST_SOURCES maps/test_shm_hash_maps.cpp maps/test_external_map_ops.cpp maps/test_bpftime_hash_map.cpp - + maps/kernel_unit_tests.cpp + test_bpftime_shm_json.cpp attach_with_ebpf/test_attach_filter_with_ebpf.cpp attach_with_ebpf/test_attach_uprobe_with_ebpf.cpp @@ -35,7 +36,7 @@ option(TEST_LCOV "option for lcov" OFF) add_executable(bpftime_runtime_tests ${TEST_SOURCES}) if (${TEST_LCOV}) - target_compile_options(bpftime_runtime_tests PRIVATE -fprofile-arcs -ftest-coverage) + target_compile_options(bpftime_runtime_tests PRIVATE -fprofile-arcs -ftest-coverage -fprofile-update=atomic) endif() set_property(TARGET bpftime_runtime_tests PROPERTY CXX_STANDARD 20) diff --git a/runtime/unit-test/maps/kernel_unit_tests.cpp b/runtime/unit-test/maps/kernel_unit_tests.cpp new file mode 100644 index 00000000..46c0cb0d --- /dev/null +++ b/runtime/unit-test/maps/kernel_unit_tests.cpp @@ -0,0 +1,805 @@ +#include "bpf_map/userspace/array_map.hpp" +#include "bpf_map/userspace/per_cpu_array_map.hpp" +#include "bpf_map/userspace/per_cpu_hash_map.hpp" +#include "catch2/catch_test_macros.hpp" +#include "linux/bpf.h" +#include "spdlog/spdlog.h" +#include "unit-test/common_def.hpp" +#include +#include +#include +#include +#include "bpf_map/userspace/var_hash_map.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +static const char *SHM_NAME = "_HASH_MAP_TEST"; + +using namespace boost::interprocess; +using namespace bpftime; + +struct shm_initializer { + std::unique_ptr remover; + std::unique_ptr mem; + shm_initializer(const shm_initializer &) = delete; + shm_initializer &operator=(const shm_initializer &) = delete; + shm_initializer(const char *memory_name, bool should_create, + size_t memory_size = 20 << 20) + { + if (should_create) { + remover = std::make_unique(memory_name); + mem = std::make_unique( + boost::interprocess::create_only, memory_name, + memory_size); + } else { + mem = std::make_unique( + boost::interprocess::open_only, memory_name); + } + } +}; + +static void test_hashmap(std::string memory_name = SHM_NAME, + bool should_create_shm = true) +{ + shm_initializer shm(memory_name.c_str(), should_create_shm); + auto &mem = *shm.mem; + + long long key, next_key, first_key, value; + var_size_hash_map_impl map(mem, sizeof(key), sizeof(value), 2, 0); + key = 1; + value = 1234; + /* Insert key=1 element. */ + REQUIRE(map.elem_update(&key, &value, BPF_ANY) == 0); + + value = 0; + /* BPF_NOEXIST means add new element if it doesn't exist. */ + REQUIRE(map.elem_update(&key, &value, BPF_NOEXIST) < 0); + REQUIRE(/* key=1 already exists. */ errno == EEXIST); + + /* -1 is an invalid flag. */ + REQUIRE(map.elem_update(&key, &value, -1) < 0); + REQUIRE(errno == EINVAL); + + /* Check that key=1 can be found. */ + { + auto ptr = map.elem_lookup(&key); + REQUIRE(ptr != nullptr); + REQUIRE(*(long long *)ptr == 1234); + } + + key = 2; + value = 1234; + /* Insert key=2 element. */ + REQUIRE(map.elem_update(&key, &value, BPF_ANY) == 0); + + /* Check that key=2 matches the value and delete it */ + REQUIRE(map.lookup_and_delete(&key, &value) == 0); + REQUIRE(value == 1234); + + /* Check that key=2 is not found. */ + REQUIRE(map.elem_lookup(&key) == nullptr); + REQUIRE(errno == ENOENT); + + /* BPF_EXIST means update existing element. */ + REQUIRE(map.elem_update(&key, &value, BPF_EXIST) < 0); + + REQUIRE(/* key=2 is not there. */ + errno == ENOENT); + + /* Insert key=2 element. */ + REQUIRE(map.elem_update(&key, &value, BPF_NOEXIST) == 0); + + /* key=1 and key=2 were inserted, check that key=0 cannot be + * inserted due to max_entries limit. + */ + key = 0; + REQUIRE(map.elem_update(&key, &value, BPF_NOEXIST) < 0); + REQUIRE(errno == E2BIG); + + /* Update existing element, though the map is full. */ + key = 1; + REQUIRE(map.elem_update(&key, &value, BPF_EXIST) == 0); + key = 2; + REQUIRE(map.elem_update(&key, &value, BPF_ANY) == 0); + key = 3; + REQUIRE(map.elem_update(&key, &value, BPF_NOEXIST) < 0); + REQUIRE(errno == E2BIG); + + /* Check that key = 0 doesn't exist. */ + key = 0; + REQUIRE(map.elem_delete(&key) < 0); + REQUIRE(errno == ENOENT); + /* Iterate over two elements. */ + REQUIRE(map.map_get_next_key(NULL, &first_key) == 0); + REQUIRE((first_key == 1 || first_key == 2)); + REQUIRE(map.map_get_next_key(&key, &next_key) == 0); + REQUIRE((next_key == first_key)); + REQUIRE(map.map_get_next_key(&next_key, &next_key) == 0); + REQUIRE(((next_key == 1 || next_key == 2) && (next_key != first_key))); + REQUIRE(map.map_get_next_key(&next_key, &next_key) < 0); + REQUIRE(errno == ENOENT); + + /* Delete both elements. */ + key = 1; + REQUIRE(map.elem_delete(&key) == 0); + key = 2; + REQUIRE(map.elem_delete(&key) == 0); + REQUIRE(map.elem_delete(&key) < 0); + REQUIRE(errno == ENOENT); + + key = 0; + /* Check that map is empty. */ + REQUIRE(map.map_get_next_key(NULL, &next_key) < 0); + REQUIRE(errno == ENOENT); + REQUIRE(map.map_get_next_key(&key, &next_key) < 0); + REQUIRE(errno == ENOENT); +} +TEST_CASE("test_hashmap (kernel)", "[kernel]") +{ + test_hashmap(); +} +static void test_hashmap_sizes(std::string memory_name = SHM_NAME, + bool should_create_shm = true) +{ + shm_initializer shm(memory_name.c_str(), should_create_shm); + auto &mem = *shm.mem; + int i, j; + + for (i = 1; i <= 512; i <<= 1) + for (j = 1; j <= 1 << 18; j <<= 1) { + var_size_hash_map_impl map(mem, i, j, 2, 0); + usleep(10); + } +} +TEST_CASE("test_hashmap_sizes (kernel)", "[kernel]") +{ + test_hashmap_sizes(); +} + +/** Macros from kernel source */ +#define __bpf_percpu_val_align __attribute__((__aligned__(8))) + +#define BPF_DECLARE_PERCPU(type, name) \ + struct { \ + type v; /* padding */ \ + } __bpf_percpu_val_align name[sysconf(_SC_NPROCESSORS_ONLN)] +#define bpf_percpu(name, cpu) name[(cpu)].v + +static void test_hashmap_percpu(std::string memory_name = SHM_NAME, + bool should_create_shm = true) +{ + shm_initializer shm(memory_name.c_str(), should_create_shm); + auto &mem = *shm.mem; + + unsigned int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); + BPF_DECLARE_PERCPU(long, value); + long long key, next_key, first_key; + int expected_key_mask = 0; + int i; + per_cpu_hash_map_impl map(mem, sizeof(key), + sizeof(bpf_percpu(value, 0)), 2); + + for (i = 0; i < (int)nr_cpus; i++) + bpf_percpu(value, i) = i + 100; + + key = 1; + /* Insert key=1 element. */ + REQUIRE(!(expected_key_mask & key)); + REQUIRE(map.elem_update_userspace(&key, value, BPF_ANY) == 0); + + /* Lookup and delete elem key=1 and check value. */ + REQUIRE((map.lookup_and_delete_userspace(&key, value) == 0 && + bpf_percpu(value, 0) == 100)); + + for (i = 0; i < (int)nr_cpus; i++) + bpf_percpu(value, i) = i + 100; + + /* Insert key=1 element which should not exist. */ + REQUIRE(map.elem_update_userspace(&key, value, BPF_NOEXIST) == 0); + expected_key_mask |= key; + + /* BPF_NOEXIST means add new element if it doesn't exist. */ + REQUIRE((map.elem_update_userspace(&key, value, BPF_NOEXIST) < 0 && + /* key=1 already exists. */ + errno == EEXIST)); + + /* -1 is an invalid flag. */ + REQUIRE((map.elem_update(&key, value, -1) < 0 && errno == EINVAL)); + + const auto lookup_helper = [&](const void *key, void *value) -> long { + auto returned_value = map.elem_lookup_userspace(key); + if (!returned_value) + return -1; + memcpy(value, returned_value, + map.get_value_size() * map.getncpu()); + return 0; + }; + + /* Check that key=1 can be found. Value could be 0 if the lookup + * was run from a different CPU. + */ + bpf_percpu(value, 0) = 1; + REQUIRE((lookup_helper(&key, value) == 0 && + bpf_percpu(value, 0) == 100)); + + key = 2; + /* Check that key=2 is not found. */ + REQUIRE((lookup_helper(&key, value) < 0 && errno == ENOENT)); + + /* BPF_EXIST means update existing element. */ + REQUIRE((map.elem_update_userspace(&key, value, BPF_EXIST) < 0 && + /* key=2 is not there. */ + errno == ENOENT)); + + /* Insert key=2 element. */ + REQUIRE(!(expected_key_mask & key)); + REQUIRE(map.elem_update_userspace(&key, value, BPF_NOEXIST) == 0); + expected_key_mask |= key; + + /* key=1 and key=2 were inserted, check that key=0 cannot be + * inserted due to max_entries limit. + */ + key = 0; + REQUIRE((map.elem_update_userspace(&key, value, BPF_NOEXIST) < 0 && + errno == E2BIG)); + + /* Check that key = 0 doesn't exist. */ + REQUIRE((map.elem_delete_userspace(&key) < 0 && errno == ENOENT)); + + /* Iterate over two elements. */ + REQUIRE((map.map_get_next_key(NULL, &first_key) == 0 && + ((expected_key_mask & first_key) == first_key))); + while (!map.map_get_next_key(&key, &next_key)) { + if (first_key) { + REQUIRE(next_key == first_key); + first_key = 0; + } + REQUIRE((expected_key_mask & next_key) == next_key); + expected_key_mask &= ~next_key; + + REQUIRE(lookup_helper(&next_key, value) == 0); + + for (i = 0; i < (int)nr_cpus; i++) + REQUIRE(bpf_percpu(value, i) == i + 100); + + key = next_key; + } + REQUIRE(errno == ENOENT); + + /* Update with BPF_EXIST. */ + key = 1; + REQUIRE(map.elem_update_userspace(&key, value, BPF_EXIST) == 0); + + /* Delete both elements. */ + key = 1; + REQUIRE(map.elem_delete_userspace(&key) == 0); + key = 2; + REQUIRE(map.elem_delete_userspace(&key) == 0); + REQUIRE((map.elem_delete_userspace(&key) < 0 && errno == ENOENT)); + + key = 0; + /* Check that map is empty. */ + REQUIRE((map.map_get_next_key(NULL, &next_key) < 0 && errno == ENOENT)); + REQUIRE((map.map_get_next_key(&key, &next_key) < 0 && errno == ENOENT)); +} + +TEST_CASE("test_hashmap_percpu (kernel)", "[kernel]") +{ + test_hashmap_percpu(); +} + +#define VALUE_SIZE 3 +static inline var_size_hash_map_impl +helper_fill_hashmap(int max_entries, managed_shared_memory &mem) +{ + int i, ret; + long long key, value[VALUE_SIZE] = {}; + + var_size_hash_map_impl map(mem, sizeof(key), sizeof(value), max_entries, + 0); + + for (i = 0; i < max_entries; i++) { + key = i; + value[0] = key; + ret = map.elem_update(&key, value, BPF_NOEXIST); + REQUIRE((ret == 0 && "Unable to update hash map")); + } + + return map; +} + +static void test_hashmap_walk(std::string memory_name = SHM_NAME, + bool should_create_shm = true) +{ + shm_initializer shm(memory_name.c_str(), should_create_shm); + auto &mem = *shm.mem; + int i, max_entries = 10000; + long long key, value[VALUE_SIZE], next_key; + bool next_key_valid = true; + + auto map = helper_fill_hashmap(max_entries, mem); + + const auto lookup_helper = [&](const void *key, void *value) -> long { + auto returned_value = map.elem_lookup(key); + if (!returned_value) + return -1; + memcpy(value, returned_value, map.get_value_size()); + return 0; + }; + + for (i = 0; map.map_get_next_key(!i ? NULL : &key, &next_key) == 0; + i++) { + key = next_key; + REQUIRE(lookup_helper(&key, value) == 0); + } + + REQUIRE(i == max_entries); + + REQUIRE(map.map_get_next_key(NULL, &key) == 0); + for (i = 0; next_key_valid; i++) { + next_key_valid = map.map_get_next_key(&key, &next_key) == 0; + REQUIRE(lookup_helper(&key, value) == 0); + value[0]++; + REQUIRE(map.elem_update(&key, value, BPF_EXIST) == 0); + key = next_key; + } + + REQUIRE(i == max_entries); + + for (i = 0; map.map_get_next_key(!i ? NULL : &key, &next_key) == 0; + i++) { + key = next_key; + REQUIRE(lookup_helper(&key, value) == 0); + REQUIRE(value[0] - 1 == key); + } + + REQUIRE(i == max_entries); +} + +TEST_CASE("test_hashmap_walk (kernel)", "[kernel]") +{ + test_hashmap_walk(); +} + +static void test_arraymap(std::string memory_name = SHM_NAME, + bool should_create_shm = true) +{ + shm_initializer shm(memory_name.c_str(), should_create_shm); + auto &mem = *shm.mem; + int key, next_key; + long long value; + auto value_size = sizeof(value); + array_map_impl map(mem, sizeof(value), 2); + const auto lookup_helper = [&](const void *key, void *value) -> long { + auto returned_value = map.elem_lookup(key); + if (!returned_value) + return -1; + memcpy(value, returned_value, sizeof(value_size)); + return 0; + }; + + key = 1; + value = 1234; + /* Insert key=1 element. */ + REQUIRE(map.elem_update(&key, &value, BPF_ANY) == 0); + + value = 0; + REQUIRE((map.elem_update(&key, &value, BPF_NOEXIST) < 0 && + errno == EEXIST)); + + /* Check that key=1 can be found. */ + REQUIRE((lookup_helper(&key, &value) == 0 && value == 1234)); + + key = 0; + /* Check that key=0 is also found and zero initialized. */ + REQUIRE((lookup_helper(&key, &value) == 0 && value == 0)); + + /* key=0 and key=1 were inserted, check that key=2 cannot be inserted + * due to max_entries limit. + */ + key = 2; + REQUIRE((map.elem_update(&key, &value, BPF_EXIST) < 0 && + errno == E2BIG)); + + /* Check that key = 2 doesn't exist. */ + REQUIRE((lookup_helper(&key, &value) < 0 && errno == ENOENT)); + + /* Iterate over two elements. */ + REQUIRE((map.map_get_next_key(NULL, &next_key) == 0 && next_key == 0)); + REQUIRE((map.map_get_next_key(&key, &next_key) == 0 && next_key == 0)); + REQUIRE((map.map_get_next_key(&next_key, &next_key) == 0 && + next_key == 1)); + REQUIRE((map.map_get_next_key(&next_key, &next_key) < 0 && + errno == ENOENT)); + + /* Delete shouldn't succeed. */ + key = 1; + REQUIRE((map.elem_delete(&key) < 0 && errno == EINVAL)); +} + +TEST_CASE("test_arraymap (kernel)", "[kernel]") +{ + test_arraymap(); +} +static void test_arraymap_percpu(std::string memory_name = SHM_NAME, + bool should_create_shm = true) +{ + shm_initializer shm(memory_name.c_str(), should_create_shm); + auto &mem = *shm.mem; + unsigned int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); + BPF_DECLARE_PERCPU(long, values); + int key, next_key, i; + per_cpu_array_map_impl map(mem, sizeof(bpf_percpu(values, 0)), 2); + const auto lookup_helper = [&](const void *key, void *value) -> long { + auto returned_value = map.elem_lookup_userspace(key); + if (!returned_value) + return -1; + memcpy(value, returned_value, sizeof(values)); + return 0; + }; + + for (i = 0; i < (int)nr_cpus; i++) + bpf_percpu(values, i) = i + 100; + + key = 1; + /* Insert key=1 element. */ + REQUIRE(map.elem_update_userspace(&key, values, BPF_ANY) == 0); + + bpf_percpu(values, 0) = 0; + REQUIRE((map.elem_update_userspace(&key, values, BPF_NOEXIST) < 0 && + errno == EEXIST)); + + /* Check that key=1 can be found. */ + REQUIRE((lookup_helper(&key, values) == 0 && + bpf_percpu(values, 0) == 100)); + + key = 0; + /* Check that key=0 is also found and zero initialized. */ + REQUIRE((lookup_helper(&key, values) == 0 && + bpf_percpu(values, 0) == 0 && + bpf_percpu(values, nr_cpus - 1) == 0)); + + /* Check that key=2 cannot be inserted due to max_entries limit. */ + key = 2; + REQUIRE((map.elem_update_userspace(&key, values, BPF_EXIST) < 0 && + errno == E2BIG)); + + /* Check that key = 2 doesn't exist. */ + REQUIRE((lookup_helper(&key, values) < 0 && errno == ENOENT)); + + /* Iterate over two elements. */ + REQUIRE((map.map_get_next_key(NULL, &next_key) == 0 && next_key == 0)); + REQUIRE((map.map_get_next_key(&key, &next_key) == 0 && next_key == 0)); + REQUIRE((map.map_get_next_key(&next_key, &next_key) == 0 && + next_key == 1)); + REQUIRE((map.map_get_next_key(&next_key, &next_key) < 0 && + errno == ENOENT)); + + /* Delete shouldn't succeed. */ + key = 1; + REQUIRE((map.elem_delete_userspace(&key) < 0 && errno == EINVAL)); +} +TEST_CASE("test_arraymap_percpu (kernel)", "[kernel]") +{ + test_arraymap_percpu(); +} +TEST_CASE("test_arraymap_percpu_many_keys (kernel)", "[kernel]") +{ + shm_initializer shm(SHM_NAME, true); + auto &mem = *shm.mem; + unsigned int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); + BPF_DECLARE_PERCPU(long, values); + /* nr_keys is not too large otherwise the test stresses percpu + * allocator more than anything else + */ + unsigned int nr_keys = 2000; + int key, i; + + per_cpu_array_map_impl map(mem, sizeof(bpf_percpu(values, 0)), nr_keys); + + for (i = 0; i < (int)nr_cpus; i++) + bpf_percpu(values, i) = i + 10; + + for (key = 0; key < (int)nr_keys; key++) + REQUIRE(map.elem_update_userspace(&key, values, BPF_ANY) == 0); + + for (key = 0; key < (int)nr_keys; key++) { + for (i = 0; i < (int)nr_cpus; i++) + bpf_percpu(values, i) = 0; + void *value_ptr; + REQUIRE((value_ptr = map.elem_lookup_userspace(&key)) != + nullptr); + memcpy(values, value_ptr, sizeof(values)); + for (i = 0; i < (int)nr_cpus; i++) + REQUIRE(bpf_percpu(values, i) == i + 10); + } +} +#define MAP_SIZE (32 * 1024) +TEST_CASE("test_map_large (kernel)", "[kernel][large_memory]") +{ + shm_initializer shm(SHM_NAME, true, 150 << 20); + auto &mem = *shm.mem; + struct bigkey { + int a; + char b[4096]; + long long c; + } key; + int i, value; + + var_size_hash_map_impl map(mem, sizeof(key), sizeof(value), MAP_SIZE, + 0); + const auto lookup_helper = [&](const void *key, void *value) -> long { + auto returned_value = map.elem_lookup(key); + if (!returned_value) + return -1; + memcpy(value, returned_value, map.get_value_size()); + return 0; + }; + + for (i = 0; i < MAP_SIZE; i++) { + key = (struct bigkey){ .c = i }; + value = i; + + REQUIRE(map.elem_update(&key, &value, BPF_NOEXIST) == 0); + } + + key.c = -1; + REQUIRE((map.elem_update(&key, &value, BPF_NOEXIST) < 0 && + errno == E2BIG)); + + /* Iterate through all elements. */ + REQUIRE(map.map_get_next_key(NULL, &key) == 0); + key.c = -1; + for (i = 0; i < MAP_SIZE; i++) + REQUIRE(map.map_get_next_key(&key, &key) == 0); + REQUIRE((map.map_get_next_key(&key, &key) < 0 && errno == ENOENT)); + + key.c = 0; + REQUIRE((lookup_helper(&key, &value) == 0 && value == 0)); + key.a = 1; + REQUIRE((lookup_helper(&key, &value) < 0 && errno == ENOENT)); +} + +static void +run_parallel(int tasks, std::function fn, + std::optional > common_shm = {}) +{ + std::vector thds; + for (int i = 0; i < tasks; i++) { + thds.push_back(std::thread(fn, i)); + } + for (auto &thd : thds) + thd.join(); +} + +TEST_CASE("test_map_stress (kernel)", "[kernel]") +{ + const auto start_and_run_task = [&](void (*fn)(std::string, bool), + const char *shm_name) { + run_parallel( + 100, + [=](int idx) { + SPDLOG_DEBUG( + "Started testing thread with memory name {}, index {}", + shm_name, idx); + fn(shm_name, false); + }, + std::make_unique(shm_name, true)); + }; + + start_and_run_task(test_hashmap, "_SHM__PARALLEL_TEST_HASHMAP"); + start_and_run_task(test_hashmap_percpu, + "_SHM__PARALLEL_TEST_HASHMAP_PERCPU"); + start_and_run_task(test_hashmap_sizes, + "_SHM__PARALLEL_TEST_HASHMAP_SIZES"); + + start_and_run_task(test_arraymap, "_SHM__PARALLEL_TEST_ARRAYMAP"); + start_and_run_task(test_arraymap_percpu, + "_SHM__PARALLEL_TEST_ARRAYMAP_PERCPU"); +} +#define MAP_RETRIES 20 +#define MAX_DELAY_US 50000 +#define MIN_DELAY_RANGE_US 5000 +#define TASKS 20 /* Kernel uses 100 here, but it will be too slow for us...*/ +#define DO_UPDATE 1 +#define DO_DELETE 0 + +struct pthread_spinlock_guard { + pthread_spinlock_t &lock; + pthread_spinlock_guard(pthread_spinlock_t &lock) : lock(lock) + { + pthread_spin_lock(&lock); + } + ~pthread_spinlock_guard() + { + pthread_spin_unlock(&lock); + } +}; + +static int map_update_retriable(var_size_hash_map_impl &map, const void *key, + const void *value, int flags, int attempts, + pthread_spinlock_t &lock) +{ + int delay = rand() % MIN_DELAY_RANGE_US; + + while (true) { + { + pthread_spinlock_guard guard(lock); + if (map.elem_update(key, value, flags) == 0) + break; + } + if (!attempts || (errno != EAGAIN && errno != EBUSY)) + return -errno; + + if (delay <= MAX_DELAY_US / 2) + delay *= 2; + + usleep(delay); + attempts--; + } + + return 0; +} + +static int map_delete_retriable(var_size_hash_map_impl &map, const void *key, + int attempts, pthread_spinlock_t &lock) +{ + int delay = rand() % MIN_DELAY_RANGE_US; + + while (true) { + { + pthread_spinlock_guard guard(lock); + if (map.elem_delete(key) == 0) + break; + } + if (!attempts || (errno != EAGAIN && errno != EBUSY)) + return -errno; + + if (delay <= MAX_DELAY_US / 2) + delay *= 2; + + usleep(delay); + attempts--; + } + + return 0; +} + +static void test_update_delete(unsigned int fn, int do_update, + var_size_hash_map_impl &map, + const char *shm_name, pthread_spinlock_t &lock) +{ + int i, key, value, err; + + if (fn & 1) + test_hashmap_walk(shm_name, false); + for (i = fn; i < MAP_SIZE; i += TASKS) { + key = value = i; + + if (do_update) { + err = map_update_retriable(map, &key, &value, + BPF_NOEXIST, MAP_RETRIES, + lock); + if (err) + SPDLOG_ERROR("error {} {}\n", err, errno); + REQUIRE(err == 0); + err = map_update_retriable(map, &key, &value, BPF_EXIST, + MAP_RETRIES, lock); + if (err) + SPDLOG_ERROR("error {} {}\n", err, errno); + REQUIRE(err == 0); + } else { + err = map_delete_retriable(map, &key, MAP_RETRIES, + lock); + if (err) + SPDLOG_ERROR("error {} {}\n", err, errno); + REQUIRE(err == 0); + } + } +} + +TEST_CASE("test_map_parallel (kernel)", "[kernel][large_memory]") +{ + shm_initializer shm(SHM_NAME, true, 150 << 20); + auto &mem = *shm.mem; + + int i, key = 0, value = 0, j = 0; + + var_size_hash_map_impl map(mem, sizeof(key), sizeof(value), MAP_SIZE, + 0); + pthread_spinlock_t lock; + pthread_spin_init(&lock, 0); +again: + /* Use the same fd in children to add elements to this map: + * child_0 adds key=0, key=1024, key=2048, ... + * child_1 adds key=1, key=1025, key=2049, ... + * child_1023 adds key=1023, ... + */ + // data[0] = fd; + // data[1] = DO_UPDATE; + // run_parallel(TASKS, test_update_delete, data); + run_parallel(TASKS, [&](int idx) { + test_update_delete(idx, DO_UPDATE, map, SHM_NAME, lock); + }); + SPDLOG_DEBUG("parallel update done"); + /* Check that key=0 is already there. */ + REQUIRE((map.elem_update(&key, &value, BPF_NOEXIST) < 0 && + errno == EEXIST)); + + /* Check that all elements were inserted. */ + REQUIRE(map.map_get_next_key(NULL, &key) == 0); + key = -1; + for (i = 0; i < MAP_SIZE; i++) + REQUIRE(map.map_get_next_key(&key, &key) == 0); + REQUIRE((map.map_get_next_key(&key, &key) < 0 && errno == ENOENT)); + + /* Another check for all elements */ + for (i = 0; i < MAP_SIZE; i++) { + key = MAP_SIZE - i - 1; + int *ptr = (int *)map.elem_lookup(&key); + REQUIRE(ptr != nullptr); + value = *ptr; + REQUIRE(value == key); + } + + /* Now let's delete all elemenets in parallel. */ + // data[1] = DO_DELETE; + run_parallel(TASKS, [&](int idx) { + test_update_delete(idx, DO_DELETE, map, SHM_NAME, lock); + }); + + SPDLOG_DEBUG("parallel delete done"); + /* Nothing should be left. */ + key = -1; + REQUIRE((map.map_get_next_key(NULL, &key) < 0 && errno == ENOENT)); + REQUIRE((map.map_get_next_key(&key, &key) < 0 && errno == ENOENT)); + + key = 0; + map.elem_delete(&key); + if (j++ < 5) + goto again; + pthread_spin_destroy(&lock); +} + +TEST_CASE("test_map_rdonly (kernel)", "[kernel]") +{ + shm_initializer shm(SHM_NAME, true); + auto &mem = *shm.mem; + int key = 0, value = 0; + + var_size_hash_map_impl map(mem, sizeof(key), sizeof(value), MAP_SIZE, + BPF_F_RDONLY); + key = 1; + value = 1234; + /* Try to insert key=1 element. */ + REQUIRE((map.elem_update(&key, &value, BPF_ANY) < 0 && errno == EPERM)); + + /* Check that key=1 is not found. */ + REQUIRE((map.elem_lookup(&key) == nullptr && errno == ENOENT)); + REQUIRE((map.map_get_next_key(&key, &value) < 0 && errno == ENOENT)); +} + +TEST_CASE("test_map_wronly (kernel)", "[kernel]") +{ + shm_initializer shm(SHM_NAME, true); + auto &mem = *shm.mem; + int key = 0, value = 0; + + var_size_hash_map_impl map(mem, sizeof(key), sizeof(value), MAP_SIZE, + BPF_F_WRONLY); + key = 1; + value = 1234; + /* Insert key=1 element. */ + REQUIRE(map.elem_update(&key, &value, BPF_ANY) == 0); + + /* Check that reading elements and keys from the map is not allowed. */ + REQUIRE((map.elem_lookup(&key) == nullptr && errno == EPERM)); + REQUIRE((map.map_get_next_key(&key, &value) < 0 && errno == EPERM)); +} diff --git a/runtime/unit-test/maps/test_bpftime_hash_map.cpp b/runtime/unit-test/maps/test_bpftime_hash_map.cpp index 6cdf8520..7f19a834 100644 --- a/runtime/unit-test/maps/test_bpftime_hash_map.cpp +++ b/runtime/unit-test/maps/test_bpftime_hash_map.cpp @@ -22,127 +22,138 @@ using namespace boost::interprocess; using namespace bpftime; -TEST_CASE("bpftime_hash_map basic operations", "[bpftime_hash_map]") { - managed_shared_memory segment(open_or_create, "bpftime_hash_map_test", 65536); - - size_t num_buckets = 10; - size_t key_size = 4; // Example key size - size_t value_size = 8; // Example value size - - bpftime_hash_map map(segment, num_buckets, key_size, value_size); - - SECTION("Insert and Lookup") { - int key1 = 1234; - int64_t value1 = 5678; - int key2 = 4321; - int64_t value2 = 8765; - - REQUIRE(map.elem_update(&key1, &value1) == true); - REQUIRE(map.elem_update(&key2, &value2) == true); - - REQUIRE(map.elem_lookup(&key1) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key1) == value1); - - REQUIRE(map.elem_lookup(&key2) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key2) == value2); - - int key3 = 9999; - REQUIRE(map.elem_lookup(&key3) == nullptr); - } - - SECTION("Update existing element") { - int key = 1234; - int64_t value1 = 5678; - int64_t value2 = 8765; - - REQUIRE(map.elem_update(&key, &value1) == true); - REQUIRE(map.elem_lookup(&key) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key) == value1); - - REQUIRE(map.elem_update(&key, &value2) == true); - REQUIRE(map.elem_lookup(&key) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key) == value2); - } - - SECTION("Delete element") { - int key1 = 1234; - int64_t value1 = 5678; - int key2 = 4321; - int64_t value2 = 8765; - - REQUIRE(map.elem_update(&key1, &value1) == true); - REQUIRE(map.elem_update(&key2, &value2) == true); - - REQUIRE(map.elem_delete(&key1) == true); - REQUIRE(map.elem_lookup(&key1) == nullptr); - REQUIRE(map.elem_lookup(&key2) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key2) == value2); - - REQUIRE(map.elem_delete(&key2) == true); - REQUIRE(map.elem_lookup(&key2) == nullptr); - } - - SECTION("Get element count") { - int key1 = 1234; - int64_t value1 = 5678; - int key2 = 4321; - int64_t value2 = 8765; - - REQUIRE(map.get_elem_count() == 0); - - REQUIRE(map.elem_update(&key1, &value1) == true); - REQUIRE(map.get_elem_count() == 1); - - REQUIRE(map.elem_update(&key2, &value2) == true); - REQUIRE(map.get_elem_count() == 2); - - REQUIRE(map.elem_delete(&key1) == true); - REQUIRE(map.get_elem_count() == 1); - - REQUIRE(map.elem_delete(&key2) == true); - REQUIRE(map.get_elem_count() == 0); - } - - SECTION("Insert more elements than num_buckets") { - int key; - int64_t value; - - for (size_t i = 0; i < num_buckets; ++i) { - key = i; - value = i * 100; - REQUIRE(map.elem_update(&key, &value) == true); - } - - key = num_buckets; - value = num_buckets * 100; - REQUIRE(map.elem_update(&key, &value) == false); // Should fail as the map is full - - for (size_t i = 0; i < num_buckets; ++i) { - key = i; - REQUIRE(map.elem_lookup(&key) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key) == i * 100); - } - } - - SECTION("Insert, delete, and re-insert elements") { - int key1 = 1234; - int64_t value1 = 5678; - int key2 = 4321; - int64_t value2 = 8765; - int key3 = 5678; - int64_t value3 = 4321; - - REQUIRE(map.elem_update(&key1, &value1) == true); - REQUIRE(map.elem_update(&key2, &value2) == true); - - REQUIRE(map.elem_delete(&key1) == true); - REQUIRE(map.elem_lookup(&key1) == nullptr); - - REQUIRE(map.elem_update(&key3, &value3) == true); - REQUIRE(map.elem_lookup(&key3) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key3) == value3); - - REQUIRE(map.elem_lookup(&key2) != nullptr); - REQUIRE(*(int64_t*)map.elem_lookup(&key2) == value2); - } +TEST_CASE("bpftime_hash_map basic operations", "[bpftime_hash_map]") +{ + managed_shared_memory segment(open_or_create, "bpftime_hash_map_test", + 65536); + + size_t num_buckets = 10; + size_t key_size = 4; // Example key size + size_t value_size = 8; // Example value size + + bpftime_hash_map map(segment, num_buckets, key_size, value_size); + + SECTION("Insert and Lookup") + { + int key1 = 1234; + int64_t value1 = 5678; + int key2 = 4321; + int64_t value2 = 8765; + + REQUIRE(map.elem_update(&key1, &value1) == true); + REQUIRE(map.elem_update(&key2, &value2) == true); + + REQUIRE(map.elem_lookup(&key1) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key1) == value1); + + REQUIRE(map.elem_lookup(&key2) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key2) == value2); + + int key3 = 9999; + REQUIRE(map.elem_lookup(&key3) == nullptr); + } + + SECTION("Update existing element") + { + int key = 1234; + int64_t value1 = 5678; + int64_t value2 = 8765; + + REQUIRE(map.elem_update(&key, &value1) == true); + REQUIRE(map.elem_lookup(&key) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key) == value1); + + REQUIRE(map.elem_update(&key, &value2) == true); + REQUIRE(map.elem_lookup(&key) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key) == value2); + } + + SECTION("Delete element") + { + int key1 = 1234; + int64_t value1 = 5678; + int key2 = 4321; + int64_t value2 = 8765; + + REQUIRE(map.elem_update(&key1, &value1) == true); + REQUIRE(map.elem_update(&key2, &value2) == true); + + REQUIRE(map.elem_delete(&key1) == true); + REQUIRE(map.elem_lookup(&key1) == nullptr); + REQUIRE(map.elem_lookup(&key2) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key2) == value2); + + REQUIRE(map.elem_delete(&key2) == true); + REQUIRE(map.elem_lookup(&key2) == nullptr); + } + + SECTION("Get element count") + { + int key1 = 1234; + int64_t value1 = 5678; + int key2 = 4321; + int64_t value2 = 8765; + + REQUIRE(map.get_elem_count() == 0); + + REQUIRE(map.elem_update(&key1, &value1) == true); + REQUIRE(map.get_elem_count() == 1); + + REQUIRE(map.elem_update(&key2, &value2) == true); + REQUIRE(map.get_elem_count() == 2); + + REQUIRE(map.elem_delete(&key1) == true); + REQUIRE(map.get_elem_count() == 1); + + REQUIRE(map.elem_delete(&key2) == true); + REQUIRE(map.get_elem_count() == 0); + } + + SECTION("Insert more elements than num_buckets") + { + int key; + int64_t value; + + for (size_t i = 0; i < num_buckets; ++i) { + key = i; + value = i * 100; + REQUIRE(map.elem_update(&key, &value) == true); + } + + key = num_buckets; + value = num_buckets * 100; + REQUIRE(map.elem_update(&key, &value) == false); // Should fail + // as the map + // is full + + for (size_t i = 0; i < num_buckets; ++i) { + key = i; + REQUIRE(map.elem_lookup(&key) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key) == + (int64_t)i * 100); + } + } + + SECTION("Insert, delete, and re-insert elements") + { + int key1 = 1234; + int64_t value1 = 5678; + int key2 = 4321; + int64_t value2 = 8765; + int key3 = 5678; + int64_t value3 = 4321; + + REQUIRE(map.elem_update(&key1, &value1) == true); + REQUIRE(map.elem_update(&key2, &value2) == true); + + REQUIRE(map.elem_delete(&key1) == true); + REQUIRE(map.elem_lookup(&key1) == nullptr); + + REQUIRE(map.elem_update(&key3, &value3) == true); + REQUIRE(map.elem_lookup(&key3) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key3) == value3); + + REQUIRE(map.elem_lookup(&key2) != nullptr); + REQUIRE(*(int64_t *)map.elem_lookup(&key2) == value2); + } } diff --git a/runtime/unit-test/maps/test_per_cpu_hash.cpp b/runtime/unit-test/maps/test_per_cpu_hash.cpp index e709a042..9c953a2a 100644 --- a/runtime/unit-test/maps/test_per_cpu_hash.cpp +++ b/runtime/unit-test/maps/test_per_cpu_hash.cpp @@ -38,7 +38,7 @@ TEST_CASE("Test basic operations of hash map") SECTION("Test writing from helpers, and read from userspace") { - per_cpu_hash_map_impl map(mem, 4, 8); + per_cpu_hash_map_impl map(mem, 4, 8, 1 << 20); for (uint32_t j = 0; j < ncpu; j++) { ensure_on_certain_cpu(j, [&]() { for (uint32_t i = 0; i < 100; i++) { @@ -64,7 +64,7 @@ TEST_CASE("Test basic operations of hash map") SECTION("Test writing from userspace, and reading & updating from helpers") { - per_cpu_hash_map_impl map(mem, 4, 8); + per_cpu_hash_map_impl map(mem, 4, 8, 1 << 20); std::vector buf(ncpu); for (uint32_t j = 0; j < ncpu; j++) { buf[j] = j;