diff --git a/runtime/extension/userspace_xdp.h b/runtime/extension/userspace_xdp.h new file mode 100644 index 00000000..ccc20c61 --- /dev/null +++ b/runtime/extension/userspace_xdp.h @@ -0,0 +1,19 @@ +#ifndef BPFTIME_EXTENSION_USERSPACE_XDP_HPP +#define BPFTIME_EXTENSION_USERSPACE_XDP_HPP + +#include + +struct xdp_md_userspace +{ + uint64_t data; + uint64_t data_end; + uint32_t data_meta; + uint32_t ingress_ifindex; + uint32_t rx_queue_index; + uint32_t egress_ifindex; + // additional fields + uint64_t buffer_start; // record the start of the available buffer + uint64_t buffer_end; // record the end of the available buffer +}; + +#endif // BPFTIME_EXTENSION_USERSPACE_XDP_HPP diff --git a/runtime/src/bpf_helper.cpp b/runtime/src/bpf_helper.cpp index d5e6559a..9bd71dd0 100644 --- a/runtime/src/bpf_helper.cpp +++ b/runtime/src/bpf_helper.cpp @@ -23,6 +23,7 @@ #include "bpftime.hpp" #include "bpftime_shm.hpp" #include "bpftime_internal.h" +#include "extension/userspace_xdp.h" #include #include #include @@ -104,9 +105,9 @@ uint64_t bpf_get_current_uid_gid(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t bpftime_ktime_get_ns(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t) { - auto now = std::chrono::steady_clock::now(); - auto ns = std::chrono::time_point_cast(now); - return ns.time_since_epoch().count(); + auto now = std::chrono::steady_clock::now(); + auto ns = std::chrono::time_point_cast(now); + return ns.time_since_epoch().count(); } uint64_t bpftime_get_current_comm(uint64_t buf, uint64_t size, uint64_t, @@ -114,7 +115,7 @@ uint64_t bpftime_get_current_comm(uint64_t buf, uint64_t size, uint64_t, { static std::string filename_buf; - if (filename_buf.empty()) { + if (unlikely(filename_buf.empty())) { char strbuf[PATH_MAX]; auto len = readlink("/proc/self/exe", strbuf, @@ -219,6 +220,7 @@ uint64_t bpf_ringbuf_submit(uint64_t data, uint64_t flags, uint64_t, uint64_t, bpftime_ringbuf_submit(fd, (void *)(uintptr_t)data, false); return 0; } + uint64_t bpf_ringbuf_discard(uint64_t data, uint64_t flags, uint64_t, uint64_t, uint64_t) { @@ -328,6 +330,7 @@ uint64_t bpftime_tail_call(uint64_t ctx, uint64_t prog_array, uint64_t index) close(to_call_fd); return run_opts.retval; } + uint64_t bpftime_get_attach_cookie(uint64_t ctx, uint64_t, uint64_t, uint64_t, uint64_t) { @@ -340,6 +343,95 @@ uint64_t bpftime_get_attach_cookie(uint64_t ctx, uint64_t, uint64_t, uint64_t, return 0; } } + +uint64_t bpftime_get_smp_processor_id() +{ + int cpu = sched_getcpu(); + if (cpu == -1) { + SPDLOG_ERROR("sched_getcpu error"); + return 0; // unlikely + } + return (uint64_t)cpu; +} + +// From https://github.com/microsoft/ebpf-for-windows +int64_t bpftime_csum_diff(const void *from, int from_size, const void *to, + int to_size, int seed) +{ + int csum_diff = -EINVAL; + + if ((from_size % 4 != 0) || (to_size % 4 != 0)) { + // size of buffers should be a multiple of 4. + goto Exit; + } + + csum_diff = seed; + if (to != NULL) { + for (int i = 0; i < to_size / 2; i++) { + csum_diff += (uint16_t)(*((uint16_t *)to + i)); + } + } + if (from != NULL) { + for (int i = 0; i < from_size / 2; i++) { + csum_diff += (uint16_t)(~*((uint16_t *)from + i)); + } + } + + // Adding 16-bit unsigned integers or their one's complement will + // produce a positive 32-bit integer, unless the length of the buffers + // is so long, that the signed 32 bit output overflows and produces a + // negative result. + if (csum_diff < 0) { + csum_diff = -EINVAL; + } +Exit: + return csum_diff; +} + +#define ETH_HLEN 14 /* Total octets in header. */ + +long bpftime_xdp_adjust_head(struct xdp_md_userspace *xdp, int offset) +{ + // We don't use xdp meta data + uint64_t data = xdp->data + offset; + if (unlikely(data > xdp->data_end - ETH_HLEN) || data > xdp->buffer_end) + return -EINVAL; + if (data < xdp->buffer_start) { + // move the data so the buffer can place the new header + memmove(reinterpret_cast(xdp->buffer_start + + (xdp->buffer_start - data)), + reinterpret_cast(xdp->data), + xdp->data_end - xdp->data); + data = xdp->buffer_start; + } + xdp->data = data; + return 0; +} + +long bpftime_xdp_adjust_tail(struct xdp_md_userspace *xdp_md, int delta) +{ + // We don't use xdp meta data + uint64_t data = xdp_md->data_end + delta; + if (data < xdp_md->data || data < xdp_md->buffer_start || + data > xdp_md->buffer_end) { + return -EINVAL; + } + xdp_md->data_end = data; + return 0; +} + +long bpftime_xdp_load_bytes(struct xdp_md_userspace *xdp_md, __u32 offset, + void *buf, __u32 len) +{ + // We don't support fragmented packets + uint64_t data = xdp_md->data + offset; + if (data + len > xdp_md->data_end) { + return -EINVAL; + } + memcpy(buf, reinterpret_cast(data), len); + return 0; +} + } // extern "C" namespace bpftime @@ -643,6 +735,30 @@ const bpftime_helper_group kernel_helper_group = { .name = "bpf_probe_read", .fn = (void *)bpftime_probe_read, } }, + { BPF_FUNC_get_smp_processor_id, + bpftime_helper_info{ + .index = BPF_FUNC_get_smp_processor_id, + .name = "bpf_get_smp_processor_id", + .fn = (void *)bpftime_get_smp_processor_id, + } }, + { BPF_FUNC_csum_diff, + bpftime_helper_info{ + .index = BPF_FUNC_csum_diff, + .name = "bpf_csum_diff", + .fn = (void *)bpftime_csum_diff, + } }, + { BPF_FUNC_xdp_adjust_head, + bpftime_helper_info{ + .index = BPF_FUNC_xdp_adjust_head, + .name = "bpf_xdp_adjust_head", + .fn = (void *)bpftime_xdp_adjust_head, + } }, + { BPF_FUNC_xdp_adjust_tail, + bpftime_helper_info{ + .index = BPF_FUNC_xdp_adjust_tail, + .name = "bpf_xdp_adjust_tail", + .fn = (void *)bpftime_xdp_adjust_tail, + } }, { BPF_FUNC_probe_read_kernel, bpftime_helper_info{ .index = BPF_FUNC_probe_read_kernel, diff --git a/runtime/src/bpf_map/userspace/map_in_maps.hpp b/runtime/src/bpf_map/userspace/map_in_maps.hpp new file mode 100644 index 00000000..5ee224bb --- /dev/null +++ b/runtime/src/bpf_map/userspace/map_in_maps.hpp @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (c) 2022, eunomia-bpf org + * All rights reserved. + */ +#ifndef _MAP_IN_MAP_HPP +#define _MAP_IN_MAP_HPP + +#include +#include "array_map.hpp" + +namespace bpftime +{ + +// implementation of array map +class array_map_of_maps_impl : public array_map_impl { + public: + array_map_of_maps_impl(boost::interprocess::managed_shared_memory &memory, uint32_t max_entries) : array_map_impl(memory, sizeof(int), max_entries){ + } + // TODO: add verify the correctness of the key + void *elem_lookup(const void *key) { + auto key_val = array_map_impl::elem_lookup(key); + int map_id = *(int *)key_val; + return (void*)((u_int64_t)map_id << 32); + } +}; + +} // namespace bpftime + +#endif // _MAP_IN_MAP_HPP diff --git a/runtime/src/handler/map_handler.cpp b/runtime/src/handler/map_handler.cpp index 1e01c131..e90fa69d 100644 --- a/runtime/src/handler/map_handler.cpp +++ b/runtime/src/handler/map_handler.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include using boost::interprocess::interprocess_sharable_mutex; @@ -134,6 +135,11 @@ const void *bpf_map_handler::map_lookup_elem(const void *key, static_cast(map_impl_ptr.get()); return do_lookup(impl); } + case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_lookup(impl); + } default: auto func_ptr = global_map_ops_table[(int)type].elem_lookup; if (func_ptr) { @@ -221,6 +227,15 @@ long bpf_map_handler::map_update_elem(const void *key, const void *value, static_cast(map_impl_ptr.get()); return do_update(impl); } + case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { + if (!from_syscall) { + // Map in maps only support update from syscall + return -EINVAL; + } + auto impl = static_cast( + map_impl_ptr.get()); + return do_update(impl); + } default: auto func_ptr = global_map_ops_table[(int)type].elem_update; if (func_ptr) { @@ -297,6 +312,11 @@ int bpf_map_handler::bpf_map_get_next_key(const void *key, void *next_key, static_cast(map_impl_ptr.get()); return do_get_next_key(impl); } + case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { + auto impl = static_cast( + map_impl_ptr.get()); + return do_get_next_key(impl); + } default: auto func_ptr = global_map_ops_table[(int)type].map_get_next_key; @@ -384,6 +404,15 @@ long bpf_map_handler::map_delete_elem(const void *key, bool from_syscall) const static_cast(map_impl_ptr.get()); return do_delete(impl); } + case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { + if (!from_syscall) { + // Map in maps only support update from syscall + return -EINVAL; + } + auto impl = static_cast( + map_impl_ptr.get()); + return do_delete(impl); + } default: auto func_ptr = global_map_ops_table[(int)type].elem_delete; if (func_ptr) { @@ -476,7 +505,11 @@ int bpf_map_handler::map_init(managed_shared_memory &memory) max_entries); return 0; } - + case bpf_map_type::BPF_MAP_TYPE_ARRAY_OF_MAPS: { + map_impl_ptr = memory.construct( + container_name.c_str())(memory, max_entries); + return 0; + } default: if (bpftime_get_agent_config().allow_non_buildin_map_types) { SPDLOG_INFO("non-builtin map type: {}", (int)type);