add xdp-counter example (#339)

* add xdp-counter example

* add build in ci

.github/workflows/test-examples.yml

@@ -44,6 +44,9 @@ jobs:
             -DBUILD_BPFTIME_DAEMON=1 \
             -DCMAKE_CXX_FLAGS="-DDEFAULT_LOGGER_OUTPUT_PATH='\"console\"'"
           cmake --build build --config RelWithDebInfo --target install -j
+      - name: Build basic examples
+        run: |
+            make -C example -j
       - name: Upload build results (without jit)
         uses: actions/upload-artifact@v3
         if: ${{!matrix.enable_jit}}

example/Makefile

@@ -0,0 +1,5 @@
+.PHONY: all xdp-counter
+all: xdp-counter
+
+xdp-counter:
+	$(MAKE) -C xdp-counter

example/xdp-counter/.gitignore

@@ -0,0 +1,11 @@
+.vscode
+package.json
+*.o
+*.skel.json
+*.skel.yaml
+package.yaml
+ecli
+malloc
+.output
+test
+xdp-counter

example/xdp-counter/Makefile

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := .output
+CLANG ?= clang
+LIBBPF_SRC := $(abspath ../../third_party/libbpf/src)
+BPFTOOL_SRC := $(abspath ../../third_party/bpftool/src)
+LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
+BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
+ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
+			 | sed 's/arm.*/arm/' \
+			 | sed 's/aarch64/arm64/' \
+			 | sed 's/ppc64le/powerpc/' \
+			 | sed 's/mips.*/mips/' \
+			 | sed 's/riscv64/riscv/' \
+			 | sed 's/loongarch64/loongarch/')
+VMLINUX := ../../third_party/vmlinux/$(ARCH)/vmlinux.h
+# Use our own libbpf API headers and Linux UAPI headers distributed with
+# libbpf to avoid dependency on system-wide headers, which could be missing or
+# outdated
+INCLUDES := -I$(OUTPUT) -I../../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
+CFLAGS := -g -Wall
+ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
+
+APPS = xdp-counter
+
+CARGO ?= $(shell which cargo)
+ifeq ($(strip $(CARGO)),)
+BZS_APPS :=
+else
+BZS_APPS := # profile
+APPS += $(BZS_APPS)
+# Required by libblazesym
+ALL_LDFLAGS += -lrt -ldl -lpthread -lm
+endif
+
+# Get Clang's default includes on this system. We'll explicitly add these dirs
+# to the includes list when compiling with `-target bpf` because otherwise some
+# architecture-specific dirs will be "missing" on some architectures/distros -
+# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
+# sys/cdefs.h etc. might be missing.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+ifeq ($(V),1)
+	Q =
+	msg =
+else
+	Q = @
+	msg = @printf '  %-8s %s%s\n'					\
+		      "$(1)"						\
+		      "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))"	\
+		      "$(if $(3), $(3))";
+	MAKEFLAGS += --no-print-directory
+endif
+
+define allow-override
+  $(if $(or $(findstring environment,$(origin $(1))),\
+            $(findstring command line,$(origin $(1)))),,\
+    $(eval $(1) = $(2)))
+endef
+
+$(call allow-override,CC,$(CROSS_COMPILE)cc)
+$(call allow-override,LD,$(CROSS_COMPILE)ld)
+
+.PHONY: all
+all: $(APPS)
+
+.PHONY: clean
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) $(APPS)
+
+$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+# Build libbpf
+$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
+	$(call msg,LIB,$@)
+	$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1		      \
+		    OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@)		      \
+		    INCLUDEDIR= LIBDIR= UAPIDIR=			      \
+		    install
+
+# Build bpftool
+$(BPFTOOL): | $(BPFTOOL_OUTPUT)
+	$(call msg,BPFTOOL,$@)
+	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
+
+
+$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
+	$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release
+
+$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB, $@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
+
+$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB,$@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
+
+# Build BPF code
+$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -Xlinker --export-dynamic -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
+		     $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES)		      \
+		     -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+	$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+
+# Generate BPF skeletons
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
+	$(call msg,GEN-SKEL,$@)
+	$(Q)$(BPFTOOL) gen skeleton $< > $@
+
+# Build user-space code
+$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
+
+$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
+	$(call msg,CC,$@)
+	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
+
+$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER)
+
+$(BZS_APPS): $(LIBBLAZESYM_OBJ)
+
+# Build application binary
+$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
+	$(call msg,BINARY,$@)
+	$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
+
+# delete failed targets
+.DELETE_ON_ERROR:
+
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:

example/xdp-counter/README.md

@@ -0,0 +1,47 @@
+# xdp in bpftime
+
+You can load the xdp program into userspace eBPF runtime, and exec it with DPDK or AF_XDP. This allows:
+
+- `Faster than kernel eBPF`: DPDK can be faster than XDP driver mode, and bpftime can be faster than kernel in some cases.
+- Compare to the ubpf in DPDK, this enable:
+  - `Control plane application support`: enable the control library to operate the eBPF map, control load and unload the eBPF program dynamically.
+  - `More maps and helpers compatible with kernel`: 
+
+## load the XDP into the userspace eBPF runtime
+
+Create a virtual network device for test:
+
+```sh
+sudo ip link add veth0 type veth peer name veth1
+```
+
+Attach the netdev:
+
+```sh
+LD_PRELOAD=build/runtime/syscall-server/libbpftime-syscall-server.so example/xdp-counter/xdp-counter example/xdp-counter/.output/xdp-counter.bpf.o veth1 example/xdp-counter/base.btf
+```
+
+- The `example/xdp-counter/base.btf` is for relocation on userspace xdp. See [runtime/extension/userspace_xdp.h](../../runtime/extension/userspace_xdp.h) for how the userspace `xdp_md` looks like.
+
+## Run XDP program in userspace
+
+See the driver progam in <https://github.com/eunomia-bpf/XDP-eBPF-in-DPDK>
+
+## TODO: support old XDP attach
+
+Currently bpftime only support attach XDP program with `bpf_link`. We need to handle the old attahc approach later.
+
+```sh
+newfstatat(1, "", {st_mode=S_IFREG|0644, st_size=0, ...}, AT_EMPTY_PATH) = 0
+socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0) = 7
+ioctl(7, SIOCGIFINDEX, {ifr_name="veth1", ifr_ifindex=5}) = 0
+close(7)                                = 0
+socket(AF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, NETLINK_ROUTE) = 7
+setsockopt(7, SOL_NETLINK, NETLINK_EXT_ACK, [1], 4) = 0
+bind(7, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 0
+getsockname(7, {sa_family=AF_NETLINK, nl_pid=405177, nl_groups=00000000}, [12]) = 0
+sendto(7, [{nlmsg_len=52, nlmsg_type=RTM_SETLINK, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=1724212008, nlmsg_pid=0}, {ifi_family=AF_UNSPEC, ifi_type=ARPHRD_NETROM, ifi_index=if_nametoindex("veth1"), ifi_flags=0, ifi_change=0}, [{nla_len=20, nla_type=NLA_F_NESTED|IFLA_XDP}, [[{nla_len=8, nla_type=IFLA_XDP_FD}, 6], [{nla_len=8, nla_type=IFLA_XDP_FLAGS}, XDP_FLAGS_UPDATE_IF_NOEXIST]]]], 52, 0, NULL, 0) = 52
+recvmsg(7, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base=[{nlmsg_len=36, nlmsg_type=NLMSG_ERROR, nlmsg_flags=NLM_F_CAPPED, nlmsg_seq=1724212008, nlmsg_pid=405177}, {error=0, msg={nlmsg_len=52, nlmsg_type=RTM_SETLINK, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=1724212008, nlmsg_pid=0}}], iov_len=4096}], msg_iovlen=1, msg_controllen=0, msg_flags=0}, MSG_PEEK|MSG_TRUNC) = 36
+recvmsg(7, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base=[{nlmsg_len=36, nlmsg_type=NLMSG_ERROR, nlmsg_flags=NLM_F_CAPPED, nlmsg_seq=1724212008, nlmsg_pid=405177}, {error=0, msg={nlmsg_len=52, nlmsg_type=RTM_SETLINK, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=1724212008, nlmsg_pid=0}}], iov_len=4096}], msg_iovlen=1, msg_controllen=0, msg_flags=0}, 0) = 36
+close(7)                                = 0
+```

example/xdp-counter/xdp-counter.bpf.c

@@ -0,0 +1,72 @@
+/* Copyright (C) 2018-present, Facebook, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+
+#define CTRL_ARRAY_SIZE 2
+#define CNTRS_ARRAY_SIZE 512
+
+// use map type define
+struct {
+  __uint(type, BPF_MAP_TYPE_ARRAY);
+  __type(key, __u32);
+  __type(value, __u32);
+  __uint(max_entries, CTRL_ARRAY_SIZE);
+} ctl_array SEC(".maps");
+
+// use global variable define
+__u64 cntrs_array[CNTRS_ARRAY_SIZE];
+
+static void swap_src_dst_mac(void *data)
+{
+	unsigned short *p = data;
+	unsigned short dst[3];
+
+	dst[0] = p[0];
+	dst[1] = p[1];
+	dst[2] = p[2];
+	p[0] = p[3];
+	p[1] = p[4];
+	p[2] = p[5];
+	p[3] = dst[0];
+	p[4] = dst[1];
+	p[5] = dst[2];
+}
+
+SEC("xdp")
+int xdp_pass(struct xdp_md* ctx) {
+  void* data_end = (void*)(long)ctx->data_end;
+  void* data = (void*)(long)ctx->data;
+  __u32 ctl_flag_pos = 0;
+  __u32 cntr_pos = 0;
+
+  // access maps with helpers
+  __u32* flag = bpf_map_lookup_elem(&ctl_array, &ctl_flag_pos);
+  if (!flag || (*flag != 0)) {
+    return XDP_PASS;
+  };
+
+  // access maps with global variables
+  cntrs_array[cntr_pos]++;
+
+	if (data + sizeof(struct ethhdr) > data_end)
+		return XDP_DROP;
+  swap_src_dst_mac(data);
+  return XDP_TX;
+}
+
+char _license[] SEC("license") = "GPL";

example/xdp-counter/xdp-counter.c

@@ -0,0 +1,73 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <linux/if_link.h>
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <arpa/inet.h>
+#include <net/if.h>
+#include <unistd.h>
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format,
+			   va_list args)
+{
+	return vfprintf(stderr, format, args);
+}
+
+int main(int argc, char **argv)
+{
+	if (argc < 3) {
+		printf("ERROR - Usage is: ./loader"
+				  " <BPF_FILE> <INTERFACE> [btf_file]\n"
+				  "\n");
+		return 1;
+	}
+    libbpf_set_print(libbpf_print_fn);
+	LIBBPF_OPTS(bpf_object_open_opts , opts,
+	);
+	if (argc == 4)
+		opts.btf_custom_path = argv[3];
+	// Open and load the BPF program
+	struct bpf_object* obj = bpf_object__open_file(argv[1], &opts);
+
+	if (bpf_object__load(obj)) {
+		printf( "Failed to load program\n");
+		return 1;
+	}
+
+	struct bpf_program* prog = bpf_object__find_program_by_name(obj, "xdp_pass");
+	if (!prog) {
+		printf("Failed to find program\n");
+		return 1;
+	}
+	bpf_program__set_type(prog, BPF_PROG_TYPE_XDP);
+	int prog_fd = bpf_program__fd(prog);
+	if (prog_fd < 0) {
+		printf("Failed to get prog FD\n");
+		return 1;
+	}
+	const char* progName = bpf_program__name(prog);
+	if (!progName) {
+		printf("Failed to get progName\n");
+		return 1;
+	}
+	printf("load prog %s\n", progName);
+
+	// Attach the XDP program to the interface
+	int ifindex = if_nametoindex(argv[2]);
+	if (!ifindex) {
+		printf("failed to if_nametoindex\n");
+		return 1;
+	}
+
+	struct bpf_link *link = bpf_program__attach_xdp(prog, ifindex);
+	if (!link) {
+		printf("attach error\n");
+		return 1;
+	}
+	printf("Attach XDP success\n");
+	while (1) {
+		sleep(1);
+		printf("wait...\n");
+	}
+	return 0;
+}