fix: log runtime output to env var "BPFTIME_LOG_OUTPUT"

.github/workflows/test-examples.yml

@@ -31,11 +31,19 @@ jobs:
       - name: Build and install runtime (with llvm-jit)
         if: ${{matrix.enable_jit}}
         run: |
-          make release-with-llvm-jit -j
+          cmake -Bbuild  -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+            -DBPFTIME_LLVM_JIT=1 \
+            -DBUILD_BPFTIME_DAEMON=1 \
+            -DCMAKE_CXX_FLAGS="-DDEFAULT_LOGGER_OUTPUT_PATH='\"console\"'"
+          cmake --build build --config RelWithDebInfo --target install -j
       - name: Build and install runtime (without llvm-jit)
         if: ${{!matrix.enable_jit}}
         run: |
-          make release -j
+          cmake -Bbuild  -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+            -DBPFTIME_LLVM_JIT=0 \
+            -DBUILD_BPFTIME_DAEMON=1 \
+            -DCMAKE_CXX_FLAGS="-DDEFAULT_LOGGER_OUTPUT_PATH='\"console\"'"
+          cmake --build build --config RelWithDebInfo --target install -j
       - name: Upload build results (without jit)
         uses: actions/upload-artifact@v3
         if: ${{!matrix.enable_jit}}

CMakeLists.txt

@@ -86,7 +86,7 @@ message(STATUS "Started CMake for ${PROJECT_NAME} v${PROJECT_VERSION}...\n")
 
 # if option to build without libbpf is set
 if(${BPFTIME_BUILD_WITH_LIBBPF})
-  add_definitions(-DUSE_LIBBPF)
+  add_definitions(-DBPFTIME_BUILD_WITH_LIBBPF=1)
 endif()
 
 if(UNIX)
@@ -135,7 +135,7 @@ endif()
 add_subdirectory(third_party/spdlog)
 if(NOT DEFINED SPDLOG_ACTIVE_LEVEL)
   if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    add_compile_definitions(SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG)
+    add_compile_definitions(SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE)
   else()
     add_compile_definitions(SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO)
   endif()

Makefile

@@ -65,7 +65,7 @@ release: ## build the release version
 
 release-with-llvm-jit: ## build the package, with llvm-jit
 	cmake -Bbuild  -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
-				   -DBPFTIME_LLVM_JIT=1
+				   -DBPFTIME_LLVM_JIT=1 \
 				   -DBUILD_BPFTIME_DAEMON=1
 	cmake --build build --config RelWithDebInfo --target install -j$(JOBS)
 

README.md

@@ -17,18 +17,18 @@
 
 ## Key Features
 
-- **Uprobe and Syscall hooks based on binary rewriting**: Run eBPF programs in userspace, attaching them to Uprobes and Syscall tracepoints: **No manual instrumentation or restart required!**. It can `trace` or `change` the execution of a function, `hook` or `filter` all syscalls of a process safely, and efficiently with an eBPF userspace runtime.
-- **Performance**: Experience up to a `10x` speedup in Uprobe overhead compared to kernel uprobe and uretprobe.
+- **Uprobe and Syscall hooks based on binary rewriting**: Run eBPF programs in userspace, attaching them to Uprobes and Syscall tracepoints: **No manual instrumentation or restart required!**. It can `trace` or `change` the execution of a function, `hook` or `filter` all syscalls of a process safely, and efficiently with an eBPF userspace runtime. Can inject eBPF runtime into any running process without the need for a restart or manual recompilation.
+- **Performance**: Experience up to a `10x` speedup in Uprobe overhead compared to kernel uprobe and uretprobe. Read/Write userspace memory is also faster than kernel eBPF.
 - **Interprocess eBPF Maps**: Implement userspace `eBPF maps` in shared userspace memory for summary aggregation or control plane communication.
-- **Compatibility**: use `existing eBPF toolchains` like clang and libbpf to develop userspace eBPF without any modifications. Supporting CO-RE via BTF, and offering userspace host function access.
-- **JIT Support**: Benefit from a cross-platform eBPF interpreter and a high-speed `JIT/AOT` compiler powered by LLVM. It also includes a handcrafted x86 JIT in C for limited resources. The vm can be built as `a standalone library` like ubpf.
-- **No instrumentation**: Can inject eBPF runtime into any running process without the need for a restart or manual recompilation.
+- **Compatibility**: use `existing eBPF toolchains` like clang, libbpf and bpftrace to develop userspace eBPF application without any modifications. Supporting CO-RE via BTF, and offering userspace `ufunc` access.
+- **Multi JIT Support**: Support [llvmbpf](https://github.com/eunomia-bpf/llvmbpf), a high-speed `JIT/AOT` compiler powered by LLVM, or using `ubpf JIT` and INTERPRETER. The vm can be built as `a standalone library` like ubpf.
 - **Run with kernel eBPF**: Can load userspace eBPF from kernel, and using kernel eBPF maps to cooperate with kernel eBPF programs like kprobes and network filters.
 
 ## Components
 
-- [`vm`](https://github.com/eunomia-bpf/bpftime/tree/master/vm): The eBPF VM and JIT for eBPF, you can choose from bpftime LLVM JIT and a simple JIT/interpreter based on ubpf. It can be built as a standalone library and integrated into other projects. The API is similar to ubpf.
-- [`runtime`](https://github.com/eunomia-bpf/bpftime/tree/master/runtime): The userspace runtime for eBPF, including the syscall server and agent, attaching eBPF programs to Uprobes and Syscall tracepoints, and eBPF maps in shared memory.
+- [`vm`](https://github.com/eunomia-bpf/bpftime/tree/master/vm): The eBPF VM and JIT compiler for bpftime, you can choose from [bpftime LLVM JIT/AOT compiler](https://github.com/eunomia-bpf/llvmbpf) and [ubpf](https://github.com/iovisor/ubpf). The [llvm-based vm](https://github.com/eunomia-bpf/llvmbpf) in bpftime can also be built as a standalone library and integrated into other projects, similar to ubpf.
+- [`runtime`](https://github.com/eunomia-bpf/bpftime/tree/master/runtime): The userspace runtime for eBPF, including the bpf-syscall loader(`syscall-server`) and agent, support attaching eBPF programs to Uprobes, Syscall tracepoints and other events, as well as eBPF maps in shared memory.
+- [verifier](https://github.com/eunomia-bpf/bpftime/tree/master/bpftime-verifier): Support using [PREVAIL](https://github.com/vbpf/ebpf-verifier) as userspace verifier, or using Linux kernel verifier as an option.
 - [`daemon`](https://github.com/eunomia-bpf/bpftime/tree/master/daemon): A daemon to make userspace eBPF working with kernel and compatible with kernel uprobe. Monitor and modify kernel eBPF events and syscalls, load eBPF in userspace from kernel.
 
 ## Quick Start

attach/text_segment_transformer/agent-transformer.cpp

@@ -69,7 +69,7 @@ extern "C" void bpftime_agent_main(const gchar *data, gboolean *stay_resident)
 			"Please set AGENT_SO to the bpftime-agent when use this tranformer");
 		return;
 	}
-	SPDLOG_INFO("Using agent {}", agent_so);
+	SPDLOG_DEBUG("Using agent {}", agent_so);
 	cs_arch_register_x86();
 	bpftime::setup_syscall_tracer();
 	SPDLOG_DEBUG("Loading dynamic library..");
@@ -98,5 +98,5 @@ extern "C" void bpftime_agent_main(const gchar *data, gboolean *stay_resident)
 		bpftime::get_call_hook();
 	entry_func(&orig_syscall_hooker_func);
 	bpftime::set_call_hook(orig_syscall_hooker_func);
-	SPDLOG_INFO("Transformer exiting, trace will be usable now");
+	SPDLOG_DEBUG("Transformer exiting, syscall trace is usable now");
 }

runtime/CMakeLists.txt

@@ -78,7 +78,7 @@ set(sources
   extension/extension_helper.cpp
 )
 
-if(UNIX AND NOT APPLE)
+if(UNIX AND NOT APPLE AND BPFTIME_BUILD_WITH_LIBBPF)
   list(APPEND sources
     src/bpf_map/shared/array_map_kernel_user.cpp
     src/bpf_map/shared/hash_map_kernel_user.cpp
@@ -94,13 +94,6 @@ set(headers
 message(INFO " Headers: ${headers}")
 message(INFO " Found the following sources: ${sources}")
 
-
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG)
-else()
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO)
-endif()
-
 add_library(
   ${PROJECT_NAME}
   ${sources}
@@ -238,16 +231,17 @@ message(DEBUG "Successfully added all dependencies and linked against them.")
 
 set(BPFTIME_RUNTIME_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
-
+if (BPFTIME_BUILD_WITH_LIBBPF)
 add_subdirectory(object)
+endif()
 add_subdirectory(agent)
 add_subdirectory(syscall-server)
 #
 # Unit testing setup
 #
-if(BPFTIME_ENABLE_UNIT_TESTING)
+if(BPFTIME_ENABLE_UNIT_TESTING AND BPFTIME_BUILD_WITH_LIBBPF)
   enable_testing()
-  message(STATUS "Build unit tests for the project. Tests should always be found in the test folder\n")
+  message(STATUS "Build unit tests for the runtime. Tests should always be found in the test folder\n")
   add_subdirectory(test)
   add_subdirectory(unit-test)
 endif()

runtime/agent/agent.cpp

@@ -6,6 +6,7 @@
 #include "spdlog/common.h"
 #include "spdlog/sinks/stdout_color_sinks.h"
 #include "spdlog/sinks/stdout_sinks.h"
+#include "bpftime_logger.hpp"
 #include <chrono>
 #include <csignal>
 #include <exception>
@@ -21,10 +22,11 @@
 #include "bpftime_shm.hpp"
 #include <spdlog/spdlog.h>
 #include <spdlog/cfg/env.h>
-#if __linux__
+#if __linux__ && BPFTIME_BUILD_WITH_LIBBPF
 #include "syscall_trace_attach_impl.hpp"
 #include "syscall_trace_attach_private_data.hpp"
 #endif
+
 using namespace bpftime;
 using namespace bpftime::attach;
 using main_func_t = int (*)(int, char **, char **);
@@ -94,18 +96,17 @@ static void sig_handler_sigusr1(int sig)
 	shm_holder.global_shared_memory.remove_pid_from_alive_agent_set(
 		getpid());
 	SPDLOG_DEBUG("Detaching done");
+	bpftime_logger_flush();
 }
 
 extern "C" void bpftime_agent_main(const gchar *data, gboolean *stay_resident)
 {
-	auto logger = spdlog::stderr_color_mt("stderr");
-	logger->set_pattern("[%Y-%m-%d %H:%M:%S][%^%l%$][%t] %v");
-	spdlog::set_default_logger(logger);
+	auto runtime_config = bpftime_get_agent_config();
+	bpftime_set_logger(runtime_config.logger_output_path);
 	SPDLOG_DEBUG("Entered bpftime_agent_main");
 	SPDLOG_DEBUG("Registering signal handler");
 	// We use SIGUSR1 to indicate the detaching
 	signal(SIGUSR1, sig_handler_sigusr1);
-	spdlog::cfg::load_env_levels();
 	try {
 		// If we are unable to initialize shared memory..
 		bpftime_initialize_global_shm(shm_open_type::SHM_OPEN_ONLY);
@@ -121,7 +122,7 @@ extern "C" void bpftime_agent_main(const gchar *data, gboolean *stay_resident)
 			getpid());
 	}
 	ctx_holder.init();
-	#if __linux__
+	#if __linux__ && BPFTIME_BUILD_WITH_LIBBPF
 	// Register syscall trace impl
 	auto syscall_trace_impl = std::make_unique<syscall_trace_attach_impl>();
 	syscall_trace_impl->set_original_syscall_function(orig_hooker);
@@ -164,7 +165,7 @@ extern "C" void bpftime_agent_main(const gchar *data, gboolean *stay_resident)
 	SPDLOG_DEBUG("Set environment variable BPFTIME_USED");
 	try {
 		res = ctx_holder.ctx.init_attach_ctx_from_handlers(
-			bpftime_get_agent_config());
+			runtime_config);
 		if (res != 0) {
 			SPDLOG_INFO("Failed to initialize attach context, exiting..");
 			return;
@@ -178,7 +179,7 @@ extern "C" void bpftime_agent_main(const gchar *data, gboolean *stay_resident)
 
 // using definition for libbpf for syscall issues
 // maybe should separate libbpf and kernel features separately
-#if __linux__
+#if __linux__ && BPFTIME_BUILD_WITH_LIBBPF
 extern "C" int64_t syscall_callback(int64_t sys_nr, int64_t arg1, int64_t arg2,
 				    int64_t arg3, int64_t arg4, int64_t arg5,
 				    int64_t arg6)

runtime/include/bpftime_config.hpp

@@ -4,6 +4,11 @@
 #include <cstdlib>
 #include <string>
 
+#ifndef DEFAULT_LOGGER_OUTPUT_PATH
+#define DEFAULT_LOGGER_OUTPUT_PATH "~/.bpftime/runtime.log"
+#endif
+#define stringize(x) #x
+
 namespace bpftime
 {
 // Configuration for the bpftime runtime
@@ -28,10 +33,16 @@ struct agent_config {
 	// available for the eBPF programs and maps
 	// The value is in MB
 	int shm_memory_size = 20; // 20MB
+
+	// specify the where the logger output should be written to
+	// It can be a file path or "console".
+	// If it is "console", the logger will output to stderr
+	std::string logger_output_path = DEFAULT_LOGGER_OUTPUT_PATH;
 };
 
 // Get the bpftime configuration from the environment variables
-const agent_config get_agent_config_from_env();
+// If the shared memory is not int, this should be called first
+const agent_config get_agent_config_from_env() noexcept;
 
 } // namespace bpftime
 

runtime/include/bpftime_logger.hpp

@@ -0,0 +1,73 @@
+#include <string>
+#include "spdlog/spdlog.h"
+#include "spdlog/cfg/env.h"
+#include "spdlog/sinks/rotating_file_sink.h"
+#include "spdlog/sinks/stdout_color_sinks.h"
+#include <cstdlib>
+#include <iostream>
+#include <filesystem>
+#include <fstream>
+
+namespace bpftime
+{
+
+inline std::string expand_user_path(const std::string &input_path)
+{
+	if (input_path.empty()) {
+		return "console";
+	}
+
+	if (input_path[0] == '~') {
+		const char *homeDir = getenv("HOME");
+		if (!homeDir) {
+			return "console";
+		}
+
+		if (input_path.size() == 1 || input_path[1] == '/') {
+			// Replace "~" with the home directory
+			std::string expandedPath = homeDir +
+				 input_path.substr(1);
+			return expandedPath;
+		} else {
+			return "console"; // Unsupported path format
+		}
+	}
+
+	// Return the original path if no tilde expansion is needed
+	return input_path;
+}
+
+inline void bpftime_set_logger(const std::string &target) noexcept
+{
+	std::string logger_target = expand_user_path(target);
+
+	if (logger_target == "console") {
+		// Set logger to stderr
+		auto logger = spdlog::stderr_color_mt("stderr");
+		logger->set_pattern("[%Y-%m-%d %H:%M:%S][%^%l%$][%t] %v");
+		logger->flush_on(spdlog::level::info);
+		spdlog::set_default_logger(logger);
+	} else {
+		// Set logger to file, with rotation 5MB and 3 files
+		auto max_size = 1048576 * 5;
+		auto max_files = 3;
+		auto logger = spdlog::rotating_logger_mt(
+			"bpftime_logger", logger_target, max_size, max_files);
+		logger->set_pattern("[%Y-%m-%d %H:%M:%S][%^%l%$][%t] %v");
+		logger->flush_on(spdlog::level::info);
+		spdlog::set_default_logger(logger);
+	}
+
+	// Load log level from environment
+	spdlog::cfg::load_env_levels();
+}
+
+/*
+    Flush the logger.
+*/
+inline void bpftime_logger_flush()
+{
+	spdlog::default_logger()->flush();
+}
+
+} // namespace bpftime

runtime/include/bpftime_shm.hpp

@@ -10,13 +10,16 @@
 #include <boost/interprocess/smart_ptr/unique_ptr.hpp>
 #include <boost/interprocess/containers/string.hpp>
 #include <cstdint>
-#include <ebpf-vm.h>
 #if __linux__
 #include <sys/epoll.h>
 #elif __APPLE__
 #include "bpftime_epoll.h"
 #endif
 
+extern "C" {
+struct ebpf_inst;
+}
+
 namespace bpftime
 {
 
@@ -153,7 +156,13 @@ enum class bpf_prog_type {
 };
 
 extern const shm_open_type global_shm_open_type;
+
+// Get the runtime config in the shared memory.
+// The shared memory should be initialized before calling this function.
+// This should be called by the agent side instead of the server side.
 const bpftime::agent_config &bpftime_get_agent_config();
+
+// Set the runtime config in the shared memory.
 void bpftime_set_agent_config(const bpftime::agent_config &cfg);
 
 // Map ops for register external map types and operations
@@ -272,7 +281,6 @@ int bpftime_link_create(int fd, struct bpf_link_create_args *args);
 int bpftime_progs_create(int fd, const ebpf_inst *insn, size_t insn_cnt,
 			 const char *prog_name, int prog_type);
 
-
 // create a bpf map in the global shared memory
 //
 // @param[fd]: fd is the fd allocated by the kernel. if fd is -1, then the
@@ -314,20 +322,27 @@ int bpftime_perf_event_enable(int fd);
 // disable the perf event
 int bpftime_perf_event_disable(int fd);
 
+// find the minimal unused fd in shared memory
+// Which is a available handler for bpf related object
 int bpftime_find_minimal_unused_fd();
 
 int bpftime_attach_perf_to_bpf(int perf_fd, int bpf_fd);
 int bpftime_attach_perf_to_bpf_with_cookie(int perf_fd, int bpf_fd,
 					   uint64_t cookie);
+
 int bpftime_add_ringbuf_fd_to_epoll(int ringbuf_fd, int epoll_fd,
 				    epoll_data_t extra_data);
+
 int bpftime_add_software_perf_event_fd_to_epoll(int swpe_fd, int epoll_fd,
 						epoll_data_t extra_data);
 
 int bpftime_epoll_create();
 void *bpftime_get_ringbuf_consumer_page(int ringbuf_fd);
 void *bpftime_get_ringbuf_producer_page(int ringbuf_fd);
 
+int bpftime_poll_from_ringbuf(int rb_fd, void *ctx,
+			      int (*cb)(void *, void *, size_t));
+
 int bpftime_is_ringbuf_map(int fd);
 int bpftime_is_array_map(int fd);
 int bpftime_is_epoll_handler(int fd);
@@ -350,7 +365,7 @@ int bpftime_add_software_perf_event(int cpu, int32_t sample_type,
 int bpftime_is_software_perf_event(int fd);
 void *bpftime_get_software_perf_event_raw_buffer(int fd, size_t expected_size);
 int bpftime_perf_event_output(int fd, const void *buf, size_t sz);
-#if __linux__
+#if __linux__ && BPFTIME_BUILD_WITH_LIBBPF
 int bpftime_shared_perf_event_output(int map_fd, const void *buf, size_t sz);
 #endif
 int bpftime_add_ureplace_or_override(int fd, int pid, const char *name,
@@ -359,9 +374,6 @@ int bpftime_add_ureplace_or_override(int fd, int pid, const char *name,
 int bpftime_get_current_thread_cookie(uint64_t *out);
 
 int bpftime_add_custom_perf_event(int type, const char *attach_argument);
-
-int bpftime_poll_from_ringbuf(int rb_fd, void *ctx,
-			      int (*cb)(void *, void *, size_t));
 }
 
 #endif // BPFTIME_SHM_CPP_H