Merge pull request #677 from nanshuaibo/develop

kvm_watcher项目：添加了一些功能，完善代码

.github/workflows/kvm_watcher.yml

@@ -20,32 +20,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install dependencies
-        run: |
-          sudo apt install clang libelf1 libelf-dev zlib1g-dev
-          sudo apt install libbpf-dev
-          sudo apt install linux-tools-$(uname -r)
-          sudo apt install linux-cloud-tools-$(uname -r)
-          sudo apt-get update && sudo apt-get install -y qemu-kvm
-      - name: Download Cirros image
-        run: |
-          wget http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img
-      - name: Load KVM module
-        run: |   
-          sudo modprobe kvm && sudo modprobe kvm-intel
-      - name: Run QEMU to start VM
-        run: |
-          sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c -nographic &
-          sleep 5
-      - name: Run kvm_watcher
+      - name: Test program execution
         run: |
           cd eBPF_Supermarket/kvm_watcher/
-          make
-          sudo ./kvm_watcher -w -t 2
-          sudo ./kvm_watcher -e -t 2 -s
-          sudo ./kvm_watcher -n -t 2 
-          sudo ./kvm_watcher -d -t 2
-          sudo ./kvm_watcher -f -m -t 2
-          sudo ./kvm_watcher -c -t 2
-          make clean
+          make test
 

.gitignore

@@ -51,3 +51,12 @@ tags
 *.iml
 
 nohup.out
+
+# Virtual machine image file
+eBPF_Supermarket/kvm_watcher/*.img
+
+eBPF_Supermarket/kvm_watcher/**/*.o
+eBPF_Supermarket/kvm_watcher/**/*.skel.h
+eBPF_Supermarket/kvm_watcher/**/vmlinux.h
+eBPF_Supermarket/kvm_watcher/kvm_watcher
+eBPF_Supermarket/kvm_watcher/**/temp*

eBPF_Supermarket/kvm_watcher/Makefile

@@ -7,7 +7,7 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
 			 | sed 's/riscv64/riscv/' \
 			 | sed 's/loongarch64/loongarch/')
 APP = src/kvm_watcher
-OPTIONS = -f -w -n -d -i '-e -s'
+OPTIONS = -f -w -n -d -c '-e -s'
 
 # 共同规则1
 define common_rules1
@@ -25,12 +25,12 @@ define common_rules2
 	clang -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) -I/usr/include/x86_64-linux-gnu -I. -c $@.bpf.c -o $@.bpf.o
 	bpftool gen skeleton $@.bpf.o > $@.skel.h
 	clang -g -O2 -Wall -I . -c $@.c -o $@.o
-	clang -Wall -O2 -g $@.o -static -lbpf -lelf -lz -o $(notdir $@)
+	clang -Wall -O2 -g $@.o -static -lbpf -lelf -lz -lzstd -o $(notdir $@) # 6.5内核编译需要lzstd库
 endef
 
 # 判断是否已安装 qemu-system-x86_64
 ifeq (,$(shell which qemu-system-x86_64))
-	INSTALL_QEMU = sudo apt update && sudo apt install qemu-system-x86_64
+	INSTALL_QEMU = sudo apt update && sudo apt install qemu-kvm
 endif
 
 bpf: $(APP) 

eBPF_Supermarket/kvm_watcher/README.md

@@ -122,7 +122,7 @@ graph TD;
     B --> C[加载 KVM 模块];
     C --> D[下载 CirrOs 镜像];
     D --> E[使用 QEMU 启动虚拟机];
-    E --> F[编译并运行程序];
+    E --> F[编译运行kvm_watcher];
     F --> G[结束虚拟机进程];
 
 ```

eBPF_Supermarket/kvm_watcher/include/kvm_exits.h

@@ -69,6 +69,7 @@ static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) {
     if (count) {
         (*count)++;
         reas.count = *count;
+        bpf_map_update_elem(&counts, &reason, count, BPF_ANY);
     } else {
         u32 new_count = 1;
         reas.count = new_count;

eBPF_Supermarket/kvm_watcher/include/kvm_irq.h

@@ -31,6 +31,13 @@ struct {
     __type(value, u64);
 } irq_set_delay SEC(".maps");
 
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, u32);
+    __type(value, u64);
+} irq_inject_delay SEC(".maps");
+
 static int entry_kvm_pic_set_irq(int irq, pid_t vm_pid) {
     CHECK_PID(vm_pid);
     if (irq < 0 || irq >= PIC_NUM_PINS) {
@@ -149,4 +156,41 @@ static int exit_kvm_set_msi_irq(
     return 0;
 }
 
+static int entry_vmx_inject_irq(struct kvm_vcpu *vcpu, pid_t vm_pid) {
+    CHECK_PID(vm_pid);
+    u32 irq_nr;
+    bool rei;
+    bpf_probe_read_kernel(&irq_nr, sizeof(u32), &vcpu->arch.interrupt.nr);
+    u64 ts = bpf_ktime_get_ns();
+    bpf_map_update_elem(&irq_inject_delay, &irq_nr, &ts, BPF_ANY);
+    return 0;
+}
+
+static int exit_vmx_inject_irq(struct kvm_vcpu *vcpu, void *rb,
+                               struct common_event *e) {
+    u32 irq_nr;
+    bpf_probe_read_kernel(&irq_nr, sizeof(u32), &vcpu->arch.interrupt.nr);
+    u64 *ts = bpf_map_lookup_elem(&irq_inject_delay, &irq_nr);
+    if (!ts) {
+        return 0;
+    }
+    u64 time = bpf_ktime_get_ns();
+    u64 delay = time - *ts;
+    bpf_map_delete_elem(&irq_inject_delay, &irq_nr);
+    bool soft;
+    bpf_probe_read_kernel(&soft, sizeof(bool), &vcpu->arch.interrupt.soft);
+    RESERVE_RINGBUF_ENTRY(rb, e);
+    e->time = *ts;
+    e->process.pid = bpf_get_current_pid_tgid() >> 32;
+    bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
+    e->irq_inject_data.delay = delay;
+    e->irq_inject_data.irq_nr = irq_nr;
+    e->irq_inject_data.soft = soft;
+    bpf_probe_read_kernel(&e->irq_inject_data.vcpu_id, sizeof(u32),
+                          &vcpu->vcpu_id);
+    bpf_probe_read_kernel(&e->irq_inject_data.injections, sizeof(u64),
+                          &vcpu->stat.irq_injections);
+    bpf_ringbuf_submit(e, 0);
+    return 0;
+}
 #endif /* __KVM_IRQ_H */

eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h

@@ -42,8 +42,8 @@ struct halt_poll_ns {
 
 struct {
     __uint(type, BPF_MAP_TYPE_HASH);
-    __uint(max_entries, 8192);
-    __type(key, u64);
+    __uint(max_entries, 128 * 1024);
+    __type(key, struct dirty_page_info);
     __type(value, u32);
 } count_dirty_map SEC(".maps");
 
@@ -114,14 +114,6 @@ static int trace_mark_page_dirty_in_slot(struct kvm *kvm,
     bpf_probe_read_kernel(&flags, sizeof(memslot->flags), &memslot->flags);
     if (slot &&
         (flags & KVM_MEM_LOG_DIRTY_PAGES)) {  // 检查memslot是否启用了脏页追踪
-        gfn_t gfnum = gfn;
-        u32 *count = bpf_map_lookup_elem(&count_dirty_map, &gfnum);
-        if (count) {
-            *count += 1;
-        } else {
-            u32 init_count = 1;
-            bpf_map_update_elem(&count_dirty_map, &gfnum, &init_count, BPF_ANY);
-        }
         u32 tid = bpf_get_current_pid_tgid();
         unsigned long base_gfn;
         RESERVE_RINGBUF_ENTRY(rb, e);
@@ -140,9 +132,25 @@ static int trace_mark_page_dirty_in_slot(struct kvm *kvm,
                               &memslot->userspace_addr);
         bpf_probe_read_kernel(&e->mark_page_dirty_data.slot_id,
                               sizeof(memslot->id), &memslot->id);
+        short int s_id;
+        bpf_probe_read_kernel(&s_id, sizeof(memslot->id), &memslot->id);
         bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
+        struct dirty_page_info dirty_page_info = {
+            .gfn = gfn, .slot_id = s_id, .rel_gfn = gfn - base_gfn, .pid = pid};
+        u32 *count;
+        count = bpf_map_lookup_elem(&count_dirty_map, &dirty_page_info);
+        if (count) {
+            *count += 1;
+            bpf_map_update_elem(&count_dirty_map, &dirty_page_info, count,
+                                BPF_ANY);
+        } else {
+            u32 init_count = 1;
+            bpf_map_update_elem(&count_dirty_map, &dirty_page_info, &init_count,
+                                BPF_ANY);
+        }
         bpf_ringbuf_submit(e, 0);
     }
+
     return 0;
 }
 #endif /* __KVM_VCPU_H */

eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h

@@ -31,9 +31,9 @@
 #define MICROSECONDS_IN_SECOND 1000000
 #define OUTPUT_INTERVAL_SECONDS 0.5
 
-#define OUTPUT_INTERVAL(us) usleep((unsigned int)(us * MICROSECONDS_IN_SECOND))
+#define OUTPUT_INTERVAL(us) usleep((__u32)(us * MICROSECONDS_IN_SECOND))
 
-#define OPTIONS_LIST "-w, -p, -d, -f, -c, or -e"
+#define OPTIONS_LIST "-w, -p, -d, -f, -c, -i, or -e"
 
 #define PFERR_PRESENT_BIT 0
 #define PFERR_WRITE_BIT 1
@@ -82,26 +82,34 @@
         e = _tmp;                                                \
     } while (0)
 
-#define CHECK_PID(vm_pid)                            \
-    unsigned pid = bpf_get_current_pid_tgid() >> 32; \
-    if ((vm_pid) > 0 && pid != (vm_pid)) {           \
-        return 0;                                    \
+#define CHECK_PID(vm_pid)                         \
+    __u32 pid = bpf_get_current_pid_tgid() >> 32; \
+    if ((vm_pid) > 0 && pid != (vm_pid)) {        \
+        return 0;                                 \
     }
 
 struct ExitReason {
-    int number;
+    __u32 number;
     const char *name;
 };
 
 struct reason_info {
-    unsigned long long time;
-    unsigned long reason;
-    int count;
+    __u64 time;
+    __u64 reason;
+    __u32 count;
+};
+
+struct dirty_page_info {
+    __u64 gfn;
+    __u64 rel_gfn;
+    __u16 slot_id;
+    __u16 pad;
+    __u32 pid;
 };
 
 struct process {
-    unsigned pid;
-    unsigned tid;
+    __u32 pid;
+    __u32 tid;
     char comm[TASK_COMM_LEN];
 };
 
@@ -113,75 +121,85 @@ enum EventType {
     MARK_PAGE_DIRTY,
     PAGE_FAULT,
     IRQCHIP,
+    IRQ_INJECT,
 } event_type;
 
 struct common_event {
     struct process process;
-    unsigned long long time;
+    __u64 time;
 
     // 成员特定于每个事件类型的数据
     union {
         struct {
-            unsigned long long dur_hlt_ns;
+            __u64 dur_hlt_ns;
             bool waited;
-            unsigned vcpu_id;
+            __u32 vcpu_id;
             bool valid;
             // VCPU_WAKEUP 特有成员
         } vcpu_wakeup_data;
 
         struct {
-            unsigned reason_number;
-            unsigned long long duration_ns;
-            int count;
-            int total;
+            __u32 reason_number;
+            __u64 duration_ns;
+            __u32 count;
+            __u32 total;
             // EXIT 特有成员
         } exit_data;
 
         struct {
             bool grow;
-            unsigned int new;
-            unsigned int old;
-            unsigned vcpu_id;
+            __u32 new;
+            __u32 old;
+            __u32 vcpu_id;
             // HALT_POLL 特有成员
         } halt_poll_data;
 
         struct {
-            unsigned long npages;
-            unsigned long userspace_addr;
-            unsigned long long rel_gfn;
-            unsigned long long gfn;
-            short slot_id;
+            __u64 npages;
+            __u64 userspace_addr;
+            __u64 rel_gfn;
+            __u64 gfn;
+            __u16 slot_id;
             // MARK_PAGE_DIRTY 特有成员
         } mark_page_dirty_data;
 
         struct {
-            unsigned long long delay;
-            unsigned long long error_code;
-            unsigned long long addr;
-            unsigned long long pfn;
-            unsigned long long hva;
-            unsigned count;
-            short memslot_id;
+            __u64 delay;
+            __u64 error_code;
+            __u64 addr;
+            __u64 pfn;
+            __u64 hva;
+            __u32 count;
+            __u16 memslot_id;
             // PAGE_FAULT 特有成员
         } page_fault_data;
 
         struct {
-            unsigned long long delay;
-            int ret;
-            int irqchip_type;
+            __u64 delay;
+            __u32 ret;
+            __u32 irqchip_type;
             /*pic*/
-            unsigned char chip;
-            unsigned pin;
-            unsigned char elcr;
-            unsigned char imr;
+            __u16 chip;
+            __u32 pin;
+            __u16 elcr;
+            __u16 imr;
             /*ioapic*/
-            unsigned long long ioapic_bits;
-            unsigned int irq_nr;
+            __u64 ioapic_bits;
+            __u32 irq_nr;
             /*msi*/
-            unsigned long long address;
-            unsigned long long data;
+            __u64 address;
+            __u64 data;
             // IRQCHIP 特有成员
         } irqchip_data;
+
+        struct {
+            __u64 delay;
+            bool soft;
+            __u32 irq_nr;
+            __u32 vcpu_id;
+            __u64 injections;
+            // IRQ_INJECT 特有成员
+        } irq_inject_data;
     };
 };
 

eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c

@@ -129,3 +129,13 @@ int BPF_PROG(fexit_kvm_set_msi_irq, struct kvm *kvm,
              struct kvm_lapic_irq *irq) {
     return exit_kvm_set_msi_irq(kvm, routing_entry, &rb, e);
 }
+
+SEC("fentry/vmx_inject_irq")
+int BPF_PROG(fentry_vmx_inject_irq, struct kvm_vcpu *vcpu, bool reinjected) {
+    return entry_vmx_inject_irq(vcpu, vm_pid);
+}
+
+SEC("fexit/vmx_inject_irq")
+int BPF_PROG(fexit_vmx_inject_irq, struct kvm_vcpu *vcpu, bool reinjected) {
+    return exit_vmx_inject_irq(vcpu, &rb, e);
+}