kvm_watcher项目:pic中断设置，处理时延记录

.github/workflows/kvm_watcher.yml

@@ -45,5 +45,6 @@ jobs:
           sudo ./kvm_watcher -n -t 10 
           sudo ./kvm_watcher -d -t 10 
           sudo ./kvm_watcher -f -m -t 10
+          sudo ./kvm_watcher -i -t 10
           make clean
 

eBPF_Supermarket/kvm_watcher/README.md

@@ -18,6 +18,10 @@
 - **vCPU相关指标分析：**
   - 记录有关vCPU的性能指标，包括唤醒时的时间戳，halt持续时间，vCPU id等相关信息。
   - 实时监控vCPU的halt-polling时间的变化信息，包括vCPU的线程tid，变化类型，变化前后的halt-polling时间等信息。
+- **kvm中中断注入时相关信息：**
+  - PIC:实时记录PIC芯片类型，中断引脚编号，中断触发方式，是否可屏蔽，处理延时，是否发生合并等信息。
+  - IOAPIC:
+  - MSI:
 
 ## 三、使用方法
 
@@ -52,6 +56,7 @@ BPF program used for monitoring KVM event
   -d, --mark_page_dirty      Monitor virtual machine dirty page information.
   -e, --vm_exit              Monitoring the event of vm exit.
   -f, --kvmmmu_page_fault    Monitoring the data of kvmmmu page fault.
+  -i, --kvm_irq              Monitor the interrupt information in KVM VM.
   -m, --mmio                 Monitoring the data of mmio page fault..(The -f option must be specified.)
   -n, --halt_poll_ns         Monitoring the variation in vCPU halt-polling time.
   -p, --vm_pid=PID           Specify the virtual machine pid to monitor.
@@ -69,6 +74,8 @@ BPF program used for monitoring KVM event
 
 `-f`：记录kvmmmu缺页信息
 
+`-i`：记录kvm中断设置相关信息
+
 `-m`：记录mmio缺页信息（需要和`-f`一同使用）
 
 `-d`：记录kvm脏页信息
@@ -77,7 +84,7 @@ BPF program used for monitoring KVM event
 
 `-w`：记录vcpu唤醒时的相关信息
 
-`-p`：指定kvm虚拟机进程pid（必须为虚拟机进程，否则会报错）
+`-p`：指定kvm虚拟机进程pid
 
 `-t`：监控时间
 
@@ -87,6 +94,7 @@ BPF program used for monitoring KVM event
 ├── include
 │   ├── kvm_exits.h           //vm exit事件相关的内核bpf程序
 │   ├── kvm_mmu.h             //kvmmmu相关的内核bpf程序
+│   ├── kvm_irq.h             //中断注入相关内核bpf程序
 │   ├── kvm_vcpu.h            //vcpu相关内核bpf程序
 │   └── kvm_watcher.h         //项目公用头文件
 ├── Makefile                  //编译脚本
@@ -131,10 +139,7 @@ BPF program used for monitoring KVM event
 
   ```
   make
-  sudo ./kvm_watcher -w -t 10
-  sudo ./kvm_watcher -e -t 10 -s
-  sudo ./kvm_watcher -n -t 10 
-  sudo ./kvm_watcher -d -t 10 
+  sudo ./kvm_watcher [options]
   make clean
   ```
 

eBPF_Supermarket/kvm_watcher/include/kvm_exits.h

@@ -80,7 +80,7 @@ static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) {
     return 0;
 }
 
-static int trace_kvm_entry(void *rb) {
+static int trace_kvm_entry(void *rb, struct common_event *e) {
     struct reason_info *reas;
     pid_t pid, tid;
     u64 id, ts, *start_ts, duration_ns = 0;
@@ -90,25 +90,22 @@ static int trace_kvm_entry(void *rb) {
     reas = bpf_map_lookup_elem(&times, &tid);
     if (reas) {
         u32 reason;
-        struct exit_event *e;
         int count = 0;
         duration_ns = bpf_ktime_get_ns() - reas->time;
         bpf_map_delete_elem(&times, &tid);
         reason = reas->reason;
         count = reas->count;
         RESERVE_RINGBUF_ENTRY(rb, e);
-        e->reason_number = reason;
+        e->exit_data.reason_number = reason;
         e->process.pid = pid;
-        e->duration_ns = duration_ns;
-        bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
         e->process.tid = tid;
-        e->total = ++total;
-        e->count = count;
+        e->exit_data.duration_ns = duration_ns;
+        bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
+        e->exit_data.total = ++total;
+        e->exit_data.count = count;
         e->time = reas->time;
         bpf_ringbuf_submit(e, 0);
-        return 0;
-    } else {
-        return 0;
     }
+    return 0;
 }
 #endif /* __KVM_EXITS_H */

eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h

@@ -50,7 +50,8 @@ static int trace_page_fault(struct trace_event_raw_kvm_page_fault *ctx,
 }
 
 static int trace_direct_page_fault(struct kvm_vcpu *vcpu,
-                                   struct kvm_page_fault *fault, void *rb) {
+                                   struct kvm_page_fault *fault, void *rb,
+                                   struct common_event *e) {
     u64 addr;
     bpf_probe_read_kernel(&addr, sizeof(u64), &fault->addr);
     u64 *ts;
@@ -66,23 +67,22 @@ static int trace_direct_page_fault(struct kvm_vcpu *vcpu,
         short memslot_id = BPF_CORE_READ(fault, slot, id);
         u64 delay = bpf_ktime_get_ns() - *ts;
         bpf_map_delete_elem(&pf_delay, &addr);
-        struct page_fault_event *e;
         RESERVE_RINGBUF_ENTRY(rb, e);
         count = bpf_map_lookup_elem(&pf_count, &addr);
         if (count) {
             (*count)++;
-            e->count = *count;
+            e->page_fault_data.count = *count;
             bpf_map_update_elem(&pf_count, &addr, count, BPF_ANY);
         } else {
-            e->count = 1;
+            e->page_fault_data.count = 1;
             bpf_map_update_elem(&pf_count, &addr, &new_count, BPF_ANY);
         }
-        e->delay = delay;
-        e->addr = addr;
-        e->error_code = error_code;
-        e->hva = hva;
-        e->pfn = pfn;
-        e->memslot_id = memslot_id;
+        e->page_fault_data.delay = delay;
+        e->page_fault_data.addr = addr;
+        e->page_fault_data.error_code = error_code;
+        e->page_fault_data.hva = hva;
+        e->page_fault_data.pfn = pfn;
+        e->page_fault_data.memslot_id = memslot_id;
         e->process.pid = bpf_get_current_pid_tgid() >> 32;
         bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
         e->time = *ts;
@@ -104,32 +104,33 @@ static int trace_kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 }
 
 static int trace_handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
-                                        bool direct, void *rb) {
+                                        bool direct, void *rb,
+                                        struct common_event *e) {
     u64 *ts;
     ts = bpf_map_lookup_elem(&pf_delay, &addr);
     if (ts) {
         u32 *count;
         u32 new_count = 1;
         u64 delay = bpf_ktime_get_ns() - *ts;
         bpf_map_delete_elem(&pf_delay, &addr);
-        struct page_fault_event *e;
         RESERVE_RINGBUF_ENTRY(rb, e);
         count = bpf_map_lookup_elem(&pf_count, &addr);
         if (count) {
             (*count)++;
-            e->count = *count;
+            e->page_fault_data.count = *count;
             bpf_map_update_elem(&pf_count, &addr, count, BPF_ANY);
         } else {
-            e->count = 1;
+            e->page_fault_data.count = 1;
             bpf_map_update_elem(&pf_count, &addr, &new_count, BPF_ANY);
         }
-        e->delay = delay;
-        e->addr = addr;
-        e->error_code = PFERR_RSVD_MASK;
+        e->page_fault_data.delay = delay;
+        e->page_fault_data.addr = addr;
+        e->page_fault_data.error_code = PFERR_RSVD_MASK;
         e->process.pid = bpf_get_current_pid_tgid() >> 32;
         bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
         e->time = *ts;
         bpf_ringbuf_submit(e, 0);
+        return 0;
     }
     return 0;
 }

eBPF_Supermarket/kvm_watcher/include/kvm_pic.h

@@ -0,0 +1,77 @@
+// Copyright 2023 The LMP Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// author: [email protected]
+//
+// Kernel space BPF program used for monitoring data for KVM PIC.
+#ifndef __KVM_PIC_H
+#define __KVM_PIC_H
+
+#include "kvm_watcher.h"
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_tracing.h>
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, u32);
+    __type(value, u64);
+} irq_delay SEC(".maps");
+
+static int trace_in_kvm_pic_set_irq(struct kvm_pic *s, int irq,
+                                    int irq_source_id, int level,
+                                    pid_t vm_pid) {
+    CHECK_PID(vm_pid) {
+        if (irq < 0 || irq >= PIC_NUM_PINS) {
+            return 0;
+        }
+        u64 ts = bpf_ktime_get_ns();
+        u32 irq_type = irq >> 3;
+        bpf_map_update_elem(&irq_delay, &irq_type, &ts, BPF_ANY);
+    }
+    return 0;
+}
+
+static int trace_out_kvm_pic_set_irq(struct kvm_pic *s, int irq,
+                                     int irq_source_id, int level, int retval,
+                                     void *rb, struct common_event *e) {
+    u64 *ts;
+    u32 irq_type = irq >> 3;
+    ts = bpf_map_lookup_elem(&irq_delay, &irq_type);
+    if (!ts) {
+        return 0;
+    }
+    u64 time = bpf_ktime_get_ns();
+    u64 delay = time - *ts;
+    bpf_map_delete_elem(&irq_delay, &irq_type);
+    RESERVE_RINGBUF_ENTRY(rb, e);
+    e->pic_data.ret = retval;
+    e->time = *ts;
+    e->pic_data.delay = delay;
+    e->process.pid = bpf_get_current_pid_tgid() >> 32;
+    e->pic_data.chip = irq_type;
+    e->pic_data.pin = irq & 7;
+    bpf_probe_read_kernel(&e->pic_data.elcr, sizeof(u8),
+                          &s->pics[irq_type].elcr);
+    bpf_probe_read_kernel(&e->pic_data.imr, sizeof(u8), &s->pics[irq_type].imr);
+    bpf_probe_read_kernel(&e->pic_data.irq_source_id, sizeof(int),
+                          &irq_source_id);
+    bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
+    bpf_ringbuf_submit(e, 0);
+    return 0;
+}
+
+#endif /* __KVM_PIC_H */

eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h

@@ -29,7 +29,7 @@ struct vcpu_wakeup {
     u64 pad;
     __u64 ns;
     bool waited;
-    bool vaild;
+    bool valid;
 };
 
 struct halt_poll_ns {
@@ -47,38 +47,59 @@ struct {
     __type(value, u32);
 } count_dirty_map SEC(".maps");
 
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, u32);
+    __type(value, u32);
+} vcpu_tid SEC(".maps");
+
+static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu, pid_t vm_pid) {
+    CHECK_PID(vm_pid) {
+        u32 tid = bpf_get_current_pid_tgid();
+        u32 vcpu_id;
+        bpf_probe_read_kernel(&vcpu_id, sizeof(vcpu->vcpu_id), &vcpu->vcpu_id);
+        bpf_map_update_elem(&vcpu_tid, &tid, &vcpu_id, BPF_ANY);
+    }
+    return 0;
+}
+
 static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx, void *rb,
-                                 pid_t vm_pid) {
+                                 struct common_event *e, pid_t vm_pid) {
     CHECK_PID(vm_pid) {
         u32 tid = bpf_get_current_pid_tgid();
-        struct vcpu_wakeup_event *e;
+        u32 *vcpu_id = bpf_map_lookup_elem(&vcpu_tid, &tid);
+        if (!vcpu_id) {
+            return 0;
+        }
         RESERVE_RINGBUF_ENTRY(rb, e);
-        u64 hlt_time = bpf_ktime_get_ns();
-        e->waited = ctx->waited;
+        u64 time = bpf_ktime_get_ns();
+        e->vcpu_wakeup_data.waited = ctx->waited;
         e->process.pid = pid;
         e->process.tid = tid;
-        e->dur_hlt_ns = ctx->ns;
-        e->hlt_time = hlt_time;
+        e->vcpu_wakeup_data.dur_hlt_ns = ctx->ns;
+        e->vcpu_wakeup_data.vcpu_id = *vcpu_id;
+        e->time = time;
+        e->vcpu_wakeup_data.valid = ctx->valid;
         bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
         bpf_ringbuf_submit(e, 0);
     }
     return 0;
 }
 
 static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb,
-                                  pid_t vm_pid) {
+                                  struct common_event *e, pid_t vm_pid) {
     CHECK_PID(vm_pid) {
         u32 tid = bpf_get_current_pid_tgid();
-        struct halt_poll_ns_event *e;
         RESERVE_RINGBUF_ENTRY(rb, e);
         u64 time = bpf_ktime_get_ns();
         e->process.pid = pid;
         e->process.tid = tid;
         e->time = time;
-        e->grow = ctx->grow;
-        e->old = ctx->old;
-        e->new = ctx->new;
-        e->vcpu_id = ctx->vcpu_id;
+        e->halt_poll_data.grow = ctx->grow;
+        e->halt_poll_data.old = ctx->old;
+        e->halt_poll_data.new = ctx->new;
+        e->halt_poll_data.vcpu_id = ctx->vcpu_id;
         bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
         bpf_ringbuf_submit(e, 0);
     }
@@ -87,7 +108,8 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb,
 
 static int trace_mark_page_dirty_in_slot(struct kvm *kvm,
                                          const struct kvm_memory_slot *memslot,
-                                         gfn_t gfn, void *rb, pid_t vm_pid) {
+                                         gfn_t gfn, void *rb,
+                                         struct common_event *e, pid_t vm_pid) {
     CHECK_PID(vm_pid) {
         u32 flags;
         struct kvm_memory_slot *slot;
@@ -107,23 +129,22 @@ static int trace_mark_page_dirty_in_slot(struct kvm *kvm,
             }
             u32 tid = bpf_get_current_pid_tgid();
             unsigned long base_gfn;
-            struct mark_page_dirty_in_slot_event *e;
             RESERVE_RINGBUF_ENTRY(rb, e);
             u64 time = bpf_ktime_get_ns();
             e->process.pid = pid;
             e->process.tid = tid;
             e->time = time;
-            e->gfn = gfn;
+            e->mark_page_dirty_data.gfn = gfn;
             bpf_probe_read_kernel(&base_gfn, sizeof(memslot->base_gfn),
                                   &memslot->base_gfn);
-            e->rel_gfn = gfn - base_gfn;
-            bpf_probe_read_kernel(&e->npages, sizeof(memslot->npages),
-                                  &memslot->npages);
-            bpf_probe_read_kernel(&e->userspace_addr,
+            e->mark_page_dirty_data.rel_gfn = gfn - base_gfn;
+            bpf_probe_read_kernel(&e->mark_page_dirty_data.npages,
+                                  sizeof(memslot->npages), &memslot->npages);
+            bpf_probe_read_kernel(&e->mark_page_dirty_data.userspace_addr,
                                   sizeof(memslot->userspace_addr),
                                   &memslot->userspace_addr);
-            bpf_probe_read_kernel(&e->slot_id, sizeof(memslot->id),
-                                  &memslot->id);
+            bpf_probe_read_kernel(&e->mark_page_dirty_data.slot_id,
+                                  sizeof(memslot->id), &memslot->id);
             bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
             bpf_ringbuf_submit(e, 0);
         }