kvm_watcher:添加监测vcpu调度过程的功能

eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h

@@ -54,6 +54,20 @@ struct {
     __type(value, u32);
 } vcpu_tid SEC(".maps");
 
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, struct load_key);
+    __type(value, struct load_value);
+} load_map SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, struct load_key);
+    __type(value, struct time_value);
+} load_time SEC(".maps");
+
 // 记录vcpu_halt的id信息
 static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu) {
     u32 tid = bpf_get_current_pid_tgid();
@@ -102,26 +116,66 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb,
     bpf_ringbuf_submit(e, 0);
     return 0;
 }
-//记录VCPU调度的信息
-static int trace_vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu, void *rb,
-                               struct common_event *e) {
-    RESERVE_RINGBUF_ENTRY(rb, e);
-    //获取pid&tid
-    pid_t pid, tid;
-    u64 id;
-    id = bpf_get_current_pid_tgid();
-    pid = id >> 32;
-    tid = (u32)id;
-    // //获取时间
-    u64 ts = bpf_ktime_get_ns();
-    e->process.pid = pid;
-    e->process.tid = tid;
-    e->time = ts;
-    bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));
-    bpf_probe_read_kernel(&e->vcpu_load_data.vcpu_id,
-                          sizeof(e->vcpu_load_data.vcpu_id), &vcpu->vcpu_id);
-    bpf_ringbuf_submit(e, 0);
-    return 1;
+//记录VCPU调度的信息--进调度
+static int trace_vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) {
+    u32 pid = bpf_get_current_pid_tgid() >> 32;
+    u32 tid = bpf_get_current_pid_tgid();
+    u64 time = bpf_ktime_get_ns();
+    u32 vcpu_id;
+    if (!vcpu) {
+        return 0;
+    }
+    bpf_probe_read_kernel(&vcpu_id, sizeof(u32), &vcpu->vcpu_id);
+    struct time_value time_value;
+    __builtin_memset(&time_value, 0, sizeof(struct time_value));
+    time_value.time = time;
+    time_value.vcpu_id = vcpu_id;
+    time_value.pcpu_id = cpu;
+    struct load_key curr_load_key;
+    __builtin_memset(&curr_load_key, 0, sizeof(struct load_key));
+    curr_load_key.pid = pid;
+    curr_load_key.tid = tid;
+    bpf_map_update_elem(&load_time, &curr_load_key, &time_value, BPF_ANY);
+    return 0;
+}
+//记录VCPU调度的信息--出调度
+static int trace_vmx_vcpu_put() {
+    u32 pid = bpf_get_current_pid_tgid() >> 32;
+    u32 tid = bpf_get_current_pid_tgid();
+    struct load_key load_key;
+    __builtin_memset(&load_key, 0, sizeof(struct load_key));
+    load_key.pid = pid;
+    load_key.tid = tid;
+    struct time_value *t_value;
+    t_value = bpf_map_lookup_elem(&load_time, &load_key);
+    if (!t_value) {
+        return 0;
+    }
+    u64 duration = bpf_ktime_get_ns() - t_value->time;
+    bpf_map_delete_elem(&load_time, &load_key);
+    struct load_value *load_value;
+    load_value = bpf_map_lookup_elem(&load_map, &load_key);
+    if (load_value) {
+        load_value->count++;
+        load_value->total_time += duration;
+        if (load_value->max_time < duration) {
+            load_value->max_time = duration;
+        }
+        if (load_value->min_time > duration) {
+            load_value->min_time = duration;
+        }
+        load_value->pcpu_id = t_value->pcpu_id;
+        load_value->vcpu_id = t_value->vcpu_id;
+    } else {
+        struct load_value new_load_value = {.count = 1,
+                                            .max_time = duration,
+                                            .total_time = duration,
+                                            .min_time = duration,
+                                            .vcpu_id = t_value->vcpu_id,
+                                            .pcpu_id = t_value->pcpu_id};
+        bpf_map_update_elem(&load_map, &load_key, &new_load_value, BPF_ANY);
+    }
+    return 0;
 }
 static int trace_mark_page_dirty_in_slot(struct kvm *kvm,
                                          const struct kvm_memory_slot *memslot,

eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h

@@ -93,6 +93,24 @@ struct exit_key {
     __u32 tid;
 };
 
+struct load_key {
+    __u32 pid;
+    __u32 tid;
+};
+struct load_value {
+    __u64 max_time;
+    __u64 total_time;
+    __u64 min_time;
+    __u32 count;
+    __u32 vcpu_id;
+    __u32 pcpu_id;
+};
+struct time_value {
+    __u32 pad;
+    __u64 time;
+    __u32 vcpu_id;
+    __u32 pcpu_id;
+};
 struct exit_value {
     __u64 max_time;
     __u64 total_time;
@@ -166,11 +184,6 @@ struct common_event {
             // VCPU_WAKEUP 特有成员
         } vcpu_wakeup_data;
 
-        struct {
-            __u32 vcpu_id;
-            // VCPU_LOAD 特有成员
-        } vcpu_load_data;
-
         struct {
             __u32 reason_number;
             __u64 duration_ns;

eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c

@@ -67,10 +67,15 @@ SEC("tp/kvm/kvm_entry")
 int tp_entry(struct exit *ctx) {
     return trace_kvm_entry();
 }
-// 记录VCPU调度的信息
+//记录VCPU调度的信息--进入
 SEC("kprobe/vmx_vcpu_load")
 int BPF_KPROBE(kp_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) {
-    return trace_vmx_vcpu_load(vcpu, cpu, &rb, e);
+    return trace_vmx_vcpu_load(vcpu, cpu);
+}
+//记录VCPU调度的信息--退出
+SEC("kprobe/vmx_vcpu_put")
+int BPF_KPROBE(kp_vmx_vcpu_put, struct kvm_vcpu *vcpu) {
+    return trace_vmx_vcpu_put();
 }
 SEC("kprobe/mark_page_dirty_in_slot")
 int BPF_KPROBE(kp_mark_page_dirty_in_slot, struct kvm *kvm,

eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c

@@ -515,9 +515,6 @@ static int handle_event(void *ctx, void *data, size_t data_sz) {
             break;
         }
         case VCPU_LOAD: {
-            printf("%-20.6f %-15s %-6d/%-8d %-10d\n", timestamp_ms,
-                   e->process.comm, e->process.pid, e->process.tid,
-                   e->vcpu_load_data.vcpu_id);
             break;
         }
         case HALT_POLL: {
@@ -716,6 +713,9 @@ static int print_event_head(struct env *env) {
         case EXIT:
             printf("Waiting vm_exit ... \n");
             break;
+        case VCPU_LOAD:
+            printf("Waiting vm_vcpu_load ... \n");
+            break;
         case HALT_POLL:
             printf("%-18s %-15s %-15s %-10s %-7s %-11s %-10s\n", "TIME(ms)",
                    "COMM", "PID/TID", "TYPE", "VCPU_ID", "OLD(ns)", "NEW(ns)");
@@ -773,6 +773,8 @@ static void set_disable_load(struct kvm_watcher_bpf *skel) {
                               env.execute_vcpu_wakeup ? true : false);
     bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_load,
                               env.execute_vcpu_load ? true : false);
+    bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_put,
+                              env.execute_vcpu_load ? true : false);
     bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt,
                               env.execute_vcpu_wakeup ? true : false);
     bpf_program__set_autoload(skel->progs.tp_exit,
@@ -927,6 +929,54 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) {
     }
     return count;
 }
+
+int print_vcpu_load_map(struct kvm_watcher_bpf *skel) {
+    int fd = bpf_map__fd(skel->maps.load_map);
+    int err;
+    struct load_key lookup_key = {};
+    struct load_key next_key = {};
+    struct load_value load_value = {};
+    int first = 1;
+    while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) {
+        if (first) {
+            first = 0;
+            printf("\nTIME:%s\n", getCurrentTimeFormatted());
+            printf("%-12s %-12s %-12s %-12s %-12s %-12s %-12s %-12s\n", "pid",
+                   "tid", "total_time", "max_time", "min_time", "counts",
+                   "vcpuid", "pcpuid");
+            printf(
+                "------------ ------------ ------------ ------------ "
+                "------------ "
+                "------------ "
+                "------------ "
+                "------------\n");
+        }
+        err = bpf_map_lookup_elem(fd, &next_key, &load_value);
+        if (err < 0) {
+            fprintf(stderr, "failed to lookup vcpu_load_value: %d\n", err);
+            return -1;
+        }
+        printf("%-12d %-12d %-12.4f %-12.4f %-12.4f %-12u %-12d %-12d\n",
+               next_key.pid, next_key.tid,
+               NS_TO_MS_WITH_DECIMAL(load_value.total_time),
+               NS_TO_MS_WITH_DECIMAL(load_value.max_time),
+               NS_TO_MS_WITH_DECIMAL(load_value.min_time), load_value.count,
+               load_value.vcpu_id, load_value.pcpu_id);
+        lookup_key = next_key;
+    }
+    // clear the maps
+    memset(&lookup_key, 0, sizeof(struct load_key));
+    while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) {
+        err = bpf_map_delete_elem(fd, &next_key);
+        if (err < 0) {
+            fprintf(stderr, "failed to cleanup counters: %d\n", err);
+            return -1;
+        }
+        lookup_key = next_key;
+    }
+    return 0;
+}
+
 void __print_exit_map(int fd, enum NameType name_type) {
     if (name_type == EXIT_NR) {
         printf(
@@ -1083,6 +1133,10 @@ int main(int argc, char **argv) {
         if (env.execute_exit) {
             print_map_and_check_error(print_exit_map, skel, "exit", err);
         }
+        if (env.execute_vcpu_load) {
+            print_map_and_check_error(print_vcpu_load_map, skel, "vcpu_load",
+                                      err);
+        }
         /* Ctrl-C will cause -EINTR */
         if (err == -EINTR) {
             err = 0;