TRex example with External Encap/Decap node

examples/trex/external_encap_node/README.md

@@ -0,0 +1,11 @@
+# TRex
+
+```
+tinet upconf | sudo sh -xe
+./do.sh
+docker exec -it -w /opt/trex/automation/trex_control_plane/interactive/trex/examples/astf T1 python3 new_connection_test.py
+```
+
+```
+docker run -it --net container:HV1 nicolaka/netshoot tcpdump -qtnni any
+```

examples/trex/external_encap_node/cfg.yaml

@@ -0,0 +1,10 @@
+- port_limit: 2
+  version: 2
+  interfaces:
+  - "net0"
+  - "net1"
+  port_info:
+  - src_mac:  52:54:00:01:00:01
+    dest_mac: 52:54:00:01:00:02
+  - src_mac:  52:54:00:02:00:01
+    dest_mac: 52:54:00:02:00:02

examples/trex/external_encap_node/do.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -xe
+sudo mkdir -p /sys/fs/bpf/xdp/globals
+sudo mfpctl bpf xdp attach common --netns L1 --interface net0 --name l1 -v --define ETH_DST=1 -f
+sudo mfpctl bpf xdp attach common --netns N1 --interface net0 --name n1 -v --define DEBUG_FUNCTION_CALL --define ETH_DST=2 -f
+sudo mfpctl bpf map set-auto -f map.yaml

examples/trex/external_encap_node/map.yaml

@@ -0,0 +1,101 @@
+kind: auto
+items:
+
+- mapfile: /sys/fs/bpf/xdp/globals/l1_encap_source
+  encap_source:
+    items:
+    - key: {index:  0}
+      val: {addr: fc00:ff00::0}
+- mapfile: /sys/fs/bpf/xdp/globals/l1_fib4
+  fib4:
+    items:
+    - key: {prefix: "142.0.0.1/32"}
+      val:
+        backend_block_index: 0
+        nat_port_hash_bit: 0x00ff
+    - key: {prefix: "142.0.0.2/32"}
+      val:
+        backend_block_index: 0
+        nat_port_hash_bit: 0x00ff
+- mapfile: /sys/fs/bpf/xdp/globals/l1_fib6
+  fib6:
+    items:
+    - key: {prefix: "fc00:ff01::/32"}
+      val:
+        action: 123
+        backend_block_index: 0
+        vip: "142.0.0.1"
+        nat_port_hash_bit: 0x00ff
+        usid_block_length: 32
+        usid_function_length: 32
+        nat_mapping: 0
+        nat_filtering: 0
+        sources:
+        - prefix: "10.0.1.0/24"
+    - key: {prefix: "fc00:ff02::/32"}
+      val:
+        action: 123
+        backend_block_index: 0
+        vip: "142.0.0.2"
+        nat_port_hash_bit: 0x00ff
+        usid_block_length: 32
+        usid_function_length: 32
+        nat_mapping: 0
+        nat_filtering: 0
+        sources:
+        - prefix: "10.0.2.0/24"
+- mapfile: /sys/fs/bpf/xdp/globals/l1_lb_backend
+  lb_backend:
+    items:
+    - {key: {index:  0}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  1}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  2}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  3}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  4}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  5}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  6}, val: {addr: "fc00:3100:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  7}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  8}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index:  9}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index: 10}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index: 11}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index: 12}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+    - {key: {index: 13}, val: {addr: "fc00:3200:0000:0000:0000:0000:0000:0000"}}
+
+- mapfile: /sys/fs/bpf/xdp/globals/n1_overlay_fib4
+  overlay_fib4:
+    items:
+    - {key: {vrf_id: 1, addr: "20.0.0.0"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.1"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.2"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.3"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.4"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.5"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.6"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.7"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.8"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.9"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.10"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.11"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.12"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.13"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.14"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.15"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+    - {key: {vrf_id: 1, addr: "20.0.0.16"}, val: {flags: 0, segs: ["fc00:1101::"]}}
+- mapfile: /sys/fs/bpf/xdp/globals/n1_fib6
+  fib6:
+    items:
+    - key:
+        prefix: "fc00:3100::/32"
+      val:
+        action: 456
+        backend_block_index: 0
+        vip: "142.0.0.1"
+        nat_port_hash_bit: 0x00ff
+        usid_block_length: 16
+        usid_function_length: 16
+        nat_mapping: 0
+        nat_filtering: 0
+        sources:
+        - prefix: "20.0.0.0/16"
+

examples/trex/external_encap_node/new_connection_test.py

@@ -0,0 +1,87 @@
+# Location
+# /opt/trex/automation/trex_control_plane/interactive/trex/examples/astf
+import astf_path
+from trex.astf.api import *
+import pprint
+import argparse
+import os
+import sys
+import time
+
+
+class Prof1():
+    def create_profile(self, cps):
+        prog_c = ASTFProgram()
+        prog_c.connect()
+        prog_c.reset()
+        prog_s = ASTFProgram()
+        prog_s.wait_for_peer_close()
+
+        # ip generator
+        ip_gen_c = ASTFIPGenDist(ip_range=["20.0.0.0", "20.0.0.16"], distribution="seq")
+        ip_gen_s = ASTFIPGenDist(ip_range=["30.0.0.0", "30.0.255.255"], distribution="seq")
+        ip_gen = ASTFIPGen(glob=ASTFIPGenGlobal(ip_offset="1.0.0.0"),
+                           dist_client=ip_gen_c,
+                           dist_server=ip_gen_s)
+
+        # template
+        temp_c = ASTFTCPClientTemplate(program=prog_c,  ip_gen=ip_gen, cps=cps)
+        temp_s = ASTFTCPServerTemplate(program=prog_s)
+        template = ASTFTemplate(client_template=temp_c, server_template=temp_s)
+
+        # profile
+        profile = ASTFProfile(default_ip_gen=ip_gen,
+                              templates=template)
+        return profile
+
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--mult', "-m", default=1, type=int)
+parser.add_argument('--duration', "-d", default=3600, type=int)
+args = parser.parse_args()
+
+c = ASTFClient()
+c.connect()
+c.reset()
+print("astfclient initialized")
+
+c.load_profile(Prof1().create_profile(1))
+c.clear_stats()
+c.start(mult=args.mult, duration=args.duration)
+print("started")
+
+def dig(d, keys):
+    for key in keys:
+        d = d.get(key, None)
+        if d is None:
+            return 0
+            break
+    return d
+
+try:
+    sv_tcps_connects = 0
+    cl_tcps_connects = 0
+    last_sv_tcps_connects = 0
+    last_cl_tcps_connects = 0
+    while True:
+        stats = c.get_stats()
+        last_sv_tcps_connects = sv_tcps_connects
+        last_cl_tcps_connects = cl_tcps_connects
+        cl_tcps_connects = dig(stats, ["traffic", "client", "tcps_connects"])
+        sv_tcps_connects = dig(stats, ["traffic", "server", "tcps_connects"])
+        rate_sv_tcps_connects = sv_tcps_connects - last_sv_tcps_connects
+        rate_cl_tcps_connects = cl_tcps_connects - last_cl_tcps_connects
+        print("global.tx_cps: {}".format(stats.get("global", {}).get("tx_cps", {})))
+        print("global.cpu_util: {}".format(stats.get("global", {}).get("cpu_util", {})))
+        print("global.rx_drop_bps: {}".format(stats.get("global", {}).get("rx_drop_bps", {})))
+        print("traffic.client.tcps_connattempt: {}".format(dig(stats, ["traffic", "client", "tcps_connattempt"])))
+        print("traffic.client.tcps_connects: {}".format(dig(stats, ["traffic", "client", "tcps_connects"])))
+        print("traffic.server.tcps_connects: {}".format(dig(stats, ["traffic", "server", "tcps_connects"])))
+        print("rate traffic.client.tcps_connects: {}".format(rate_cl_tcps_connects))
+        print("rate traffic.server.tcps_connects: {}".format(rate_sv_tcps_connects))
+        print("---")
+        time.sleep(1)
+except KeyboardInterrupt:
+    c.stop()
+    print("bye...")

examples/trex/external_encap_node/spec.yaml

@@ -0,0 +1,134 @@
+#   (net0) ---+
+# T1         HV1 -- CLOS----BR
+#   (net1) ---+             |
+#                     L1----+
+#                           |
+#                     N1----+
+nodes:
+- name: T1
+  image: tinynetwork/trex:develop
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: HV1#net1 }
+  - { name: net1, type: direct, args: HV1#net2 }
+  sysctls:
+  - sysctl: net.ipv6.conf.all.disable_ipv6=0
+  - sysctl: net.ipv6.conf.default.disable_ipv6=0
+- name: HV1
+  image: slankdev/vpp:19.04
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: BR#hv1 }
+  - { name: net1, type: direct, args: T1#net0 }
+  - { name: net2, type: direct, args: T1#net1 }
+  sysctls:
+  - sysctl: net.ipv6.conf.all.disable_ipv6=0
+  - sysctl: net.ipv6.conf.default.disable_ipv6=0
+- name: BR
+  image: nicolaka/netshoot
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: hv1, type: direct, args: HV1#net0 }
+  - { name: l1, type: direct, args: L1#net0 }
+  - { name: n1, type: direct, args: N1#net0 }
+  sysctls: []
+- name: L1
+  image: ghcr.io/slankdev/mfplane-mfpctl:branch-main
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: BR#l1 }
+  sysctls:
+  - sysctl: net.ipv4.ip_forward=1
+  - sysctl: net.ipv6.conf.all.disable_ipv6=0
+  - sysctl: net.ipv6.conf.default.disable_ipv6=0
+- name: N1
+  image: ghcr.io/slankdev/mfplane-mfpctl:branch-main
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: BR#n1 }
+  sysctls:
+  - sysctl: net.ipv4.ip_forward=1
+  - sysctl: net.ipv6.conf.all.disable_ipv6=0
+  - sysctl: net.ipv6.conf.default.disable_ipv6=0
+
+node_configs:
+- name: T1
+  cmds:
+  - cmd: ip link set net0 down
+  - cmd: ip link set net0 address 52:54:00:01:00:01
+  - cmd: ip link set net0 up
+  - cmd: ip link set net1 down
+  - cmd: ip link set net1 address 52:54:00:02:00:01
+  - cmd: ip link set net1 up
+  - cmd: nohup ./_t-rex-64 -i --astf --cfg ./cfg.yaml &
+- name: BR
+  cmds:
+  - cmd: ip link add br0 type bridge
+  - cmd: ip link set br0 up
+  - cmd: ip link set hv1 master br0
+  - cmd: ip link set l1 master br0
+  - cmd: ip link set n1 master br0
+- name: HV1
+  cmds:
+  - cmd: ip link set net0 down
+  - cmd: ip link set net0 address 52:54:00:ff:00:01
+  - cmd: ip link set net0 up
+  - cmd: nohup vpp -c /etc/vpp/startup.conf &
+- name: L1
+  cmds:
+  - cmd: ip link set net0 down
+  - cmd: ip link set net0 address 52:54:00:ff:00:02
+  - cmd: ip link set net0 up
+- name: N1
+  cmds:
+  - cmd: ip link set net0 down
+  - cmd: ip link set net0 address 52:54:00:ff:00:03
+  - cmd: ip link set net0 up
+
+postinit:
+  cmds:
+  - cmd: docker cp cfg.yaml T1:/opt/trex
+  - cmd: >-
+      docker cp new_connection_test.py
+      T1:/opt/trex/automation/trex_control_plane/interactive/trex/examples/astf
+  - cmd: |
+      cat <<EOF > /tmp/hv1_exec.vpp
+      create host-interface name net0
+      create host-interface name net1
+      create host-interface name net2
+      set int state host-net0 up
+      set int state host-net1 up
+      set int state host-net2 up
+      ip table add 10
+      set int ip table host-net1 10
+
+      set interface mac address host-net0 52:54:00:ff:00:01
+      set interface mac address host-net1 52:54:00:01:00:02
+      set interface mac address host-net2 52:54:00:02:00:02
+
+      set int ip addr host-net0 2001::1/64
+      set int ip addr host-net1 50.0.0.1/8
+      set int ip addr host-net2 40.0.0.1/8
+      ip route add table 10 20.0.0.0/8 via 50.0.0.2 host-net1
+      ip route add 30.0.0.0/8 via 40.0.0.2 host-net2
+      ip route add 142.0.0.0/8 via fe80::5054:ff:feff:2 host-net0
+      ip route add fc00:ff00::/24 via fe80::5054:ff:feff:2 host-net0
+      ip route add fc00:3100::/24 via fe80::5054:ff:feff:3 host-net0
+
+      set ip arp host-net2 40.0.0.2 52:54:00:02:00:01
+      set ip arp host-net1 50.0.0.2 52:54:00:01:00:01
+      set int feature host-net0 ip4-not-enabled arc ip4-unicast disable
+
+      sr localsid address fc00:1100:: behavior end
+      sr localsid address fc00:1101:: behavior end.dt4 10
+      set sr encaps source addr fc00:1100::
+
+      sr policy add bsid cafe::10 next fc00:ff01:: fib-table 0
+      sr steer l3 30.0.0.0/8 via bsid cafe::10 fib-table 10
+      EOF
+
+  - cmd: docker exec HV1 mkdir -p /etc/vpp
+  - cmd: docker cp /tmp/hv1_exec.vpp HV1:/etc/vpp/exec.vpp
+  - cmd: mkdir -p /var/run/netns
+  - cmd: ln -s /proc/$(docker inspect L1 -f {{.State.Pid}})/ns/net /var/run/netns/L1
+  - cmd: ln -s /proc/$(docker inspect N1 -f {{.State.Pid}})/ns/net /var/run/netns/N1