vpp: cnat performance

Signed-off-by: Nathan Skrzypczak <[email protected]>
projectcalico · Apr 7, 2023 · 546f545 · 546f545
1 parent fa5cbd2
commit 546f545
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 55 deletions.
diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go
@@ -55,13 +55,15 @@ var (
 	log *logrus.Logger
 )
 
-func Go(f func(t *tomb.Tomb) error) {
+func Go(f func(t *tomb.Tomb) error, name string) {
 	if t.Alive() {
+		log.Infof("STARTING %s", name)
 		t.Go(func() error {
 			err := f(&t)
 			if err != nil {
 				log.Warnf("Tomb function errored with %s", err)
 			}
+			log.Infof("STOPPED %s", name)
 			return err
 		})
 	}
@@ -160,7 +162,7 @@ func main() {
 	serviceServer.SetBGPConf(bgpConf)
 
 	watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t)
-	Go(policyServer.ServePolicy)
+	Go(policyServer.ServePolicy, "policyServer.ServePolicy")
 	felixConfig := watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod")
 	ourBGPSpec := watchDog.Wait(policyServer.GotOurNodeBGPchan, "Waiting for bgp spec to be provided on node add")
 	if ourBGPSpec != nil {
@@ -173,7 +175,7 @@ func main() {
 	}
 
 	if *config.GetCalicoVppFeatureGates().MultinetEnabled {
-		Go(netWatcher.WatchNetworks)
+		Go(netWatcher.WatchNetworks, "netWatcher.WatchNetworks")
 		watchDog.Wait(netWatcher.InSync, "Waiting for networks to be listed and synced")
 	}
 
@@ -182,20 +184,20 @@ func main() {
 		connectivityServer.SetFelixConfig(felixConfig.(*felixconfig.Config))
 	}
 
-	Go(routeWatcher.WatchRoutes)
-	Go(linkWatcher.WatchLinks)
-	Go(bgpConfigurationWatcher.WatchBGPConfiguration)
-	Go(prefixWatcher.WatchPrefix)
-	Go(peerWatcher.WatchBGPPeers)
-	Go(connectivityServer.ServeConnectivity)
-	Go(routingServer.ServeRouting)
-	Go(serviceServer.ServeService)
-	Go(cniServer.ServeCNI)
-	Go(prometheusServer.ServePrometheus)
+	Go(routeWatcher.WatchRoutes, "routeWatcher.WatchRoutes")
+	Go(linkWatcher.WatchLinks, "linkWatcher.WatchLinks")
+	Go(bgpConfigurationWatcher.WatchBGPConfiguration, "bgpConfigurationWatcher.WatchBGPConfiguration")
+	Go(prefixWatcher.WatchPrefix, "prefixWatcher.WatchPrefix")
+	Go(peerWatcher.WatchBGPPeers, "peerWatcher.WatchBGPPeers")
+	Go(connectivityServer.ServeConnectivity, "connectivityServer.ServeConnectivity")
+	Go(routingServer.ServeRouting, "routingServer.ServeRouting")
+	Go(serviceServer.ServeService, "serviceServer.ServeService")
+	Go(cniServer.ServeCNI, "cniServer.ServeCNI")
+	Go(prometheusServer.ServePrometheus, "prometheusServer.ServePrometheus")
 
 	// watch LocalSID if SRv6 is enabled
 	if *config.GetCalicoVppFeatureGates().SRv6Enabled {
-		Go(localSIDWatcher.WatchLocalSID)
+		Go(localSIDWatcher.WatchLocalSID, "localSIDWatcher.WatchLocalSID")
 	}
 
 	log.Infof("Agent started")

diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go
@@ -373,7 +373,8 @@ func (s *Server) cniServerEventLoop(t *tomb.Tomb) error {
 	for {
 		select {
 		case <-t.Dying():
-			break
+			s.log.Warnf("CNI server asked to stop")
+			return nil
 		case evt := <-s.cniEventChan:
 			switch evt.Type {
 			case common.FelixConfChanged:
@@ -482,12 +483,14 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error {
 	}
 
 	s.log.Infof("CNI Server returned")
+	s.grpcServer.Stop()
+	s.log.Infof("GRPC stopped")
 
-	s.grpcServer.GracefulStop()
 	err = syscall.Unlink(config.CNIServerSocket)
 	if err != nil {
 		return err
 	}
+	s.log.Infof("Socket CNI unlink")
 
 	return nil
 }

diff --git a/vpplink/cnat.go b/vpplink/cnat.go
@@ -18,16 +18,19 @@ package vpplink
 import (
 	"fmt"
 	"net"
+	"github.com/pkg/errors"
 
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/cnat"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/interface_types"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
 
 const (
-	FeatureArcCnatInput  = "ip?-unicast cnat-input-ip?"
-	FeatureArcCnatOutput = "ip?-output cnat-output-ip?"
-	FeatureArcSnat       = "ip?-unicast cnat-snat-ip?"
+	FeatureArcCnatLookup    = "ip?-unicast cnat-lookup-ip?"
+	FeatureArcCnatInput     = "ip?-unicast cnat-input-ip?"
+	FeatureArcCnatOutput    = "ip?-output cnat-output-ip?"
+	FeatureArcCnatWriteBack = "ip?-output cnat-writeback-ip?"
+	FeatureArcSnat          = "ip?-unicast cnat-snat-ip?"
 )
 
 const InvalidID = ^uint32(0)
@@ -124,6 +127,14 @@ func (v *VppLink) CnatDelSnatPrefix(prefix *net.IPNet) error {
 }
 
 func (v *VppLink) CnatEnableFeatures(swIfIndex uint32) (err error) {
+	err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatLookup)
+	if err != nil {
+		return errors.Wrap(err, "Error enabling arc dnat in")
+	}
+	err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatWriteBack)
+	if err != nil {
+		return errors.Wrap(err, "Error enabling arc dnat out")
+	}
 	err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatInput)
 	if err != nil {
 		return fmt.Errorf("enabling arc dnat input failed: %w", err)

diff --git a/vpplink/generated/generate.log b/vpplink/generated/generate.log
@@ -1,10 +1,15 @@
-VPP Version                 : 23.06-rc0~206-gbc006bb5a
+VPP Version                 : 23.06-rc0~211-gf559a7816
 Binapi-generator version    : govpp v0.8.0-dev
-VPP Base commit             : 28d74a396 gerrit:34726/3 interface: add buffer stats api
+VPP Base commit             : 004ebac6a gerrit:34726/3 interface: add buffer stats api
 ------------------ Cherry picked commits --------------------
 capo: Calico Policies plugin
 acl: acl-plugin custom policies
 cnat: [WIP] no k8s maglev from pods
 pbl: Port based balancer
+gerrit:34552/8 cnat: add single lookup
+gerrit:29748/5 cnat: remove rwlock on ts
+gerrit:32821/6 cnat: add ip/client bihash
+gerrit:34108/5 cnat: flag to disable rsession
+gerrit:31449/10 cnat: dont compute offloaded cksums
 gerrit:34726/3 interface: add buffer stats api
 -------------------------------------------------------------
diff --git a/vpplink/generated/patches/0002-cnat-WIP-no-k8s-maglev-from-pods.patch b/vpplink/generated/patches/0002-cnat-WIP-no-k8s-maglev-from-pods.patch
@@ -1,49 +1,30 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Nathan Skrzypczak <[email protected]>
-Date: Mon, 8 Mar 2021 19:00:04 +0100
+Date: Fri, 7 Apr 2023 16:57:30 +0200
 Subject: [PATCH 2/4] cnat: [WIP] no k8s maglev from pods
 
-Type: improvement
-
-Change-Id: If0702dbc51c308f0bb0ed16149c293d7adf9a984
+Change-Id: Id262a97986b6de01a42019287377486787f2e606
 Signed-off-by: Nathan Skrzypczak <[email protected]>
 ---
- src/plugins/cnat/cnat_node_feature.c | 8 +++++++-
- 1 file changed, 7 insertions(+), 1 deletion(-)
+ src/plugins/cnat/cnat_node_feature.c | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
 
 diff --git a/src/plugins/cnat/cnat_node_feature.c b/src/plugins/cnat/cnat_node_feature.c
-index 76aa89398..fd3b90a1a 100644
+index f6d4850f8..f2b65b2ff 100644
 --- a/src/plugins/cnat/cnat_node_feature.c
 +++ b/src/plugins/cnat/cnat_node_feature.c
-@@ -43,6 +43,7 @@ cnat_input_feature_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
- 		       int session_not_found, cnat_session_t *session)
- {
-   vlib_combined_counter_main_t *cntm = &cnat_translation_counters;
-+  cnat_snat_policy_main_t *cpm = &cnat_snat_policy_main;
-   const cnat_translation_t *ct = NULL;
-   ip4_header_t *ip4 = NULL;
-   ip_protocol_t iproto;
-@@ -53,6 +54,9 @@ cnat_input_feature_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
-   index_t cti;
-   u8 trace_flags = 0;
+@@ -112,7 +112,10 @@ cnat_input_feature_new_flow_inline (vlib_main_t *vm, vlib_buffer_t *b,
+ 	    clib_host_to_net_u16 (trk0->ct_ep[VLIB_TX].ce_port) :
+ 	    rw->tuple.port[VLIB_TX];
 
+-  if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT)
 +  u32 in_if = vnet_buffer (b)->sw_if_index[VLIB_RX];
-+  int ispod;
-+
-   /* By default follow arc default next */
-   vnet_feature_next (&next0, b);
-
-@@ -127,7 +131,9 @@ cnat_input_feature_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
-       session->value.cs_port[VLIB_RX] = udp0->src_port;
-       session->value.flags = 0;
-
--      if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT)
-+      ispod = clib_bitmap_get (
-+	cpm->interface_maps[CNAT_SNAT_IF_MAP_INCLUDE_POD], in_if);
-+      if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT && !ispod)
- 	{
- 	  const dpo_id_t *dpo0;
- 	  const load_balance_t *lb1;
++  int ispod = clib_bitmap_get (
++    cnat_snat_policy_main.interface_maps[CNAT_SNAT_IF_MAP_INCLUDE_POD], in_if);
++  if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT && !ispod)
+     {
+       const dpo_id_t *dpo0;
+       const load_balance_t *lb1;
 -- 
 2.39.2
 
diff --git a/vpplink/generated/vpp_clone_current.sh b/vpplink/generated/vpp_clone_current.sh
@@ -96,9 +96,20 @@ git_clone_cd_and_reset "$1" 156d452750ab37a51984938bf4a8ab0e16650d82
 
 git_cherry_pick refs/changes/26/34726/3 # 34726: interface: add buffer stats api | https://gerrit.fd.io/r/c/vpp/+/34726
 
+git_cherry_pick refs/changes/49/31449/10 # 31449: cnat: dont compute offloaded cksums | https://gerrit.fd.io/r/c/vpp/+/31449
+git_cherry_pick refs/changes/08/34108/5 # 34108: cnat: flag to disable rsession | https://gerrit.fd.io/r/c/vpp/+/34108
+git_cherry_pick refs/changes/21/32821/6 # 32821: cnat: add ip/client bihash | https://gerrit.fd.io/r/c/vpp/+/32821
+git_cherry_pick refs/changes/48/29748/5 # 29748: cnat: remove rwlock on ts | https://gerrit.fd.io/r/c/vpp/+/29748
+git_cherry_pick refs/changes/52/34552/8 # 34552: cnat: add single lookup | https://gerrit.fd.io/r/c/vpp/+/34552
+
 # --------------- private plugins ---------------
 # Generated with 'git format-patch --zero-commit -o ./patches/ HEAD^^^'
 git_apply_private 0001-pbl-Port-based-balancer.patch
 git_apply_private 0002-cnat-WIP-no-k8s-maglev-from-pods.patch
 git_apply_private 0003-acl-acl-plugin-custom-policies.patch
 git_apply_private 0004-capo-Calico-Policies-plugin.patch
+
+
+# git cp 236423e261385e58556b75c9f9eb56006a2b9d67 # perf WIP
+# git cp 7603fd367739e74c47349a1dd9662c78dcf9377d # capo+cnat
+