WIP: Correlate Interface connections

cmd/discover.go

@@ -39,12 +39,14 @@ func NewDiscoverCmd(cli Cli) *cobra.Command {
 	flags := cmd.Flags()
 	flags.StringVarP(&opts.Format, "format", "f", "", "Output format (json, yaml, go-template..)")
 	flags.BoolVar(&opts.IPsecAgg, "ipsec-agg", false, "Print aggregated IPSec info")
+	flags.BoolVar(&opts.IfInfoAgg, "ifinfo-agg", false, "Print aggregated interface info")
 	return cmd
 }
 
 type DiscoverOptions struct {
 	Format   string
 	IPsecAgg bool
+	IfInfoAgg bool
 }
 
 func RunDiscover(cli Cli, opts DiscoverOptions) error {
@@ -96,6 +98,17 @@ func RunDiscover(cli Cli, opts DiscoverOptions) error {
 		}
 	}
 
+	if opts.IfInfoAgg {
+		logrus.Infof("Aggregating Interface info for instances")
+
+		forwarderConnInfo, err := agent.CorrelateNsmForwarderConnections(vppInstances)
+		if err != nil {
+			logrus.Warnf("correlating IPSec failed: %v", err)
+		} else {
+			printDiscoverForwarderConnInfo(cli.Out(), forwarderConnInfo)
+		}
+	}
+
 	return nil
 }
 
@@ -153,3 +166,13 @@ func printDiscoverIPSecAggr(out io.Writer, ipsecCorrelations *agent.IPSecCorrela
 
 	fmt.Fprint(out, renderColor(buf.String()))
 }
+
+func printDiscoverForwarderConnInfo(out io.Writer, forwarderConnCorrelations *agent.ForwarderConnCorrelations) {
+	var buf bytes.Buffer
+
+	printSectionHeader(&buf, []string{"Aggregated Interface info"})
+
+	PrintCorrelatedIfInfo(prefixWriter(&buf), forwarderConnCorrelations)
+
+	fmt.Fprint(out, renderColor(buf.String()))
+}

cmd/print.go

@@ -281,6 +281,50 @@ func PrintCorrelatedIpSec(out io.Writer, correlations *agent.IPSecCorrelations)
 	fmt.Fprint(out, buf.String())
 }
 
+func PrintCorrelatedIfInfo(out io.Writer, connCorrelations *agent.ForwarderConnCorrelations) {
+	var buf bytes.Buffer
+	w := tabwriter.NewWriter(&buf, 0, 8, 1, '\t', tabwriter.StripEscape|tabwriter.FilterHTML|tabwriter.DiscardEmptyColumns)
+
+	header := []string{
+		"Connection Chain -- [Abbrev. pod name]/[intf] <-[connect type]-> ...",
+	}
+	for i, h := range header {
+		if h != "" {
+			header[i] = colorize(color.Bold, h)
+		}
+	}
+	fmt.Fprintln(w, strings.Join(header, "\t"))
+
+	// Print connection chains
+	for _, connChain := range connCorrelations.Connections {
+		var chain string
+		for i, intf := range connChain.IntfPath {
+			if i > 0 {
+				conStr := intf.NormalizedType.String()
+				if intf.NormalizedType != connChain.IntfPath[i-1].NormalizedType {
+					conStr = "XCON"
+				}
+				chain += fmt.Sprintf(" <-%s-> ", conStr)
+			}
+			if intf.IfContextType == agent.ForwarderContextLinux {
+				chain += fmt.Sprintf("LinuxPod/%s", intf.InternalIfName)
+			} else {
+				podNameParts := strings.Split(intf.Owner.Pod, "-")
+				displayPodInfo := strings.Join(podNameParts[0:2], "-") + "-" + podNameParts[len(podNameParts)-1]
+				chain += fmt.Sprintf("%s/%s", displayPodInfo, intf.IfName)
+			}
+		}
+		fmt.Fprintln(w, chain)
+	}
+
+	if err := w.Flush(); err != nil {
+		log.Println(err)
+		return
+	}
+
+	fmt.Fprint(out, buf.String())
+}
+
 func linuxInterfaceType(iface agent.LinuxInterface) string {
 	return iface.Value.Type.String()
 }

docs/CORRELATE.md

@@ -0,0 +1,51 @@
+# Correlation Implementation
+
+## Interface Connection Relationships
+
+### Kubernetes Inter-Pod/Node Connections
+
+![Kubernetes Pod Interconnect](kubernetes_pod_interconnect.png)
+
+1. **Inter-connect** is a reference to the relationships between interfaces that are interconnecting 2 pods.
+   - Tap interfaces--host network interconnect via Linux kernel tap interface to VPP tap interface
+   - Memif interfaces--interconnect via memif interfaces using the same socketfile/shared-mem between VPP pods
+   - VXLAN interfaces--interconnect across nodes via the same src/dst IP pair and VNI
+
+1. **Cross-connect (Xcon)** refers to the cross-connect of interfaces within a VPP instance.  This is how traffic from interfaces of different types are directly connected.
+
+#### Correlation: Inter-connect Mapping Implementation
+
+The attributes and configuration for all interfaces for discovered VPP instances are normalized to a common format.  For each `NormalizedIfType`, an implementation of the `ForwarderIfNormalizedConfig` interface's `MatchKey()` method generates a string that matches with the value of its inter-connected peer interface.
+
+This is implemented as follows for each `NormalizedIfType`:
+
+- `TAP` -- matching VPP and Linux interface names
+- `MEMIF` -- matching socket file path and name for last 2 portions of file path.
+- `VXLAN` -- original src/dst IP config is mapped to the full list of k8s node addresses.  The `MatchKey` format is `<srcIP>,<dstIP>/<VNI>` and uses the first IPs in each of the node objects' addresses list, with the src IP equal to the lexigraphically smallest of the 2 node addresses.
+  - NOTE: the reason this is necessary is to work for public cloud nodes with Node SNAT functionality to public addresses.  Using the k8s node object info in this way should work for all known k8s cases.
+
+To correlate interfaces' configuration to their inter-connect peers, the `IfInterconnects` map is used:
+
+```go
+// pairs of interconnect intfs map[type][IfMatchkey]{ forwarderIf1, forwarderIf2 }
+IfInterconnects map[NormalizedIfType]map[string][]*ForwarderIf
+```
+
+Interfaces with matching `MatchKey()` output are inter-connect pairs and get mapped to the same slice. 
+
+#### Correlation: Cross-connect Mappings
+
+The `IfXconnects` map associates interfaces to their crossconnect peer interface:
+
+```go
+// map[pod][ifName] = xconn peer ifName
+IfXconnects map[string]map[string]string
+```
+
+The VPP instance config's xconnect info is used to build this map.
+
+#### Correlation: Full Connection Path
+
+The endpoints to the full connection paths are Linux pod TAP interfaces or VPP MEMIF interfaces that are not part of a VPP crossconnect.  The correlation starts at interfaces fitting these criteria and traverses the inter-connect and cross-connect relationships at each interface to creates a `ForwarderConnection` object for the full paths.  While traversing the interfaces, they are associated (marked) with the connection they are a part of.
+
+Any "leftover" VXLAN interfaces are considered dangling but are also attempted to be associated to their inter-connect peer for display.

providers/kube/client/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -140,3 +141,13 @@ func (k *Client) ListPods(namespace string, labelSelector, fieldSelector string)
 	}
 	return list, nil
 }
+
+// GetNode calls the API to get node with name.
+func (k *Client) GetNode(name string) (*corev1.Node, error) {
+
+	node, err := k.client.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return node, nil
+}

providers/kube/client/pod.go

@@ -26,9 +26,11 @@ type Pod struct {
 	URL       string
 	Image     string
 	ImageID   string
+	HostNetwork bool
 
 	pod    *corev1.Pod
 	client *Client
+	Node   *corev1.Node
 }
 
 func newPod(client *Client, pod *corev1.Pod) *Pod {
@@ -44,6 +46,7 @@ func newPod(client *Client, pod *corev1.Pod) *Pod {
 		URL:       pod.GetSelfLink(),
 		Image:     getPodFirstContainer(pod).Image,
 		ImageID:   getPodFirstContainerStatus(pod).ImageID,
+		HostNetwork: pod.Spec.HostNetwork,
 		pod:       pod,
 		client:    client,
 	}

providers/kube/handler.go

@@ -2,6 +2,7 @@ package kube
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	govppapi "git.fd.io/govpp.git/api"
@@ -37,6 +38,13 @@ func (h *PodHandler) ID() string {
 }
 
 func (h *PodHandler) Metadata() map[string]string {
+	var addresses []string
+	if h.pod.HostNetwork {
+		for _, nodeAddress := range h.pod.Node.Status.Addresses {
+			addresses = append(addresses, nodeAddress.Address)
+		}
+	}
+
 	return map[string]string{
 		"env":       providers.Kube,
 		"pod":       h.pod.Name,
@@ -50,6 +58,7 @@ func (h *PodHandler) Metadata() map[string]string {
 		"image_id":  h.pod.ImageID,
 		"uid":       string(h.pod.UID),
 		"created":   h.pod.Created.Format(time.UnixDate),
+		"hostnet_addresses": strings.Join(addresses, ","),
 	}
 }
 

providers/kube/provider.go

@@ -83,6 +83,12 @@ func (p *Provider) Query(params ...map[string]string) ([]probe.Handler, error) {
 
 	var handlers []probe.Handler
 	for _, pod := range pods {
+		node, err := p.client.GetNode(pod.NodeName)
+		if err != nil {
+			logrus.Errorf("Unable to get node %s for pod %s: %v", pod.NodeName, pod.Name, err)
+		} else {
+			pod.Node = node
+		}
 		handlers = append(handlers, NewHandler(pod))
 	}