Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move all DNS proxy related rules into single nftables table. #179

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Move all DNS proxy related rules into single nftables table. #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
23901b8
Move all DNS proxy related rules into single nftables table.
Gerrit91 Apr 22, 2024
9f13af8
Move back SNAT.
Gerrit91 Apr 23, 2024
72f9872
Try optimize.
Gerrit91 Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions pkg/nftables/networkpolicy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package nftables

import (
"fmt"
"strconv"
"strings"

networkingv1 "k8s.io/api/networking/v1"
Expand Down Expand Up @@ -60,16 +61,16 @@ func clusterwideNetworkPolicyIngressRules(np firewallv1.ClusterwideNetworkPolicy
return uniqueSorted(rules)
}

func clusterwideNetworkPolicyEgressDNSCacheRules(cache FQDNCache, logAcceptedConnections bool) (nftablesRules, error) {
func clusterwideNetworkPolicyEgressDNSCacheRules(cache FQDNCache, port int, logAcceptedConnections bool) (nftablesRules, error) {
addr, err := cache.CacheAddr()
if err != nil {
return nil, err
}
base := []string{"ip saddr == @cluster_prefixes", fmt.Sprintf("ip daddr { %s }", addr)}
comment := fmt.Sprintf("accept intercepted traffic for dns cache")
return nftablesRules{
assembleDestinationPortRule(base, "tcp", []string{"53"}, logAcceptedConnections, comment+" tcp"),
assembleDestinationPortRule(base, "udp", []string{"53"}, logAcceptedConnections, comment+" udp"),
assembleDestinationPortRule(base, "tcp", []string{strconv.Itoa(port)}, logAcceptedConnections, comment+" tcp"),
assembleDestinationPortRule(base, "udp", []string{strconv.Itoa(port)}, logAcceptedConnections, comment+" udp"),
}, nil
}

Expand Down
221 changes: 133 additions & 88 deletions pkg/nftables/nftables.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,96 +1,141 @@
table inet firewall {
# internal prefixes, which are not leaving the partition or the partition interconnect
set internal_prefixes {
type ipv4_addr
flags interval
auto-merge
{{ if gt (len .InternalPrefixes) 0 }}
elements = { {{ .InternalPrefixes }} }
{{ end }}
}

# Prefixes in the cluster, typically 10.x.x.x
# FIXME Should be filled with nodeCidr
set cluster_prefixes {
type ipv4_addr
flags interval
auto-merge
elements = { 10.0.0.0/8 }
}
{{- range .Sets }}

set {{ .SetName }} {
type {{ .Version }}
{{ if gt (len .IPs) 0 }}
elements = { {{ StringsJoin .IPs ", " }} }
{{ end }}
}
{{- end }}

# counters
counter internal_in { }
counter internal_out { }
counter external_in { }
counter external_out { }
counter drop_total { }
counter drop_ratelimit { }

chain forward {
type filter hook forward priority 1; policy drop;

# network traffic accounting for external traffic
ip saddr != @internal_prefixes oifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_in comment "count external traffic incomming"
ip daddr != @internal_prefixes iifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_out comment "count external traffic outgoing"

# network traffic accounting for internal traffic
ip saddr @internal_prefixes oifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name internal_in comment "count internal traffic incomming"
ip daddr @internal_prefixes iifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name internal_out comment "count internal traffic outgoing"

# rate limits
{{- range .RateLimitRules }}
{{ . }}
{{- end }}

# state dependent rules
ct state established,related counter accept comment "accept established connections"
ct state invalid counter drop comment "drop packets with invalid ct state"

# icmp
ip protocol icmp icmp type echo-request limit rate over 10/second burst 4 packets counter drop comment "drop ping floods"
ip protocol icmp icmp type { destination-unreachable, router-solicitation, router-advertisement, time-exceeded, parameter-problem } counter log prefix "nftables-firewall-accepted: " accept comment "accept icmp"

# dynamic ingress rules
{{- range .ForwardingRules.Ingress }}
{{ . }}
{{- end }}

# dynamic egress rules
{{- range .ForwardingRules.Egress }}
{{ . }}
{{- end }}

counter comment "count and log dropped packets"
limit rate 10/second counter name drop_total log prefix "nftables-firewall-dropped: "
}
# internal prefixes, which are not leaving the partition or the partition interconnect
set internal_prefixes {
type ipv4_addr
flags interval
auto-merge
{{- if gt (len .InternalPrefixes) 0 }}
elements = { {{ .InternalPrefixes }} }
{{- end }}
}

# Prefixes in the cluster, typically 10.x.x.x
# FIXME Should be filled with nodeCidr
set cluster_prefixes {
type ipv4_addr
flags interval
auto-merge
elements = { 10.0.0.0/8 }
}
{{- range .Sets }}

set {{ .SetName }} {
type {{ .Version }}
{{- if gt (len .IPs) 0 }}
elements = { {{ StringsJoin .IPs ", " }} }
{{- end }}
}
{{- end }}

# counters
counter internal_in { }
counter internal_out { }
counter external_in { }
counter external_out { }
counter drop_total { }
counter drop_ratelimit { }

chain forward {
type filter hook forward priority 1; policy drop;

# network traffic accounting for external traffic
ip saddr != @internal_prefixes oifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_in comment "count external traffic incomming"
ip daddr != @internal_prefixes iifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_out comment "count external traffic outgoing"

# network traffic accounting for internal traffic
ip saddr @internal_prefixes oifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name internal_in comment "count internal traffic incomming"
ip daddr @internal_prefixes iifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name internal_out comment "count internal traffic outgoing"

# rate limits
{{- range .RateLimitRules }}
{{ . }}
{{- end }}

# state dependent rules
ct state established,related counter accept comment "accept established connections"
ct state invalid counter drop comment "drop packets with invalid ct state"

# icmp
ip protocol icmp icmp type echo-request limit rate over 10/second burst 4 packets counter drop comment "drop ping floods"
ip protocol icmp icmp type { destination-unreachable, router-solicitation, router-advertisement, time-exceeded, parameter-problem } counter log prefix "nftables-firewall-accepted: " accept comment "accept icmp"

# dynamic ingress rules
{{- range .ForwardingRules.Ingress }}
{{ . }}
{{- end }}

# dynamic egress rules
{{- range .ForwardingRules.Egress }}
{{ . }}
{{- end }}

counter comment "count and log dropped packets"
limit rate 10/second counter name drop_total log prefix "nftables-firewall-dropped: "
}
{{- if gt (len .SnatRules) 0 }}

chain postrouting {
type nat hook postrouting priority -1; policy accept;
{{- range .SnatRules }}
{{ . }}
chain postrouting {
type nat hook postrouting priority -1; policy accept;
{{- range .SnatRules }}
{{ . }}
{{- end }}
}
}
{{- end }}
}
{{- if .AdditionalDNSAddrs }}

# Add additional DNS addresses for dnat redirection for the dns proxy
table inet nat {
set proxy_dns_servers {
type ipv4_addr
flags interval
auto-merge
elements = { {{ StringsJoin .AdditionalDNSAddrs ", " }} }
}

{{- if and .DnsProxy .DnsProxy.Enabled }}
{{ $dnsProxy := .DnsProxy }}

table inet dnsproxy {
set node_network_prefixes {
type ipv4_addr
flags interval
auto-merge
elements = { {{ StringsJoin $dnsProxy.NodeCidrs ", " }} }
}

set proxy_dns_servers {
type ipv4_addr
flags interval
auto-merge
elements = { {{ StringsJoin $dnsProxy.DNSAddrs ", " }} }
}

chain prerouting {
type nat hook prerouting priority 0; policy accept;
{{- range $ip := $dnsProxy.ExternalIPs }}
ip daddr @proxy_dns_servers iifname "{{ $dnsProxy.PrimaryIface }}" tcp dport domain dnat ip to {{ $ip }} comment "dnat intercept traffic for dns proxy"
ip daddr @proxy_dns_servers iifname "{{ $dnsProxy.PrimaryIface }}" udp dport domain dnat ip to {{ $ip }} comment "dnat intercept traffic for dns proxy"
{{- end }}
}

chain prerouting_ct {
# Set up additional conntrack zone for DNS traffic.
# There was a problem that duplicate packets were registered by conntrack
# when packet was leaking from private VRF to the internet VRF.
# Isolating traffic to special zone solves the problem.
# Zone number(3) was obtained by experiments.
type filter hook prerouting priority raw; policy accept;
iifname "{{ $dnsProxy.PrimaryIface }}" tcp dport domain ct zone set 3
iifname "{{ $dnsProxy.PrimaryIface }}" udp dport domain ct zone set 3
}

chain input {
{{- range $dnsProxy.ExternalIPs }}
ip saddr @node_network_prefixes tcp dport domain ip daddr {{ . }} accept comment "allow dnatted packages to reach dns proxy"
ip saddr @node_network_prefixes udp dport domain ip daddr {{ . }} accept comment "allow dnatted packages to reach dns proxy"
{{- end }}
}

chain forward {
ip saddr == @node_network_prefixes ip daddr @proxy_dns_servers tcp dport { {{ $dnsProxy.DNSPort }} } counter accept comment "accept traffic for dns cache tcp"
ip saddr == @node_network_prefixes ip daddr @proxy_dns_servers udp dport { {{ $dnsProxy.DNSPort }} } counter accept comment "accept traffic for dns cache udp"
}

chain output_ct {
type nat hook output priority raw; policy accept;
iifname "{{ $dnsProxy.PrimaryIface }}" tcp dport domain ct zone set 3
iifname "{{ $dnsProxy.PrimaryIface }}" udp dport domain ct zone set 3
}
}
{{- end }}
61 changes: 47 additions & 14 deletions pkg/nftables/rendering.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,22 @@ import (

// firewallRenderingData holds the data available in the nftables template
type firewallRenderingData struct {
ForwardingRules forwardingRules
RateLimitRules nftablesRules
SnatRules nftablesRules
Sets []dns.RenderIPSet
InternalPrefixes string
PrivateVrfID uint
AdditionalDNSAddrs []string
ForwardingRules forwardingRules
RateLimitRules nftablesRules
SnatRules nftablesRules
Sets []dns.RenderIPSet
InternalPrefixes string
PrivateVrfID uint
DnsProxy *dnsProxyData
}

type dnsProxyData struct {
Enabled bool
DNSAddrs []string
DNSPort int
ExternalIPs []string
PrimaryIface string
NodeCidrs []string
}

func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) {
Expand Down Expand Up @@ -58,24 +67,48 @@ func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) {
}

var (
sets []dns.RenderIPSet
dnsAddrs = []string{}
sets []dns.RenderIPSet

dnsProxy = &dnsProxyData{
Enabled: false,
DNSAddrs: []string{"8.8.8.8", "8.8.4.4", "1.1.1.1", "1.0.0.1"},
DNSPort: 53,
}
)
if f.cache.IsInitialized() {
sets = f.cache.GetSetsForRendering(f.clusterwideNetworkPolicies.GetFQDNs())
rules, err := clusterwideNetworkPolicyEgressDNSCacheRules(f.cache, f.logAcceptedConnections)

if f.firewall.Spec.DNSPort != nil {
dnsProxy.DNSPort = int(*f.firewall.Spec.DNSPort)
}

rules, err := clusterwideNetworkPolicyEgressDNSCacheRules(f.cache, dnsProxy.DNSPort, f.logAcceptedConnections)
if err != nil {
return &firewallRenderingData{}, err
}

if f.firewall.Spec.DNSServerAddress != "" {
dnsAddrs = append(dnsAddrs, f.firewall.Spec.DNSServerAddress)
dnsProxy.DNSAddrs = strings.Split(f.firewall.Spec.DNSServerAddress, ",")
}

for _, nw := range f.networkMap {
if nw.NetworkType == nil || *nw.NetworkType != "external" {
continue
}

dnsProxy.ExternalIPs = append(dnsProxy.ExternalIPs, nw.IPs...)
}

dnsProxy.PrimaryIface = fmt.Sprintf("%d", *f.primaryPrivateNet.Vrf)
dnsProxy.NodeCidrs = append(dnsProxy.NodeCidrs, f.primaryPrivateNet.Prefixes...)

egress = append(egress, rules...)
}

return &firewallRenderingData{
AdditionalDNSAddrs: dnsAddrs,
PrivateVrfID: uint(*f.primaryPrivateNet.Vrf),
InternalPrefixes: strings.Join(f.firewall.Spec.InternalPrefixes, ", "),
DnsProxy: dnsProxy,
PrivateVrfID: uint(*f.primaryPrivateNet.Vrf),
InternalPrefixes: strings.Join(f.firewall.Spec.InternalPrefixes, ", "),
ForwardingRules: forwardingRules{
Ingress: ingress,
Egress: egress,
Expand Down
18 changes: 13 additions & 5 deletions pkg/nftables/rendering_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,18 @@ func TestFirewallRenderingData_renderString(t *testing.T) {
Egress: []string{"egress rule 1", "egress rule 2"},
Ingress: []string{"ingress rule 1", "ingress rule 2"},
},
InternalPrefixes: "1.2.3.0/24, 2.3.4.0/8",
RateLimitRules: []string{"meta iifname \"eth0\" limit rate over 10 mbytes/second counter name drop_ratelimit drop"},
SnatRules: []string{"ip saddr { 10.0.0.0/8 } oifname \"vlan104009\" counter snat 185.1.2.3 comment \"snat internet\""},
PrivateVrfID: uint(42),
AdditionalDNSAddrs: []string{"8.9.10.11", "4.5.6.7"},
InternalPrefixes: "1.2.3.0/24, 2.3.4.0/8",
RateLimitRules: []string{"meta iifname \"eth0\" limit rate over 10 mbytes/second counter name drop_ratelimit drop"},
SnatRules: []string{"ip saddr { 10.0.0.0/8 } oifname \"vlan104009\" counter snat 185.1.2.3 comment \"snat internet\""},
PrivateVrfID: uint(42),
DnsProxy: &dnsProxyData{
Enabled: true,
DNSAddrs: []string{"1.1.1.1"},
DNSPort: 53,
ExternalIPs: []string{"212.34.83.19"},
PrimaryIface: "vlan20",
NodeCidrs: []string{"10.130.184.0/22"},
},
},
wantErr: false,
},
Expand Down Expand Up @@ -99,6 +106,7 @@ func TestFirewallRenderingData_renderString(t *testing.T) {
rendered, _ := os.ReadFile(path.Join("test_data", tt.name+".nftable.v4"))
want := string(rendered)
if got != want {
t.Log(got)
t.Errorf("Firewall.renderString() diff: %v", cmp.Diff(want, got))
}
})
Expand Down
Loading
Loading