Support wgkex loadbalancing/client steering mechanism

ffmuc-mesh-vpn-wireguard-vxlan/Makefile

@@ -1,7 +1,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ffmuc-mesh-vpn-wireguard-vxlan
-PKG_VERSION:=1
+PKG_VERSION:=2
 PKG_RELEASE:=1
 
 PKG_MAINTAINER:=Annika Wickert <[email protected]>
@@ -11,7 +11,7 @@ include $(TOPDIR)/../package/gluon.mk
 
 define Package/ffmuc-mesh-vpn-wireguard-vxlan
   TITLE:=Support for connecting meshes via wireguard
-  DEPENDS:=+gluon-mesh-vpn-core +micrond +kmod-wireguard +wireguard-tools +ip-full
+  DEPENDS:=+gluon-mesh-vpn-core +micrond +kmod-wireguard +wireguard-tools +ip-full +lua-jsonc
 endef
 
 $(eval $(call BuildPackageGluon,ffmuc-mesh-vpn-wireguard-vxlan))

ffmuc-mesh-vpn-wireguard-vxlan/README.md

@@ -1,6 +1,24 @@
 # ffmuc-mesh-vpn-wireguard-vxlan
 
-You can use this package for connecting with wireguard to the Freifunk Munich network.
+This package adds support for WireGuard+VXLAN as Mesh VPN protocol stack as it is used in the Freifunk Munich network.
+
+### Dependencies
+
+This relies on [wgkex](https://github.com/freifunkMUC/wgkex), the FFMUC WireGuard key exchange broker running on the configured broker address. The broker programms the gateway to accept the WireGuard key which is transmitted during connection.
+Starting with the key exchange API v2, the wgkex broker also returns WireGuard peer data for a gateway selected by the broker, which this package then configures as mesh VPN peer/endpoint. This can be enabled by setting the `loadbalancing` option accordingly.
+
+For the health-checks a webserver of some kind needs to listen to `HTTP GET` requests on the gateways.
+
+### How it works
+
+When `checkuplink` gets called (which happens every minute via cronjob), it checks if the gateway connection is still alive by calling `wget` and connecting to the WireGuard peer link address. If this address replies, we also start a `batctl ping` to the same address. If both checks succeed the connection just stays alive.
+
+If one of the checks above bails out with an error the reconnect cycle is started. This means `checkuplink` registers itself with `wireguard.broker` by sending the WireGuard public key over either HTTP or HTTPS (depending on the device support).
+The broker responds with JSON data containing the gateway peer data (pubkey, address, port, allowed IPs aka link address). `checkuplink` adds the peer to the wg interface using this data, and sets up the VXLAN interface with the peer link address as remote endpoint.
+
+This script prefers to establish connections over IPv6 and falls back to IPv4 **only if there is no IPv6 default route**.
+
+### Configuration
 
 You should use something like the following in the site.conf:
 
@@ -10,8 +28,16 @@ You should use something like the following in the site.conf:
 		enabled = true,
 		iface = 'wg_mesh_vpn', -- not 'mesh-vpn', this is used for the VXLAN interface
 		mtu = 1406,
-		broker = 'broker.ffmuc.net/api/v1/wg/key/exchange',
-		peers = {
+		broker = 'broker.ffmuc.net', -- base path of broker, will be combined with API path
+
+		-- loadbalancing controls whether the client can enable the loadbalancing/gateway assignment feature of the broker
+		-- on: the client will always use loadbalancing
+		-- off: the client cannot enable loadbalancing
+		-- on-by-default: the client can enable/disable loadbalancing and will use loadbalancing by default
+		-- off-by-default: the client can enable/disable loadbalancing and will not use loadbalancing by default
+		loadbalancing = 'on-by-default', -- optional
+
+		peers = { -- not needed if loadbalancing = 'on'
 			{
 				publickey = 'TszFS3oFRdhsJP3K0VOlklGMGYZy+oFCtlaghXJqW2g=',
 				endpoint = 'gw04.ext.ffmuc.net:40011',
@@ -24,22 +50,16 @@ You should use something like the following in the site.conf:
 			},
 		},
 	},
-
 ```
 
-And you should include the package in the site.mk of course!
-
-### Dependencies
-
-This relies on [wgkex](https://github.com/freifunkMUC/wgkex) the FFMUC wireguard broker running on the configured broker address. The broker programms the gateway to accept the WireGuard key which is transmitted during connection.
+If permitted via `on/off-by-default`, a user can override loadbalancing with `uci`:
 
-For the health-checks a webserver of some kind needs to listen to `HTTP GET` requests on the gateways.
-
-### How it works
-
-When `checkuplink` gets called (which happens every minute via cronjob), it checks if the gateway connection is still alive by calling `wget` and connecting to `wireguard.peer.peer_[number].link_address`. If this address replies we also start a `batctl ping` to the same address. If both checks succeed the connection just stays alive.
+```sh
+uci set wireguard.mesh_vpn.loadbalancing=0 # override loadbalancing to be always off
+uci set wireguard.mesh_vpn.loadbalancing=1 # override loadbalancing to be always on
+```
 
-If one of the checks above bails out with an error the reconnect cycle is started. Which means `checkuplink` registers itself with `wireguard.broker` by sending the WireGuard public_key over either http or https (depending on the device support). After the key was sent the script tries to randomely connect to one of the `wireguard.peer`. This script prefers to establish connections over IPv6 and falls back to IPv4 only if there is no IPv6 default route.
+And you should include the package in the site.mk of course!
 
 ### Interesting Links
 

ffmuc-mesh-vpn-wireguard-vxlan/check_site.lua

@@ -6,6 +6,11 @@ local function check_peer(k)
 	need_string(in_domain(extend(k, {'link_address'})))
 end
 
-need_table({'mesh_vpn', 'wireguard', 'peers'}, check_peer)
 need_number({'mesh_vpn', 'wireguard', 'mtu'})
 need_string({'mesh_vpn', 'wireguard', 'broker'})
+
+local loadbalancing = need_one_of({ 'mesh_vpn', 'wireguard', 'loadbalancing' },
+	{ 'on', 'off', 'on-by-default', 'off-by-default' }, false)
+if loadbalancing ~= 'on' then -- peers are not required when loadbalancing is enforced
+	need_table({'mesh_vpn', 'wireguard', 'peers'}, check_peer)
+end

ffmuc-mesh-vpn-wireguard-vxlan/files/lib/gluon/gluon-mesh-wireguard-vxlan/checkuplink

@@ -11,6 +11,28 @@ else
 	exit
 fi
 
+get_site_string() {
+	local path="$1"
+
+	lua <<EOF
+local site = require 'gluon.site'
+print(site.${path}())
+EOF
+}
+
+get_site_bool() {
+	local path="$1"
+
+	lua <<EOF
+local site = require 'gluon.site'
+if site.${path}() then
+    print("true")
+else
+	print("false")
+end
+EOF
+}
+
 interface_linklocal() {
 	# We generate a predictable v6 address
 	local macaddr oldIFS
@@ -27,28 +49,132 @@ clean_port() {
 	echo "$1" | sed -r 's/:[0-9]+$|\[|\]//g'
 }
 
-check_address_family() {
-	local peer_endpoint="$2"
-	local gateway
-	gateway="$(clean_port "$peer_endpoint")"
+extract_port() {
+    echo "$1" | awk -F: '{print $NF}'
+}
+
+combine_ip_port() {
+    local ip="$1"
+    local port="$2"
+
+    # Add brackets in case the IP is an IPv6
+    case $ip in
+        *":"*)
+            ip="[${ip}]"
+            ;;
+    esac
+
+    echo "$ip:$port"
+}
+
+resolve_host() {
+	local gateway="$1"
 	# Check if we have a default route for v6 if not fallback to v4
 	if ip -6 route show table 1 | grep -q 'default via'
 	then
 		local ipv6
-		ipv6="$(gluon-wan nslookup "$gateway" | grep 'Address:\? [0-9]' | grep -E -o '([a-f0-9:]+:+)+[a-f0-9]+')"
-		echo "[$ipv6]$(echo "$peer_endpoint" | grep -E -oe ":[0-9]+$")"
+		ipv6="$(gluon-wan nslookup "$gateway" | grep 'Address:\? [0-9]' | grep -oE '([a-f0-9:]+:+)+[a-f0-9]+')"
+		echo "$ipv6"
 	else
 		local ipv4
-		ipv4="$(gluon-wan nslookup "$gateway" | grep 'Address:\? [0-9]' | grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b")"
-		echo "$ipv4$(echo "$peer_endpoint" | grep -E -oe ":[0-9]+$")"
+		ipv4="$(gluon-wan nslookup "$gateway" | grep 'Address:\? [0-9]' | grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b')"
+		echo "$ipv4"
 	fi
-
 }
 
 force_wan_connection() {
 	LD_PRELOAD=libpacketmark.so LIBPACKETMARK_MARK=1 gluon-wan "$@"
 }
 
+is_loadbalancing_enabled() {
+	local lb_default
+	local lb_overwrite
+	lb_default=$(get_site_string mesh_vpn.wireguard.loadbalancing)
+
+	if [[ $lb_default == "on" ]]; then
+		return 0 # true
+	elif [[ $lb_default == "off" ]]; then
+		return 1 # false
+	fi
+
+	# check if an overwrite was specified
+	if lb_overwrite=$(uci -q get wireguard.mesh_vpn.loadbalancing); then
+		logger -p info -t checkuplink "Loadbalancing overwrite detected: ${lb_overwrite}"
+		if [[ $lb_overwrite == "1" ]]; then
+			return 0 # true
+		elif [[ $lb_overwrite == "0" ]]; then
+			return 1 # false
+		fi
+	fi
+
+	if [[ $lb_default == "on-by-default" ]]; then
+		return 0 # true
+	elif [[ $lb_default == "off-by-default" ]]; then
+		return 1 # false
+	fi
+
+	logger -p err -t checkuplink "Invalid loadbalancing parameter '${lb_default}', assuming 'off'"
+	return 0
+}
+
+get_wgkex_data(){
+	local version="$1"
+	WGKEX_BROKER="$PROTO://$WGKEX_BROKER_BASE_PATH/api/$version/wg/key/exchange"
+
+	logger -p info -t checkuplink "Contacting wgkex broker $WGKEX_BROKER"
+
+	if ! WGKEX_DATA=$(force_wan_connection wget -q -O- --post-data='{"domain": "'"$SEGMENT"'","public_key": "'"$PUBLICKEY"'"}' "$WGKEX_BROKER"); then
+		logger -p err -t checkuplink "Contacting wgkex broker failed, response: $WGKEX_DATA"
+		return 1
+	fi
+
+	logger -p info -t checkuplink "Got data from wgkex broker: $WGKEX_DATA"
+	echo "$WGKEX_DATA"
+}
+
+use_api_v1(){
+	WGKEX_DATA=$(get_wgkex_data v1)
+
+	# Get the number of configured peers and randomly select one
+	NUMBER_OF_PEERS=$(uci -q show wireguard | grep -E -ce "peer_[0-9]+.endpoint")
+	PEER="$(awk -v min=1 -v max="$NUMBER_OF_PEERS" 'BEGIN{srand(); print int(min+rand()*(max-min+1))}')"
+
+	logger -p info -t checkuplink "Selected peer $PEER"
+	PEER_HOSTPORT="$(uci get wireguard.peer_"$PEER".endpoint)"
+	PEER_HOST="$(clean_port "$PEER_HOSTPORT")"
+	PEER_ADDRESS="$(resolve_host "$PEER_HOST")"
+	PEER_PORT="$(extract_port "$PEER_HOSTPORT")"
+	PEER_ENDPOINT="$(combine_ip_port "$PEER_ADDRESS" "$PEER_PORT")"
+
+	PEER_PUBLICKEY="$(uci get wireguard.peer_"$PEER".publickey)"
+	PEER_LINKADDRESS="$(uci get wireguard.peer_"$PEER".link_address)"
+}
+
+
+
+use_api_v2() {
+	WGKEX_DATA=$(get_wgkex_data v2)
+
+	# Parse the returned JSON in a Lua script, returning the endpoint address, port, pubkey and first allowed IP, separated by newlines
+	if ! data=$(lua /lib/gluon/gluon-mesh-wireguard-vxlan/parse-wgkex-response.lua "$WGKEX_DATA"); then
+		logger -p err -t checkuplink "Parsing wgkex broker data failed"
+		logger -p info -t checkuplink "Falling back to API v1"
+		use_api_v1
+		return
+	fi
+
+	logger -p debug -t checkuplink "Successfully parsed wgkex broker data"
+	PEER_HOST="$(echo "$data" | sed -n 1p)"
+	PEER_PORT="$(echo "$data" | sed -n 2p)"
+	PEER_PUBLICKEY="$(echo "$data" | sed -n 3p)"
+	PEER_LINKADDRESS=$(echo "$data" | sed -n 4p)
+
+	PEER_ADDRESS="$(resolve_host "$PEER_HOST")"
+	PEER_ENDPOINT="$(combine_ip_port "$PEER_ADDRESS" "$PEER_PORT")"
+}
+
+
+
 
 mesh_vpn_enabled="$(uci get wireguard.mesh_vpn.enabled)"
 
@@ -76,7 +202,7 @@ fi
 #We assume we are not connected by default
 CONNECTED=0
 
-MESH_VPN_IFACE=$(uci get wireguard.mesh_vpn.iface)
+MESH_VPN_IFACE=$(get_site_string mesh_vpn.wireguard.iface)
 
 # Check connectivity to supernode
 
@@ -125,17 +251,6 @@ then
 	exit 3
 fi
 
-# Get the number of configured peers and randomly select one
-NUMBER_OF_PEERS=$(uci -q show wireguard | grep -E -ce "peer_[0-9]+.endpoint")
-PEER="$(awk -v min=1 -v max="$NUMBER_OF_PEERS" 'BEGIN{srand(); print int(min+rand()*(max-min+1))}')"
-PEER_PUBLICKEY="$(uci get wireguard.peer_"$PEER".publickey)"
-
-logger -t checkuplink "Selected peer $PEER"
-
-endpoint="$(check_address_family "$PEER_PUBLICKEY" "$(uci get wireguard.peer_"$PEER".endpoint)")"
-
-logger -t checkuplink "Connecting to $endpoint"
-
 # Delete Interfaces
 {
 	ip link set nomaster dev mesh-vpn >/dev/null 2>&1
@@ -146,7 +261,7 @@ ip link delete dev "${MESH_VPN_IFACE}" >/dev/null 2>&1 || true
 PUBLICKEY=$(uci get wireguard.mesh_vpn.privatekey | wg pubkey)
 SEGMENT=$(uci get gluon.core.domain)
 
-# Push public key to broker, test for https and use if supported
+# Push public key to broker and receive gateway data, test for https and use if supported
 ret=0
 wget -q "https://[::1]" || ret=$?
 # returns Network Failure =4 if https exists
@@ -156,7 +271,22 @@ if [ "$ret" -eq 1 ]; then
 else
 	PROTO=https
 fi
-force_wan_connection wget -q  -O- --post-data='{"domain": "'"$SEGMENT"'","public_key": "'"$PUBLICKEY"'"}' "$PROTO://$(uci get wireguard.mesh_vpn.broker)"
+
+# Remove API path suffix if still present in config
+WGKEX_BROKER_BASE_PATH="$(get_site_string mesh_vpn.wireguard.broker | sed 's|/api/v1/wg/key/exchange||')"
+
+if is_loadbalancing_enabled; then
+	# Use /api/v2, get gateway peer details from broker response
+	logger -p info -t checkuplink "Loadbalancing enabled."
+	use_api_v2
+
+else
+	# Use /api/v1, get gateway peer details from config
+	logger -p info -t checkuplink "Loadbalancing disabled."
+	use_api_v1
+fi
+
+logger -p info -t checkuplink "Connecting to $PEER_HOST($PEER_ENDPOINT)"
 
 # Bring up the wireguard interface
 ip link add dev "$MESH_VPN_IFACE" type wireguard
@@ -168,10 +298,7 @@ LINKLOCAL="$(interface_linklocal)"
 
 # Add link-address and Peer
 ip address add "${LINKLOCAL}"/64 dev "$MESH_VPN_IFACE"
-if [ "$endpoint" = "" ]; then
-	endpoint=$(uci get wireguard.peer_"$PEER".endpoint)
-fi
-gluon-wan wg set "$MESH_VPN_IFACE" peer "$(uci get wireguard.peer_"$PEER".publickey)" persistent-keepalive 25 allowed-ips "$(uci get wireguard.peer_"$PEER".link_address)/128" endpoint "$endpoint"
+gluon-wan wg set "$MESH_VPN_IFACE" peer "$PEER_PUBLICKEY" persistent-keepalive 25 allowed-ips "$PEER_LINKADDRESS/128" endpoint "$PEER_ENDPOINT"
 
 # We need to allow incoming vxlan traffic on mesh iface
 sleep 10
@@ -184,7 +311,7 @@ then
 fi
 
 # Bring up VXLAN
-if ! ip link add mesh-vpn type vxlan id "$(lua -e 'print(tonumber(require("gluon.util").domain_seed_bytes("gluon-mesh-vpn-vxlan", 3), 16))')" local "${LINKLOCAL}" remote "$(uci get wireguard.peer_"$PEER".link_address)" dstport 8472 dev "$MESH_VPN_IFACE"
+if ! ip link add mesh-vpn type vxlan id "$(lua -e 'print(tonumber(require("gluon.util").domain_seed_bytes("gluon-mesh-vpn-vxlan", 3), 16))')" local "${LINKLOCAL}" remote "$PEER_LINKADDRESS" dstport 8472 dev "$MESH_VPN_IFACE"
 then
 	logger -p err -t checkuplink "Unable to create mesh-vpn interface"
 	exit 2

ffmuc-mesh-vpn-wireguard-vxlan/luasrc/lib/gluon/gluon-mesh-wireguard-vxlan/parse-wgkex-response.lua

@@ -0,0 +1,12 @@
+local json = require 'jsonc'
+
+local input = assert(arg[1])
+local data = assert(json.parse(input))
+if not data.Endpoint or not data.Endpoint.Address or not data.Endpoint.Port
+    or not data.Endpoint.PublicKey or not data.Endpoint.AllowedIPs or not data.Endpoint.AllowedIPs[1] then
+    error("Malformed JSON response, missing required value")
+end
+print(data.Endpoint.Address)
+print(data.Endpoint.Port)
+print(data.Endpoint.PublicKey)
+print(data.Endpoint.AllowedIPs[1])

ffmuc-mesh-vpn-wireguard-vxlan/luasrc/lib/gluon/upgrade/400-mesh-vpn-wireguard

@@ -18,22 +18,22 @@ end
 uci:delete_all('wireguard', 'peer', function(peer)
 	return peer.preserve ~= '1'
 end)
--- Clean up previous configuration
-uci:delete_all('wireguard', 'wireguard', function(peer)
-	return peer.preserve ~= '1'
-end)
+
+-- Delete unused configurations from older versions
+uci:delete("wireguard", "iface")
+uci:delete("wireguard", "limit")
+uci:delete("wireguard", "broker")
 
 local mesh_enabled = uci:get_bool('gluon', 'mesh_vpn', 'enabled') -- default
 	or uci:get_bool('fastd', 'mesh_vpn', 'enabled') --migration
 	or wg_enabled -- specific config
 
 uci:section("wireguard", "wireguard", "mesh_vpn", {
-	iface = site.mesh_vpn.wireguard.iface(),
-	broker = site.mesh_vpn.wireguard.broker(),
 	enabled = mesh_enabled,
 	privatekey = privkey,
 })
 
+-- TODO: consider removing wireguard.peer and using site directly
 for name, peer in pairs(site.mesh_vpn.wireguard.peers()) do
 	uci:section("wireguard", "peer", "peer_" .. name, {
 		enabled = true,