Merge pull request #127 from freifunkMUC/freifunk-net-domains

_modules/ddns.py

@@ -231,9 +231,9 @@ def update(
             replace = True
         # If there is no entry usual dns_update.add() happens
 
-        dns_update = dns.update.Update(
-            zone, keyring=keyring, keyname=keyname, keyalgorithm=keyalgorithm
-        )
+    dns_update = dns.update.Update(
+        zone, keyring=keyring, keyname=keyname, keyalgorithm=keyalgorithm
+    )
     if replace:
         dns_update.replace(name, ttl, rdata)
     elif not is_exist:

_modules/netbox_vms.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python
+"""WIP module to get virtual machine data from netbox, e.g. to get all VMs with a certain tag..
+"""
+
+import requests
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def get_vms_by_filter(netbox_api, netbox_token, filter):
+    # Example filter: 'tag=authorative-dns'
+    headers = {
+        "Authorization": "Token {}".format(netbox_token),
+        "Accept": "application/json",
+    }
+    url = f"{netbox_api}/virtualization/virtual-machines/?{filter}"
+    auth_servers = []
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+        response = response.json()
+        log.info(response)
+        for auth in response["results"]:
+            auth_servers.append(auth["name"])
+    except Exception as e:
+        log.error(str(e))
+        __context__["retcode"] = 1
+        return e
+    return auth_servers

certs/files/cleanup-dns.sh.jinja

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Script to nsupdate all our authoritative servers, supposed to be run by certbot.
+# Requires bind9-utils installed.
+
+# The following environment variables are passed to the script by certbot:
+# CERTBOT_DOMAIN, CERTBOT_VALIDATION, CERTBOT_TOKEN (HTTP-01 only), CERTBOT_REMAINING_CHALLENGES, CERTBOT_ALL_DOMAINS, CERTBOT_AUTH_OUTPUT
+
+HOST="_acme-challenge"
+
+{%- set update_key = salt['pillar.get']('netbox:config_context:dns_zones:update_keys:letsencrypt:key') %}
+UPDATE_KEY="{{ update_key }}"
+
+AUTH_SERVERS=("webfrontend03.ov.ffmuc.net" "webfrontend04.ov.ffmuc.net" "webfrontend05.ov.ffmuc.net" "webfrontend06.ov.ffmuc.net")
+
+for AUTH in ${AUTH_SERVERS[@]}; do
+	nsupdate -y "hmac-sha512:letsencrypt:${UPDATE_KEY}" << EOM
+server ${AUTH} 553
+zone ${CERTBOT_DOMAIN}
+update delete ${HOST}.${CERTBOT_DOMAIN} TXT "${CERTBOT_VALIDATION}"
+send
+EOM
+	echo ""
+done

certs/files/dns_plugin_credentials.ini

@@ -0,0 +1 @@
+dns_cloudflare_api_token = {{ cloudflare_token }}

certs/files/update-dns.sh.jinja

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Script to nsupdate all our authoritative servers, supposed to be run by certbot.
+# Requires bind-tools installed.
+
+# The following environment variables are passed to the script by certbot:
+# CERTBOT_DOMAIN, CERTBOT_VALIDATION, CERTBOT_TOKEN (HTTP-01 only), CERTBOT_REMAINING_CHALLENGES, CERTBOT_ALL_DOMAINS
+
+HOST="_acme-challenge"
+
+{%- set update_key = salt['pillar.get']('netbox:config_context:dns_zones:update_keys:letsencrypt:key') %}
+UPDATE_KEY="{{ update_key }}"
+
+{#- TODO: use netbox_vms:get_vms_by_filter to get authoritative DNS servers
+{%- set auth_servers = salt['netbox_vms:get_vms_by_filter'](
+    salt['pillar.get']('netbox:config_context:netbox:api_url'),
+    salt['pillar.get']('netbox:config_context:dns_zones:netbox_token'),
+    'tag=authorative-dns'
+) %}
+AUTH_SERVERS=({{ auth_servers | join(" ") }})
+#}
+
+AUTH_SERVERS=("webfrontend03.ov.ffmuc.net" "webfrontend04.ov.ffmuc.net" "webfrontend05.ov.ffmuc.net" "webfrontend06.ov.ffmuc.net")
+
+for AUTH in ${AUTH_SERVERS[@]}; do
+	nsupdate -y "hmac-sha512:letsencrypt:${UPDATE_KEY}" << EOM
+server ${AUTH} 553
+zone ${CERTBOT_DOMAIN}
+{#- Don't delete existing records as they might be of other webfrontends renewing simultaneously. #}
+update add ${HOST}.${CERTBOT_DOMAIN} 5 TXT "${CERTBOT_VALIDATION}"
+send
+EOM
+	echo ""
+done

certs/init.sls

@@ -17,7 +17,7 @@ update_ca_certificates:
 
 generate-dhparam:
   cmd.run:
-    - name: openssl dhparam -out /etc/ssl/dhparam.pem 2048
+    - name: openssl dhparam -out /etc/ssl/dhparam.pem 4096
     - creates: /etc/ssl/dhparam.pem
 
 # Install FFMUC internal CA into Debian CA certificate mangling mechanism so
@@ -129,9 +129,12 @@ certbot-dns-cloudflare:
 dns_credentials:
   file.managed:
     - name: /var/lib/cache/salt/dns_plugin_credentials.ini
+    - source: salt://certs/files/dns_plugin_credentials.ini
     - makedirs: True
-    - contents: "dns_cloudflare_api_token = {{ cloudflare_token }}"
     - mode: "0600"
+    - template: jinja
+    - defaults:
+        cloudflare_token: {{ cloudflare_token }}
 
 ffmuc-wildcard-cert:
   acme.cert:
@@ -173,6 +176,49 @@ ffmuc-wildcard-cert:
         - pip: acme-client
         - file: dns_credentials
 
+
+{% if "webserver-external" in role %}
+# Required for running nsupdate with certbot
+bind9-utils:
+  pkg.installed
+
+update-dns.sh:
+  file.managed:
+    - name: /var/lib/cache/salt/update-dns.sh
+    - source: salt://certs/files/update-dns.sh.jinja
+    - makedirs: True
+    - mode: "0700"
+    - template: jinja
+
+cleanup-dns.sh:
+  file.managed:
+    - name: /var/lib/cache/salt/cleanup-dns.sh
+    - source: salt://certs/files/cleanup-dns.sh.jinja
+    - makedirs: True
+    - mode: "0700"
+    - template: jinja
+
+# Salt's acme module doesn't support any DNS plugin besides Cloudflare, not even manual. Thus use cmd.run.
+# TODO add 'unless' condition which checks whether cert needs renewal.
+# Expiration date is not enough, should check revocation status as well. As of 2023-06 Cerbot has no command exposed for this.
+muenchen.freifunk.net-wildcard-cert:
+  cmd.run:
+    - name: >
+        certbot certonly -n --agree-tos -m [email protected]
+        --manual --manual-auth-hook /var/lib/cache/salt/update-dns.sh --manual-cleanup-hook /var/lib/cache/salt/cleanup-dns.sh
+        --preferred-challenges dns --expand
+        -d "muenchen.freifunk.net" -d "*.muenchen.freifunk.net"
+        -d "xn--mnchen-3ya.freifunk.net" -d "*.xn--mnchen-3ya.freifunk.net"
+        -d "wertingen.freifunk.net" -d "*.wertingen.freifunk.net"
+        -d "donau-ries.freifunk.net" -d "*.donau-ries.freifunk.net"
+{#-        -d "augsburg.freifunk.net" -d "*.augsburg.freifunk.net" #}
+    - require:
+        - cmd: certbot
+        - pip: acme-client
+        - file: update-dns.sh
+{% endif %}
+
+
 /etc/letsencrypt/renewal-hooks/deploy/01-reload-nginx.sh:
   file.managed:
     - contents: |

cloudflare/init.sls

@@ -6,7 +6,6 @@
 # Get all nodes for DNS records
 {% set nodes = salt['mine.get']('netbox:platform:slug:linux', 'minion_id', tgt_type='pillar') %}
 {% set cnames = salt['pillar.get']('netbox:config_context:dns_zones:cnames') %}
-{% set custom_records = salt['pillar.get']('netbox:config_context:dns_zones:custom_records', []) %}
 
 ffmuc.net:
   cloudflare.manage_zone_records:

dns-server/auth/db.x.freifunk.net.jinja

@@ -0,0 +1,19 @@
+$ORIGIN .
+$TTL 3600	; 1 week
+{{ domain }}		IN	SOA	anycast01.ffmuc.net. hostmaster.ffmuc.net. (
+					2023071001 ; serial
+					300        ; refresh (5 minutes)
+					100        ; retry (1 minute 40 seconds)
+					6000       ; expire (1 hour 40 minutes)
+					600        ; minimum (10 minutes)
+					)
+			IN	NS	anycast01.ffmuc.net.
+			IN	NS	anycast02.ffmuc.net.
+
+			IN	AAAA	2001:678:ed0:f000::
+			IN	AAAA	2001:678:e68:f000::
+
+			IN	A	5.1.66.255
+			IN	A	185.150.99.255
+
+$ORIGIN {{ domain }}.

dns-server/auth/init.sls

@@ -53,71 +53,82 @@ python-dnspython:
       - pkg: bind9
 
 
-{% if not salt['file.file_exists' ]('/etc/bind/zones/db.in.ffmuc.net') %}
 /etc/bind/zones/db.in.ffmuc.net:
   file.managed:
     - source: salt://dns-server/auth/db.in.ffmuc.net
     - user: bind
     - group: bind
     - mode: "0775"
+    - replace: False
     - require:
       - file: /etc/bind/zones
     - watch_in:
       - cmd: rndc-reload
-{% endif %}
 
-{% if not salt['file.file_exists' ]('/etc/bind/zones/db.ov.ffmuc.net') %}
 /etc/bind/zones/db.ov.ffmuc.net:
   file.managed:
     - source: salt://dns-server/auth/db.ov.ffmuc.net
     - user: bind
     - group: bind
     - mode: "0775"
+    - replace: False
     - require:
       - file: /etc/bind/zones
     - watch_in:
       - cmd: rndc-reload
-{% endif %}
 
-{% if not salt['file.file_exists' ]('/etc/bind/zones/db.ext.ffmuc.net') %}
 /etc/bind/zones/db.ext.ffmuc.net:
   file.managed:
     - source: salt://dns-server/auth/db.ext.ffmuc.net
     - user: bind
     - group: bind
     - mode: "0775"
+    - replace: False
     - require:
       - file: /etc/bind/zones
     - watch_in:
       - cmd: rndc-reload
-{% endif %}
 
-{% if not salt['file.file_exists' ]('/etc/bind/zones/db.80.10.in-addr.arpa') %}
 /etc/bind/zones/db.80.10.in-addr.arpa:
   file.managed:
     - source: salt://dns-server/auth/db.80.10.in-addr.arpa
     - user: bind
     - group: bind
     - mode: "0775"
+    - replace: False
     - require:
       - file: /etc/bind/zones
     - watch_in:
       - cmd: rndc-reload
-{% endif %}
 
-{% if not salt['file.file_exists' ]('/etc/bind/zones/db.1.0.a.0.8.0.6.0.1.0.0.2.ip6.arpa') %}
 /etc/bind/zones/db.1.0.a.0.8.0.6.0.1.0.0.2.ip6.arpa:
   file.managed:
     - source: salt://dns-server/auth/db.1.0.a.0.8.0.6.0.1.0.0.2.ip6.arpa
     - user: bind
     - group: bind
     - mode: "0775"
+    - replace: False
     - require:
       - file: /etc/bind/zones
     - watch_in:
       - cmd: rndc-reload
-{% endif %}
 
+{% set freifunk_net_zones = salt['pillar.get']('netbox:config_context:dns_zones:freifunk_net_zones') %}
+{% for domain in freifunk_net_zones %}
+{% set zonefile_path = '/etc/bind/zones/db.'+domain %}
+/etc/bind/zones/db.{{ domain }}:
+  file.managed:
+    - source: salt://dns-server/auth/db.x.freifunk.net.jinja
+    - user: bind
+    - group: bind
+    - mode: "0644"
+    - template: jinja
+    - defaults:
+        domain: {{ domain }}
+    - replace: False
+    - require:
+      - file: /etc/bind/zones
+{% endfor %}
 
 dns-key:
   file.managed:
@@ -130,6 +141,7 @@ dns-key:
     - require:
       - pkg: bind9
 
+
 # Create DNS records for each node
 {% for node_id in nodes %}
   {%- if 'meet.ffmuc.net' not in node_id and 'lighthouse' not in node_id %}
@@ -388,4 +400,26 @@ record-AAAA-extra-{{ dns_entry }}:
   {%- endif %}
 
 {%- endfor %}{# for dns_entry in extra_dns_entries #}
-{%- endif %}{# if 'dnsserver' in role #}
+
+# Additional DNS records
+{%- set custom_records = salt['pillar.get']('netbox:config_context:dns_zones:custom_records', []) %}
+{%- for record in custom_records %}
+record-{{ record.get('type') }}-{{ record.get('name') }}.{{ record.get('zone') }}:
+  ddns.present:
+    - name: {{ record.get('name') }}
+    - zone: {{ record.get('zone') }}
+    - ttl: 60
+    - data: {{ record.get('content') }}
+    - rdtype: {{ record.get('type') }}
+    - nameserver: 127.0.0.1
+    - port: {{ listening_port }}
+    - keyfile: /etc/bind/salt-master.key
+    - keyalgorithm: hmac-sha512
+    - replace_on_change: True
+    - require:
+      - pkg: python-dnspython
+      - file: dns-key
+{%- endfor %}{# for record in custom_records #}
+
+
+{%- endif %}{# if 'authorative-dns' in salt['pillar.get']('netbox:tag_list', []) #}

dns-server/auth/named.conf.local

@@ -7,6 +7,7 @@
 //include "/etc/bind/zones.rfc1918";
 {%- set update_keys = salt['pillar.get']('netbox:config_context:dns_zones:update_keys') %}
 {%- set zones = salt['pillar.get']('netbox:config_context:dns_zones:zones') %}
+{%- set freifunk_net_zones = salt['pillar.get']('netbox:config_context:dns_zones:freifunk_net_zones') %}
 
 
 {%- for zone_key in update_keys | sort %}
@@ -21,7 +22,18 @@ key "{{ zone_key }}" {
 zone "{{ zone }}" {
 	type master;
 	file "/etc/bind/zones/db.{{ zone }}";
-        update-policy  { 
+	update-policy  {
+	{%- for zone_key in update_keys | sort %}
+		grant {{ zone_key }} zonesub {{ update_keys[zone_key]['type'] }};
+	{%- endfor %}
+	};
+};
+{%- endfor %}
+{%- for zone in freifunk_net_zones %}
+zone "{{ zone }}" {
+	type master;
+	file "/etc/bind/zones/db.{{ zone }}";
+	update-policy  {
 	{%- for zone_key in update_keys | sort %}
 		grant {{ zone_key }} zonesub {{ update_keys[zone_key]['type'] }}; 
 	{%- endfor %}

nebula/files/config.yml.jinja

@@ -325,9 +325,13 @@ firewall:
   - port: 53
     proto: tcp
     host: any
+  # Bind
   - port: 553
     proto: udp
     host: any
+  - port: 553
+    proto: tcp
+    host: any
   # access pdns-recursor as dnsdist is listening on 53
   - port: 1653
     proto: udp