Merge pull request #11 from UKCloud/feature/resilientDns

Added resilient DNS and fixed some keepalived bugs

all.yml

@@ -0,0 +1,3 @@
+- hosts: all, localhost
+  roles:
+    - common

ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+host_key_checking = False
+log_path = ~/ansible.log
+pipelining = True

bastion.yml

@@ -1,4 +1,3 @@
 - hosts: localhost
   roles:
-    - common
     - initalisation

deploy-openshift.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-ansible-playbook --private-key ../id_rsa_jenkins -i localhost bastion.yml
-ansible-playbook --private-key ../id_rsa_jenkins -i openshift-ansible-hosts site.yml
+ansible-playbook --private-key ../id_rsa_jenkins -i openshift-ansible-hosts bastion.yml
+ansible-playbook --private-key ../id_rsa_jenkins -i openshift-ansible-hosts site.yml

dns.yml

@@ -0,0 +1,3 @@
+- hosts: dns
+  roles:
+    - dns

loadbalancers.yml

@@ -1,5 +1,5 @@
 - hosts: loadbalancers
   roles:
-    - common
+    - firewalld
     - keepalived
     - haproxy

roles/common/tasks/main.yml

@@ -0,0 +1,42 @@
+- name: Include environment vairables for infrastructure hosts
+  include_vars:
+    file: /etc/ansible/group_vars/all.yml
+    name: env_details
+- name: Check if we have already set resolv.conf
+  stat: 
+    path: .resolv_conf_set
+  register: status
+- name: Set network manager DNS
+  shell: nmcli c modify 'System eth0' ipv4.ignore-auto-dns yes ipv4.dns {{ hostvars[groups.dns[0]].ansible_default_ipv4.address }},{{ hostvars[groups.dns[1]].ansible_default_ipv4.address }} && nmcli c up "System eth0"  && touch .set_network_manager_dns
+  args:
+    creates: .set_network_manager_dns
+  become: yes
+  become_method: sudo
+- name: Tidy up hosts file on bastion
+  blockinfile:
+    block: |
+      127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+      ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+    path: /etc/hosts
+    backup: yes
+  when: inventory_hostname == 'localhost' and not status.stat.exists
+  become: yes
+  become_method: sudo
+- name: disable TTY for sudo on all hosts
+  lineinfile:
+    path: /etc/sudoers
+    regexp: '^Defaults.*requiretty$'
+    line: '#Defaults    requiretty'
+    backup: yes
+  become: yes
+  become_method: sudo
+- name: Create a file so we only run once
+  file:
+    path: .resolv_conf_set
+    state: touch
+- name: Restart keepalived now the network has been reloaded to bring up the VIP again
+  systemd:
+    name: keepalived
+    state: restarted
+    daemon_reload: yes
+  when: inventory_hostname in groups.loadbalancers

roles/common/templates/resolv.j2

@@ -0,0 +1,4 @@
+search {{ env_details.localDomainSuffix }} localdomain
+{% for hostname in groups.dns %}
+nameserver {{ hostvars[hostname].ansible_default_ipv4.address }}
+{% endfor %}

roles/dns/tasks/dns_common.yml

@@ -0,0 +1,35 @@
+- name: Load vairables
+  include_vars:
+    file: /etc/ansible/group_vars/all.yml
+    name: env_details
+- name: Install bind packages
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - bind
+    - bind-utils
+- name: setup firewall - dns
+  firewalld:
+    service: dns
+    immediate: true
+    permanent: true
+    zone: public
+    state: enabled
+- name: Configure bind to listen on all addresses
+  replace:
+    path: /etc/named.conf
+    regexp: "listen-on port 53 { 127.0.0.1; }"
+    replace: "listen-on port 53 { any; }"
+    backup: yes
+- name: Configure bind to allow queries
+  replace:
+    path: /etc/named.conf
+    regexp: "allow-query     { localhost; };"
+    replace: "allow-query     { any; };"
+    backup: yes
+- name: Enable bind and start service
+  systemd:
+    name: named
+    state: started
+    enabled: true

roles/dns/tasks/dns_master.yml

@@ -0,0 +1,44 @@
+- name: Load vairables
+  include_vars:
+    file: /etc/ansible/group_vars/all.yml
+    name: env_details
+- name: inv hostname
+  debug:
+    var: inventory_hostname
+- name: group hostname
+  debug:
+    var: groups.dns[0]
+- name: Generate DNS signing key
+  shell: dnssec-keygen -a HMAC-MD5 -n HOST -b 128 -r /dev/urandom openshift_dns_key && touch .dns_key_created
+  args:
+    creates: .dns_key_created
+  register: dnsKeyFile
+  when: inventory_hostname == groups.dns[0]
+- name: Extract DNS Key
+  shell: cat "{{ dnsKeyFile.stdout }}".key
+  register: dnsKeyOutout
+  when: inventory_hostname == groups.dns[0] and dnsKeyFile.changed
+- set_fact:
+     dnsKey: "{{ dnsKeyOutout }}"
+- name: Setup master key
+  template:
+    src: templates/master_key_config.j2
+    dest: /etc/openshift-tsig.key
+  when: inventory_hostname == groups.dns[0] and dnsKeyOutout.changed
+- set_fact:
+- name: Add master DNS configuration
+  blockinfile:
+    block: "{{ lookup('template', 'templates/master_zone_config.j2') }}"
+    dest: "/etc/named.conf"
+    backup: yes
+  when: inventory_hostname == groups.dns[0]
+- name: Copy across zone
+  template:
+    src: templates/zonefile.j2
+    dest: /var/named/{{ env_details.localDomainSuffix }}.zone
+  when: inventory_hostname == groups.dns[0]
+- name: Restart bind
+  systemd:
+    name: named
+    state: restarted
+  when: inventory_hostname == groups.dns[0]

roles/dns/tasks/dns_slave.yml

@@ -0,0 +1,26 @@
+- name: Load vairables
+  include_vars:
+    file: /etc/ansible/group_vars/all.yml
+    name: env_details
+- name: Setup slave key
+  template:
+    src: templates/slave_key_config.j2
+    dest: /etc/openshift-tsig.key
+  when: inventory_hostname == groups.dns[1] and hostvars[groups.dns[0]].dnsKeyOutout.changed
+- name: Add slave DNS configuration
+  blockinfile:
+    block: "{{ lookup('template', 'templates/slave_zone_config.j2') }}"
+    dest: "/etc/named.conf"
+    backup: yes
+  when: inventory_hostname == groups.dns[1]
+- name: Set corect selinux permissions
+  seboolean:
+    name: named_write_master_zones
+    state: yes
+    persistent: yes
+  when: inventory_hostname == groups.dns[1]
+- name: Restart bind
+  systemd:
+    name: named
+    state: restarted
+  when: inventory_hostname == groups.dns[1]

roles/dns/tasks/main.yml

@@ -0,0 +1,3 @@
+- include: dns_common.yml
+- include: dns_master.yml
+- include: dns_slave.yml

roles/dns/templates/master_key_config.j2

@@ -0,0 +1,10 @@
+key "TRANSFER" {
+    algorithm hmac-md5;
+    secret "{{ dnsKey.stdout.split(' ')[-1] }}";
+};
+# Slave server IP # 1
+server {{ hostvars[groups.dns[1]].ansible_default_ipv4.address }} {
+    keys {
+        TRANSFER;
+    };
+};

roles/dns/templates/master_zone_config.j2

@@ -0,0 +1,8 @@
+zone "{{ env_details.localDomainSuffix }}" IN {
+    type master;
+    file "{{ env_details.localDomainSuffix }}.zone";
+    allow-update { none; };
+    allow-transfer { key TRANSFER; };
+};
+
+include "/etc/openshift-tsig.key";

roles/dns/templates/slave_key_config.j2

@@ -0,0 +1,10 @@
+key "TRANSFER" {
+    algorithm hmac-md5;
+    secret "{{ hostvars[groups.dns[0]].dnsKey.stdout.split(' ')[-1] }}";
+};
+# Master server IP # 1
+server {{ hostvars[groups.dns[0]].ansible_default_ipv4.address }} {
+    keys {
+        TRANSFER;
+    };
+};

roles/dns/templates/slave_zone_config.j2

@@ -0,0 +1,9 @@
+zone"{{ env_details.localDomainSuffix }}" IN {
+    type slave;
+    masters { {{ hostvars[groups.dns[0]].ansible_default_ipv4.address }}; };
+    file "{{ env_details.localDomainSuffix }}.zone";
+    allow-update { none; };
+    allow-transfer { none; };
+};
+
+include "/etc/openshift-tsig.key";

roles/dns/templates/zonefile.j2

@@ -0,0 +1,32 @@
+$ORIGIN {{ env_details.localDomainSuffix }}. 
+$TTL 60s 
+@    IN    SOA    dns1.{{ env_details.localDomainSuffix }}.    hostmaster.{{ env_details.localDomainSuffix }}. (
+            2001062501 ; serial                     
+            21600      ; refresh after 6 hours                     
+            3600       ; retry after 1 hour                     
+            604800     ; expire after 1 week                     
+            86400 )    ; minimum TTL of 1 day  
+
+
+    IN    NS    dns1.{{ env_details.localDomainSuffix }}.
+    IN    NS    dns2.{{ env_details.localDomainSuffix }}.        
+
+{% set count = 1 %}
+{% for hostname in groups.dns %}
+dns{{ count }}	IN	A	{{ hostvars[hostname].ansible_default_ipv4.address }}
+{% set count = count + 1 %}
+{% endfor %}
+
+{% for ip, hostname in env_details.haproxy_details.iteritems() %}
+{{ hostname }}	IN	A	{{ ip }}
+{% endfor %}
+
+{% for ip, hostname in env_details.worker_details.iteritems() %}
+{{ hostname }}	IN	A	{{ ip }}
+{% endfor %}
+
+{% for ip, hostname in env_details.master_details.iteritems() %}
+{{ hostname }}	IN	A	{{ ip }}
+{% endfor %}
+
+console		IN 	A	{{ env_details.haproxy_vip }}

roles/firewalld/tasks/main.yml

@@ -0,0 +1,13 @@
+- name: Install firewalld
+  yum:
+    name: firewalld
+    state: latest
+- name: Start firewalld
+  systemd:
+    name: firewalld
+    state: started
+    enabled: yes
+- name: FIX BUG WITH NMCLI
+  systemd:
+    name: NetworkManager
+    state: restarted

roles/haproxy/tasks/main.yml

@@ -28,16 +28,6 @@
   command: systemctl restart haproxy
 - name: Restart rsyslog
   command: systemctl restart rsyslog
-- name: Install firewalld
-  yum:
-    name: firewalld
-    state: latest
-- name: Start firewalld
-  command: systemctl unmask firewalld
-- name: Start firewalld
-  command: systemctl start firewalld
-- name: Enable firewalld
-  command: systemctl enable firewalld
 - name: setup firewall - http
   firewalld:
     service: http
@@ -59,10 +49,3 @@
     permanent: true
     state: enabled
     zone: public
-- name: setup firewall - Allow VRRP Adverts
-  firewalld:
-    rich_rule: 'rule protocol value="vrrp" accept'
-    immediate: true
-    permanent: true
-    state: enabled
-    zone: public

roles/initalisation/tasks/main.yml

@@ -1,3 +1,7 @@
+- name: Install bind-utils
+  yum:
+    name: bind-utils
+    state: latest
 - name: Include environment vairables for infrastructure hosts
   include_vars:
     file: /etc/ansible/group_vars/all.yml
@@ -7,4 +11,4 @@
     src: templates/ansible-hosts-multimaster.j2
     dest: ./openshift-ansible-hosts
     force: yes
-    backup: yes
+    backup: yes

roles/initalisation/templates/ansible-hosts-multimaster.j2

@@ -25,7 +25,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 openshift_master_htpasswd_users={'admin': '$apr1$mSg5YHIt$TSWms4a24FdbZbqiGPJvc/', 'demo': '$apr1$kxv5vr/4$Y6gc7iyLEa7kvG.DaYwrc/'}
 
 openshift_master_cluster_method=native
-openshift_master_cluster_hostname=console.{{ env_details.domainSuffix }}
+openshift_master_cluster_hostname=console.{{ env_details.localDomainSuffix }}
 openshift_master_cluster_public_hostname=ocp.{{ env_details.domainSuffix }}
 
 openshift_set_hostname=true
@@ -102,6 +102,11 @@ openshift_metrics_cassandra_pvc_size=50Gi
 {{ hostname }}.openstacklocal
 {% endfor %}
 
+[dns]
+{% for ip, hostname in env_details.haproxy_details.items() %}
+{{ hostname }}.openstacklocal
+{% endfor %}
+
 # host group for nodes, includes region info
 # Routers are placed only on first 3 workers
 [nodes]