Portworx node changes (#111)

* adding config to deploy extra volumes for portworx * adding portworx deployment ansible code and modification * adding config to iron out network and disk ordering issues * adding port for lighthouse comms * adjusting storage volume sizes * Remove vol2 from portworx nodes and specify new Flavor with Ephemeral device for journal * Initial changes for net2 * Move the change of medium worker flavor for pwx into all-portworx.yaml * Revert default net2 worker flavor (now modified in add-portworx.yaml * Fix to make sure pwx flavor is used (will need enhancement * Reworking of portworx heat (#102) * First try replacing keys with comments, need to move more changes into add-portworx.yaml * Add automatic wiping of ephemeral disk * Edit deploy.sh to avoid error if deploy_portworx_storage is not defined in environment file * Update environment_example * Add kvdb volume for portworx workers * Remove commented-out old code * Tweak to wiping code to get around inconsistent behaviour of cloud-init... * Tweak to wiping code to get around inconsistent behaviour of cloud-init... * Append controlplane and internet gateway to net2NodeRoutes to fix oauth proxy issue * Add CIDR to controlplane net2NodeRoute Co-authored-by: Steve Mulholland <[email protected]> Co-authored-by: Ben Bacon <[email protected]> Co-authored-by: gellner <[email protected]>
UKCloud · Jan 20, 2020 · 6b4a138 · 6b4a138
1 parent 498d205
commit 6b4a138
Show file tree

Hide file tree

Showing 12 changed files with 410 additions and 11 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ rhel_reg_creds.yaml
 # auto-generated from openshift-template.yaml: see deploy.sh
 openshift.yaml
 bastion.yaml
+*_pwx.yaml
 
 # Backup filenames as generated by ansible playbooks
 *.yaml.*~

diff --git a/add-portworx.yaml b/add-portworx.yaml
@@ -0,0 +1,337 @@
+- hosts: localhost
+  vars:
+    deploy_portworx_storage: false
+  tasks:
+# This inserts the parameters and resources necessary for portworx
+# It searchs for comments in the base files which reference
+# add-portworx.yaml
+#
+# We make a copy of each yaml file that is modified - 
+# ... the *_pwx.yaml files are in gitignore.
+#
+
+# Create copies of files to be changed
+  - name: create portworx network file
+    copy:
+      src: ./network.yaml
+      dest: ./network_pwx.yaml
+    when: deploy_portworx_storage|bool
+
+  - name: create portworx node_group file
+    copy:
+      src: ./node_group.yaml
+      dest: ./node_group_pwx.yaml
+    when: deploy_portworx_storage|bool
+
+  - name: create portworx server_atomic file
+    copy:
+      src: ./server_atomic.yaml
+      dest: ./server_atomic_pwx.yaml
+    when: deploy_portworx_storage|bool
+
+# Replace references to point to copied files
+  - name: specify pwx network template
+    replace:
+      path: openshift.yaml
+      regexp: 'network.yaml'
+      replace: 'network_pwx.yaml'
+      backup: yes
+    when: deploy_portworx_storage|bool
+
+  - name: specify pwx node_group template
+    replace:
+      path: openshift.yaml
+      regexp: 'node_group'
+      replace: 'node_group_pwx'
+      after: '  worker_small_nodes_deployment'
+      backup: yes
+    when: deploy_portworx_storage|bool
+
+  - name: specify pwx server_atomic template
+    replace:
+      path: node_group_pwx.yaml
+      regexp: 'server_atomic.yaml'
+      replace: 'server_atomic_pwx.yaml'
+      backup: yes
+    when: deploy_portworx_storage|bool
+
+# Make edits in openshift.yaml
+  - name: add parameters to network stack
+    blockinfile:
+      dest: openshift.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts deploy_storage_networks parameter here"
+      marker: "        # ANSIBLE MANAGED BLOCK - parameters for portworx deployment"
+      block: |2
+                # deploy portworx?
+                deploy_storage_networks: { get_param: deploy_portworx_storage }
+    when: deploy_portworx_storage|bool
+
+  - name: add storage networks parameters
+    blockinfile:
+      dest: openshift.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts {{ item }} tenant worker storage networks here"
+      marker: "        # ANSIBLE MANAGED BLOCK - add_portworx_networks_parameters - worker-{{ item }}"
+      block: |2
+                extra_volumes: { get_param: deploy_portworx_storage }
+                storage_management_network: { get_attr: [internal_network, outputs, storage_management_network] }
+                storage_data_network: { get_attr: [internal_network, outputs, storage_data_network] }
+    loop: [ small, medium, large ]
+    when: deploy_portworx_storage|bool
+
+  - name: add storage networks parameters Net2
+    blockinfile:
+      dest: openshift.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts {{ item }} net2 worker storage networks here"
+      marker: "        # ANSIBLE MANAGED BLOCK - add_portworx_networks_parameters Net2 - {{ purpose_ident }}-{{ item }}"
+      block: |2
+                extra_volumes: { get_param: deploy_portworx_storage }
+                storage_management_network: { get_attr: [internal_network, outputs, storage_management_network] }
+                storage_data_network: { get_attr: [internal_network, outputs, storage_data_network] }
+    loop: [ small, medium, large ]
+    when: deploy_portworx_storage|bool and multinetwork|bool
+
+    #  - name: Change small worker size
+    #    replace:
+    #      path: openshift.yaml
+    #      regexp: 'ocp.t1.xxlarge # add-portworx.yaml replaces this with pwx.t1.xxlarge'
+    #      replace: 'pwx.t1.xxlarge'
+    #      backup: yes
+    #    when: deploy_portworx_storage|bool
+
+  - name: Change medium worker size
+    replace:
+      path: openshift.yaml
+      regexp: 'ocp.m1.large # add-portworx.yaml replaces this with pwx.m1.medium'
+      replace: 'pwx.m1.medium'
+      backup: yes
+    when: deploy_portworx_storage|bool
+
+    #  - name: Change large worker size
+    #    replace:
+    #      path: openshift.yaml
+    #      regexp: 'ocp.r1.large # add-portworx.yaml replaces this with pwx.r1.large'
+    #      replace: 'pwx.r1.large'
+    #      backup: yes
+    #    when: deploy_portworx_storage|bool
+
+# Make edits in network_pwx.yaml
+  - name: add storage networks parameters
+    blockinfile:
+      dest: network_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts deploy_storage_networks parameter here"
+      marker: "  # ANSIBLE MANAGED BLOCK - add_portworx_networks_parameters"
+      block: |2
+          deploy_storage_networks:
+            type: boolean
+            default: false
+    when: deploy_portworx_storage|bool
+
+  - name: add storage networks
+    blockinfile:
+      dest: network_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts storage_networks resources here"
+      marker: "  # ANSIBLE MANAGED BLOCK - add_portworx_networks_resources"
+      block: |2
+          storage_management_network:
+            type: OS::Neutron::Net
+            condition: deploy_storage_networks
+            properties:
+              admin_state_up: true
+              name: storage_management_network
+        
+          storage_management_subnet:
+            type: OS::Neutron::Subnet
+            condition: deploy_storage_networks
+            properties:
+              name: storage_management_subnet
+              network: { get_resource: storage_management_network }
+              cidr: "10.254.253.0/24"
+              ip_version: 4
+              gateway_ip: ""
+              dns_nameservers: []
+        
+          storage_data_network:
+            type: OS::Neutron::Net
+            condition: deploy_storage_networks
+            properties:
+              admin_state_up: true
+              name: storage_data_network
+        
+          storage_data_subnet:
+            type: OS::Neutron::Subnet
+            condition: deploy_storage_networks
+            properties:
+              name: storage_data_subnet
+              network: { get_resource: storage_data_network }
+              cidr: "10.254.254.0/24"
+              ip_version: 4
+              gateway_ip: ""
+              dns_nameservers: []
+        conditions:
+          deploy_storage_networks:
+            get_param: deploy_storage_networks
+    when: deploy_portworx_storage|bool
+
+  - name: add storage networks outputs
+    blockinfile:
+      dest: network_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts storage_networks outputs here"
+      marker: "  # ANSIBLE MANAGED BLOCK - add_portwox_networks_outputs"
+      block: |2
+          storage_data_network: 
+            value: { get_resource: storage_data_network }
+            condition: deploy_storage_networks
+          storage_management_network: 
+            value: { get_resource: storage_management_network }
+            condition: deploy_storage_networks
+    when: deploy_portworx_storage|bool
+
+# Make edits in node_group_pwx.yaml
+  - name: node_group_parameters
+    blockinfile:
+      dest: node_group_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts network and volume parameters here"
+      marker: "  # ANSIBLE MANAGED BLOCK - add parameters to node_group_pwx.yaml"
+      block: |2
+          extra_volumes:
+            type: boolean
+            description: Deploy extra volumes to non-infra nodes
+            default: false
+          storage_management_network:
+            type: string
+            description: Network for storage management traffic 
+          storage_data_network:
+            type: string
+            description: Network for storage data traffic
+    when: deploy_portworx_storage|bool
+
+  - name: node_group_resources
+    blockinfile:
+      dest: node_group_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts server group parameters here"
+      marker: "          # ANSIBLE MANAGED BLOCK - add properties to node_group server resources"
+      block: |2
+                  extra_volumes: { get_param: extra_volumes }
+                  storage_management_network: { get_param: storage_management_network }
+                  storage_data_network: { get_param: storage_data_network }
+
+    when: deploy_portworx_storage|bool
+
+# Make changes in server_atomic_pwx.yaml
+  - name: server_atomic parameters
+    blockinfile:
+      dest: server_atomic_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts atomic network and volume parameters here"
+      marker: "  # ANSIBLE MANAGED BLOCK - add parameters to server_atomic_pwx.yaml"
+      block: |2
+          extra_volumes:
+            type: boolean
+            description: Deploy extra volumes to non-infra nodes
+            default: false
+          storage_management_network:
+            type: string
+            description: Network for storage management traffic 
+          storage_data_network:
+            type: string
+            description: Network for storage data traffic
+    when: deploy_portworx_storage|bool
+
+  - name: server_atomic resources
+    blockinfile:
+      dest: server_atomic_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts server_atomic network resources here"
+      marker: "        # ANSIBLE MANAGED BLOCK - add properties to server_atomic resources"
+      block: |2
+                - network: { get_param: storage_management_network }
+                - network: { get_param: storage_data_network }
+    when: deploy_portworx_storage|bool
+
+  - name: server_atomic storage resources
+    blockinfile:
+      dest: server_atomic_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts server_atomic volumes here"
+      marker: "  # ANSIBLE MANAGED BLOCK - add properties to server_atomic resources"
+      block: |2
+          portworx_vol1:
+            type: OS::Cinder::Volume
+            condition: deploy_extra_volumes
+            properties:
+              description: Volume for kvdb
+              name:
+                str_replace:
+                  template: vol_kvdb_servername_1
+                  params:
+                    servername: { get_param: server_name }
+              size: 100
+              volume_type: TIER2
+        
+          vol_attachment_vol1:
+            type: OS::Cinder::VolumeAttachment
+            depends_on: [ server, portworx_vol1 ]
+            condition: deploy_extra_volumes
+            properties:
+              instance_uuid: { get_resource: server }
+              mountpoint: /dev/vdc
+              volume_id: { get_resource: portworx_vol1 }
+
+          portworx_vol2:
+            type: OS::Cinder::Volume
+            condition: deploy_extra_volumes
+            properties:
+              description: Volume for data
+              name:
+                str_replace:
+                  template: vol_data_servername_1
+                  params:
+                    servername: { get_param: server_name }
+              size: 100
+              volume_type: TIER2
+
+          vol_attachment_vol2:
+            type: OS::Cinder::VolumeAttachment
+            depends_on: [ server, portworx_vol2, vol_attachment_vol1 ]
+            condition: deploy_extra_volumes
+            properties:
+              instance_uuid: { get_resource: server }
+              mountpoint: /dev/vdd
+              volume_id: { get_resource: portworx_vol2 }
+
+        conditions:
+          not_infra:
+            not:
+              equals:
+              - get_param: purpose_ident
+              - infra
+          deploy_volumes:
+            equals:
+            - get_param: extra_volumes
+            - true
+          deploy_extra_volumes:
+            and:
+            - not_infra
+            - deploy_volumes
+    when: deploy_portworx_storage|bool
+
+  - name: server_atomic - add code to wipe ephemeral disk
+    blockinfile:
+      dest: server_atomic_pwx.yaml
+      backup: yes
+      insertafter: "# add-portworx.yaml inserts extra code to wipe ephemeral disk here"
+      marker: "             # ANSIBLE MANAGED BLOCK - add code to wipe ephemeral disk"
+      block: |2
+                     EPHDEV="/dev/"`lsblk | grep /var/mnt | awk '{print $1}' | sed 's/[^a-z]*//g'`
+                     sudo umount /mnt; sudo wipefs -af $EPHDEV
+                     grep -v "${EPHDEV}" /etc/fstab > /tmp/fstab; sudo mv /tmp/fstab /etc/fstab
+    when: deploy_portworx_storage|bool
+
diff --git a/bastion-template.yaml b/bastion-template.yaml
@@ -151,6 +151,10 @@ parameters:
   multinetwork:
     type: boolean
     description: Deployment is multi-network?
+  deploy_portworx_storage:
+    type: boolean
+    description: Deploy Portworx?
+    default: false
   deploy_extra_gateway:
     type: boolean
     description: Deployment includes extra-gateway?
@@ -198,6 +202,9 @@ parameters:
   ansible_vault_password:
     type: string
     description: Ansible Vault password for decrypting secrets
+  internet_gateway_ip:
+    type: string
+    description: Internet gateway for internal network
 
 resources:
   bastion_port:
@@ -310,6 +317,7 @@ resources:
             __ocp_branding__ : { get_param: ocp_branding }
             __ocp_branding_url__ : { get_param: ocp_branding_url }
             __multinetwork__ : { get_param: multinetwork }
+            __deploy_portworx_storage__ : { get_param: deploy_portworx_storage }
             __deploy_extra_gateway__ : { get_param: deploy_extra_gateway }
             __external_dns__ : { get_param: external_dns }
             __set_node_routes__ : { get_param: set_node_routes }
@@ -330,6 +338,7 @@ resources:
             __neustar_ultradns_password__ : { get_param: neustar_ultradns_password }
             __slack_webhook_url_acme_sh__ : { get_param: slack_webhook_url_acme_sh }
             __ansible_vault_password__: { get_param: ansible_vault_password }
+            __internet_gateway_ip__: { get_param: internet_gateway_ip }
           template: { get_file: 'files/setup_bastion.yaml' }
       outputs:
       - name: result