Merge branch 'PMM-10680-enable-pbm-collector' into PMM-10680-enable-p…

…bm-collector-v3
percona · Aug 26, 2024 · a9ee989 · a9ee989
2 parents 5883e93 + a2b545d
commit a9ee989
Show file tree

Hide file tree

Showing 16 changed files with 363 additions and 66 deletions.
diff --git a/.gitignore b/.gitignore
@@ -42,3 +42,5 @@ cli-tests/playwright-report/
 
 api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
+
+packer.log
diff --git a/build/packer/ansible/agent-aws.yml b/build/packer/ansible/agent-aws.yml
@@ -22,10 +22,36 @@
     - name: Add a repository for NodeJS
       shell: "curl -fsSL https://rpm.nodesource.com/setup_{{ node_version }} | bash -"
 
+    - name: Install Oracle EPEL Repo
+      yum:
+        name: oracle-epel-release-el9
+        state: latest
+
+    - name: Install yum-utils
+      yum:
+        name: yum-utils
+        state: latest
+
+    - name: Add Docker repository
+      command: yum-config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo
+
+    - name: Install Docker packages
+      yum:
+        name:
+          - docker-ce
+          - docker-ce-cli
+          - containerd.io
+        state: latest
+
+    - name: Enable Docker service
+      systemd:
+        name: docker
+        enabled: yes
+        state: started
+
     - name: Install packages
-      package:
+      yum:
         name:
-          - docker
           - git
           - jq
           - gettext
@@ -41,22 +67,22 @@
           - gcc
           - make
           - perl
-          # - p7zip
+            #- p7zip
           - nodejs
-        state: present
+        state: latest
 
     - name: Install Java17
-      package:
-        name: jdk-17-headless
-        state: present
+      yum:
+        name: java-17-openjdk-headless
+        state: latest
 
     - name: Install pip3 and ansible
-      package:
+      yum:
         name:
           - python3-pip
           - ansible-core
           - glibc-langpack-en
-        state: present
+        state: latest
 
     - name: Install ansible collections
       shell: "ansible-galaxy collection install {{ item }}"
@@ -110,8 +136,7 @@
         mode: "555"
 
     - name: Install Python's virtualenv
-      pip:
-        name: virtualenv
+      command: python3 -m venv /home/ec2-user/venv
 
     - name: Install Python modules
       pip:
@@ -276,16 +301,6 @@
         name: fs.inotify.max_user_watches
         value: 1048576
 
-    - name: Get disk UUID
-      command: blkid -s UUID -o value /dev/nvme0n1p1
-      register: disk_uuid
-
-    - name: Replace fstab entities with new mount options
-      lineinfile:
-        path: /etc/fstab
-        regexp: "^UUID={{ disk_uuid.stdout }}"
-        line: "UUID={{ disk_uuid.stdout }}     /           xfs    defaults,noatime,nobarrier,discard  1   1"
-
     - name: Add repo.ci.percona.com to /etc/hosts
       lineinfile:
         dest: /etc/hosts

diff --git a/build/packer/ansible/agent-do.yml b/build/packer/ansible/agent-do.yml
@@ -14,6 +14,11 @@
         # Alternative architecture name just because we never can have only one name in computers
         ansible_architecture_alt: "{% if ansible_architecture == 'x86_64' %}amd64{% elif ansible_architecture == 'aarch64' %}arm64{% else %}none{% endif %}"
 
+    - name: Add hashicorp repository
+      get_url:
+        url: https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+        dest: /etc/yum.repos.d/hashicorp.repo
+
     - name: Enable epel repo
       yum:
         name: epel-release
@@ -27,10 +32,26 @@
     - name: Add a repository for NodeJS
       shell: "curl -fsSL https://rpm.nodesource.com/setup_{{ node_version }} | bash -"
 
+    - name: Install docker
+      block:
+        - name: Add docker repository
+          get_url:
+            url: https://download.docker.com/linux/centos/docker-ce.repo
+            dest: /etc/yum.repos.d/docker-ce.repo
+
+        - name: Install docker packages
+          package:
+            name:
+              - docker-ce
+              - docker-ce-cli
+              - containerd.io
+              - docker-buildx-plugin
+              - docker-compose-plugin
+            state: present
+
     - name: Install packages
       package:
         name:
-          - docker
           - git
           - jq
           - gettext
@@ -49,12 +70,14 @@
           - perl
           - p7zip
           - nodejs
+          - packer
           # - xmlstarlet
         state: present
 
+
     - name: Install Java17
       package:
-        name: jdk-17-headless
+        name: java-17-openjdk-headless
         state: present
 
     - name: Install docker-compose
@@ -105,6 +128,11 @@
         group: "root"
         owner: "root"
 
+    - name: Install pip3
+      package:
+        name: python3-pip
+        state: present
+
     - name: Install Python modules
       pip:
         name: "{{ item }}"
@@ -124,16 +152,26 @@
             description: VirtualBox
             baseurl: "http://download.virtualbox.org/virtualbox/rpm/el/$releasever/$basearch"
             gpgcheck: true
-            gpgkey: https://www.virtualbox.org/download/oracle_vbox.asc
+            gpgkey: https://www.virtualbox.org/download/oracle_vbox_2016.asc
             enabled: true
             state: present
 
+        ##- name: Add base repository for Hashicorp
+        ##  yum_repository:
+        ##    name: hashicorp
+        ##    description: HashiCorp Repository
+        ##    baseurl: "https://rpm.releases.hashicorp.com/RHEL/$releasever/$basearch/stable"
+        ##    gpgcheck: true
+        ##    gpgkey: https://rpm.releases.hashicorp.com/gpg
+        ##    enabled: true
+        ##    state: present
+
         - name: Install packages for virtualization
           yum:
             state: present
             name: "{{ item }}"
           loop:
-            - VirtualBox-6.1
+            - VirtualBox-7.0
             - vagrant
 
         - name: Reboot to the latest kernel
@@ -142,6 +180,47 @@
         - name: Recompile kernel
           command: /sbin/vboxconfig
 
+    - name: Install Packer Plugins
+      block:
+        - name: Create Packer plugins directory
+          file:
+            path: ~/.packer.d/plugins/github.com
+            state: directory
+            mode: '0755'
+            recurse: yes
+
+        - name: Install packer virtualbox plugin
+          command: >
+            /usr/bin/packer plugins install github.com/hashicorp/virtualbox
+          register: packer_plugin_install
+          changed_when: "'Installed plugin successfully' in packer_plugin_install.stdout"
+
+        - name: Install packer ansible plugin
+          command: >
+            /usr/bin/packer plugins install github.com/hashicorp/ansible
+          register: packer_ansible_plugin_install
+          changed_when: "'Installing' in packer_ansible_plugin_install.stdout"
+
+        - name: Verify packer virtualbox plugin installation
+          stat:
+            path: ~/.packer.d/plugins/github.com/hashicorp/virtualbox
+          register: packer_virtualbox_plugin
+
+        - name: Verify packer ansible plugin installation
+          stat:
+            path: ~/.packer.d/plugins/github.com/hashicorp/ansible
+          register: packer_ansible_plugin
+
+        - name: Output virtualbox plugin installation status
+          debug:
+            msg: "Packer VirtualBox plugin is installed"
+          when: packer_virtualbox_plugin.stat.exists
+
+        - name: Output ansible plugin installation status
+          debug:
+            msg: "Packer Ansible plugin is installed"
+          when: packer_ansible_plugin.stat.exists
+
     - name: Copy docker daemon config
       copy:
         src: daemon.json
@@ -153,19 +232,14 @@
         state: started
         enabled: yes
 
-    - name: Pull and cache docker images
-      docker_image:
-        name: "{{ item }}"
-        source: pull
-      loop:
-        - debian:buster
-        - debian:stretch
-        - debian:bullseye
-        - ubuntu:jammy
-        - ubuntu:bionic
-        - ubuntu:focal
-        - rockylinux:8
-        - oraclelinux:9
+    - name: Download and unarchive AWS installer
+      unarchive:
+        src: https://awscli.amazonaws.com/awscli-exe-linux-{{ ansible_architecture }}.zip
+        dest: /tmp
+        remote_src: yes
+
+    - name: Run awscli installer
+      shell: /tmp/aws/install
 
     - name: Turn off swap
       sysctl:

diff --git a/build/packer/aws.pkr.hcl b/build/packer/aws.pkr.hcl
@@ -39,9 +39,9 @@ source "amazon-ebs" "agent" {
     iit-billing-tag = "pmm-worker"
   }
   launch_block_device_mappings {
-    device_name           = "/dev/xvda"
-    volume_size           = 50
-    volume_type           = "gp3"
+    device_name = "/dev/sda1"
+    volume_size = 50
+    volume_type = "gp3"
     delete_on_termination = true
   }
   vpc_filter {
@@ -114,7 +114,7 @@ build {
     user                   = "ec2-user"
     ansible_env_vars       = ["ANSIBLE_NOCOLOR=True"]
     extra_arguments = [
-      "--ssh-extra-args", "-o HostKeyAlgorithms=+ssh-rsa -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null", "-v"
+      "--ssh-extra-args", "-o HostKeyAlgorithms=+ssh-rsa -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null", "-vvv"
     ]
     playbook_file          = "./ansible/agent-aws.yml"
   }

diff --git a/build/packer/do.pkr.hcl b/build/packer/do.pkr.hcl
@@ -1,9 +1,13 @@
 packer {
   required_plugins {
     digitalocean = {
-      version = "=1.0.4"
+      version = "=1.4.0"
       source  = "github.com/digitalocean/digitalocean"
     }
+    ansible = {
+      version = "~> 1"
+      source  = "github.com/hashicorp/ansible"
+    }
   }
 }
 

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
@@ -103,7 +103,6 @@ main() {
     gobuild_component "proxysql_exporter"
     gobuild_component "rds_exporter"
     gobuild_component "azure_metrics_exporter"
-    gobuild_component "clickhouse_exporter" "" "github.com/Percona-Lab/clickhouse_exporter"
 
     gobuild_component "pmm-admin" "pmm" "" "admin"
     gobuild_component "pmm-agent" "pmm" "" "agent"

diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
@@ -475,8 +475,6 @@ if [ "$TARBALL" = '1' -o "$SOURCE" = '1' ]; then
     ProxysqlExp_REPO=$(get_repos "proxysql_exporter")
     NodeExp_BRANCH_NAME=$(get_branches "node_exporter")
     NodeExp_REPO=$(get_repos "node_exporter")
-    ClickhouseExp_BRANCH_NAME=$(get_branches "clickhouse_exporter")
-    ClickhouseExp_REPO=$(get_repos "clickhouse_exporter")
     AzureExp_BRANCH_NAME=$(get_branches "azure_metrics_exporter")
     AzureExp_REPO=$(get_repos "azure_metrics_exporter")
 fi

diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
@@ -69,7 +69,6 @@ main() {
     prepare_tarball postgres_exporter
     prepare_tarball proxysql_exporter
     prepare_tarball node_exporter
-    prepare_tarball clickhouse_exporter
     prepare_tarball rds_exporter
     prepare_tarball azure_metrics_exporter