From c8c61114e27e8b845204fe06455b8d9af95353e6 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Tue, 15 Oct 2024 17:26:16 +0300 Subject: [PATCH] Switch BBM deployment to Host Networking --- docker-compose/.env.dev | 2 +- docker-compose/docker-compose.yaml | 126 +++--------------- docker-compose/generate-config.py | 55 ++------ docker-compose/nginx/conf.d/monitoring.conf | 2 +- docker-compose/nginx/proxy_params | 4 - .../nginx/templates/bb.conf.template | 36 ++--- .../nginx/templates/ci.conf.template | 4 - docker-compose/start-bbm-web.sh | 21 +-- docker-compose/start.sh | 7 +- 9 files changed, 60 insertions(+), 197 deletions(-) delete mode 100644 docker-compose/nginx/proxy_params diff --git a/docker-compose/.env.dev b/docker-compose/.env.dev index 72abc994..9717e8b1 100644 --- a/docker-compose/.env.dev +++ b/docker-compose/.env.dev @@ -3,7 +3,7 @@ TITLE_URL=https://github.com/MariaDB/server BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ CR_HOST_WG_ADDR="hz-bbw5=127.0.0.1" BUILDMASTER_WG_IP=100.64.101.1 -MQ_ROUTER_URL=ws://crossbar:8080/ws +MQ_ROUTER_URL=ws://127.0.0.1:8080/ws MASTER_PACKAGES_DIR="/mnt/autofs/master_dev_packages" GALERA_PACKAGES_DIR="/mnt/autofs/galera_dev_packages" ARTIFACTS_URL="https://ci.dev.mariadb.org" diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 2d819940..b4394e7e 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -13,15 +13,11 @@ services: - MARIADB_DATABASE=buildbot - MARIADB_USER=buildmaster - MARIADB_PASSWORD=password - networks: - net_back: + network_mode: host healthcheck: test: ['CMD', "mariadb-admin", "--password=password", "--protocol", "tcp", "ping"] volumes: - # Only needed during GSOC - # - ./db:/docker-entrypoint-initdb.d:ro - ./mariadb:/var/lib/mysql:rw - # command: --tmpdir=/var/lib/mysql/tmp logging: driver: journald options: @@ -32,8 +28,7 @@ services: restart: unless-stopped container_name: crossbar hostname: crossbar - networks: - net_back: + network_mode: host logging: driver: journald options: @@ -46,7 +41,6 @@ services: hostname: nginx volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./nginx/proxy_params:/etc/nginx/proxy_params:ro - ./nginx/conf.d/:/etc/nginx/conf.d/ - ./nginx/templates/:/etc/nginx/templates/:ro - /srv/buildbot/packages:/srv/buildbot/packages:ro @@ -55,15 +49,10 @@ services: - ./logs/nginx:/var/log/nginx - ./certbot/www/:/var/www/certbot/:ro - ./certbot/conf/:/etc/nginx/ssl/:ro - ports: - - "443:443" - - "80:80" environment: - NGINX_ARTIFACTS_VHOST - NGINX_BUILDBOT_VHOST - networks: - net_front: - net_back: + network_mode: host logging: driver: journald options: @@ -71,11 +60,13 @@ services: certbot: image: certbot/certbot:latest + restart: "no" + container_name: certbot + hostname: certbot volumes: - ./certbot/www/:/var/www/certbot/:rw - ./certbot/conf/:/etc/letsencrypt/:rw - networks: - net_front: + network_mode: host master-web: image: quay.io/mariadb-foundation/bb-master:master-web @@ -105,9 +96,7 @@ services: - ./buildbot/:/srv/buildbot/master entrypoint: - /srv/buildbot/master/docker-compose/start-bbm-web.sh - networks: - net_back: - net_front: + network_mode: host depends_on: - mariadb - crossbar @@ -143,11 +132,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-nonlatent" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:9996:9996" + network_mode: host depends_on: - mariadb - crossbar @@ -182,11 +167,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-libvirt" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:9997:9997" + network_mode: host depends_on: - mariadb - crossbar @@ -221,11 +202,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh autogen/aarch64-master-0" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:9998:9998" + network_mode: host depends_on: - mariadb - crossbar @@ -260,11 +237,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh autogen/amd64-master-0" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:9999:9999" + network_mode: host depends_on: - mariadb - crossbar @@ -299,11 +272,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh autogen/amd64-master-1" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10000:10000" + network_mode: host depends_on: - mariadb - crossbar @@ -338,11 +307,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh autogen/ppc64le-master-0" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10001:10001" + network_mode: host depends_on: - mariadb - crossbar @@ -377,11 +342,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh autogen/s390x-master-0" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10002:10002" + network_mode: host depends_on: - mariadb - crossbar @@ -416,11 +377,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh autogen/x86-master-0" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10003:10003" + network_mode: host depends_on: - mariadb - crossbar @@ -455,11 +412,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-docker-nonstandard" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10004:10004" + network_mode: host depends_on: - mariadb - crossbar @@ -494,11 +447,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-galera" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10005:10005" + network_mode: host depends_on: - mariadb - crossbar @@ -533,11 +482,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-protected-branches" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10006:10006" + network_mode: host depends_on: - mariadb - crossbar @@ -572,11 +517,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-docker-nonstandard-2" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10007:10007" + network_mode: host depends_on: - mariadb - crossbar @@ -611,32 +552,7 @@ services: - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh master-bintars" - networks: - net_front: - net_back: - ports: - - "100.64.101.1:10008:10008" + network_mode: host depends_on: - mariadb - crossbar - -networks: - net_front: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.200.0.0/24 - driver_opts: - com.docker.network.enable_ipv6: "false" - com.docker.network.bridge.name: "br_bb_front" - net_back: - driver: bridge - internal: true - ipam: - driver: default - config: - - subnet: 172.16.201.0/24 - driver_opts: - com.docker.network.enable_ipv6: "false" - com.docker.network.bridge.name: "br_bb_back" diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index 3ab129ea..35c9f3b7 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -39,15 +39,11 @@ - MARIADB_DATABASE=buildbot - MARIADB_USER=buildmaster - MARIADB_PASSWORD=password - networks: - net_back: + network_mode: host healthcheck: test: ['CMD', "mariadb-admin", "--password=password", "--protocol", "tcp", "ping"] volumes: - # Only needed during GSOC - # - ./db:/docker-entrypoint-initdb.d:ro - ./mariadb:/var/lib/mysql:rw - # command: --tmpdir=/var/lib/mysql/tmp logging: driver: journald options: @@ -58,8 +54,7 @@ restart: unless-stopped container_name: crossbar hostname: crossbar - networks: - net_back: + network_mode: host logging: driver: journald options: @@ -72,7 +67,6 @@ hostname: nginx volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./nginx/proxy_params:/etc/nginx/proxy_params:ro - ./nginx/conf.d/:/etc/nginx/conf.d/ - ./nginx/templates/:/etc/nginx/templates/:ro - /srv/buildbot/packages:/srv/buildbot/packages:ro @@ -81,15 +75,10 @@ - ./logs/nginx:/var/log/nginx - ./certbot/www/:/var/www/certbot/:ro - ./certbot/conf/:/etc/nginx/ssl/:ro - ports: - - "443:443" - - "80:80" environment: - NGINX_ARTIFACTS_VHOST - NGINX_BUILDBOT_VHOST - networks: - net_front: - net_back: + network_mode: host logging: driver: journald options: @@ -97,11 +86,13 @@ certbot: image: certbot/certbot:latest + restart: "no" + container_name: certbot + hostname: certbot volumes: - ./certbot/www/:/var/www/certbot/:rw - ./certbot/conf/:/etc/letsencrypt/:rw - networks: - net_front: + network_mode: host master-web: image: quay.io/mariadb-foundation/bb-master:master-web @@ -113,9 +104,7 @@ - ./buildbot/:/srv/buildbot/master entrypoint: - /srv/buildbot/master/docker-compose/start-bbm-web.sh - networks: - net_back: - net_front: + network_mode: host depends_on: - mariadb - crossbar @@ -132,37 +121,13 @@ - /bin/bash - -c - "/srv/buildbot/master/docker-compose/start.sh {master_directory}" - networks: - net_front: - net_back: - ports: - - "{buildmaster_wg_ip}:{port}:{port}" + network_mode: host depends_on: - mariadb - crossbar """ END_TEMPLATE = """ -networks: - net_front: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.200.0.0/24 - driver_opts: - com.docker.network.enable_ipv6: "false" - com.docker.network.bridge.name: "br_bb_front" - net_back: - driver: bridge - internal: true - ipam: - driver: default - config: - - subnet: 172.16.201.0/24 - driver_opts: - com.docker.network.enable_ipv6: "false" - com.docker.network.bridge.name: "br_bb_back" """ @@ -246,13 +211,11 @@ def main(args): master_name=master_name, master_directory=master_directory, port=port, - buildmaster_wg_ip=env_vars["BUILDMASTER_WG_IP"], volumes=generate_volumes(master_volumes[master_name]), ) port += 1 file.write(docker_compose_piece) - file.write(END_TEMPLATE) if __name__ == "__main__": diff --git a/docker-compose/nginx/conf.d/monitoring.conf b/docker-compose/nginx/conf.d/monitoring.conf index c432e5b2..4ba6c75b 100644 --- a/docker-compose/nginx/conf.d/monitoring.conf +++ b/docker-compose/nginx/conf.d/monitoring.conf @@ -7,7 +7,7 @@ server { # this is for monitoring location = /basic_status { stub_status; - allow 172.200.0.0/24; + allow 127.0.0.1; deny all; } } \ No newline at end of file diff --git a/docker-compose/nginx/proxy_params b/docker-compose/nginx/proxy_params deleted file mode 100644 index 11c0f2c4..00000000 --- a/docker-compose/nginx/proxy_params +++ /dev/null @@ -1,4 +0,0 @@ -proxy_set_header Host $http_host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; \ No newline at end of file diff --git a/docker-compose/nginx/templates/bb.conf.template b/docker-compose/nginx/templates/bb.conf.template index 5129eed5..521e6802 100644 --- a/docker-compose/nginx/templates/bb.conf.template +++ b/docker-compose/nginx/templates/bb.conf.template @@ -25,10 +25,6 @@ server { server_name ${NGINX_BUILDBOT_VHOST}; - # logging - access_log /var/log/nginx/buildbot.access.log; - error_log /var/log/nginx/buildbot.error.log error; - # SSL configuration # ssl on; Deprecated in newer versions of NGINX (yields nginx: [emerg] unknown directive "ssl ) ssl_certificate /etc/nginx/ssl/live/${NGINX_BUILDBOT_VHOST}/fullchain.pem; # managed by Certbot @@ -52,21 +48,19 @@ server { # Set mime types gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header HOST $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Host $host; # Use default zone for rate limiting, allow burst of 10 requests with # no delay limit_req zone=default burst=10 nodelay; location / { - # Reverse proxy settings - include proxy_params; - proxy_pass http://master-web:8010; + proxy_pass http://127.0.0.1:8010; } # disable logging for wsgi_dashboards/styles.css since it's generated @@ -84,7 +78,7 @@ server { # Server sent event (sse) settings location /sse { proxy_buffering off; - proxy_pass http://master-web:8010/sse; + proxy_pass http://127.0.0.1:8010/sse; } # Websocket settings @@ -92,18 +86,14 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_pass http://master-web:8010/ws; + proxy_pass http://127.0.0.1:8010/ws; proxy_read_timeout 6000s; } - #FIXME: CrossReference not in DEV. ENABLE me when CR is deployed. + #FIXME: CrossReference not in DEV. # Cross-reference - # location /cr/static { - # alias /srv/cr/static; - # } - - # location /cr/ { - # include proxy_params; - # proxy_pass http://hz-bbw5:8080; - # } + location /cr/static { + alias /srv/cr/static; + } + } diff --git a/docker-compose/nginx/templates/ci.conf.template b/docker-compose/nginx/templates/ci.conf.template index 014a87d6..007ec1ce 100644 --- a/docker-compose/nginx/templates/ci.conf.template +++ b/docker-compose/nginx/templates/ci.conf.template @@ -47,10 +47,6 @@ server { autoindex on; - # logging - access_log /var/log/nginx/ci.access.log; - error_log /var/log/nginx/ci.error.log error; - # SSL configuration # ssl on; ssl_certificate /etc/nginx/ssl/live/${NGINX_BUILDBOT_VHOST}/fullchain.pem; diff --git a/docker-compose/start-bbm-web.sh b/docker-compose/start-bbm-web.sh index 2e55b046..f8404103 100755 --- a/docker-compose/start-bbm-web.sh +++ b/docker-compose/start-bbm-web.sh @@ -10,23 +10,24 @@ err() { exit 1 } +cd /srv/buildbot/master/master-web || err "cd /srv/buildbot/master/master-web" + +# # loop for debug +# while true; do date && sleep 30; done + +# shellcheck disable=SC2226 +[[ -f master-private.cfg ]] || ln -s ../master-private.cfg +VAR_DB_HOST=$(grep db_host master-private.cfg | awk '{print $3}' | sed s/\"//g) echo "Waiting for MariaDB to start..." -while ! nc -z mariadb 3306; do +while ! nc -z $VAR_DB_HOST 3306; do sleep 0.1 done echo "MariaDB started" echo "Waiting for Crossbar to start..." -while ! nc -z crossbar 8080; do +while ! nc -z 127.0.0.1 8080; do sleep 0.1 done -echo "MariaDB started" - -# # loop for debug -# while true; do date && sleep 30; done - -cd /srv/buildbot/master/master-web || err "cd /srv/buildbot/master/master-web" -# shellcheck disable=SC2226 -[[ -f master-private.cfg ]] || ln -s ../master-private.cfg +echo "Crossbar started" buildbot upgrade-master /srv/buildbot/master/master-web buildbot start --nodaemon diff --git a/docker-compose/start.sh b/docker-compose/start.sh index f78f9dea..36524a77 100755 --- a/docker-compose/start.sh +++ b/docker-compose/start.sh @@ -16,14 +16,15 @@ cd "/srv/buildbot/master/$1" || err "cd /srv/buildbot/master/$1" # shellcheck disable=SC2226 [[ -f master-private.cfg ]] || ln -s ../master-private.cfg +VAR_DB_HOST=$(grep db_host master-private.cfg | awk '{print $3}' | sed s/\"//g) echo "Waiting for MariaDB to start..." -while ! nc -z mariadb 3306; do +while ! nc -z $VAR_DB_HOST 3306; do sleep 0.1 done echo "MariaDB started" echo "Waiting for Crossbar to start..." -while ! nc -z crossbar 8080; do +while ! nc -z 127.0.0.1 8080; do sleep 0.1 done echo "Crossbar started" @@ -40,4 +41,4 @@ fi # loop for debug # while true; do date && sleep 30; done -buildbot start --nodaemon +buildbot start --nodaemon \ No newline at end of file