From 2ae726fab1e015042a4bc18132a42d7562058be7 Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Thu, 3 Aug 2023 16:29:25 +0200 Subject: [PATCH] Restart work on docker-compose for BBM --- define_masters.py | 2 + docker-compose/README.md | 2 +- .../config/master-private.cfg-sample | 88 ----------- docker-compose/config/master-web/master.cfg | 118 -------------- docker-compose/docker-compose.yaml | 149 ++++++++---------- docker-compose/generate-config.py | 11 +- docker-compose/start-bbm-web.sh | 21 +-- docker-compose/start.sh | 10 +- master-galera/master.cfg | 2 +- master-libvirt/master.cfg | 2 +- master-nonlatent/master.cfg | 2 +- master-web/master.cfg | 58 +++---- master-web/master.cfg.sample | 107 +++++++++++++ master.cfg | 2 +- 14 files changed, 218 insertions(+), 356 deletions(-) mode change 100644 => 100755 define_masters.py delete mode 100644 docker-compose/config/master-private.cfg-sample delete mode 100644 docker-compose/config/master-web/master.cfg mode change 100644 => 100755 docker-compose/generate-config.py create mode 100644 master-web/master.cfg.sample diff --git a/define_masters.py b/define_masters.py old mode 100644 new mode 100755 index 11450fb4..d5a38ab7 --- a/define_masters.py +++ b/define_masters.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import yaml import os import shutil diff --git a/docker-compose/README.md b/docker-compose/README.md index f0c866f0..5988627d 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -16,7 +16,7 @@ To run buildbot locally with a pre-initialized DB to populate some fields git clone https://github.com/MariaDB/buildbot.git cd buildbot/docker-compose ln -s config/master-private.cfg-sample config/master-private.cfg -ln -s ../buildbot +ln -s .. buildbot mkdir -p logs db mariadb/tmp cd db wget https://ci.mariadb.org/helper_files/buildbot_dump.sql.gz diff --git a/docker-compose/config/master-private.cfg-sample b/docker-compose/config/master-private.cfg-sample deleted file mode 100644 index 4204dde3..00000000 --- a/docker-compose/config/master-private.cfg-sample +++ /dev/null @@ -1,88 +0,0 @@ -private["db_url"] = "mysql://buildmaster:password@mariadb/buildbot?max_idle=300&storage_engine=InnoDB" -private["db_host"] = "mariadb" -private["db_user"] = "buildmaster" -private["db_password"] = "password" -private["db_mtr_db"] = "buildbot" -private["gh_secret"] = "secret" -private["zabbix_server"] = "https://zabbix.server" -private["zabbix_token"] = "zabbix_token" -private["user_pass"]= { - "admin":"user_pass", -} -private["master-variables"] = { - "max_builds": 15, - "starting_port": 9996, - "workers": { - "amd64": [ - 'amd-bbw1', - 'amd-bbw2', - 'intel-bbw1', - ], - "aarch64": [ - 'aarch64-bbw1', - 'aarch64-bbw2', - 'aarch64-bbw3', - 'aarch64-bbw4', - ], - "ppc64le": [ - 'ppc64le-rhel8-bbw1', - 'ppc64le-rhel7-bbw1', - 'ppc64le-db-bbw1', - ], - "s390x": [ - 's390x-bbw1', - 's390x-bbw2', - 's390x-bbw3', - ], - "x86": [ - 'hz-bbw2', - ], - }, -} -private["worker_pass"]= { - "hz-bbw2-ubuntu1804":"1234", - "hz-bbw2-libvirt-debian-10":"1234", - "bm-bbw1-ubuntu1804":"1234", - "shinnok-bbw1-macos":"1234", - "aix-worker":"1234", - "bbw1-windows":"1234", - "bbw2-windows":"1234", - "bb-rhel8-docker":"1234", - "s390x-rhel8":"1234", - "libvirt": "1234", -} -private["docker_workers"]= { - "hz-dev-bbm-bbw1-docker":"tcp://IP:2375", -} - -private["worker_name_mapping"] = { - "s390x-bbw1": "ibm-s390x-ubuntu20.04", - "s390x-bbw2": "ibm-s390x-sles15", - "s390x-bbw3": "ibm-s390x-rhel8", -} - -private["gh_mdbci"]= { - "username":"username", - "name":"username", - "email":"user@domain.com", - "access_token":"access_token", - "push_access_token":"push_access_token" -} - -# github.com/fauust/mariadb-server/ auth app -private["gh_mdbauth"]= { - "client":"user", - "secret":"secret" -} - -# RHEL subscription -private["rhel_sub"]= { - "user":"user", - "password": "password" -} - -private["libvirt_workers"] = { - "amd64": ("bb-hz-bbw5", "qemu+ssh://user@IP_address/system"), - "aarch64": ("bb-eq-arm1", "qemu+ssh://user@IP_addres/system"), - "ppc64le": ("bb-db-p9-bbw1", "qemu+ssh://user@IP_adress/system"), -} diff --git a/docker-compose/config/master-web/master.cfg b/docker-compose/config/master-web/master.cfg deleted file mode 100644 index 1c6cb1bf..00000000 --- a/docker-compose/config/master-web/master.cfg +++ /dev/null @@ -1,118 +0,0 @@ -# -*- python -*- -# ex: set filetype=python: - -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from twisted.internet import defer -import os -import sys -import docker -from datetime import timedelta - -sys.setrecursionlimit(10000) - -sys.path.append(os.getcwd() + '/..') -from constants import * - -c = BuildmasterConfig = {} - -# Load the slave, database passwords and 3rd-party tokens from an external -# private file, so that the rest of the configuration can be public. -config = { "private": { } } -exec(open("../master-private.cfg").read(), config, { }) - -####### PROJECT IDENTITY - -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" -c['buildbotURL'] = "http://localhost:8010/" - -# minimalistic config to activate web UI -c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={}), custom_templates_dir='templates') - -# Github Auth, allow control for MariaDB affiliated accounts -c['www']['authz'] = util.Authz( - allowRules=[ - util.AnyControlEndpointMatcher(role="MariaDB", defaultDeny=True) - ], - roleMatchers=[ - util.RolesFromGroups() - ] -) -c['www']['auth'] = util.GitHubAuth(config["private"]["gh_mdbauth"]["client"], config["private"]["gh_mdbauth"]["secret"]) - -# Sponsor plugin -exec(open("../sponsor.py").read()) - -####### DB URL -c['db'] = { - 'db_url' : config["private"]["db_url"] -} - -####### Disable net usage reports from being sent to buildbot.net -c['buildbotNetUsageData'] = None - -####### GitHub hooks - -# GitHub webhook receiver -c['www']['change_hook_dialects'] = { - 'github': { - 'secret': config["private"]["gh_secret"], - 'strict': True, - } -} - -c['www']['ui_default_config'] = { - 'Grid.changeFetchLimit': 5, - 'Grid.buildFetchLimit': 50, -} - -c['logEncoding'] = 'utf-8' - -c['multiMaster'] = True - -# Need to enable multimaster aware mq. Wamp is the only option for now. -c['mq'] = { - 'type' : 'wamp', - 'router_url': 'ws://crossbar:8080/ws', - 'realm': 'realm1', - # valid are: none, critical, error, warn, info, debug, trace - 'wamp_debug_level' : 'info' -} - -# git branch filter using fnmatch -import fnmatch -def upstream_branch_fn(branch): - return branch in branches_main or fnmatch.fnmatch(branch, 'bb-*') or fnmatch.fnmatch(branch, 'st-*') or fnmatch.fnmatch(branch, 'prot-*') or fnmatch.fnmatch(branch, "refs/pull/*") or fnmatch.fnmatch(branch, "preview-10.*") -def staging_branch_fn(branch): - return fnmatch.fnmatch(branch, 'st-*') -def fnmatch_any(s, list_of_patterns): - return any(fnmatch.fnmatch(s, p) for p in list_of_patterns) - -c['schedulers'] = [] - -# upstream scheduling -schedulerTarball = schedulers.AnyBranchScheduler( - name="s_upstream_tarball", - change_filter=util.ChangeFilter(repository="https://github.com/MariaDB/server", branch_fn=upstream_branch_fn), - treeStableTimer=60, - builderNames=["tarball-docker"]) -c['schedulers'].append(schedulerTarball) - -# Other schedulers -# github.com/cvicentiu/server -c['schedulers'].append(schedulers.SingleBranchScheduler( name="s_vicentiu", - change_filter=util.ChangeFilter(repository="https://github.com/cvicentiu/server", branch_fn=upstream_branch_fn), - treeStableTimer=60, - builderNames=["tarball-docker"])) - -# github.com/fauust/mariadb-server -c['schedulers'].append(schedulers.SingleBranchScheduler( name="s_fauust", - change_filter=util.ChangeFilter(repository="https://github.com/fauust/mariadb-server", branch_fn=upstream_branch_fn), - treeStableTimer=60, - builderNames=["tarball-docker"])) - diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index d4945ed9..7239e7d6 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -2,8 +2,9 @@ version: "3.7" services: mariadb: - image: mariadb:10.5 + image: mariadb:10.6 restart: unless-stopped + container_name: mariadb environment: - MARIADB_ROOT_PASSWORD=password - MARIADB_DATABASE=buildbot @@ -14,26 +15,26 @@ services: healthcheck: test: ['CMD', "mariadb-admin", "--password=password", "--protocol", "tcp", "ping"] volumes: - - ./db/buildbot_dump.sql:/docker-entrypoint-initdb.d/buildbot.sql - - ./mariadb:/var/lib/mysql - command: --tmpdir=/var/lib/mysql/tmp + - ./db:/docker-entrypoint-initdb.d:ro + - ./mariadb:/var/lib/mysql:rw + # command: --tmpdir=/var/lib/mysql/tmp crossbar: image: crossbario/crossbar restart: unless-stopped + container_name: crossbar networks: net_back: master-web: image: quay.io/mariadb-foundation/bb-master:master-web restart: unless-stopped + container_name: master-web volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start-bbm-web.sh:/usr/local/bin/start-bbm-web.sh - ./buildbot/:/srv/buildbot/master entrypoint: - - /usr/local/bin/start-bbm-web.sh + - /srv/buildbot/master/docker-compose/start-bbm-web.sh networks: net_front: net_back: @@ -46,15 +47,14 @@ services: autogen_aarch64-master-0: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: autogen_aarch64-master-0 volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh autogen/aarch64-master-0" + - "/srv/buildbot/master/docker-compose/start.sh autogen/aarch64-master-0" networks: net_front: net_back: @@ -64,19 +64,17 @@ services: - mariadb - crossbar - autogen_amd64-master-0: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: autogen_amd64-master-0 volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh autogen/amd64-master-0" + - "/srv/buildbot/master/docker-compose/start.sh autogen/amd64-master-0" networks: net_front: net_back: @@ -86,19 +84,17 @@ services: - mariadb - crossbar - autogen_amd64-master-1: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: autogen_amd64-master-1 volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh autogen/amd64-master-1" + - "/srv/buildbot/master/docker-compose/start.sh autogen/amd64-master-1" networks: net_front: net_back: @@ -108,19 +104,17 @@ services: - mariadb - crossbar - autogen_ppc64le-master-0: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: autogen_ppc64le-master-0 volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh autogen/ppc64le-master-0" + - "/srv/buildbot/master/docker-compose/start.sh autogen/ppc64le-master-0" networks: net_front: net_back: @@ -130,19 +124,17 @@ services: - mariadb - crossbar - autogen_s390x-master-0: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: autogen_s390x-master-0 volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh autogen/s390x-master-0" + - "/srv/buildbot/master/docker-compose/start.sh autogen/s390x-master-0" networks: net_front: net_back: @@ -152,19 +144,17 @@ services: - mariadb - crossbar - autogen_x86-master-0: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: autogen_x86-master-0 volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh autogen/x86-master-0" + - "/srv/buildbot/master/docker-compose/start.sh autogen/x86-master-0" networks: net_front: net_back: @@ -174,19 +164,17 @@ services: - mariadb - crossbar - master-docker-nonstandard: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: master-docker-nonstandard volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh master-docker-nonstandard" + - "/srv/buildbot/master/docker-compose/start.sh master-docker-nonstandard" networks: net_front: net_back: @@ -196,19 +184,17 @@ services: - mariadb - crossbar - master-galera: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: master-galera volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh master-galera" + - "/srv/buildbot/master/docker-compose/start.sh master-galera" networks: net_front: net_back: @@ -218,63 +204,57 @@ services: - mariadb - crossbar + # master-libvirt: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # container_name: master-libvirt + # volumes: + # - ./logs:/var/log/buildbot + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/srv/buildbot/master/docker-compose/start.sh master-libvirt" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8019:8019" + # depends_on: + # - mariadb + # - crossbar - master-libvirt: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-libvirt" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8019:8019" - depends_on: - - mariadb - - crossbar - - - master-nonlatent: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-nonlatent" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8020:8020" - depends_on: - - mariadb - - crossbar - + # master-nonlatent: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # container_name: master-nonlatent + # volumes: + # - ./logs:/var/log/buildbot + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/srv/buildbot/master/docker-compose/start.sh master-nonlatent" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8020:8020" + # depends_on: + # - mariadb + # - crossbar master-protected-branches: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: master-protected-branches volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh master-protected-branches" + - "/srv/buildbot/master/docker-compose/start.sh master-protected-branches" networks: net_front: net_back: @@ -284,7 +264,6 @@ services: - mariadb - crossbar - networks: net_front: driver: bridge diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py old mode 100644 new mode 100755 index ffe49264..709ab3bd --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -1,3 +1,9 @@ +#!/usr/bin/env python + +# TODO vlad: +# - generate directly into docker-compose.yaml file +# - generate only 1 blank lines between block (not 2) + master_directories = [ "autogen/aarch64-master-0", "autogen/amd64-master-0", @@ -16,15 +22,14 @@ {master_name}: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped + container_name: {master_name} volumes: - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - ./buildbot/:/srv/buildbot/master entrypoint: - /bin/bash - -c - - "/usr/local/bin/start.sh {master_directory}" + - "/srv/buildbot/master/docker-compose/start.sh {master_directory}" networks: net_front: net_back: diff --git a/docker-compose/start-bbm-web.sh b/docker-compose/start-bbm-web.sh index 0f507f9b..5b127805 100755 --- a/docker-compose/start-bbm-web.sh +++ b/docker-compose/start-bbm-web.sh @@ -6,22 +6,10 @@ set -o pipefail set -o posix err() { - echo_red >&2 "ERROR: $*" + echo >&2 "ERROR: $*" exit 1 } -cd /srv/buildbot/master || err "cd /srv/buildbot/master" -ln -sf /srv/buildbot-config/master-private.cfg master-private.cfg - -# config buildbot master-web -cd /srv/buildbot/master/master-web || err "cd" -for file in /srv/buildbot-config/master-web/*; do - # shellcheck disable=SC2226 - ln -sf "$file" -done -# # loop for debug -# while true; do date && sleep 30; done - echo "Waiting for MariaDB to start..." while ! nc -z mariadb 3306; do sleep 0.1 @@ -32,6 +20,11 @@ echo "Waiting for Crossbar to start..." while ! nc -z crossbar 8080; do sleep 0.1 done -echo "Crossbar started" +echo "MariaDB started" + +# # loop for debug +# while true; do date && sleep 30; done +cd /srv/buildbot/master/master-web || err "cd /srv/buildbot/master/master-web" +buildbot upgrade-master /srv/buildbot/master/master-web buildbot start --nodaemon diff --git a/docker-compose/start.sh b/docker-compose/start.sh index e629ee99..cf6faefb 100755 --- a/docker-compose/start.sh +++ b/docker-compose/start.sh @@ -6,13 +6,10 @@ set -o pipefail set -o posix err() { - echo_red >&2 "ERROR: $*" + echo >&2 "ERROR: $*" exit 1 } -cd /srv/buildbot/master || err "cd /srv/buildbot/master" -ln -sf /srv/buildbot-config/master-private.cfg master-private.cfg - cd "/srv/buildbot/master/" # Generate master configs /opt/buildbot/.venv/bin/python define_masters.py @@ -32,4 +29,7 @@ while ! nc -z crossbar 8080; do done echo "Crossbar started" -buildbot start --nodaemon \ No newline at end of file +# loop for debug +# while true; do date && sleep 30; done + +buildbot start --nodaemon diff --git a/master-galera/master.cfg b/master-galera/master.cfg index 62008c7d..f15753ff 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -245,7 +245,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': 'ws://crossbar:8085/ws', 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 55e3a0c6..ac52dd15 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -287,7 +287,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': 'ws://crossbar:8085/ws', 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index f95488a6..dd2c82c9 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -344,7 +344,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': 'ws://crossbar:8085/ws', 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-web/master.cfg b/master-web/master.cfg index c5577a6f..df817154 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -20,32 +20,20 @@ from constants import * c = BuildmasterConfig = {} -config = { "private": { }} +# Load the slave, database passwords and 3rd-party tokens from an external +# private file, so that the rest of the configuration can be public. +config = { "private": { } } exec(open("../master-private.cfg").read(), config, { }) ####### PROJECT IDENTITY -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). - c['title'] = "MariaDB CI" c['titleURL'] = "https://github.com/MariaDB/server" - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" +c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" # minimalistic config to activate web UI c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={}), custom_templates_dir='templates') -# Plain username auth -#c['www']['authz'] = util.Authz(allowRules=[util.AnyControlEndpointMatcher(role="admins")], -# roleMatchers=[util.RolesFromUsername(roles=["admins"], usernames=["admin"])]) -#c['www']['auth'] = util.UserPasswordAuth({'admin': config["private"]["user_pass"]["admin"]}) - # Github Auth, allow control for MariaDB affiliated accounts c['www']['authz'] = util.Authz( allowRules=[ @@ -60,10 +48,13 @@ c['www']['auth'] = util.GitHubAuth(config["private"]["gh_mdbauth"]["client"], co # Sponsor plugin exec(open("../sponsor.py").read()) -####### CHANGESOURCES +####### DB URL +c['db'] = { + 'db_url' : config["private"]["db_url"] +} -# the 'change_source' setting tells the buildmaster how it should find out -# about source code changes. +####### Disable net usage reports from being sent to buildbot.net +c['buildbotNetUsageData'] = None ####### GitHub hooks @@ -82,30 +73,21 @@ c['www']['ui_default_config'] = { c['logEncoding'] = 'utf-8' -#c['www']['plugins']['profiler'] = True -#c['services'] = [util.ProfilerService(frequency=100, gatherperiod=30 * 60, mode='prof', basepath='/srv/buildbot/master/master-web/prof', wantBuilds=100)] - c['multiMaster'] = True -c['db'] = { - # This specifies what database buildbot uses to store its state. - 'db_url' : config["private"]["db_url"] -} - -####### Disable net usage reports from being sent to buildbot.net -c['buildbotNetUsageData'] = None +# Need to enable multimaster aware mq. Wamp is the only option for now. c['mq'] = { 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': 'ws://crossbar:8080/ws', 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace - 'wamp_debug_level' : 'warn' + 'wamp_debug_level' : 'info' } # git branch filter using fnmatch import fnmatch def upstream_branch_fn(branch): - return branch in branches_main or fnmatch.fnmatch(branch, 'bb-*') or fnmatch.fnmatch(branch, 'st-*') or fnmatch.fnmatch(branch, 'prot-*') or fnmatch.fnmatch(branch, "refs/pull/*") or fnmatch.fnmatch(branch, "preview-*") + return branch in branches_main or fnmatch.fnmatch(branch, 'bb-*') or fnmatch.fnmatch(branch, 'st-*') or fnmatch.fnmatch(branch, 'prot-*') or fnmatch.fnmatch(branch, "refs/pull/*") or fnmatch.fnmatch(branch, "preview-10.*") def staging_branch_fn(branch): return fnmatch.fnmatch(branch, 'st-*') def fnmatch_any(s, list_of_patterns): @@ -121,15 +103,15 @@ schedulerTarball = schedulers.AnyBranchScheduler( builderNames=["tarball-docker"]) c['schedulers'].append(schedulerTarball) -#-- - # Other schedulers - -# github.com/shinnok/server +# github.com/cvicentiu/server c['schedulers'].append(schedulers.SingleBranchScheduler( name="s_vicentiu", change_filter=util.ChangeFilter(repository="https://github.com/cvicentiu/server", branch_fn=upstream_branch_fn), treeStableTimer=60, builderNames=["tarball-docker"])) -#-- - +# github.com/fauust/mariadb-server +c['schedulers'].append(schedulers.SingleBranchScheduler( name="s_fauust", + change_filter=util.ChangeFilter(repository="https://github.com/fauust/mariadb-server", branch_fn=upstream_branch_fn), + treeStableTimer=60, + builderNames=["tarball-docker"])) diff --git a/master-web/master.cfg.sample b/master-web/master.cfg.sample new file mode 100644 index 00000000..3b7ba685 --- /dev/null +++ b/master-web/master.cfg.sample @@ -0,0 +1,107 @@ +# -*- python -*- +# ex: set filetype=python: + +from buildbot.plugins import * + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +c['workers'] = [worker.Worker("example-worker", "pass")] + +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +c['protocols'] = {'pb': {'port': 9989}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot version of a python hello-world project. + +c['change_source'] = [] +c['change_source'].append(changes.GitPoller( + 'git://github.com/buildbot/hello-world.git', + workdir='gitpoller-workdir', branch='master', + pollInterval=300)) + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +c['schedulers'] = [] +c['schedulers'].append(schedulers.SingleBranchScheduler( + name="all", + change_filter=util.ChangeFilter(branch='master'), + treeStableTimer=None, + builderNames=["runtests"])) +c['schedulers'].append(schedulers.ForceScheduler( + name="force", + builderNames=["runtests"])) + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. + +factory = util.BuildFactory() +# check out the source +factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) +# run the tests (note that this will require that 'trial' is installed) +factory.addStep(steps.ShellCommand(command=["trial", "hello"], + env={"PYTHONPATH": "."})) + +c['builders'] = [] +c['builders'].append( + util.BuilderConfig(name="runtests", + workernames=["example-worker"], + factory=factory)) + +####### BUILDBOT SERVICES + +# 'services' is a list of BuildbotService items like reporter targets. The +# status of each build will be pushed to these targets. buildbot/reporters/*.py +# has a variety to choose from, like IRC bots. + +c['services'] = [] + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). + +c['title'] = "Hello World CI" +c['titleURL'] = "https://buildbot.github.io/hello-world/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = "http://localhost:8010/" + +# minimalistic config to activate new web UI +c['www'] = dict(port=8010, + plugins=dict(waterfall_view={}, console_view={}, grid_view={})) + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. + # It's easy to start with sqlite, but it's recommended to switch to a dedicated + # database, such as PostgreSQL or MySQL, for use in production environments. + # http://docs.buildbot.net/current/manual/configuration/global.html#database-specification + 'db_url' : "sqlite:///state.sqlite", +} diff --git a/master.cfg b/master.cfg index 68dac7d2..75895872 100644 --- a/master.cfg +++ b/master.cfg @@ -262,7 +262,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': 'ws://crossbar:8085/ws', 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info'