diff --git a/docker-compose/README.md b/docker-compose/README.md index f0c866f0..5988627d 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -16,7 +16,7 @@ To run buildbot locally with a pre-initialized DB to populate some fields git clone https://github.com/MariaDB/buildbot.git cd buildbot/docker-compose ln -s config/master-private.cfg-sample config/master-private.cfg -ln -s ../buildbot +ln -s .. buildbot mkdir -p logs db mariadb/tmp cd db wget https://ci.mariadb.org/helper_files/buildbot_dump.sql.gz diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index d4945ed9..1a04601e 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3.7" services: mariadb: - image: mariadb:10.5 + image: mariadb:10.6 restart: unless-stopped environment: - MARIADB_ROOT_PASSWORD=password @@ -14,9 +14,9 @@ services: healthcheck: test: ['CMD', "mariadb-admin", "--password=password", "--protocol", "tcp", "ping"] volumes: - - ./db/buildbot_dump.sql:/docker-entrypoint-initdb.d/buildbot.sql - - ./mariadb:/var/lib/mysql - command: --tmpdir=/var/lib/mysql/tmp + - ./db:/docker-entrypoint-initdb.d:ro + - ./mariadb:/var/lib/mysql:rw + # command: --tmpdir=/var/lib/mysql/tmp crossbar: image: crossbario/crossbar @@ -43,26 +43,26 @@ services: - mariadb - crossbar - autogen_aarch64-master-0: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh autogen/aarch64-master-0" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8011:8011" - depends_on: - - mariadb - - crossbar + # autogen_aarch64-master-0: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh autogen/aarch64-master-0" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8011:8011" + # depends_on: + # - mariadb + # - crossbar autogen_amd64-master-0: @@ -87,202 +87,202 @@ services: - crossbar - autogen_amd64-master-1: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh autogen/amd64-master-1" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8013:8013" - depends_on: - - mariadb - - crossbar + # autogen_amd64-master-1: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh autogen/amd64-master-1" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8013:8013" + # depends_on: + # - mariadb + # - crossbar - autogen_ppc64le-master-0: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh autogen/ppc64le-master-0" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8014:8014" - depends_on: - - mariadb - - crossbar + # autogen_ppc64le-master-0: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh autogen/ppc64le-master-0" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8014:8014" + # depends_on: + # - mariadb + # - crossbar - autogen_s390x-master-0: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh autogen/s390x-master-0" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8015:8015" - depends_on: - - mariadb - - crossbar + # autogen_s390x-master-0: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh autogen/s390x-master-0" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8015:8015" + # depends_on: + # - mariadb + # - crossbar - autogen_x86-master-0: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh autogen/x86-master-0" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8016:8016" - depends_on: - - mariadb - - crossbar + # autogen_x86-master-0: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh autogen/x86-master-0" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8016:8016" + # depends_on: + # - mariadb + # - crossbar - master-docker-nonstandard: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-docker-nonstandard" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8017:8017" - depends_on: - - mariadb - - crossbar + # master-docker-nonstandard: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh master-docker-nonstandard" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8017:8017" + # depends_on: + # - mariadb + # - crossbar - master-galera: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-galera" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8018:8018" - depends_on: - - mariadb - - crossbar + # master-galera: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh master-galera" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8018:8018" + # depends_on: + # - mariadb + # - crossbar - master-libvirt: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-libvirt" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8019:8019" - depends_on: - - mariadb - - crossbar + # master-libvirt: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh master-libvirt" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8019:8019" + # depends_on: + # - mariadb + # - crossbar - master-nonlatent: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-nonlatent" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8020:8020" - depends_on: - - mariadb - - crossbar + # master-nonlatent: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh master-nonlatent" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8020:8020" + # depends_on: + # - mariadb + # - crossbar - master-protected-branches: - image: quay.io/mariadb-foundation/bb-master:master - restart: unless-stopped - volumes: - - ./logs:/var/log/buildbot - - ./config:/srv/buildbot-config - - ./start.sh:/usr/local/bin/start.sh - - ./buildbot/:/srv/buildbot/master - entrypoint: - - /bin/bash - - -c - - "/usr/local/bin/start.sh master-protected-branches" - networks: - net_front: - net_back: - ports: - - "127.0.0.1:8021:8021" - depends_on: - - mariadb - - crossbar + # master-protected-branches: + # image: quay.io/mariadb-foundation/bb-master:master + # restart: unless-stopped + # volumes: + # - ./logs:/var/log/buildbot + # - ./config:/srv/buildbot-config + # - ./start.sh:/usr/local/bin/start.sh + # - ./buildbot/:/srv/buildbot/master + # entrypoint: + # - /bin/bash + # - -c + # - "/usr/local/bin/start.sh master-protected-branches" + # networks: + # net_front: + # net_back: + # ports: + # - "127.0.0.1:8021:8021" + # depends_on: + # - mariadb + # - crossbar networks: diff --git a/docker-compose/start-bbm-web.sh b/docker-compose/start-bbm-web.sh index 0f507f9b..f337128e 100755 --- a/docker-compose/start-bbm-web.sh +++ b/docker-compose/start-bbm-web.sh @@ -6,19 +6,20 @@ set -o pipefail set -o posix err() { - echo_red >&2 "ERROR: $*" + echo >&2 "ERROR: $*" exit 1 } cd /srv/buildbot/master || err "cd /srv/buildbot/master" -ln -sf /srv/buildbot-config/master-private.cfg master-private.cfg +[[ -f master-private.cfg ]] || ln -sf /srv/buildbot-config/master-private.cfg master-private.cfg # config buildbot master-web -cd /srv/buildbot/master/master-web || err "cd" +cd /srv/buildbot/master/master-web || err "cd /srv/buildbot/master/master-web" for file in /srv/buildbot-config/master-web/*; do # shellcheck disable=SC2226 ln -sf "$file" done + # # loop for debug # while true; do date && sleep 30; done @@ -32,6 +33,7 @@ echo "Waiting for Crossbar to start..." while ! nc -z crossbar 8080; do sleep 0.1 done -echo "Crossbar started" +echo "MariaDB started" +buildbot upgrade-master /srv/buildbot/master/master-web buildbot start --nodaemon diff --git a/docker-compose/start.sh b/docker-compose/start.sh index e629ee99..a0ac1605 100755 --- a/docker-compose/start.sh +++ b/docker-compose/start.sh @@ -32,4 +32,8 @@ while ! nc -z crossbar 8080; do done echo "Crossbar started" -buildbot start --nodaemon \ No newline at end of file +# while true; do +# sleep 10 +# done + +buildbot start --nodaemon diff --git a/master-web/master.cfg b/master-web/master.cfg deleted file mode 100644 index c5577a6f..00000000 --- a/master-web/master.cfg +++ /dev/null @@ -1,135 +0,0 @@ -# -*- python -*- -# ex: set filetype=python: - -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from twisted.internet import defer -import os -import sys -import docker -from datetime import timedelta - -sys.setrecursionlimit(10000) - -sys.path.append(os.getcwd() + '/..') -from constants import * - -c = BuildmasterConfig = {} - -config = { "private": { }} -exec(open("../master-private.cfg").read(), config, { }) - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). - -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# minimalistic config to activate web UI -c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={}), custom_templates_dir='templates') - -# Plain username auth -#c['www']['authz'] = util.Authz(allowRules=[util.AnyControlEndpointMatcher(role="admins")], -# roleMatchers=[util.RolesFromUsername(roles=["admins"], usernames=["admin"])]) -#c['www']['auth'] = util.UserPasswordAuth({'admin': config["private"]["user_pass"]["admin"]}) - -# Github Auth, allow control for MariaDB affiliated accounts -c['www']['authz'] = util.Authz( - allowRules=[ - util.AnyControlEndpointMatcher(role="MariaDB", defaultDeny=True) - ], - roleMatchers=[ - util.RolesFromGroups() - ] -) -c['www']['auth'] = util.GitHubAuth(config["private"]["gh_mdbauth"]["client"], config["private"]["gh_mdbauth"]["secret"]) - -# Sponsor plugin -exec(open("../sponsor.py").read()) - -####### CHANGESOURCES - -# the 'change_source' setting tells the buildmaster how it should find out -# about source code changes. - -####### GitHub hooks - -# GitHub webhook receiver -c['www']['change_hook_dialects'] = { - 'github': { - 'secret': config["private"]["gh_secret"], - 'strict': True, - } -} - -c['www']['ui_default_config'] = { - 'Grid.changeFetchLimit': 5, - 'Grid.buildFetchLimit': 50, -} - -c['logEncoding'] = 'utf-8' - -#c['www']['plugins']['profiler'] = True -#c['services'] = [util.ProfilerService(frequency=100, gatherperiod=30 * 60, mode='prof', basepath='/srv/buildbot/master/master-web/prof', wantBuilds=100)] - -c['multiMaster'] = True -c['db'] = { - # This specifies what database buildbot uses to store its state. - 'db_url' : config["private"]["db_url"] -} - -####### Disable net usage reports from being sent to buildbot.net -c['buildbotNetUsageData'] = None - -c['mq'] = { - 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', - 'realm': 'realm1', - # valid are: none, critical, error, warn, info, debug, trace - 'wamp_debug_level' : 'warn' -} - -# git branch filter using fnmatch -import fnmatch -def upstream_branch_fn(branch): - return branch in branches_main or fnmatch.fnmatch(branch, 'bb-*') or fnmatch.fnmatch(branch, 'st-*') or fnmatch.fnmatch(branch, 'prot-*') or fnmatch.fnmatch(branch, "refs/pull/*") or fnmatch.fnmatch(branch, "preview-*") -def staging_branch_fn(branch): - return fnmatch.fnmatch(branch, 'st-*') -def fnmatch_any(s, list_of_patterns): - return any(fnmatch.fnmatch(s, p) for p in list_of_patterns) - -c['schedulers'] = [] - -# upstream scheduling -schedulerTarball = schedulers.AnyBranchScheduler( - name="s_upstream_tarball", - change_filter=util.ChangeFilter(repository="https://github.com/MariaDB/server", branch_fn=upstream_branch_fn), - treeStableTimer=60, - builderNames=["tarball-docker"]) -c['schedulers'].append(schedulerTarball) - -#-- - -# Other schedulers - -# github.com/shinnok/server -c['schedulers'].append(schedulers.SingleBranchScheduler( name="s_vicentiu", - change_filter=util.ChangeFilter(repository="https://github.com/cvicentiu/server", branch_fn=upstream_branch_fn), - treeStableTimer=60, - builderNames=["tarball-docker"])) - -#-- - diff --git a/master-web/master.cfg b/master-web/master.cfg new file mode 120000 index 00000000..80dcb1da --- /dev/null +++ b/master-web/master.cfg @@ -0,0 +1 @@ +/srv/buildbot-config/master-web/master.cfg \ No newline at end of file diff --git a/master-web/master.cfg.sample b/master-web/master.cfg.sample new file mode 100644 index 00000000..3b7ba685 --- /dev/null +++ b/master-web/master.cfg.sample @@ -0,0 +1,107 @@ +# -*- python -*- +# ex: set filetype=python: + +from buildbot.plugins import * + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +c['workers'] = [worker.Worker("example-worker", "pass")] + +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +c['protocols'] = {'pb': {'port': 9989}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot version of a python hello-world project. + +c['change_source'] = [] +c['change_source'].append(changes.GitPoller( + 'git://github.com/buildbot/hello-world.git', + workdir='gitpoller-workdir', branch='master', + pollInterval=300)) + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +c['schedulers'] = [] +c['schedulers'].append(schedulers.SingleBranchScheduler( + name="all", + change_filter=util.ChangeFilter(branch='master'), + treeStableTimer=None, + builderNames=["runtests"])) +c['schedulers'].append(schedulers.ForceScheduler( + name="force", + builderNames=["runtests"])) + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. + +factory = util.BuildFactory() +# check out the source +factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) +# run the tests (note that this will require that 'trial' is installed) +factory.addStep(steps.ShellCommand(command=["trial", "hello"], + env={"PYTHONPATH": "."})) + +c['builders'] = [] +c['builders'].append( + util.BuilderConfig(name="runtests", + workernames=["example-worker"], + factory=factory)) + +####### BUILDBOT SERVICES + +# 'services' is a list of BuildbotService items like reporter targets. The +# status of each build will be pushed to these targets. buildbot/reporters/*.py +# has a variety to choose from, like IRC bots. + +c['services'] = [] + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). + +c['title'] = "Hello World CI" +c['titleURL'] = "https://buildbot.github.io/hello-world/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = "http://localhost:8010/" + +# minimalistic config to activate new web UI +c['www'] = dict(port=8010, + plugins=dict(waterfall_view={}, console_view={}, grid_view={})) + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. + # It's easy to start with sqlite, but it's recommended to switch to a dedicated + # database, such as PostgreSQL or MySQL, for use in production environments. + # http://docs.buildbot.net/current/manual/configuration/global.html#database-specification + 'db_url' : "sqlite:///state.sqlite", +}