diff --git a/master.cfg b/master.cfg index b55be454..06d17f28 100644 --- a/master.cfg +++ b/master.cfg @@ -16,7 +16,7 @@ import sys sys.setrecursionlimit(10000) -sys.path.insert(0, '/srv/buildbot/master') +sys.path.insert(0, "/srv/buildbot/master") from common_factories import * from constants import * @@ -24,7 +24,7 @@ from locks import * from schedulers_definition import * from utils import * -with open('master-config.yaml', 'r') as f: +with open("master-config.yaml", "r") as f: master_config = yaml.safe_load(f) # This is the dictionary that the buildmaster pays attention to. We also use @@ -33,8 +33,8 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. -config = { "private": { } } -exec(open("master-private.cfg").read(), config, { }) +config = {"private": {}} +exec(open("master-private.cfg").read(), config, {}) ####### BUILDBOT SERVICES @@ -43,236 +43,347 @@ exec(open("master-private.cfg").read(), config, { }) # has a variety to choose from, like IRC bots. -c['services'] = [] +c["services"] = [] context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush(token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription='Build started.', - endDescription='Build done.', - verbose=True, - builders=github_status_builders) -c['services'].append(gs) +gs = reporters.GitHubStatusPush( + token=config["private"]["gh_mdbci"]["access_token"], + context=context, + startDescription="Build started.", + endDescription="Build done.", + verbose=True, + builders=github_status_builders, +) +c["services"].append(gs) ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = os.getenv('TITLE', default="MariaDB CI") -c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") +c["title"] = os.getenv("TITLE", default="MariaDB CI") +c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. -c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") +c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers # could connect to your master with this protocol. # 'port' must match the value configured into the workers (with their # --master option) -c['protocols'] = {'pb': {'port': os.getenv('PORT', default=master_config['port'])}} +c["protocols"] = {"pb": {"port": os.getenv("PORT", default=master_config["port"])}} ####### DB URL -c['db'] = { +c["db"] = { # This specifies what database buildbot uses to store its state. - 'db_url' : config["private"]["db_url"] + "db_url": config["private"]["db_url"] } -mtrDbPool = util.EqConnectionPool("MySQLdb", config["private"]["db_host"], config["private"]["db_user"], config["private"]["db_password"], config["private"]["db_mtr_db"]) +mtrDbPool = util.EqConnectionPool( + "MySQLdb", + config["private"]["db_host"], + config["private"]["db_user"], + config["private"]["db_password"], + config["private"]["db_mtr_db"], +) ####### Disable net usage reports from being sent to buildbot.net -c['buildbotNetUsageData'] = None +c["buildbotNetUsageData"] = None ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. -c['schedulers'] = getSchedulers() +c["schedulers"] = getSchedulers() ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is # a Worker object, specifying a unique worker name and password. The same # worker name and password must be configured on the worker. -c['workers'] = [] +c["workers"] = [] # Docker workers -workers={} -def addWorker(worker_name_prefix, worker_id, worker_type, dockerfile, jobs=5, save_packages=False, shm_size='15G'): +workers = {} + + +def addWorker( + worker_name_prefix, + worker_id, + worker_type, + dockerfile, + jobs=5, + save_packages=False, + shm_size="15G", +): name, instance = createWorker( - worker_name_prefix, - worker_id, - worker_type, - dockerfile, - jobs, - save_packages, - shm_size, - ) + worker_name_prefix, + worker_id, + worker_type, + dockerfile, + jobs, + save_packages, + shm_size, + ) if name[0] not in workers: workers[name[0]] = [name[1]] else: workers[name[0]].append(name[1]) - c['workers'].append(instance) + c["workers"].append(instance) + -for w_name in master_config['workers']: +for w_name in master_config["workers"]: jobs = 7 - for builder in master_config['builders']: + for builder in master_config["builders"]: worker_name = w_name[:-1] worker_id = w_name[-1] - os_name = '-'.join(builder.split('-')[1:]) - image_tag = ''.join(os_name.split('-')) + os_name = "-".join(builder.split("-")[1:]) + image_tag = "".join(os_name.split("-")) # Skip s390x non-SLES builders on SLES host (bbw2) - if ('s390x' in builder) and (worker_id == '2') and ('sles' not in os_name): + if ("s390x" in builder) and (worker_id == "2") and ("sles" not in os_name): continue if image_tag.startswith("ubuntu"): - image_tag = image_tag[:-2] + '.' + image_tag[-2:] - - quay_name = 'quay.io/mariadb-foundation/bb-worker:' + image_tag - if builder.startswith('x86'): - os_name += '-i386' - quay_name += '-386' - addWorker(worker_name, worker_id, '-' + os_name, quay_name, jobs=jobs, save_packages=True) + image_tag = image_tag[:-2] + "." + image_tag[-2:] + + quay_name = "quay.io/mariadb-foundation/bb-worker:" + image_tag + if builder.startswith("x86"): + os_name += "-i386" + quay_name += "-386" + addWorker( + worker_name, + worker_id, + "-" + os_name, + quay_name, + jobs=jobs, + save_packages=True, + ) ####### FACTORY CODE -def dpkgDeb(): - return ShellCommand( - name="dpkg-scanpackages/sources", - haltOnFailure=True, - command=["sh", "-xc", util.Interpolate(""" - mkdir -p debs/ - find .. -maxdepth 1 -type f -exec cp {} debs/ \; - cd debs - ( dpkg-scanpackages . /dev/null && dpkg-scanpackages --type ddeb . /dev/null )| gzip -9c > Packages.gz - dpkg-scansources . /dev/null | gzip -9c > Sources.gz - cd .. - find debs -type f -exec sha256sum {} \; | sort > sha256sums.txt -""")], - doStepIf=lambda step: hasFiles(step) and savePackage(step), - descriptionDone=util.Interpolate(""" - Repository available with: echo "deb [trusted=yes] """ + os.getenv('ARTIFACTS_URL', default='https://ci.mariadb.org') + """/%(prop:tarbuildnum)s/%(prop:buildername)s/debs ./" >> /etc/apt/sources.list - """) - ) - - f_quick_build = getQuickBuildFactory(mtrDbPool) f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) ## f_deb_autobake f_deb_autobake = util.BuildFactory() -f_deb_autobake.addStep(steps.ShellCommand(name="Environment details", command=['bash', '-c', 'date -u && uname -a && ulimit -a'])) -f_deb_autobake.addStep(steps.SetProperty(property="dockerfile", value=util.Interpolate("%(kw:url)s", url=dockerfile), description="dockerfile")) +f_deb_autobake.addStep( + steps.ShellCommand( + name="Environment details", + command=["bash", "-c", "date -u && uname -a && ulimit -a"], + ) +) +f_deb_autobake.addStep( + steps.SetProperty( + property="dockerfile", + value=util.Interpolate("%(kw:url)s", url=dockerfile), + description="dockerfile", + ) +) f_deb_autobake.addStep(downloadSourceTarball()) -f_deb_autobake.addStep(steps.ShellCommand(command=util.Interpolate("tar -xvzf /mnt/packages/%(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz --strip-components=1"))) +f_deb_autobake.addStep( + steps.ShellCommand( + command=util.Interpolate( + "tar -xvzf /mnt/packages/%(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz --strip-components=1" + ) + ) +) # build steps -f_deb_autobake.addStep(steps.Compile(logfiles={'CMakeCache.txt': './builddir/CMakeCache.txt'}, command=["debian/autobake-deb.sh"], - env={'CCACHE_DIR':'/mnt/ccache', 'DEB_BUILD_OPTIONS':util.Interpolate('parallel=%(kw:jobs)s', jobs=util.Property('jobs', default='$(getconf _NPROCESSORS_ONLN)'))}, description="autobake-deb.sh")) -# upload binaries -f_deb_autobake.addStep(steps.SetPropertyFromCommand(command="find .. -maxdepth 1 -type f", extract_fn=ls2string)) -f_deb_autobake.addStep(dpkgDeb()) -f_deb_autobake.addStep(steps.ShellCommand(name='save_packages', timeout=7200, haltOnFailure=True, - command=util.Interpolate(""" - . /etc/os-release && - mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s && - cp -r debs/ sha256sums.txt /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ && - cat << EOF > /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/mariadb.sources -X-Repolib-Name: MariaDB -Types: deb -URIs: https://ci.mariadb.org/%(prop:tarbuildnum)s/%(prop:buildername)s/ -Suites: $VERSION_CODENAME -Components: main main/debug -Trusted: yes -EOF -ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/mariadb.sources /packages/%(prop:branch)s-latest-%(prop:buildername)s.sources && -sync /packages/%(prop:tarbuildnum)s - """), doStepIf=lambda step: hasFiles(step) and savePackage(step))) -f_deb_autobake.addStep(steps.Trigger(name='dockerlibrary', schedulerNames=['s_dockerlibrary'], waitForFinish=False, updateSourceStamp=False, - set_properties={"tarbuildnum" : Property("tarbuildnum"), "mariadb_version" : Property("mariadb_version"), "master_branch" : Property("master_branch"), "parentbuildername": Property("buildername")}, doStepIf=lambda step: hasDockerLibrary(step))) -f_deb_autobake.addStep(steps.Trigger(name='install', schedulerNames=['s_install'], waitForFinish=False, updateSourceStamp=False, - set_properties={"tarbuildnum" : Property("tarbuildnum"), "mariadb_version" : Property("mariadb_version"), "master_branch" : Property("master_branch"), "parentbuildername": Property("buildername"), "sst_mode": "off"}, doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step))) -f_deb_autobake.addStep(steps.Trigger(name='galera-sst-mariabackup', schedulerNames=['s_install'], waitForFinish=False, updateSourceStamp=False, - set_properties={"tarbuildnum" : Property("tarbuildnum"), "mariadb_version" : Property("mariadb_version"), "master_branch" : Property("master_branch"), "parentbuildername": Property("buildername"), "sst_mode": "mariabackup"}, doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step))) -f_deb_autobake.addStep(steps.Trigger(name='galera-sst-mysqldump', schedulerNames=['s_install'], waitForFinish=False, updateSourceStamp=False, - set_properties={"tarbuildnum" : Property("tarbuildnum"), "mariadb_version" : Property("mariadb_version"), "master_branch" : Property("master_branch"), "parentbuildername": Property("buildername"), "sst_mode": "mysqldump"}, doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step))) -f_deb_autobake.addStep(steps.Trigger(name='galera-sst-rsync', schedulerNames=['s_install'], waitForFinish=False, updateSourceStamp=False, - set_properties={"tarbuildnum" : Property("tarbuildnum"), "mariadb_version" : Property("mariadb_version"), "master_branch" : Property("master_branch"), "parentbuildername": Property("buildername"), "sst_mode": "rsync"}, doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step))) -f_deb_autobake.addStep(steps.Trigger(name='major-minor-upgrade', schedulerNames=['s_upgrade'], waitForFinish=False, updateSourceStamp=False, - set_properties={"tarbuildnum" : Property("tarbuildnum"), "mariadb_version" : Property("mariadb_version"), "master_branch" : Property("master_branch"), "parentbuildername": Property("buildername")}, doStepIf=lambda step: hasUpgrade(step) and savePackage(step) and hasFiles(step))) -f_deb_autobake.addStep(steps.ShellCommand(name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True)) +f_deb_autobake.addStep( + steps.Compile( + logfiles={"CMakeCache.txt": "./builddir/CMakeCache.txt"}, + command=["debian/autobake-deb.sh"], + env={ + "CCACHE_DIR": "/mnt/ccache", + "DEB_BUILD_OPTIONS": util.Interpolate( + "parallel=%(kw:jobs)s", + jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"), + ), + }, + description="autobake-deb.sh", + ) +) +# upload artifacts +f_deb_autobake.addStep(createDebRepo()) +f_deb_autobake.addStep(uploadDebArtifacts()) + +f_deb_autobake.addStep( + steps.Trigger( + name="dockerlibrary", + schedulerNames=["s_dockerlibrary"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + }, + doStepIf=lambda step: hasDockerLibrary(step), + ) +) +f_deb_autobake.addStep( + steps.Trigger( + name="install", + schedulerNames=["s_install"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + "sst_mode": "off", + }, + doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), + ) +) +f_deb_autobake.addStep( + steps.Trigger( + name="galera-sst-mariabackup", + schedulerNames=["s_install"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + "sst_mode": "mariabackup", + }, + doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), + ) +) +f_deb_autobake.addStep( + steps.Trigger( + name="galera-sst-mysqldump", + schedulerNames=["s_install"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + "sst_mode": "mysqldump", + }, + doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), + ) +) +f_deb_autobake.addStep( + steps.Trigger( + name="galera-sst-rsync", + schedulerNames=["s_install"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + "sst_mode": "rsync", + }, + doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), + ) +) +f_deb_autobake.addStep( + steps.Trigger( + name="major-minor-upgrade", + schedulerNames=["s_upgrade"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + }, + doStepIf=lambda step: hasUpgrade(step) and savePackage(step) and hasFiles(step), + ) +) +f_deb_autobake.addStep( + steps.ShellCommand( + name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True + ) +) ####### BUILDERS LIST -c['builders'] = [] +c["builders"] = [] -for builder in master_config['builders']: - splits = builder.split('-') +for builder in master_config["builders"]: + splits = builder.split("-") arch = splits[0] - os_name = '-'.join(splits[1:]) + os_name = "-".join(splits[1:]) mtr_additional_args = None - if 'mtr_additional_args' in os_info[os_name]: - if arch in os_info[os_name]['mtr_additional_args']: - mtr_additional_args = os_info[os_name]['mtr_additional_args'][arch] + if "mtr_additional_args" in os_info[os_name]: + if arch in os_info[os_name]["mtr_additional_args"]: + mtr_additional_args = os_info[os_name]["mtr_additional_args"][arch] - if arch == 'amd64': - arch = 'x64' - worker_name = arch + '-bbw-docker-' + os_name + if arch == "amd64": + arch = "x64" + worker_name = arch + "-bbw-docker-" + os_name - if arch == 'x86': - worker_name = 'x64-bbw-docker-' + os_name + '-i386' + if arch == "x86": + worker_name = "x64-bbw-docker-" + os_name + "-i386" - build_type = os_info[os_name]['type'] + build_type = os_info[os_name]["type"] # Add builder only if it's not a protected branches one if builder not in github_status_builders: - c['builders'].append( - util.BuilderConfig(name=builder, - workernames=workers[worker_name], - tags=[os_name], - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - locks=getLocks, - factory=f_quick_build)) + c["builders"].append( + util.BuilderConfig( + name=builder, + workernames=workers[worker_name], + tags=[os_name], + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + locks=getLocks, + factory=f_quick_build, + ) + ) factory_instance = f_deb_autobake properties = {} - if arch == 'ppc64le': - properties['verbose_build'] = 'VERBOSE=1' + if arch == "ppc64le": + properties["verbose_build"] = "VERBOSE=1" if mtr_additional_args is not None: - properties['mtr_additional_args'] = mtr_additional_args - if build_type == 'rpm': - properties['rpm_type'] = ''.join(os_name.split('-')) + properties["mtr_additional_args"] = mtr_additional_args + if build_type == "rpm": + properties["rpm_type"] = "".join(os_name.split("-")) factory_instance = f_rpm_autobake - c['builders'].append( - util.BuilderConfig(name=builder + "-" + build_type + "-autobake", - workernames=workers[worker_name], - tags=[os_name, build_type, "autobake"], - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - locks=getLocks, - properties=properties, - factory=factory_instance)) - -c['logEncoding'] = 'utf-8' - -c['multiMaster'] = True - -c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - 'type' : 'wamp', - 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), - 'realm': 'realm1', + c["builders"].append( + util.BuilderConfig( + name=builder + "-" + build_type + "-autobake", + workernames=workers[worker_name], + tags=[os_name, build_type, "autobake"], + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + locks=getLocks, + properties=properties, + factory=factory_instance, + ) + ) + +c["logEncoding"] = "utf-8" + +c["multiMaster"] = True + +c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. + "type": "wamp", + "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), + "realm": "realm1", # valid are: none, critical, error, warn, info, debug, trace - 'wamp_debug_level' : 'info' + "wamp_debug_level": "info", } - diff --git a/utils.py b/utils.py index 1c001e6e..3d461521 100644 --- a/utils.py +++ b/utils.py @@ -178,6 +178,77 @@ def downloadSourceTarball(output_dir="/mnt/packages/"): ) +def createDebRepo(): + return ShellCommand( + name="Create repository", + haltOnFailure=True, + command=[ + "bash", + "-exc", + util.Interpolate( + """ + . /etc/os-release + cd .. + mkdir -p conf/ + cat << EOF > conf/distributions +Origin: MariaDB +Label: MariaDB +Codename: bullseye +Architectures: %(prop:arch)s source +Components: main +Description: MariaDB Repository +EOF + reprepro -b . --ignore=wrongsourceversion include $VERSION_CODENAME *.changes + ls -l +""" + ), + ], + doStepIf=lambda step: hasFiles(step) and savePackage(step), + descriptionDone=util.Interpolate( + """ + Repository available with: echo "deb [trusted=yes] """ + + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org") + + """/%(prop:tarbuildnum)s/%(prop:buildername)s/ ./" >> /etc/apt/sources.list + """ + ), + ) + + +def uploadDebArtifacts(): + return ShellCommand( + name="Upload artifacts", + timeout=7200, + haltOnFailure=True, + command=[ + "bash", + "-exc", + util.Interpolate( + """ + . /etc/os-release + if [[ $ID == "debian" ]]; then + COMPONENTS="main" + else + COMPONENTS="main main/debug" + fi + mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s + cd .. && cp -r conf db dists pool /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ + cat << EOF > /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/mariadb.sources +X-Repolib-Name: MariaDB +Types: deb +URIs: $ARTIFACTS_URL/%(prop:tarbuildnum)s/%(prop:buildername)s/ +Suites: $VERSION_CODENAME +Components: $COMPONENTS +Trusted: yes +EOF + ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/mariadb.sources /packages/%(prop:branch)s-latest-%(prop:buildername)s.sources + sync /packages/%(prop:tarbuildnum)s + """ + ), + ], + doStepIf=lambda step: hasFiles(step) and savePackage(step), + ) + + # git branch filter using fnmatch import fnmatch