WIP: Cleanup define_masters

define_masters.py

@@ -21,44 +21,50 @@
         builder_name = arch + "-" + os_name
         if arch not in platforms:
             platforms[arch] = []
-        platforms[arch].append(builder_name)
+        platforms[arch].append({
+            os_name: {
+                'image_tag': OS_INFO[os_name]['image_tag'],
+                # TODO(cvicentiu) load tags from os_info.
+                'tags': OS_INFO[os_name]['tags'] if 'tags' in OS_INFO[os_name] else []
+            }
+        })
 
 # Clear old configurations
 if os.path.exists(BASE_PATH):
     shutil.rmtree(BASE_PATH)
 
 for arch in platforms:
+    master_variables = config["private"]["master-variables"]
     # Create the directory for the architecture that is handled by each master
     # If for a given architecture there are more than "max_builds" builds,
     # create multiple masters
     # "max_builds" is defined is master-private.py
     num_masters = (
-        int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"])
-        + 1
+        int(len(platforms[arch]) / master_variables["max_builds"]) + 1
     )
 
     for master_id in range(num_masters):
-        dir_path = BASE_PATH + arch + "-master-" + str(master_id)
+        dir_path = f'{BASE_PATH}{arch}-master-{master_id}'
         os.makedirs(dir_path)
 
-        master_config = {}
-        master_config["builders"] = platforms[arch]
-        master_config["workers"] = config["private"]["master-variables"]["workers"][
-            arch
-        ]
-        master_config["log_name"] = (
-            "master-docker-" + arch + "-" + str(master_id) + ".log"
-        )
+        master_config = {
+            'builders': {arch: platforms[arch]},
+            'workers': master_variables["workers"][arch],
+            'log_name': f'master-docker-{arch}-{master_id}.log'
+        }
 
-        with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file:
+        with open(f"{dir_path}/master-config.yaml", mode="w",
+                  encoding="utf-8") as file:
             yaml.dump(master_config, file)
 
         shutil.copyfile("master.cfg", dir_path + "/master.cfg")
         shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg")
 
-        buildbot_tac = (
-            open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"]
-        )
+        # TODO(cvicentiu) fix this through environment variables, not this
+        # weird hardcoding and code generation.
+        with open("buildbot.tac", encoding="utf-8") as f:
+            buildbot_tac = f.read() % master_config["log_name"]
+
         with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f:
             f.write(buildbot_tac)
     print(arch, len(master_config["builders"]))

master-config.yaml-sample

@@ -1,12 +1,47 @@
 builders:
-- aarch64-centos-stream9
-- aarch64-debian-11
-- aarch64-debian-12
-- aarch64-debian-sid
-- aarch64-rhel-8
-- aarch64-rhel-9
-- aarch64-ubuntu-2004
-- aarch64-ubuntu-2204
+  aarch64:
+    centos-stream9:
+      image_tag: centosstream9
+      tags:
+        - release_packages
+        - autobake
+        - bleeding-edge
+    debian-11:
+      image_tag: debian11
+      tags:
+        - release_packages
+        - autobake
+    debian-12:
+      image_tag: debian12
+      tags:
+        - release_packages
+        - autobake
+    debian-sid:
+      image_tag: debiansid
+      tags:
+        - release_packages
+        - autobake
+        - bleeding-edge
+    rhel-8:
+      image_tag: rhel8
+      tags:
+        - release_packages
+        - autobake
+    rhel-9:
+      image_tag: rhel9
+      tags:
+        - release_packages
+        - autobake
+    ubuntu-2004:
+      image_tag: ubuntu20.04
+      tags:
+        - release_packages
+        - autobake
+    ubuntu-2204:
+      image_tag: ubuntu22.04
+      tags:
+        - release_packages
+        - autobake
 log_name: master-docker-aarch64-0.log
 port: 9998
 workers:

master.cfg

@@ -14,7 +14,6 @@ from common_factories import (
     getSourceTarball,
 )
 from constants import (
-    BUILDERS_GALERA_MTR,
     GITHUB_STATUS_BUILDERS,
     OS_INFO,
     SAVED_PACKAGE_BRANCHES,
@@ -79,40 +78,52 @@ c["workers"] = []
 
 workers = defaultdict(list)
 
+print('Loading workers')
+print(os.environ)
+# For each worker in master_config ['aarch64-bbw1', 2, 3, 4]
 for w_name in master_config["workers"]:
     jobs = 7
+    print(w_name)
+    worker_name = w_name[:-1]  # aarch64-bbw
+    worker_id = w_name[-1]     # 1, 2, 3, 4
+
+    for arch in master_config["builders"]:
+        builders = master_config["builders"][arch]
+        for os_name in builders:
+            os_definition = builders[os_name]
+            print(f'Loading builder {os_name}')
+            image_tag = os_definition['image_tag']
+
+            # Skip s390x non-SLES builders on SLES host (bbw2)
+            if ("s390x" in arch
+                    and (worker_id == "2")
+                    and ("sles" not in os_name)):
+                continue
+
+            if image_tag.startswith("ubuntu"):
+                image_tag = image_tag[:-2] + "." + image_tag[-2:]
+
+            quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}'
+            if arch.startswith("x86"):
+                os_name += "-i386"
+                quay_name += "-386"
+
+            base_name, name, worker_instance = createWorker(
+                worker_name,
+                worker_id,
+                os_name,
+                quay_name,
+                jobs=jobs,
+                save_packages=True,
+                shm_size="15G",
+            )
 
-    for builder in master_config["builders"]:
-        worker_name = w_name[:-1]
-        worker_id = w_name[-1]
-
-        os_name = "-".join(builder.split("-")[1:])
-        image_tag = "".join(os_name.split("-"))
-
-        # Skip s390x non-SLES builders on SLES host (bbw2)
-        if ("s390x" in builder) and (worker_id == "2") and ("sles" not in os_name):
-            continue
-
-        if image_tag.startswith("ubuntu"):
-            image_tag = image_tag[:-2] + "." + image_tag[-2:]
-
-        quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag
-        if builder.startswith("x86"):
-            os_name += "-i386"
-            quay_name += "-386"
-
-        base_name, name, worker_instance = createWorker(
-            worker_name,
-            worker_id,
-            os_name,
-            quay_name,
-            jobs=jobs,
-            save_packages=True,
-            shm_size="15G",
-        )
+            print(base_name)
+
+            workers[base_name].append(name)
+            c["workers"].append(worker_instance)
 
-        workers[base_name].append(name)
-        c["workers"].append(worker_instance)
+print('WORKERS LOADED: ', workers)
 
 ####### FACTORY CODE
 
@@ -230,81 +241,63 @@ f_deb_autobake.addStep(
 ####### BUILDERS LIST
 
 c["builders"] = []
+print('workers: ', workers)
 
-for builder in master_config["builders"]:
-    splits = builder.split("-")
-    arch = splits[0]
-    os_name = "-".join(splits[1:])
+for arch in master_config["builders"]:
+    builders_group = master_config["builders"][arch]
+    print('Builders group: ', builders_group)
+    for os_name in builders_group:
+        worker_prefix = arch
+        worker_suffix = ''
 
-    if arch == "amd64":
-        arch = "x64"
-    worker_name = arch + "-bbw-docker-" + os_name
+        if arch == "amd64":
+            worker_prefix = "x64"
 
-    if arch == "x86":
-        worker_name = "x64-bbw-docker-" + os_name + "-i386"
+        if arch == "x86":
+            worker_prefix = 'x64'
+            worker_suffix = '-i386'
+        worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}'
 
-    build_type = OS_INFO[os_name]["type"]
+        build_type = OS_INFO[os_name]["type"]
+
+        builder = f'{arch}-{os_name}'
+        print('builder: ', builder)
+        print(f'Looking for builder the worker: {worker_name}')
 
-    # Add builder only if it's not a protected branches one
-    if builder not in GITHUB_STATUS_BUILDERS:
         tags = [os_name]
-        if arch == "s390x" and builder in BUILDERS_GALERA_MTR:
-            tags += ["experimental"]
-        if "sid" in builder or "stream-9" in builder:
-            tags += ["bleeding-edge"]
+        # Add builder only if it's not a protected branches one
+        if builder not in GITHUB_STATUS_BUILDERS:
+            c["builders"].append(
+                util.BuilderConfig(
+                    name=builder,
+                    workernames=workers[worker_name],
+                    tags=tags,
+                    collapseRequests=True,
+                    nextBuild=nextBuild,
+                    canStartBuild=canStartBuild,
+                    locks=getLocks,
+                    factory=f_quick_build,
+                )
+            )
+
+        factory_instance = f_deb_autobake if build_type != "rpm" else f_rpm_autobake
+        properties = {
+            "verbose_build": "VERBOSE=1" if arch == "ppc4le" else None,
+            "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else None
+        }
+
+        tags += [build_type, "autobake"]
+
         c["builders"].append(
             util.BuilderConfig(
-                name=builder,
+                name=builder + "-" + build_type + "-autobake",
                 workernames=workers[worker_name],
                 tags=tags,
                 collapseRequests=True,
                 nextBuild=nextBuild,
                 canStartBuild=canStartBuild,
                 locks=getLocks,
-                factory=f_quick_build,
+                properties=properties,
+                factory=factory_instance,
             )
         )
-
-    factory_instance = f_deb_autobake
-    properties = {}
-
-    if arch == "ppc64le":
-        properties["verbose_build"] = "VERBOSE=1"
-    if build_type == "rpm":
-        properties["rpm_type"] = "".join(os_name.split("-"))
-        factory_instance = f_rpm_autobake
-    tags = [os_name, build_type, "autobake"]
-    # From mariadb.org-tools/release/prep - under
-    # Dirs for buildbot.mariadb.org
-    if builder in [
-        "aarch64-openeuler-2403",
-        "amd64-openeuler-2403",
-        "s390x-ubuntu-2004",
-        "s390x-rhel-8",
-        "s390x-sles-15",
-        "ppc64le-rhel-9",
-        "s390x-rhel-9",
-        "ppc64le-ubuntu-2204",
-        "s390x-ubuntu-2204",
-        "amd64-debian-sid",
-        "aarch64-debian-sid",
-        "ppc64le-debian-sid",
-        "amd64-opensuse-1505",
-        "amd64-opensuse-1506",
-        "amd64-sles-1505",
-        "s390x-sles-1505",
-    ]:
-        tags += ["release_packages"]
-    c["builders"].append(
-        util.BuilderConfig(
-            name=builder + "-" + build_type + "-autobake",
-            workernames=workers[worker_name],
-            tags=tags,
-            collapseRequests=True,
-            nextBuild=nextBuild,
-            canStartBuild=canStartBuild,
-            locks=getLocks,
-            properties=properties,
-            factory=factory_instance,
-        )
-    )