From 2d9683c98544534d3b4d2f97a9c353073cb6ec24 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 11:10:13 +0200 Subject: [PATCH 01/10] Use apt search patterns a little more Let's use apt search patterns more to match packages that are only available on specific ubuntu/debian releases. --- mkosi.conf.d/30-debian-ubuntu/mkosi.conf | 2 ++ .../mkosi.conf.d/20-systemd-extra.conf | 9 --------- .../mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 4 ++++ .../10-debian-ubuntu/mkosi.conf.d/grub.conf | 8 -------- .../mkosi.conf.d/systemd-boot.conf | 9 --------- .../mkosi.conf.d/ubuntu-keyring.conf | 9 --------- .../10-debian-ubuntu/mkosi.conf.d/virtiofsd.conf | 15 --------------- 7 files changed, 6 insertions(+), 50 deletions(-) delete mode 100644 mkosi.conf.d/30-debian-ubuntu/mkosi.conf.d/20-systemd-extra.conf delete mode 100644 mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/grub.conf delete mode 100644 mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/systemd-boot.conf delete mode 100644 mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/ubuntu-keyring.conf delete mode 100644 mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/virtiofsd.conf diff --git a/mkosi.conf.d/30-debian-ubuntu/mkosi.conf b/mkosi.conf.d/30-debian-ubuntu/mkosi.conf index 7f91220ee..6b7813c6a 100644 --- a/mkosi.conf.d/30-debian-ubuntu/mkosi.conf +++ b/mkosi.conf.d/30-debian-ubuntu/mkosi.conf @@ -6,6 +6,8 @@ Distribution=|ubuntu [Content] Packages= + ?exact-name(systemd-boot) + ?exact-name(systemd-resolved) ^libtss2-esys-[0-9.]+-0$ ^libtss2-mu[0-9.-]+$ bash diff --git a/mkosi.conf.d/30-debian-ubuntu/mkosi.conf.d/20-systemd-extra.conf b/mkosi.conf.d/30-debian-ubuntu/mkosi.conf.d/20-systemd-extra.conf deleted file mode 100644 index 4fa991635..000000000 --- a/mkosi.conf.d/30-debian-ubuntu/mkosi.conf.d/20-systemd-extra.conf +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later - -[Match] -Release=!jammy - -[Content] -Packages= - systemd-boot - systemd-resolved diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index 69c4bdf5a..d7c4612dd 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -6,8 +6,12 @@ Distribution=|ubuntu [Content] Packages= + ?exact-name(grub-pc-bin) + ?exact-name(systemd-boot) ?exact-name(systemd-repart) ?exact-name(systemd-ukify) + ?exact-name(ubuntu-keyring) + ?exact-name(virtiofsd) apt archlinux-keyring btrfs-progs diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/grub.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/grub.conf deleted file mode 100644 index 2ab4230d4..000000000 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/grub.conf +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later - -[Match] -Architecture=x86-64 - -[Content] -Packages= - grub-pc-bin diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/systemd-boot.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/systemd-boot.conf deleted file mode 100644 index a31acd147..000000000 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/systemd-boot.conf +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later - -[Match] -Distribution=|!ubuntu -Release=|!jammy - -[Content] -Packages= - systemd-boot diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/ubuntu-keyring.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/ubuntu-keyring.conf deleted file mode 100644 index a94ac4ab4..000000000 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/ubuntu-keyring.conf +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later - -[Match] -Distribution=|!debian -Release=|!bookworm - -[Content] -Packages= - ubuntu-keyring diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/virtiofsd.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/virtiofsd.conf deleted file mode 100644 index 8c2b285bd..000000000 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/virtiofsd.conf +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later - -# Install virtiofsd except on Ubuntu Jammy and Debian Bookworm. - -[Match] -Distribution=|!ubuntu -Release=|!jammy - -[Match] -Distribution=|!debian -Release=|!bookworm - -[Content] -Packages= - virtiofsd From 244448155b2165d6929658b1dcc177242ab620b9 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 11:11:16 +0200 Subject: [PATCH 02/10] Install distribution-gpg-keys in Debian/Ubuntu tools tree if available It recently got into debian testing so let's add it to the tools tree. --- .../mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index d7c4612dd..575187f73 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -6,6 +6,7 @@ Distribution=|ubuntu [Content] Packages= + ?exact-name(distribution-gpg-keys) ?exact-name(grub-pc-bin) ?exact-name(systemd-boot) ?exact-name(systemd-repart) From 9df0f75801204816548a7aca61b49100123d1269 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 11:44:28 +0200 Subject: [PATCH 03/10] Fix RepositoryKeyCheck docs --- mkosi/resources/mkosi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index ffd026d3a..41ee2b61d 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -450,7 +450,7 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `RepositoryKeyCheck=`, `--repository-key-check=` : Controls signature/key checks when using repositories, enabled by default. Useful to disable checks when combined with `--local-mirror=` and using only - a repository from a local filesystem. Not used for DNF-based distros yet. + a repository from a local filesystem. `Repositories=`, `--repositories=` : Enable package repositories that are disabled by default. This can be used to enable the EPEL repos for From 84396477f10a8644f17d609d73c7688885e954ee Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 12:32:13 +0200 Subject: [PATCH 04/10] opensuse: Rename gpgurls to gpgkeys --- mkosi/distributions/opensuse.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 438450e5f..382f54eb3 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -94,7 +94,7 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: mirror = context.config.mirror or "https://download.opensuse.org" if context.config.release == "tumbleweed" or context.config.release.isdigit(): - gpgurls = ( + gpgkeys = ( *([p] if (p := find_rpm_gpgkey(context, key="RPM-GPG-KEY-openSUSE-Tumbleweed")) else []), *([p] if (p := find_rpm_gpgkey(context, key="RPM-GPG-KEY-openSUSE")) else []), ) @@ -115,7 +115,7 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: yield RpmRepository( id=repo, url=f"baseurl={url}", - gpgurls=gpgurls or (fetch_gpgurls(context, url) if not zypper else ()), + gpgurls=gpgkeys or (fetch_gpgurls(context, url) if not zypper else ()), enabled=repo == "oss", ) @@ -125,7 +125,7 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: yield RpmRepository( id=f"{repo}-{d}", url=f"baseurl={url}", - gpgurls=gpgurls or (fetch_gpgurls(context, url) if not zypper else ()), + gpgurls=gpgkeys or (fetch_gpgurls(context, url) if not zypper else ()), enabled=False, ) @@ -134,14 +134,14 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: yield RpmRepository( id="oss-update", url=f"baseurl={url}", - gpgurls=gpgurls or (fetch_gpgurls(context, url) if not zypper else ()), + gpgurls=gpgkeys or (fetch_gpgurls(context, url) if not zypper else ()), ) url = join_mirror(mirror, f"{subdir}/update/tumbleweed-non-oss") yield RpmRepository( id="non-oss-update", url=f"baseurl={url}", - gpgurls=gpgurls or (fetch_gpgurls(context, url) if not zypper else ()), + gpgurls=gpgkeys or (fetch_gpgurls(context, url) if not zypper else ()), enabled=False, ) else: From a242141fe4259cfe5b6d7df3a8eb43b6f151ce54 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 15:13:24 +0200 Subject: [PATCH 05/10] tests: Drop workaround for older Ubuntu --- tests/test_initrd.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/tests/test_initrd.py b/tests/test_initrd.py index 461a3a821..9a5825873 100644 --- a/tests/test_initrd.py +++ b/tests/test_initrd.py @@ -5,7 +5,6 @@ import subprocess import tempfile import textwrap -import time from collections.abc import Iterator from pathlib import Path from typing import Any @@ -13,13 +12,10 @@ import pytest from mkosi.distributions import Distribution -from mkosi.log import die from mkosi.mounts import mount -from mkosi.run import find_binary, run +from mkosi.run import run from mkosi.tree import copy_tree -from mkosi.types import PathString from mkosi.user import INVOKING_USER -from mkosi.versioncomp import GenericVersion from . import Image, ImageConfig, ci_group @@ -73,23 +69,6 @@ def test_initrd(initrd: Image) -> None: image.qemu() -def wait_for_device(device: PathString) -> None: - if ( - find_binary("udevadm") and - GenericVersion(run(["udevadm", "--version"], stdout=subprocess.PIPE).stdout.strip()) >= 251 - ): - run(["udevadm", "wait", "--timeout=30", "/dev/vg_mkosi/lv0"]) - return - - for i in range(30): - if Path(device).exists(): - return - - time.sleep(1) - - die(f"Device {device} did not appear within 30 seconds") - - @pytest.mark.skipif(os.getuid() != 0, reason="mkosi-initrd LVM test can only be executed as root") def test_initrd_lvm(initrd: Image) -> None: with Image( @@ -123,7 +102,7 @@ def test_initrd_lvm(initrd: Image) -> None: stack.callback(lambda: run(["vgchange", "-an", "vg_mkosi"])) run(["lvm", "lvcreate", "-l", "100%FREE", "-n", "lv0", "vg_mkosi"]) run(["lvm", "lvs"]) - wait_for_device("/dev/vg_mkosi/lv0") + run(["udevadm", "wait", "--timeout=30", "/dev/vg_mkosi/lv0"]) run([f"mkfs.{image.config.distribution.filesystem()}", "-L", "root", "/dev/vg_mkosi/lv0"]) with tempfile.TemporaryDirectory() as mnt, mount(Path("/dev/vg_mkosi/lv0"), Path(mnt)): @@ -243,7 +222,7 @@ def test_initrd_luks_lvm(config: ImageConfig, initrd: Image, passphrase: Path) - stack.callback(lambda: run(["vgchange", "-an", "vg_mkosi"])) run(["lvm", "lvcreate", "-l", "100%FREE", "-n", "lv0", "vg_mkosi"]) run(["lvm", "lvs"]) - wait_for_device("/dev/vg_mkosi/lv0") + run(["udevadm", "wait", "--timeout=30", "/dev/vg_mkosi/lv0"]) run([f"mkfs.{image.config.distribution.filesystem()}", "-L", "root", "/dev/vg_mkosi/lv0"]) with tempfile.TemporaryDirectory() as mnt, mount(Path("/dev/vg_mkosi/lv0"), Path(mnt)): From 79539371d0b6d240c2f4e9c45ba68824ca748ecc Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 15:26:32 +0200 Subject: [PATCH 06/10] tests: Simplify initrd tests Let's get rid of the fixtures and just rely on the default initrd built as part of the image itself. This also means any settings picked up from mkosi.local.conf are applied to the initrd build. --- tests/test_initrd.py | 68 ++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/tests/test_initrd.py b/tests/test_initrd.py index 9a5825873..b07e2ae8c 100644 --- a/tests/test_initrd.py +++ b/tests/test_initrd.py @@ -7,7 +7,6 @@ import textwrap from collections.abc import Iterator from pathlib import Path -from typing import Any import pytest @@ -17,7 +16,7 @@ from mkosi.tree import copy_tree from mkosi.user import INVOKING_USER -from . import Image, ImageConfig, ci_group +from . import Image, ImageConfig pytestmark = pytest.mark.integration @@ -35,30 +34,10 @@ def passphrase() -> Iterator[Path]: yield Path(passphrase.name) -@pytest.fixture(scope="module") -def initrd(request: Any, config: ImageConfig) -> Iterator[Image]: - with ( - ci_group(f"Initrd image {config.distribution}/{config.release}"), - Image( - config, - options=[ - "--directory", "", - "--include=mkosi-initrd/", - ], - ) as initrd - ): - if initrd.config.distribution == Distribution.rhel_ubi: - pytest.skip("Cannot build RHEL-UBI initrds") - - initrd.build() - yield initrd - - -def test_initrd(initrd: Image) -> None: +def test_initrd(config: ImageConfig) -> None: with Image( - initrd.config, + config, options=[ - "--initrd", Path(initrd.output_dir) / "initrd", "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", "--incremental", "--ephemeral", @@ -70,11 +49,10 @@ def test_initrd(initrd: Image) -> None: @pytest.mark.skipif(os.getuid() != 0, reason="mkosi-initrd LVM test can only be executed as root") -def test_initrd_lvm(initrd: Image) -> None: +def test_initrd_lvm(config: ImageConfig) -> None: with Image( - initrd.config, + config, options=[ - "--initrd", Path(initrd.output_dir) / "initrd", "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", # LVM confuses systemd-repart so we mask it for this test. "--kernel-command-line=systemd.mask=systemd-repart.service", @@ -115,7 +93,7 @@ def test_initrd_lvm(initrd: Image) -> None: image.qemu(["--format=disk"]) -def test_initrd_luks(initrd: Image, passphrase: Path) -> None: +def test_initrd_luks(config: ImageConfig, passphrase: Path) -> None: with tempfile.TemporaryDirectory() as repartd: os.chown(repartd, INVOKING_USER.uid, INVOKING_USER.gid) @@ -151,7 +129,7 @@ def test_initrd_luks(initrd: Image, passphrase: Path) -> None: f"""\ [Partition] Type=root - Format={initrd.config.distribution.filesystem()} + Format={config.distribution.filesystem()} Minimize=guess Encrypt=key-file CopyFiles=/ @@ -160,9 +138,8 @@ def test_initrd_luks(initrd: Image, passphrase: Path) -> None: ) with Image( - initrd.config, + config, options=[ - "--initrd", Path(initrd.output_dir) / "initrd", "--repart-dir", repartd, "--passphrase", passphrase, "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", @@ -177,11 +154,10 @@ def test_initrd_luks(initrd: Image, passphrase: Path) -> None: @pytest.mark.skipif(os.getuid() != 0, reason="mkosi-initrd LUKS+LVM test can only be executed as root") -def test_initrd_luks_lvm(config: ImageConfig, initrd: Image, passphrase: Path) -> None: +def test_initrd_luks_lvm(config: ImageConfig, passphrase: Path) -> None: with Image( config, options=[ - "--initrd", Path(initrd.output_dir) / "initrd", "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", "--kernel-command-line=root=LABEL=root", "--kernel-command-line=rw", @@ -238,14 +214,20 @@ def test_initrd_luks_lvm(config: ImageConfig, initrd: Image, passphrase: Path) - ]) -def test_initrd_size(initrd: Image) -> None: - # The fallback value is for CentOS and related distributions. - maxsize = 1024**2 * { - Distribution.fedora: 46, - Distribution.debian: 40, - Distribution.ubuntu: 36, - Distribution.arch: 67, - Distribution.opensuse: 39, - }.get(initrd.config.distribution, 48) +def test_initrd_size(config: ImageConfig) -> None: + with Image( + config, + options=["--incremental", "--format=directory"], + ) as image: + image.build() + + # The fallback value is for CentOS and related distributions. + maxsize = 1024**2 * { + Distribution.fedora: 55, + Distribution.debian: 55, + Distribution.ubuntu: 50, + Distribution.arch: 80, + Distribution.opensuse: 55, + }.get(config.distribution, 55) - assert (Path(initrd.output_dir) / "initrd").stat().st_size <= maxsize + assert (Path(image.output_dir) / "image.initrd").stat().st_size <= maxsize From 5c1b3d99fe87fa77bbcbe97cc4b1534f3cf2fe35 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 16:35:36 +0200 Subject: [PATCH 07/10] Only use unshare to become root if we're actually going to use a scope If the relevant environment variables are not set, scope_cmd() will return an empty list and we won't use a scope after all. In that case we don't need to use unshare either to become root and can rely on our own become_root() function so check whether we're actually going to use a scope or not. --- mkosi/qemu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkosi/qemu.py b/mkosi/qemu.py index bd1ddcf3f..058cbc1d8 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -351,8 +351,9 @@ def start_virtiofsd(config: Config, directory: PathString, *, name: str, selinux gid = INVOKING_USER.gid if os.getgid() != INVOKING_USER.gid else None scope = scope_cmd(name=name, description=description, user=uid, group=gid) elif not uidmap and (os.getuid() == 0 or unshare_version() >= "2.38"): - runas = become_root_cmd() scope = scope_cmd(name=name, description=description) + if scope: + runas = become_root_cmd() with spawn( cmdline, From 7bf1855c1738c582f3f7eba5ff79756fcaa93c32 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 16:37:14 +0200 Subject: [PATCH 08/10] tests: Pass host environment when invoking mkosi --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index c1488cca6..75b475663 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -96,7 +96,7 @@ def mkosi( "--qemu-mem=2G", verb, *args, - ], check=check, stdin=stdin, stdout=sys.stdout, user=user, group=group) + ], check=check, stdin=stdin, stdout=sys.stdout, user=user, group=group, env=os.environ) def build(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess: return self.mkosi( From a94f3d0e20ac2b2e3fcce1a4d7c164f1b8d20cac Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 16:38:30 +0200 Subject: [PATCH 09/10] tests: Reduce duplication Let's reduce the amount of duplication in the tests. --- mkosi.conf | 7 ++++++- tests/__init__.py | 13 ++++--------- tests/test_boot.py | 12 +----------- tests/test_initrd.py | 26 ++------------------------ tests/test_sysext.py | 2 +- 5 files changed, 14 insertions(+), 46 deletions(-) diff --git a/mkosi.conf b/mkosi.conf index 6977bd19e..cb9fe1308 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -35,7 +35,12 @@ RemoveFiles= /usr/lib/kernel/install.d/50-dracut.install # Make sure that SELinux doesn't run in enforcing mode even if it's pulled in as a dependency. -KernelCommandLine=enforcing=0 +KernelCommandLine= + enforcing=0 + systemd.log_ratelimit_kmsg=0 + systemd.crash_shell + printk.devkmsg=on + systemd.early_core_pattern=/core KernelModulesInitrdExclude=.* KernelModulesInitrdInclude=default diff --git a/tests/__init__.py b/tests/__init__.py index 75b475663..3900198a5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,7 +13,6 @@ import pytest -from mkosi.config import finalize_term from mkosi.distributions import Distribution from mkosi.run import run from mkosi.types import _FILE, CompletedProcess, PathString @@ -58,18 +57,14 @@ def mkosi( check: bool = True, ) -> CompletedProcess: kcl = [ - f"TERM={finalize_term()}", "loglevel=6", - "systemd.crash_shell", "systemd.log_level=debug", "udev.log_level=info", - "systemd.log_ratelimit_kmsg=0", "systemd.show_status=false", "systemd.journald.forward_to_console", "systemd.journald.max_level_console=info", - "printk.devkmsg=on", - "systemd.early_core_pattern=/core", "systemd.firstboot=no", + "systemd.unit=mkosi-check-and-shutdown.service", ] return run([ @@ -83,12 +78,12 @@ def mkosi( else [] ), *(["--tools-tree-release", self.config.tools_tree_release] if self.config.tools_tree_release else []), + "--incremental", + "--ephemeral", + "--runtime-build-sources=no", *self.options, *options, "--output-dir", self.output_dir, - # Some tests ignore the default image config but we still want them to reuse the cache directory for the - # tools tree cache. - "--cache-dir", "mkosi.cache", *(f"--kernel-command-line={i}" for i in kcl), "--qemu-vsock=yes", # TODO: Drop once both Hyper-V bugs are fixed in Github Actions. diff --git a/tests/test_boot.py b/tests/test_boot.py index 3e4517831..fa245725a 100644 --- a/tests/test_boot.py +++ b/tests/test_boot.py @@ -26,14 +26,7 @@ def have_vmspawn() -> bool: @pytest.mark.parametrize("format", [f for f in OutputFormat if f not in (OutputFormat.confext, OutputFormat.sysext)]) def test_format(config: ImageConfig, format: OutputFormat) -> None: - with Image( - config, - options=[ - "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", - "--incremental", - "--ephemeral", - ], - ) as image: + with Image(config) as image: if image.config.distribution == Distribution.rhel_ubi and format in (OutputFormat.esp, OutputFormat.uki): pytest.skip("Cannot build RHEL-UBI images with format 'esp' or 'uki'") @@ -91,9 +84,6 @@ def test_bootloader(config: ImageConfig, bootloader: Bootloader) -> None: with Image( config, options=[ - "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", - "--incremental", - "--ephemeral", "--format=disk", "--bootloader", str(bootloader), "--qemu-firmware", str(firmware) diff --git a/tests/test_initrd.py b/tests/test_initrd.py index b07e2ae8c..59833fba2 100644 --- a/tests/test_initrd.py +++ b/tests/test_initrd.py @@ -35,15 +35,7 @@ def passphrase() -> Iterator[Path]: def test_initrd(config: ImageConfig) -> None: - with Image( - config, - options=[ - "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", - "--incremental", - "--ephemeral", - "--format=disk", - ] - ) as image: + with Image(config, options=["--format=disk"]) as image: image.build() image.qemu() @@ -53,13 +45,9 @@ def test_initrd_lvm(config: ImageConfig) -> None: with Image( config, options=[ - "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", # LVM confuses systemd-repart so we mask it for this test. "--kernel-command-line=systemd.mask=systemd-repart.service", "--kernel-command-line=root=LABEL=root", - "--kernel-command-line=rw", - "--incremental", - "--ephemeral", "--qemu-firmware=linux", ] ) as image, contextlib.ExitStack() as stack: @@ -142,10 +130,7 @@ def test_initrd_luks(config: ImageConfig, passphrase: Path) -> None: options=[ "--repart-dir", repartd, "--passphrase", passphrase, - "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", "--credential=cryptsetup.passphrase=mkosi", - "--incremental", - "--ephemeral", "--format=disk", ] ) as image: @@ -158,12 +143,8 @@ def test_initrd_luks_lvm(config: ImageConfig, passphrase: Path) -> None: with Image( config, options=[ - "--kernel-command-line=systemd.unit=mkosi-check-and-shutdown.service", "--kernel-command-line=root=LABEL=root", - "--kernel-command-line=rw", "--credential=cryptsetup.passphrase=mkosi", - "--incremental", - "--ephemeral", "--qemu-firmware=linux", ] ) as image, contextlib.ExitStack() as stack: @@ -215,10 +196,7 @@ def test_initrd_luks_lvm(config: ImageConfig, passphrase: Path) -> None: def test_initrd_size(config: ImageConfig) -> None: - with Image( - config, - options=["--incremental", "--format=directory"], - ) as image: + with Image(config) as image: image.build() # The fallback value is for CentOS and related distributions. diff --git a/tests/test_sysext.py b/tests/test_sysext.py index df33bd6ab..6b5cf244e 100644 --- a/tests/test_sysext.py +++ b/tests/test_sysext.py @@ -13,7 +13,6 @@ def test_sysext(config: ImageConfig) -> None: with Image( config, options=[ - "--incremental", "--clean-package-metadata=no", "--format=directory", ], @@ -24,6 +23,7 @@ def test_sysext(config: ImageConfig) -> None: image.config, options=[ "--directory", "", + "--incremental=no", "--base-tree", Path(image.output_dir) / "image", "--overlay", "--package=dnsmasq", From 35889a9f63785db1c4e210f8e11a344bb9d363aa Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 1 Aug 2024 12:37:50 +0200 Subject: [PATCH 10/10] Introduce RepositoryKeyFetch= This setting controls whether we'll fetch GPG keys remotely or not. We disable it by default so that we only rely on locally available GPG keys for checking package and repository metadata signatures. This new setting only affects dnf/zypper based distributions as apt and pacman do not support retrieving GPG keys remotely in the first place. zypper does not trust GPG keys listed in gpgkey= by default so we import local GPG keys manually with rpm to work around that. Fixes #757 --- .github/workflows/ci.yml | 8 ++++++ mkosi/__init__.py | 2 ++ mkosi/config.py | 12 +++++++++ mkosi/distributions/alma.py | 3 ++- mkosi/distributions/centos.py | 7 +++--- mkosi/distributions/fedora.py | 10 ++++++-- mkosi/distributions/mageia.py | 3 ++- mkosi/distributions/openmandriva.py | 3 ++- mkosi/distributions/opensuse.py | 23 ++++++++++++++--- mkosi/distributions/rhel.py | 3 ++- mkosi/distributions/rhel_ubi.py | 3 ++- mkosi/distributions/rocky.py | 3 ++- mkosi/installer/rpm.py | 38 +++++++++++++++++++++++++++-- mkosi/installer/zypper.py | 3 ++- mkosi/resources/mkosi.md | 13 ++++++++++ tests/test_json.py | 2 ++ tests/test_sysext.py | 1 + 17 files changed, 120 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f71fe4f6..9ee7a9827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,14 @@ jobs: QemuKvm=yes EOF + # TODO: Drop once distribution-gpg-keys is in noble-backports. + if [[ "${{ matrix.tools }}" =~ opensuse|fedora|ubuntu ]]; then + tee --append mkosi.local.conf < Co "--repositories", ",".join(config.tools_tree_repositories), "--package-manager-tree", ",".join(str(t) for t in config.tools_tree_package_manager_trees), "--repository-key-check", str(config.repository_key_check), + "--repository-key-fetch", str(config.repository_key_fetch), "--cache-only", str(config.cacheonly), *(["--output-dir", str(config.output_dir)] if config.output_dir else []), *(["--workspace-dir", str(config.workspace_dir)] if config.workspace_dir else []), diff --git a/mkosi/config.py b/mkosi/config.py index 2e17be0b4..96285ee73 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1402,6 +1402,7 @@ class Config: mirror: Optional[str] local_mirror: Optional[str] repository_key_check: bool + repository_key_fetch: bool repositories: list[str] cacheonly: Cacheonly package_manager_trees: list[ConfigTree] @@ -1961,6 +1962,16 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple help="Controls signature and key checks on repositories", universal=True, ), + ConfigSetting( + dest="repository_key_fetch", + metavar="BOOL", + nargs="?", + section="Distribution", + default=False, + parse=config_parse_boolean, + help="Controls whether distribution GPG keys can be fetched remotely", + universal=True, + ), ConfigSetting( dest="repositories", metavar="REPOS", @@ -4066,6 +4077,7 @@ def bold(s: Any) -> str: Mirror: {none_to_default(config.mirror)} Local Mirror (build): {none_to_none(config.local_mirror)} Repo Signature/Key check: {yes_no(config.repository_key_check)} + Fetch Repository Keys: {yes_no(config.repository_key_fetch)} Repositories: {line_join_list(config.repositories)} Use Only Package Cache: {config.cacheonly} Package Manager Trees: {line_join_list(config.package_manager_trees)} diff --git a/mkosi/distributions/alma.py b/mkosi/distributions/alma.py index ac6285cac..16345ebb7 100644 --- a/mkosi/distributions/alma.py +++ b/mkosi/distributions/alma.py @@ -16,7 +16,8 @@ def gpgurls(context: Context) -> tuple[str, ...]: find_rpm_gpgkey( context, f"RPM-GPG-KEY-AlmaLinux-{context.config.release}", - ) or f"https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux-{context.config.release}", + f"https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux-{context.config.release}", + ), ) @classmethod diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index e6cd985de..e324620bc 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -100,7 +100,7 @@ def architecture(cls, arch: Architecture) -> str: def gpgurls(context: Context) -> tuple[str, ...]: rel = "RPM-GPG-KEY-CentOS-Official" if context.config.release == "9" else "RPM-GPG-KEY-CentOS-Official-SHA256" return tuple( - find_rpm_gpgkey(context, key) or f"https://www.centos.org/keys/{key}" + find_rpm_gpgkey(context, key, f"https://www.centos.org/keys/{key}") for key in (rel, "RPM-GPG-KEY-CentOS-SIG-Extras") ) @@ -196,7 +196,8 @@ def epel_repositories(cls, context: Context) -> Iterable[RpmRepository]: find_rpm_gpgkey( context, f"RPM-GPG-KEY-EPEL-{context.config.release}", - ) or f"https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{context.config.release}", + f"https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{context.config.release}", + ), ) if context.config.local_mirror: @@ -299,7 +300,7 @@ def sig_repositories(cls, context: Context) -> Iterable[RpmRepository]: ) for sig, components, keys in sigs: - gpgurls = tuple(find_rpm_gpgkey(context, key) or f"https://www.centos.org/keys/{key}" for key in keys) + gpgurls = tuple(find_rpm_gpgkey(context, key, f"https://www.centos.org/keys/{key}") for key in keys) for c in components: if mirror := context.config.mirror: diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 551d31905..97008311a 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -20,8 +20,8 @@ @tuplify def find_fedora_rpm_gpgkeys(context: Context) -> Iterable[str]: - key1 = find_rpm_gpgkey(context, key=f"RPM-GPG-KEY-fedora-{context.config.release}-primary") - key2 = find_rpm_gpgkey(context, key=f"RPM-GPG-KEY-fedora-{context.config.release}-secondary") + key1 = find_rpm_gpgkey(context, key=f"RPM-GPG-KEY-fedora-{context.config.release}-primary", required=False) + key2 = find_rpm_gpgkey(context, key=f"RPM-GPG-KEY-fedora-{context.config.release}-secondary", required=False) if key1: # During branching, there is always a kerfuffle with the key transition. @@ -38,9 +38,15 @@ def find_fedora_rpm_gpgkeys(context: Context) -> Iterable[str]: yield key3 yield key1 + if key2: yield key2 + if not key1 and not key2: + if not context.config.repository_key_fetch: + die("Fedora GPG keys not found in /usr/share/distribution-gpg-keys", + hint="Make sure the distribution-gpg-keys package is installed") + yield "https://fedoraproject.org/fedora.gpg" diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index c0e3eb2b7..4b817fb51 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -34,7 +34,8 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: find_rpm_gpgkey( context, "RPM-GPG-KEY-Mageia", - ) or "https://mirrors.kernel.org/mageia/distrib/$releasever/$basearch/media/core/release/media_info/pubkey", + "https://mirrors.kernel.org/mageia/distrib/$releasever/$basearch/media/core/release/media_info/pubkey", + ), ) if context.config.local_mirror: diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index 15a76c8c4..5238864a3 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -36,7 +36,8 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: find_rpm_gpgkey( context, "RPM-GPG-KEY-OpenMandriva", - ) or "https://raw.githubusercontent.com/OpenMandrivaAssociation/openmandriva-repos/master/RPM-GPG-KEY-OpenMandriva", + "https://raw.githubusercontent.com/OpenMandrivaAssociation/openmandriva-repos/master/RPM-GPG-KEY-OpenMandriva", + ), ) if context.config.local_mirror: diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 382f54eb3..f089a97eb 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -94,11 +94,28 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]: mirror = context.config.mirror or "https://download.opensuse.org" if context.config.release == "tumbleweed" or context.config.release.isdigit(): - gpgkeys = ( - *([p] if (p := find_rpm_gpgkey(context, key="RPM-GPG-KEY-openSUSE-Tumbleweed")) else []), - *([p] if (p := find_rpm_gpgkey(context, key="RPM-GPG-KEY-openSUSE")) else []), + gpgkeys = tuple( + p + for key in ("RPM-GPG-KEY-openSUSE-Tumbleweed", "RPM-GPG-KEY-openSUSE") + if (p := find_rpm_gpgkey(context, key, required=False)) ) + if not gpgkeys and not context.config.repository_key_fetch: + die("OpenSUSE GPG keys not found in /usr/share/distribution-gpg-keys", + hint="Make sure the distribution-gpg-keys package is installed") + + if zypper and gpgkeys: + run( + ["rpm", "--root=/buildroot", "--import", *(key.removeprefix("file://") for key in gpgkeys)], + sandbox=context.sandbox( + binary="rpm", + mounts=[ + Mount(context.root, "/buildroot"), + *finalize_crypto_mounts(context.config) + ], + ) + ) + if context.config.release == "tumbleweed": if context.config.architecture == Architecture.x86_64: subdir = "" diff --git a/mkosi/distributions/rhel.py b/mkosi/distributions/rhel.py index a70875199..ccc1e4a27 100644 --- a/mkosi/distributions/rhel.py +++ b/mkosi/distributions/rhel.py @@ -24,7 +24,8 @@ def gpgurls(context: Context) -> tuple[str, ...]: find_rpm_gpgkey( context, f"RPM-GPG-KEY-redhat{major}-release", - ) or "https://access.redhat.com/security/data/fd431d51.txt", + "https://access.redhat.com/security/data/fd431d51.txt", + ), ) @staticmethod diff --git a/mkosi/distributions/rhel_ubi.py b/mkosi/distributions/rhel_ubi.py index 78bfc233b..c921ecd1f 100644 --- a/mkosi/distributions/rhel_ubi.py +++ b/mkosi/distributions/rhel_ubi.py @@ -21,7 +21,8 @@ def gpgurls(context: Context) -> tuple[str, ...]: find_rpm_gpgkey( context, f"RPM-GPG-KEY-redhat{major}-release", - ) or "https://access.redhat.com/security/data/fd431d51.txt", + "https://access.redhat.com/security/data/fd431d51.txt", + ), ) @classmethod diff --git a/mkosi/distributions/rocky.py b/mkosi/distributions/rocky.py index af2c4d186..0a2cdfd08 100644 --- a/mkosi/distributions/rocky.py +++ b/mkosi/distributions/rocky.py @@ -16,7 +16,8 @@ def gpgurls(context: Context) -> tuple[str, ...]: find_rpm_gpgkey( context, f"RPM-GPG-KEY-Rocky-{context.config.release}", - ) or f"https://download.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-{context.config.release}", + f"https://download.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-{context.config.release}", + ), ) @classmethod diff --git a/mkosi/installer/rpm.py b/mkosi/installer/rpm.py index 94d07697a..185db91c6 100644 --- a/mkosi/installer/rpm.py +++ b/mkosi/installer/rpm.py @@ -4,9 +4,10 @@ import subprocess import textwrap from pathlib import Path -from typing import Optional +from typing import Literal, Optional, overload from mkosi.context import Context +from mkosi.log import die from mkosi.run import run from mkosi.types import PathString @@ -23,7 +24,33 @@ class RpmRepository: priority: Optional[int] = None -def find_rpm_gpgkey(context: Context, key: str) -> Optional[str]: +@overload +def find_rpm_gpgkey( + context: Context, + key: str, + fallback: Optional[str] = None, + *, + required: Literal[True] = True, +) -> str: ... + + +@overload +def find_rpm_gpgkey( + context: Context, + key: str, + fallback: Optional[str] = None, + *, + required: Literal[False] +) -> Optional[str]: ... + + +def find_rpm_gpgkey( + context: Context, + key: str, + fallback: Optional[str] = None, + *, + required: bool = True +) -> Optional[str]: root = context.config.tools() if context.config.tools_tree_certificates else Path("/") if gpgpath := next((root / "usr/share/distribution-gpg-keys").rglob(key), None): @@ -32,6 +59,13 @@ def find_rpm_gpgkey(context: Context, key: str) -> Optional[str]: if gpgpath := next(Path(context.pkgmngr / "etc/pki/rpm-gpg").rglob(key), None): return (Path("/") / gpgpath.relative_to(context.pkgmngr)).as_uri() + if context.config.repository_key_fetch: + return fallback + + if required: + die(f"{key} GPG key not found in /usr/share/distribution-gpg-keys", + hint="Make sure the distribution-gpg-keys package is installed") + return None diff --git a/mkosi/installer/zypper.py b/mkosi/installer/zypper.py index d58c6a6b8..90ef6ad34 100644 --- a/mkosi/installer/zypper.py +++ b/mkosi/installer/zypper.py @@ -109,9 +109,10 @@ def cmd(cls, context: Context) -> list[PathString]: "zypper", "--installroot=/buildroot", "--cache-dir=/var/cache/zypp", - "--gpg-auto-import-keys" if context.config.repository_key_check else "--no-gpg-checks", "--non-interactive", "--no-refresh", + *(["--gpg-auto-import-keys"] if context.config.repository_key_fetch else []), + *(["--no-gpg-checks"] if not context.config.repository_key_check else []), *([f"--plus-content={repo}" for repo in context.config.repositories]), ] diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 41ee2b61d..02fdd1fa6 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -452,6 +452,19 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, Useful to disable checks when combined with `--local-mirror=` and using only a repository from a local filesystem. +`RepositoryKeyFetch=`, `--repository-key-fetch=` +: Controls whether mkosi will fetch distribution GPG keys remotely. Disabled + by default. When disabled, the distribution GPG keys for the target distribution + have to be installed locally on the host system alongside the package manager for + that distribution. + + This setting is only implemented for distributions using dnf or zypper as their + package manager. For other distributions the distribution GPG keys are always looked + up locally regardless of the value of this setting. To make the distribution GPG keys + for distributions available without enabling this setting, the corresponding package + has to be installed on the host. This is usually one of `archlinux-keyring`, + `debian-keyring`, `ubuntu-keyring` or `distribution-gpg-keys` (for rpm-based distributions). + `Repositories=`, `--repositories=` : Enable package repositories that are disabled by default. This can be used to enable the EPEL repos for CentOS or different components of the Debian/Ubuntu repositories. diff --git a/tests/test_json.py b/tests/test_json.py index 43a51f1ab..783790412 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -264,6 +264,7 @@ def test_config() -> None: "RepartOffline": true, "Repositories": [], "RepositoryKeyCheck": false, + "RepositoryKeyFetch": true, "RootPassword": [ "test1234", false @@ -465,6 +466,7 @@ def test_config() -> None: repart_offline=True, repositories=[], repository_key_check=False, + repository_key_fetch=True, root_password=("test1234", False), root_shell="/bin/tcsh", runtime_build_sources=True, diff --git a/tests/test_sysext.py b/tests/test_sysext.py index 6b5cf244e..3d07c8ced 100644 --- a/tests/test_sysext.py +++ b/tests/test_sysext.py @@ -24,6 +24,7 @@ def test_sysext(config: ImageConfig) -> None: options=[ "--directory", "", "--incremental=no", + "--repository-key-fetch=yes", "--base-tree", Path(image.output_dir) / "image", "--overlay", "--package=dnsmasq",