From a5c8dcbb34b1dcce282f95722b62edad32ef029c Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Fri, 6 Dec 2024 00:28:13 +0000 Subject: [PATCH] Add mkosi-addon and kernel-install plugin Add new mkosi-addon and kernel-install plugin to build local customizations into an EFI addon. This allows us to move closer to the desired goal of having universal UKIs, built by vendors, used together with locally built enhancements. --- .gitignore | 1 + README.md | 26 ++- bin/mkosi-addon | 1 + kernel-install/50-mkosi.install | 86 +--------- kernel-install/51-mkosi-addon.install | 54 +++++++ mkosi-addon | 1 + mkosi/__init__.py | 1 + mkosi/addon.py | 92 +++++++++++ mkosi/initrd.py | 209 ++++++++++++++++++------- mkosi/resources/man/mkosi-addon.1.md | 50 ++++++ mkosi/resources/mkosi-addon/mkosi.conf | 30 ++++ mkosi/util.py | 7 + pyproject.toml | 2 + 13 files changed, 417 insertions(+), 143 deletions(-) create mode 120000 bin/mkosi-addon create mode 100755 kernel-install/51-mkosi-addon.install create mode 120000 mkosi-addon create mode 100644 mkosi/addon.py create mode 100644 mkosi/resources/man/mkosi-addon.1.md create mode 100644 mkosi/resources/mkosi-addon/mkosi.conf diff --git a/.gitignore b/.gitignore index b6e54df8a..eef237602 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.cache-pre-inst .cache .mkosi.1 +.mkosi-addon.1 .mkosi-initrd.1 .mkosi-sandbox.1 .mypy_cache/ diff --git a/README.md b/README.md index 3ff31bf3d..0288dfaa3 100644 --- a/README.md +++ b/README.md @@ -79,17 +79,31 @@ when not installed as a zipapp. Please note, that the python module exists solely for the usage of the mkosi binary and is not to be considered a public API. -## kernel-install plugin +## kernel-install plugins -mkosi can also be used as a kernel-install plugin to build initrds. To -enable this feature, install `kernel-install/50-mkosi.install` +mkosi can also be used as a kernel-install plugin to build initrds and addons. +It is recommended to use only one of these two plugins at a given time. + +## UKI plugin +To enable this feature, install `kernel-install/50-mkosi.install` into `/usr/lib/kernel/install.d`. Extra distro configuration for the initrd can be configured in `/usr/lib/mkosi-initrd`. Users can add their -own customizations in `/etc/mkosi-initrd`. +own customizations in `/etc/mkosi-initrd`. A full self-contained UKI will +be built and installed. Once installed, the mkosi plugin can be enabled by writing -`initrd_generator=mkosi-initrd` to `/usr/lib/kernel/install.conf` or to -`/etc/kernel/install.conf`. +`initrd_generator=mkosi-initrd` and `layout=uki` to `/usr/lib/kernel/install.conf` +or to `/etc/kernel/install.conf`. + +## Addon plugin +To enable this feature, install `kernel-install/51-mkosi-addon.install` into +`/usr/lib/kernel/install.d`. Extra distro configuration for the addon can be +configured in `/usr/lib/mkosi-addon`. Users can add their own customizations in +`/etc/mkosi-addon` and `/run/mkosi-addon`. Note that unless at least one of the +last two directories are present, the plugin will not operate. + +This plugin is useful to enhance a vendor-provided UKI with local-only +modifications. # Hacking on mkosi diff --git a/bin/mkosi-addon b/bin/mkosi-addon new file mode 120000 index 000000000..b5f44fa8e --- /dev/null +++ b/bin/mkosi-addon @@ -0,0 +1 @@ +mkosi \ No newline at end of file diff --git a/kernel-install/50-mkosi.install b/kernel-install/50-mkosi.install index 1f93d1419..68b371b62 100755 --- a/kernel-install/50-mkosi.install +++ b/kernel-install/50-mkosi.install @@ -1,10 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: LGPL-2.1-or-later -import argparse -import dataclasses import logging -import os import sys import tempfile from pathlib import Path @@ -13,38 +10,17 @@ from typing import Optional from mkosi import identify_cpu from mkosi.archive import make_cpio from mkosi.config import OutputFormat -from mkosi.log import die, log_setup +from mkosi.initrd import KernelInstallContext +from mkosi.log import log_setup from mkosi.run import run, uncaught_exception_handler -from mkosi.sandbox import __version__, umask +from mkosi.sandbox import umask from mkosi.types import PathString -@dataclasses.dataclass(frozen=True) -class Context: - command: str - kernel_version: str - entry_dir: Path - kernel_image: Path - initrds: list[Path] - staging_area: Path - layout: str - image_type: str - initrd_generator: Optional[str] - uki_generator: Optional[str] - verbose: bool - - -def we_are_wanted(context: Context) -> bool: +def we_are_wanted(context: KernelInstallContext) -> bool: return context.uki_generator == "mkosi" or context.initrd_generator in ("mkosi", "mkosi-initrd") -def mandatory_variable(name: str) -> str: - try: - return os.environ[name] - except KeyError: - die(f"${name} must be set in the environment") - - def build_microcode_initrd(output: Path) -> Optional[Path]: vendor, ucode = identify_cpu(Path("/")) @@ -75,57 +51,9 @@ def build_microcode_initrd(output: Path) -> Optional[Path]: def main() -> None: log_setup() - parser = argparse.ArgumentParser( - description="kernel-install plugin to build initrds or Unified Kernel Images using mkosi", - allow_abbrev=False, - usage="50-mkosi.install COMMAND KERNEL_VERSION ENTRY_DIR KERNEL_IMAGE INITRD…", - ) - - parser.add_argument( - "command", - metavar="COMMAND", - help="The action to perform. Only 'add' is supported.", - ) - parser.add_argument( - "kernel_version", - metavar="KERNEL_VERSION", - help="Kernel version string", - ) - parser.add_argument( - "entry_dir", - metavar="ENTRY_DIR", - type=Path, - nargs="?", - help="Type#1 entry directory (ignored)", - ) - parser.add_argument( - "kernel_image", - metavar="KERNEL_IMAGE", - type=Path, - nargs="?", - help="Kernel image", - ) - parser.add_argument( - "initrds", - metavar="INITRD…", - type=Path, - nargs="*", - help="Initrd files", - ) - parser.add_argument( - "--version", - action="version", - version=f"mkosi {__version__}", - ) - - context = Context( - **vars(parser.parse_args()), - staging_area=Path(mandatory_variable("KERNEL_INSTALL_STAGING_AREA")), - layout=mandatory_variable("KERNEL_INSTALL_LAYOUT"), - image_type=mandatory_variable("KERNEL_INSTALL_IMAGE_TYPE"), - initrd_generator=os.getenv("KERNEL_INSTALL_INITRD_GENERATOR"), - uki_generator=os.getenv("KERNEL_INSTALL_UKI_GENERATOR"), - verbose=int(os.getenv("KERNEL_INSTALL_VERBOSE", 0)) > 0, + context = KernelInstallContext.parse( + "kernel-install plugin to build initrds or Unified Kernel Images using mkosi", + "50-mkosi.install COMMAND KERNEL_VERSION ENTRY_DIR KERNEL_IMAGE INITRD…", ) if context.command != "add" or not we_are_wanted(context): diff --git a/kernel-install/51-mkosi-addon.install b/kernel-install/51-mkosi-addon.install new file mode 100755 index 000000000..d16e4abf3 --- /dev/null +++ b/kernel-install/51-mkosi-addon.install @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: LGPL-2.1-or-later + +import logging +import sys +from pathlib import Path + +from mkosi.initrd import KernelInstallContext +from mkosi.log import log_setup +from mkosi.run import run, uncaught_exception_handler +from mkosi.types import PathString + + +@uncaught_exception_handler() +def main() -> None: + log_setup() + + context = KernelInstallContext.parse( + "kernel-install plugin to build local addon for initrd/cmdline", + "51-mkosi-addon.install COMMAND KERNEL_VERSION ENTRY_DIR KERNEL_IMAGE…", + ) + + # No local configuration? Then nothing to do + if not Path("/etc/mkosi-addon").exists() and not Path("/run/mkosi-addon").exists(): + if context.verbose: + logging.info("No local configuration defined, skipping mkosi-addon") + return + + if context.command != "add" or context.layout != "uki": + if context.verbose: + logging.info("Not an UKI layout 'add' step, skipping mkosi-addon") + return + + if not context.kernel_image or not context.kernel_image.exists(): + if context.verbose: + logging.info("No kernel image provided, skipping mkosi-addon") + return + + cmdline: list[PathString] = [ + "mkosi-addon", + "--output", "mkosi-local.addon.efi", + "--output-dir", context.staging_area / "uki.efi.extra.d", + ] # fmt: skip + + if context.verbose: + cmdline += ["--debug"] + + logging.info("Building mkosi-local.addon.efi") + + run(cmdline, stdin=sys.stdin, stdout=sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/mkosi-addon b/mkosi-addon new file mode 120000 index 000000000..c8442ca02 --- /dev/null +++ b/mkosi-addon @@ -0,0 +1 @@ +mkosi/resources/mkosi-addon \ No newline at end of file diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 9f3d934ff..55c8ccb9b 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -4665,6 +4665,7 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None: if args.verb == Verb.documentation: if args.cmdline: manual = { + "addon": "mkosi-addon", "initrd": "mkosi-initrd", "sandbox": "mkosi-sandbox", "news": "mkosi.news", diff --git a/mkosi/addon.py b/mkosi/addon.py new file mode 100644 index 000000000..819f13774 --- /dev/null +++ b/mkosi/addon.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +import argparse +import os +import sys +import tempfile +from pathlib import Path + +import mkosi.resources +from mkosi.config import DocFormat +from mkosi.documentation import show_docs +from mkosi.initrd import initrd_common_args, initrd_finalize, process_crypttab +from mkosi.log import log_setup +from mkosi.run import run, uncaught_exception_handler +from mkosi.types import PathString +from mkosi.util import resource_path + + +@uncaught_exception_handler() +def main() -> None: + log_setup() + + parser = argparse.ArgumentParser( + prog="mkosi-addon", + description="Build initrd/cmdline/ucode addon for the current system using mkosi", + allow_abbrev=False, + usage="mkosi-addon [options...]", + ) + parser.add_argument( + "-o", + "--output", + metavar="NAME", + help="Output name", + default="mkosi-local.addon.efi", + ) + + initrd_common_args(parser) + + args = parser.parse_args() + + if args.show_documentation: + with resource_path(mkosi.resources) as r: + show_docs("mkosi-addon", DocFormat.all(), resources=r) + return + + with tempfile.TemporaryDirectory() as staging_dir: + cmdline: list[PathString] = [ + "mkosi", + "--force", + "--directory", "", + "--output", args.output, + "--output-directory", staging_dir, + "--build-sources", "", + "--include=mkosi-addon", + "--extra-tree", + f"/usr/lib/modules/{args.kernel_version}:/usr/lib/modules/{args.kernel_version}", + "--extra-tree=/usr/lib/firmware:/usr/lib/firmware", + "--kernel-modules-exclude=.*", + ] # fmt: skip + + if args.debug: + cmdline += ["--debug"] + if args.debug_shell: + cmdline += ["--debug-shell"] + + if os.getuid() == 0: + cmdline += [ + "--workspace-dir=/var/tmp", + "--output-mode=600", + ] + + for d in ( + "/usr/lib/mkosi-addon", + "/usr/local/lib/mkosi-addon", + "/run/mkosi-addon", + "/etc/mkosi-addon", + ): + if Path(d).exists(): + cmdline += ["--include", d] + + cmdline += process_crypttab(staging_dir) + + if Path("/etc/kernel/cmdline").exists(): + cmdline += ["--kernel-command-line", Path("/etc/kernel/cmdline").read_text()] + + run(cmdline, stdin=sys.stdin, stdout=sys.stdout) + + initrd_finalize(staging_dir, args.output, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/mkosi/initrd.py b/mkosi/initrd.py index 6673605a0..34294a5e0 100644 --- a/mkosi/initrd.py +++ b/mkosi/initrd.py @@ -2,13 +2,14 @@ import argparse import contextlib +import dataclasses import os import platform import shutil import sys import tempfile from pathlib import Path -from typing import cast +from typing import Optional, cast import mkosi.resources from mkosi.config import DocFormat, OutputFormat @@ -18,46 +19,130 @@ from mkosi.sandbox import __version__, umask from mkosi.tree import copy_tree from mkosi.types import PathString -from mkosi.util import resource_path +from mkosi.util import mandatory_variable, resource_path -@uncaught_exception_handler() -def main() -> None: - log_setup() +@dataclasses.dataclass(frozen=True) +class KernelInstallContext: + command: str + kernel_version: str + entry_dir: Path + kernel_image: Path + initrds: list[Path] + staging_area: Path + layout: str + image_type: str + initrd_generator: Optional[str] + uki_generator: Optional[str] + verbose: bool - parser = argparse.ArgumentParser( - prog="mkosi-initrd", - description="Build initrds or unified kernel images for the current system using mkosi", - allow_abbrev=False, - usage="mkosi-initrd [options...]", + @staticmethod + def parse(description: str, usage: str) -> "KernelInstallContext": + parser = argparse.ArgumentParser( + description=description, + allow_abbrev=False, + usage=usage, + ) + + parser.add_argument( + "command", + metavar="COMMAND", + help="The action to perform. Only 'add' is supported.", + ) + parser.add_argument( + "kernel_version", + metavar="KERNEL_VERSION", + help="Kernel version string", + ) + parser.add_argument( + "entry_dir", + metavar="ENTRY_DIR", + type=Path, + nargs="?", + help="Type#1 entry directory (ignored)", + ) + parser.add_argument( + "kernel_image", + metavar="KERNEL_IMAGE", + type=Path, + nargs="?", + help="Kernel image", + ) + parser.add_argument( + "initrds", + metavar="INITRD…", + type=Path, + nargs="*", + help="Initrd files", + ) + parser.add_argument( + "--version", + action="version", + version=f"mkosi {__version__}", + ) + + args = parser.parse_args() + + return KernelInstallContext( + command=args.command, + kernel_version=args.kernel_version, + entry_dir=args.entry_dir, + kernel_image=args.kernel_image, + initrds=args.initrds, + staging_area=Path(mandatory_variable("KERNEL_INSTALL_STAGING_AREA")), + layout=mandatory_variable("KERNEL_INSTALL_LAYOUT"), + image_type=mandatory_variable("KERNEL_INSTALL_IMAGE_TYPE"), + initrd_generator=os.getenv("KERNEL_INSTALL_INITRD_GENERATOR"), + uki_generator=os.getenv("KERNEL_INSTALL_UKI_GENERATOR"), + verbose=int(os.getenv("KERNEL_INSTALL_VERBOSE", 0)) > 0, + ) + + +def process_crypttab(staging_dir: str) -> list[str]: + cmdline = [] + + # Generate crypttab with all the x-initrd.attach entries + if Path("/etc/crypttab").exists(): + crypttab = [ + line + for line in Path("/etc/crypttab").read_text().splitlines() + if ( + len(entry := line.split()) >= 4 + and not entry[0].startswith("#") + and "x-initrd.attach" in entry[3] + ) + ] + if crypttab: + with (Path(staging_dir) / "crypttab").open("w") as f: + f.write("# Automatically generated by mkosi-initrd\n") + f.write("\n".join(crypttab)) + cmdline += ["--extra-tree", f"{staging_dir}/crypttab:/etc/crypttab"] + + return cmdline + + +def initrd_finalize(staging_dir: str, output: str, output_dir: str) -> None: + if output_dir: + with umask(~0o700) if os.getuid() == 0 else cast(umask, contextlib.nullcontext()): + Path(output_dir).mkdir(parents=True, exist_ok=True) + else: + output_dir = str(Path.cwd()) + + log_notice(f"Copying {staging_dir}/{output} to {output_dir}/{output}") + # mkosi symlinks the expected output image, so dereference it + copy_tree( + Path(f"{staging_dir}/{output}").resolve(), + Path(f"{output_dir}/{output}"), ) + +def initrd_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--kernel-version", metavar="KERNEL_VERSION", help="Kernel version string", default=platform.uname().release, ) - parser.add_argument( - "--kernel-image", - metavar="KERNEL_IMAGE", - help="Kernel image", - type=Path, - ) - parser.add_argument( - "-t", - "--format", - choices=[str(OutputFormat.cpio), str(OutputFormat.uki), str(OutputFormat.directory)], - help="Output format (CPIO archive, UKI or local directory)", - default="cpio", - ) - parser.add_argument( - "-o", - "--output", - metavar="NAME", - help="Output name", - default="initrd", - ) parser.add_argument( "-O", "--output-dir", @@ -90,6 +175,40 @@ def main() -> None: version=f"mkosi {__version__}", ) + +@uncaught_exception_handler() +def main() -> None: + log_setup() + + parser = argparse.ArgumentParser( + prog="mkosi-initrd", + description="Build initrds or unified kernel images for the current system using mkosi", + allow_abbrev=False, + usage="mkosi-initrd [options...]", + ) + parser.add_argument( + "-o", + "--output", + metavar="NAME", + help="Output name", + default="initrd", + ) + parser.add_argument( + "--kernel-image", + metavar="KERNEL_IMAGE", + help="Kernel image", + type=Path, + ) + parser.add_argument( + "-t", + "--format", + choices=[str(OutputFormat.cpio), str(OutputFormat.uki), str(OutputFormat.directory)], + help="Output format (CPIO archive, UKI or local directory)", + default="cpio", + ) + + initrd_common_args(parser) + args = parser.parse_args() if args.show_documentation: @@ -180,22 +299,7 @@ def main() -> None: cmdline += ["--sandbox-tree", sandbox_tree] - # Generate crypttab with all the x-initrd.attach entries - if Path("/etc/crypttab").exists(): - crypttab = [ - line - for line in Path("/etc/crypttab").read_text().splitlines() - if ( - len(entry := line.split()) >= 4 - and not entry[0].startswith("#") - and "x-initrd.attach" in entry[3] - ) - ] - if crypttab: - with (Path(staging_dir) / "crypttab").open("w") as f: - f.write("# Automatically generated by mkosi-initrd\n") - f.write("\n".join(crypttab)) - cmdline += ["--extra-tree", f"{staging_dir}/crypttab:/etc/crypttab"] + cmdline += process_crypttab(staging_dir) if Path("/etc/kernel/cmdline").exists(): cmdline += ["--kernel-command-line", Path("/etc/kernel/cmdline").read_text()] @@ -210,18 +314,7 @@ def main() -> None: env={"MKOSI_DNF": dnf.resolve().name} if (dnf := find_binary("dnf")) else {}, ) - if args.output_dir: - with umask(~0o700) if os.getuid() == 0 else cast(umask, contextlib.nullcontext()): - Path(args.output_dir).mkdir(parents=True, exist_ok=True) - else: - args.output_dir = Path.cwd() - - log_notice(f"Copying {staging_dir}/{args.output} to {args.output_dir}/{args.output}") - # mkosi symlinks the expected output image, so dereference it - copy_tree( - Path(f"{staging_dir}/{args.output}").resolve(), - Path(f"{args.output_dir}/{args.output}"), - ) + initrd_finalize(staging_dir, args.output, args.output_dir) if __name__ == "__main__": diff --git a/mkosi/resources/man/mkosi-addon.1.md b/mkosi/resources/man/mkosi-addon.1.md new file mode 100644 index 000000000..7e5572a3d --- /dev/null +++ b/mkosi/resources/man/mkosi-addon.1.md @@ -0,0 +1,50 @@ +% mkosi-addon(1) +% +% + +# NAME + +mkosi-addon — Build addons for unified kernel images for the current system +using mkosi + +# SYNOPSIS + +`mkosi-addon [options…]` + +# DESCRIPTION + +`mkosi-addon` is a wrapper on top of `mkosi` to simplify the generation of PE +addons containing customizations for unified kernel images specific to the +running or local system. Will include entries in `/etc/crypttab` marked with +`x-initrd.attach`, and `/etc/kernel/cmdline`. Kernel modules and firmwares for the +running hardware can be included if a local configuration with the option +`KernelModulesIncludeHost=` is provided. + +# OPTIONS + +`--kernel-version=` +: Kernel version where to look for the kernel modules to include. Defaults to + the kernel version of the running system (`uname -r`). + +`--output=`, `-o` +: Name to use for the generated output addon. Defaults to + `mkosi-local.addon.efi`. + +`--output-dir=`, `-O` +: Path to a directory where to place all generated artifacts. Defaults to the + current working directory. + +`--debug=` +: Enable additional debugging output. + +`--debug-shell=` +: Spawn debug shell in sandbox if a sandboxed command fails. + +`--version` +: Show package version. + +`--help`, `-h` +: Show brief usage information. + +# SEE ALSO +`mkosi(1)` diff --git a/mkosi/resources/mkosi-addon/mkosi.conf b/mkosi/resources/mkosi-addon/mkosi.conf new file mode 100644 index 000000000..158e1c143 --- /dev/null +++ b/mkosi/resources/mkosi-addon/mkosi.conf @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Distribution] +Distribution=custom + +[Output] +Output=addon +Format=addon +ManifestFormat= +SplitArtifacts= + +[Content] +Bootable=no +# Needs to be available for the addon stub, but don't want it in the initrd +ExtraTrees=/usr/lib/systemd/boot/efi:/usr/lib/systemd/boot/efi +RemoveFiles=/usr/lib/systemd/boot/efi/ +RemoveFiles= + # Including kernel images in the initrd is generally not useful. + # This also stops mkosi from extracting the kernel image out of the image as a separate output. + /usr/lib/modules/*/vmlinuz* + /usr/lib/modules/*/vmlinux* + /usr/lib/modules/*/System.map + # This is an addon so drop all modules files as these would override the ones from the base image. + /usr/lib/modules/*/modules.* + # Arch Linux specific file. + /usr/lib/modules/*/pkgbase + # Drop microcode directories explicitly as these are not dropped by the kernel modules processing + # logic. + /usr/lib/firmware/intel-ucode + /usr/lib/firmware/amd-ucode diff --git a/mkosi/util.py b/mkosi/util.py index 9b6d7e377..2ce615aef 100644 --- a/mkosi/util.py +++ b/mkosi/util.py @@ -256,3 +256,10 @@ def current_home_dir() -> Optional[Path]: def unique(seq: Sequence[T]) -> list[T]: return list(dict.fromkeys(seq)) + + +def mandatory_variable(name: str) -> str: + try: + return os.environ[name] + except KeyError: + die(f"${name} must be set in the environment") diff --git a/pyproject.toml b/pyproject.toml index d7f9524f3..c2a204344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ bootable = [ mkosi = "mkosi.__main__:main" mkosi-initrd = "mkosi.initrd:main" mkosi-sandbox = "mkosi.sandbox:main" +mkosi-addon = "mkosi.addon:main" [tool.setuptools] packages = [ @@ -35,6 +36,7 @@ packages = [ "mkosi.resources" = [ "completion.*", "man/*", + "mkosi-addon/**/*", "mkosi-initrd/**/*", "mkosi-tools/**/*", "mkosi-vm/**/*",