Add LogsDirectory= to enable log forwarding of VMs and containers

In systemd v256, journald will support forwarding to systemd-journal-remote
via the new journal.forward_to_socket credential. Let's expose this
functionality via a new LogsDirectory= setting, which specifies a directory
to which logs should be forwarded.

mkosi/__init__.py

@@ -12,6 +12,7 @@
 import resource
 import shlex
 import shutil
+import socket
 import stat
 import subprocess
 import sys
@@ -59,7 +60,7 @@
 from mkosi.mounts import finalize_source_mounts, mount_overlay
 from mkosi.pager import page
 from mkosi.partition import Partition, finalize_root, finalize_roothash
-from mkosi.qemu import KernelType, copy_ephemeral, run_qemu, run_ssh
+from mkosi.qemu import KernelType, copy_ephemeral, run_qemu, run_ssh, start_journal_remote
 from mkosi.run import (
     find_binary,
     fork_and_wait,
@@ -75,6 +76,7 @@
     flock,
     flock_or_die,
     format_rlimit,
+    getxattr,
     make_executable,
     one_zero,
     read_env_file,
@@ -3777,6 +3779,19 @@ def run_shell(args: Args, config: Config) -> None:
             os.chmod(scratch, 0o1777)
             cmdline += ["--bind", f"{scratch}:/var/tmp"]
 
+        if args.verb == Verb.boot and config.logs_directory:
+            with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+                addr = Path(os.getenv("TMPDIR", "/tmp")) / f"mkosi-journal-remote-unix-{uuid.uuid4().hex[:16]}"
+                sock.bind(os.fspath(addr))
+                sock.listen()
+                if config.output_format == OutputFormat.directory and (stat := os.stat(fname)).st_uid != 0:
+                    os.chown(addr, stat.st_uid, stat.st_gid)
+                stack.enter_context(start_journal_remote(config, sock.fileno()))
+                cmdline += [
+                    "--bind", f"{addr}:/run/host/journal/socket",
+                    "--set-credential=journal.forward_to_socket:/run/host/journal/socket",
+                ]
+
         if args.verb == Verb.boot:
             # Add nspawn options first since systemd-nspawn ignores all options after the first argument.
             cmdline += args.cmdline
@@ -4114,6 +4129,16 @@ def run_clean(args: Args, config: Config, *, resources: Path) -> None:
         ):
             rmtree(*outputs)
 
+    if config.logs_directory:
+        logs = [
+            p
+            for p in config.logs_directory.iterdir()
+            if getxattr(p, "user.image") == config.name()
+        ]
+        if logs:
+            with complete_step(f"Removing logs of {config.name()} image"):
+                rmtree(*logs)
+
     if remove_build_cache:
         if config.cache_dir:
             initrd = (

mkosi/config.py

@@ -1415,6 +1415,7 @@ class Config:
     ssh_key: Optional[Path]
     ssh_certificate: Optional[Path]
     machine: Optional[str]
+    logs_directory: Optional[Path]
     vmm: Vmm
 
     # QEMU-specific options
@@ -2757,6 +2758,14 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple
         section="Host",
         help="Set the machine name to use when booting the image",
     ),
+    ConfigSetting(
+        dest="logs_directory",
+        metavar="PATH",
+        section="Host",
+        parse=config_make_path_parser(),
+        paths=("mkosi.logs",),
+        help="Set the logs directory used to store forwarded machine logs",
+    ),
     ConfigSetting(
         dest="qemu_gui",
         metavar="BOOL",
@@ -3880,6 +3889,7 @@ def bold(s: Any) -> str:
                     SSH Signing Key: {none_to_none(config.ssh_key)}
                     SSH Certificate: {none_to_none(config.ssh_certificate)}
                             Machine: {config.machine_or_name()}
+                      LogsDirectory: {none_to_none(config.logs_directory)}
 
             Virtual Machine Monitor: {config.vmm}
                            QEMU GUI: {yes_no(config.qemu_gui)}

mkosi/qemu.py

@@ -399,6 +399,48 @@ async def notify() -> None:
             logging.debug(f"- {k}={v}")
 
 
+@contextlib.contextmanager
+def start_journal_remote(config: Config, sockfd: int) -> Iterator[None]:
+    assert config.logs_directory
+
+    bin = find_binary("systemd-journal-remote", "/usr/lib/systemd/systemd-journal-remote", root=config.tools())
+    if not bin:
+        die("systemd-journal-remote must be installed to forward logs from the virtual machine")
+
+    output = config.logs_directory / f"{config.machine_or_name()}.journal"
+    INVOKING_USER.mkdir(output.parent)
+
+    # Make sure COW is disabled for files created in this directory so systemd-journal-remote doesn't complain on btrfs
+    # filesystems.
+    run(["chattr", "+C", output.parent], check=False, stderr=subprocess.DEVNULL if not ARG_DEBUG.get() else None)
+
+    with spawn(
+        [bin, "--output", output, "--split-mode=none"],
+        pass_fds=(sockfd,),
+        sandbox=config.sandbox(mounts=[Mount(config.logs_directory, config.logs_directory)]),
+        user=output.parent.stat().st_uid,
+        group=output.parent.stat().st_gid,
+        # All logs go into a single file so we disable compact mode to allow for journal files exceeding 4G.
+        env={"SYSTEMD_JOURNAL_COMPACT": "0"},
+    ) as proc:
+        try:
+            yield
+        finally:
+            os.setxattr(output, "user.image", config.name().encode())
+
+        proc.terminate()
+
+
+@contextlib.contextmanager
+def start_journal_remote_vsock(config: Config) -> Iterator[str]:
+    with socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) as sock:
+        sock.bind((socket.VMADDR_CID_ANY, socket.VMADDR_PORT_ANY))
+        sock.listen()
+
+        with start_journal_remote(config, sock.fileno()):
+            yield f"vsock-stream:{socket.VMADDR_CID_HOST}:{sock.getsockname()[1]}"
+
+
 @contextlib.contextmanager
 def copy_ephemeral(config: Config, src: Path) -> Iterator[Path]:
     if not config.ephemeral or config.output_format in (OutputFormat.cpio, OutputFormat.uki):
@@ -947,6 +989,9 @@ def run_qemu(args: Args, config: Config) -> None:
             addr, notifications = stack.enter_context(vsock_notify_handler())
             credentials["vmm.notify_socket"] = addr
 
+        if config.logs_directory:
+            credentials["journal.forward_to_socket"] = stack.enter_context(start_journal_remote_vsock(config))
+
         for k, v in credentials.items():
             payload = base64.b64encode(v.encode()).decode()
             if config.architecture.supports_smbios(firmware):

mkosi/resources/mkosi-tools/mkosi.conf.d/10-centos-fedora/mkosi.conf

@@ -24,6 +24,7 @@ Packages=
         squashfs-tools
         swtpm-tools
         systemd-container
+        systemd-journal-remote
         systemd-udev
         ubu-keyring
         virt-firmware

mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu.conf

@@ -31,6 +31,7 @@ Packages=
         systemd-boot
         systemd-container
         systemd-coredump
+        systemd-journal-remote
         ubuntu-keyring
         uidmap
         xz-utils

mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf

@@ -27,5 +27,6 @@ Packages=
         systemd-container
         systemd-coredump
         systemd-experimental
+        systemd-journal-remote
         xz
         zypper

mkosi/resources/mkosi.md

@@ -1837,6 +1837,16 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
 : Note that `Ephemeral=` has to be enabled to start multiple instances
   of the same image.
 
+`LogsDirectory=`, `--logs-directory=`
+
+: Specify the logs directory to which journal logs from containers and
+  virtual machines should be forwarded. A subdirectory in this directory
+  is created named after the machine name from `Machine=` if available
+  or the image name otherwise.
+
+: Note that systemd v256 or newer is required in the virtual machine for
+  log forwarding to work.
+
 ## Specifiers
 
 The current value of various settings can be accessed when parsing

mkosi/util.py

@@ -195,6 +195,16 @@ def parents_below(path: Path, below: Path) -> list[Path]:
     return parents[:parents.index(below)]
 
 
+def getxattr(path: Path, attribute: str) -> Optional[str]:
+    try:
+        return os.getxattr(path, attribute).decode()
+    except OSError as e:
+        if e.errno != errno.ENODATA:
+            raise
+
+        return None
+
+
 @contextlib.contextmanager
 def resource_path(mod: ModuleType) -> Iterator[Path]:
 

mkosi/vmspawn.py

@@ -19,6 +19,7 @@
     copy_ephemeral,
     finalize_qemu_firmware,
     find_ovmf_firmware,
+    start_journal_remote_vsock,
 )
 from mkosi.run import run
 from mkosi.types import PathString
@@ -93,6 +94,10 @@ def run_vmspawn(args: Args, config: Config) -> None:
         else:
             cmdline += ["--image", fname]
 
+        if config.logs_directory:
+            addr = stack.enter_context(start_journal_remote_vsock(config))
+            cmdline += [f"--set-credential=journal.forward_to_socket:{addr}"]
+
         cmdline += [*args.cmdline, *config.kernel_command_line_extra]
 
         run(cmdline, stdin=sys.stdin, stdout=sys.stdout, env=os.environ | config.environment, log=False)

tests/test_json.py

@@ -184,6 +184,7 @@ def test_config() -> None:
             "LocalMirror": null,
             "Locale": "en_C.UTF-8",
             "LocaleMessages": "",
+            "LogsDirectory": "/logs",
             "Machine": "machine",
             "MakeInitrd": false,
             "ManifestFormat": [
@@ -393,6 +394,7 @@ def test_config() -> None:
         local_mirror = None,
         locale = "en_C.UTF-8",
         locale_messages = "",
+        logs_directory = Path("/logs"),
         machine = "machine",
         make_initrd = False,
         manifest_format = [ManifestFormat.json, ManifestFormat.changelog],