From b2e7af7b4d5cd287dafa8cec62cd2e03b90a3ede Mon Sep 17 00:00:00 2001 From: Yasin Turan Date: Fri, 27 Jan 2023 22:49:34 +0000 Subject: [PATCH] [WIP] added container export functionality. Signed-off-by: Yasin Turan --- cmd/nerdctl/container.go | 1 + cmd/nerdctl/container_export.go | 85 +++++++++++++++++++++ cmd/nerdctl/main.go | 1 + pkg/cmd/container/export.go | 68 +++++++++++++++++ pkg/tarutil/tarutil.go | 129 ++++++++++++++++++++++++++++++++ pkg/tarutil/tarutil_unix.go | 38 ++++++++++ pkg/tarutil/tarutil_windows.go | 31 ++++++++ 7 files changed, 353 insertions(+) create mode 100644 cmd/nerdctl/container_export.go create mode 100644 pkg/cmd/container/export.go create mode 100644 pkg/tarutil/tarutil_unix.go create mode 100644 pkg/tarutil/tarutil_windows.go diff --git a/cmd/nerdctl/container.go b/cmd/nerdctl/container.go index 6c0a763f14b..b0d9db60121 100644 --- a/cmd/nerdctl/container.go +++ b/cmd/nerdctl/container.go @@ -49,6 +49,7 @@ func newContainerCommand() *cobra.Command { newCommitCommand(), newRenameCommand(), newContainerPruneCommand(), + newExportCommand(), ) addCpCommand(containerCommand) return containerCommand diff --git a/cmd/nerdctl/container_export.go b/cmd/nerdctl/container_export.go new file mode 100644 index 00000000000..f65e190a3fa --- /dev/null +++ b/cmd/nerdctl/container_export.go @@ -0,0 +1,85 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + + "github.com/containerd/nerdctl/pkg/clientutil" + "github.com/containerd/nerdctl/pkg/cmd/container" + "github.com/mattn/go-isatty" + "github.com/spf13/cobra" +) + +func newExportCommand() *cobra.Command { + var exportCommand = &cobra.Command{ + Use: "export CONTAINER", + Args: cobra.MinimumNArgs(1), + Short: "Export a containers filesystem as a tar archive", + Long: "Export a containers filesystem as a tar archive", + RunE: exportAction, + ValidArgsFunction: exportShellComplete, + SilenceUsage: true, + SilenceErrors: true, + } + exportCommand.Flags().StringP("output", "o", "", "Write to a file, instead of STDOUT") + + return exportCommand +} + +func exportAction(cmd *cobra.Command, args []string) error { + globalOptions, err := processRootCmdFlags(cmd) + if err != nil { + return err + } + if len(args) == 0 { + return fmt.Errorf("requires at least 1 argument") + } + + output, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + + client, ctx, cancel, err := clientutil.NewClient(cmd.Context(), globalOptions.Namespace, globalOptions.Address) + if err != nil { + return err + } + defer cancel() + + writer := cmd.OutOrStdout() + if output != "" { + f, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + writer = f + } else { + if isatty.IsTerminal(os.Stdout.Fd()) { + return fmt.Errorf("cowardly refusing to save to a terminal. Use the -o flag or redirect") + } + } + return container.Export(ctx, client, args, writer) + +} + +func exportShellComplete(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // show container names + return shellCompleteContainerNames(cmd, nil) +} diff --git a/cmd/nerdctl/main.go b/cmd/nerdctl/main.go index 6113790c8e0..8c763aa846e 100644 --- a/cmd/nerdctl/main.go +++ b/cmd/nerdctl/main.go @@ -259,6 +259,7 @@ Config file ($NERDCTL_TOML): %s newCommitCommand(), newWaitCommand(), newRenameCommand(), + newExportCommand(), // #endregion // Build diff --git a/pkg/cmd/container/export.go b/pkg/cmd/container/export.go new file mode 100644 index 00000000000..4ec61beb1b2 --- /dev/null +++ b/pkg/cmd/container/export.go @@ -0,0 +1,68 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package container + +import ( + "context" + "fmt" + "io" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/mount" + "github.com/containerd/nerdctl/pkg/idutil/containerwalker" + "github.com/containerd/nerdctl/pkg/tarutil" +) + +func Export(ctx context.Context, client *containerd.Client, args []string, w io.Writer) error { + walker := &containerwalker.ContainerWalker{ + Client: client, + OnFound: func(ctx context.Context, found containerwalker.Found) error { + if found.MatchCount > 1 { + return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req) + } + container := found.Container + c, err := container.Info(ctx) + if err != nil { + return err + } + return performWithBaseFS(ctx, client, c, func(root string) error { + tb := tarutil.NewTarballer(w) + return tb.Tar(root) + }) + + }, + } + req := args[0] + n, err := walker.Walk(ctx, req) + if err != nil { + return fmt.Errorf("failed to export container %s: %w", req, err) + } else if n == 0 { + return fmt.Errorf("no such container %s", req) + } + return nil +} + +// performWithBaseFS will execute a given function with respect to the root filesystem of a container. +// copied over from: https://github.com/moby/moby/blob/master/daemon/containerd/image_exporter.go#L24 +func performWithBaseFS(ctx context.Context, client *containerd.Client, c containers.Container, fn func(root string) error) error { + mounts, err := client.SnapshotService(c.Snapshotter).Mounts(ctx, c.SnapshotKey) + if err != nil { + return err + } + return mount.WithTempMount(ctx, mounts, fn) +} diff --git a/pkg/tarutil/tarutil.go b/pkg/tarutil/tarutil.go index 74019144d49..5b7bb2cc96c 100644 --- a/pkg/tarutil/tarutil.go +++ b/pkg/tarutil/tarutil.go @@ -17,14 +17,28 @@ package tarutil import ( + "archive/tar" + "bufio" "fmt" + "io" "os" "os/exec" + "path/filepath" "strings" + "time" + "github.com/containerd/containerd/archive/tarheader" + cfs "github.com/containerd/continuity/fs" + "github.com/docker/docker/pkg/pools" + "github.com/moby/sys/sequential" "github.com/sirupsen/logrus" ) +const ( + paxSchilyXattr = "SCHILY.xattr." + securityCapabilityXattr = "security.capability" +) + // FindTarBinary returns a path to the tar binary and whether it is GNU tar. func FindTarBinary() (string, bool, error) { isGNU := func(exe string) bool { @@ -55,3 +69,118 @@ func FindTarBinary() (string, bool, error) { } return "", false, fmt.Errorf("failed to find `tar` binary") } + +type Tarballer struct { + Buffer *bufio.Writer + TarWriter *tar.Writer + seenFiles map[uint64]string +} + +// TODO: Add tar options for compression, whiteout files, chown ..etc + +func NewTarballer(writer io.Writer) *Tarballer { + return &Tarballer{ + Buffer: pools.BufioWriter32KPool.Get(nil), + TarWriter: tar.NewWriter(writer), + seenFiles: make(map[uint64]string), + } +} + +// TODO: Add unit test + +// Tar creates an archive from the directory at `root`. +// Mostly copied over from https://github.com/containerd/containerd/blob/main/archive/tar.go#L552 +func (tb *Tarballer) Tar(root string) error { + defer func() error { + pools.BufioWriter32KPool.Put(tb.Buffer) + return tb.TarWriter.Close() + }() + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("failed to Lstat: %w", err) + } + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + info, err := d.Info() + if err != nil { + return err + } + var link string + if info.Mode()&os.ModeSymlink != 0 { + link, err = os.Readlink(path) + if err != nil { + return err + } + } + header, err := FileInfoHeader(info, relPath, link) + if err != nil { + return err + } + inode, isHardlink := cfs.GetLinkInfo(info) + + if isHardlink { + if oldpath, ok := tb.seenFiles[inode]; ok { + header.Typeflag = tar.TypeLink + header.Linkname = oldpath + header.Size = 0 + } else { + tb.seenFiles[inode] = relPath + } + } + if capability, err := getxattr(path, securityCapabilityXattr); err != nil { + return fmt.Errorf("failed to get capabilities xattr: %w", err) + } else if len(capability) > 0 { + if header.PAXRecords == nil { + header.PAXRecords = map[string]string{} + } + header.PAXRecords[paxSchilyXattr+securityCapabilityXattr] = string(capability) + } + + // TODO: Currently not setting UID/GID. Handle remapping UID/GID in container to that of host + + err = tb.TarWriter.WriteHeader(header) + if err != nil { + return err + } + if info.Mode().IsRegular() && header.Size > 0 { + f, err := sequential.Open(path) + if err != nil { + return err + } + tb.Buffer.Reset(tb.TarWriter) + defer tb.Buffer.Reset(tb.TarWriter) + if _, err = io.Copy(tb.Buffer, f); err != nil { + return err + } + if err = f.Close(); err != nil { + return err + } + if err = tb.Buffer.Flush(); err != nil { + return err + } + } + return nil + }) +} + +func FileInfoHeader(info os.FileInfo, path, link string) (*tar.Header, error) { + header, err := tarheader.FileInfoHeaderNoLookups(info, link) + if err != nil { + return nil, err + } + header.Mode = int64(chmodTarEntry(os.FileMode(header.Mode))) + header.Format = tar.FormatPAX + header.ModTime = header.ModTime.Truncate(time.Second) + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + + name := filepath.ToSlash(path) + if info.IsDir() && !strings.HasSuffix(path, "/") { + name += "/" + } + header.Name = name + + return header, nil +} diff --git a/pkg/tarutil/tarutil_unix.go b/pkg/tarutil/tarutil_unix.go new file mode 100644 index 00000000000..e4b5d01ab48 --- /dev/null +++ b/pkg/tarutil/tarutil_unix.go @@ -0,0 +1,38 @@ +//go:build freebsd || linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tarutil + +import ( + "os" + + "github.com/containerd/continuity/sysx" + "golang.org/x/sys/unix" +) + +func chmodTarEntry(perm os.FileMode) os.FileMode { + return perm +} + +func getxattr(path, attr string) ([]byte, error) { + b, err := sysx.LGetxattr(path, attr) + if err == unix.ENOTSUP || err == sysx.ENODATA { + return nil, nil + } + return b, err +} diff --git a/pkg/tarutil/tarutil_windows.go b/pkg/tarutil/tarutil_windows.go new file mode 100644 index 00000000000..df02dc372c0 --- /dev/null +++ b/pkg/tarutil/tarutil_windows.go @@ -0,0 +1,31 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tarutil + +import "os" + +func chmodTarEntry(perm os.FileMode) os.FileMode { + perm &= 0755 + // Add the x bit: make everything +x from windows + perm |= 0111 + + return perm +} + +func getxattr(path, attr string) ([]byte, error) { + return nil, nil +}