feat(import): allow importing of container images

pkg/oci/oci.go

@@ -2,10 +2,18 @@ package oci
 
 import (
 	"context"
+	"os"
+	"path"
+	"runtime"
 
+	"github.com/klauspost/pgzip"
+	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/umoci"
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/layer"
 	"github.com/pkg/errors"
+	"stackerbuild.io/stacker/pkg/squashfs"
 )
 
 func LookupManifest(oci casext.Engine, tag string) (ispec.Manifest, error) {
@@ -76,3 +84,82 @@ func UpdateImageConfig(oci casext.Engine, name string, newConfig ispec.Image, ne
 
 	return desc, nil
 }
+
+func UnpackOne(ociDir string, bundlePath string, digest digest.Digest, isSquashfs bool) error {
+	if isSquashfs {
+		return squashfs.ExtractSingleSquash(
+			path.Join(ociDir, "blobs", "sha256", digest.Encoded()),
+			bundlePath, "overlay")
+	}
+
+	oci, err := umoci.OpenLayout(ociDir)
+	if err != nil {
+		return err
+	}
+	defer oci.Close()
+
+	compressed, err := oci.GetBlob(context.Background(), digest)
+	if err != nil {
+		return err
+	}
+	defer compressed.Close()
+
+	uncompressed, err := pgzip.NewReader(compressed)
+	if err != nil {
+		return err
+	}
+
+	return layer.UnpackLayer(bundlePath, uncompressed, nil)
+}
+
+// Unpack an image with "tag" from "ociLayout" into paths returned by "pathfunc"
+func Unpack(ociLayout, tag string, pathfunc func(digest.Digest) string) (int, error) {
+	oci, err := umoci.OpenLayout(ociLayout)
+	if err != nil {
+		return -1, err
+	}
+	defer oci.Close()
+
+	manifest, err := LookupManifest(oci, tag)
+	if err != nil {
+		return -1, err
+	}
+
+	pool := NewThreadPool(runtime.NumCPU())
+
+	for _, layer := range manifest.Layers {
+		digest := layer.Digest
+		contents := pathfunc(digest)
+		if squashfs.IsSquashfsMediaType(layer.MediaType) {
+			// don't really need to do this in parallel, but what
+			// the hell.
+			pool.Add(func(ctx context.Context) error {
+				return UnpackOne(ociLayout, contents, digest, true)
+			})
+		} else {
+			switch layer.MediaType {
+			case ispec.MediaTypeImageLayer:
+				fallthrough
+			case ispec.MediaTypeImageLayerGzip:
+				// don't extract things that have already been
+				// extracted
+				if _, err := os.Stat(contents); err == nil {
+					continue
+				}
+
+				// TODO: when the umoci API grows support for uid
+				// shifting, we can use the fancier features of context
+				// cancelling in the thread pool...
+				pool.Add(func(ctx context.Context) error {
+					return UnpackOne(ociLayout, contents, digest, false)
+				})
+			default:
+				return -1, errors.Errorf("unknown media type %s", layer.MediaType)
+			}
+		}
+	}
+
+	pool.DoneAddingJobs()
+
+	return len(manifest.Layers), pool.Run()
+}

pkg/overlay/pool.go → pkg/oci/pool.go

@@ -1,4 +1,4 @@
-package overlay
+package oci
 
 import (
 	"context"

pkg/overlay/overlay.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 	"stackerbuild.io/stacker/pkg/mount"
+	"stackerbuild.io/stacker/pkg/oci"
 	"stackerbuild.io/stacker/pkg/squashfs"
 	"stackerbuild.io/stacker/pkg/types"
 )
@@ -184,7 +185,7 @@ func (o *overlay) snapshot(source string, target string) error {
 			// We have done an unsquashfs of this atom
 			continue
 		}
-		if err := unpackOne(cacheDir, contents, digest, true); err != nil {
+		if err := oci.UnpackOne(cacheDir, contents, digest, true); err != nil {
 			return errors.Wrapf(err, "Failed mounting %#v", layer)
 		}
 	}

pkg/overlay/pack.go

@@ -8,11 +8,9 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"time"
 
-	"github.com/klauspost/pgzip"
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/umoci"
@@ -44,57 +42,21 @@ func overlayPath(rootfs string, d digest.Digest, subdirs ...string) string {
 
 func (o *overlay) Unpack(tag, name string) error {
 	cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci")
-	oci, err := umoci.OpenLayout(cacheDir)
-	if err != nil {
-		return err
+
+	pathfunc := func(digest digest.Digest) string {
+		return overlayPath(o.config.RootFSDir, digest, "overlay")
 	}
-	defer oci.Close()
 
-	manifest, err := stackeroci.LookupManifest(oci, tag)
+	_, err := stackeroci.Unpack(cacheDir, tag, pathfunc)
 	if err != nil {
 		return err
 	}
 
-	pool := NewThreadPool(runtime.NumCPU())
-
-	for _, layer := range manifest.Layers {
-		digest := layer.Digest
-		contents := overlayPath(o.config.RootFSDir, digest, "overlay")
-		if squashfs.IsSquashfsMediaType(layer.MediaType) {
-			// don't really need to do this in parallel, but what
-			// the hell.
-			pool.Add(func(ctx context.Context) error {
-				return unpackOne(cacheDir, contents, digest, true)
-			})
-		} else {
-			switch layer.MediaType {
-			case ispec.MediaTypeImageLayer:
-				fallthrough
-			case ispec.MediaTypeImageLayerGzip:
-				// don't extract things that have already been
-				// extracted
-				if _, err := os.Stat(contents); err == nil {
-					continue
-				}
-
-				// TODO: when the umoci API grows support for uid
-				// shifting, we can use the fancier features of context
-				// cancelling in the thread pool...
-				pool.Add(func(ctx context.Context) error {
-					return unpackOne(cacheDir, contents, digest, false)
-				})
-			default:
-				return errors.Errorf("unknown media type %s", layer.MediaType)
-			}
-		}
-	}
-
-	pool.DoneAddingJobs()
-
-	err = pool.Run()
+	oci, err := umoci.OpenLayout(cacheDir)
 	if err != nil {
 		return err
 	}
+	defer oci.Close()
 
 	err = o.Create(name)
 	if err != nil {
@@ -654,30 +616,3 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L
 
 	return ovl.write(config, name)
 }
-
-func unpackOne(ociDir string, bundlePath string, digest digest.Digest, isSquashfs bool) error {
-	if isSquashfs {
-		return squashfs.ExtractSingleSquash(
-			path.Join(ociDir, "blobs", "sha256", digest.Encoded()),
-			bundlePath, "overlay")
-	}
-
-	oci, err := umoci.OpenLayout(ociDir)
-	if err != nil {
-		return err
-	}
-	defer oci.Close()
-
-	compressed, err := oci.GetBlob(context.Background(), digest)
-	if err != nil {
-		return err
-	}
-	defer compressed.Close()
-
-	uncompressed, err := pgzip.NewReader(compressed)
-	if err != nil {
-		return err
-	}
-
-	return layer.UnpackLayer(bundlePath, uncompressed, nil)
-}

pkg/stacker/import.go

@@ -13,6 +13,7 @@ import (
 	"github.com/vbatts/go-mtree"
 	"stackerbuild.io/stacker/pkg/lib"
 	"stackerbuild.io/stacker/pkg/log"
+	"stackerbuild.io/stacker/pkg/oci"
 	"stackerbuild.io/stacker/pkg/types"
 )
 
@@ -271,6 +272,38 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st
 		}
 
 		return p, nil
+	} else if url.Scheme == "docker" {
+		if idest != "" && idest[len(idest)-1:] != "/" {
+			return "", errors.Errorf("The destination path must be directory: %s", idest)
+		}
+
+		is := types.ImageSource{Type: "docker", Url: i}
+		if err := importContainersImage(is, c, false); err != nil {
+			return "", err
+		}
+
+		tag, err := is.ParseTag()
+		if err != nil {
+			return "", err
+		}
+
+		pathfunc := func(digest digest.Digest) string {
+			_ = os.Remove(cache)
+			return cache
+		}
+
+		ociDir := path.Join(c.StackerDir, "layer-bases", "oci")
+
+		n, err := oci.Unpack(ociDir, tag, pathfunc)
+		if err != nil {
+			return "", err
+		}
+
+		if n > 1 {
+			return "", errors.Errorf("Currently supporting single-layer container image imports")
+		}
+
+		return cache, nil
 	}
 
 	return "", errors.Errorf("unsupported url scheme %s", i)

test/import.bats

@@ -393,3 +393,33 @@ EOF
 
     stacker build
 }
+
+@test "importing container images" {
+    cat > stacker.yaml <<EOF
+cimg-import:
+    from:
+      type: oci
+      url: $CENTOS_OCI
+    import:
+    - path: docker://alpine:edge
+      dest: /
+    run: |
+      [ -d /var/lib/apk ]
+
+compose-img:
+    from:
+      type: oci
+      url: $CENTOS_OCI
+    import:
+    - path: docker://ghcr.io/homebrew/core/openssl/1.1:1.1.1k
+      dest: /
+    - path: docker://ghcr.io/homebrew/core/curl:8.0.1
+      dest: /
+    - path: docker://ghcr.io/homebrew/core/ca-certificates:2022-10-11
+      dest: /
+    run: |
+      [ -f /curl/8.0.1/bin/curl ]
+      [ -f /ca-certificates/2022-10-11/share/ca-certificates/cacert.pem ]
+EOF
+    stacker build
+}