Add APKFS

Create an fs.FS implementation that lets us see into
an APK file.

* Add tests for apkfs
* Add test file
* Move ExpandApk into its own package so that it may
 be referenced from both the apk and fs packages

pkg/apk/implementation.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/chainguard-dev/go-apk/pkg/expandapk"
 	"io"
 	"io/fs"
 	"net/http"
@@ -261,12 +262,12 @@ func (a *APK) InitDB(ctx context.Context, alpineVersions ...string) error {
 
 	// add scripts.tar with nothing in it
 	scriptsTarPerms := 0o644
-	tarfile, err := a.fs.OpenFile(scriptsFilePath, os.O_CREATE|os.O_WRONLY, fs.FileMode(scriptsTarPerms))
+	TarFile, err := a.fs.OpenFile(scriptsFilePath, os.O_CREATE|os.O_WRONLY, fs.FileMode(scriptsTarPerms))
 	if err != nil {
 		return fmt.Errorf("could not create tarball file '%s', got error '%w'", scriptsFilePath, err)
 	}
-	defer tarfile.Close()
-	tarWriter := tar.NewWriter(tarfile)
+	defer TarFile.Close()
+	tarWriter := tar.NewWriter(TarFile)
 	defer tarWriter.Close()
 
 	// nothing to add to it; scripts.tar should be empty
@@ -492,7 +493,7 @@ func (a *APK) FixateWorld(ctx context.Context, sourceDateEpoch *time.Time) error
 	g, gctx := errgroup.WithContext(ctx)
 	g.SetLimit(jobs + 1)
 
-	expanded := make([]*APKExpanded, len(allpkgs))
+	expanded := make([]*expandapk.APKExpanded, len(allpkgs))
 
 	// A slice of pseudo-promises that get closed when expanded[i] is ready.
 	done := make([]chan struct{}, len(allpkgs))
@@ -637,7 +638,7 @@ func (a *APK) fetchAlpineKeys(ctx context.Context, alpineVersions []string) erro
 	return nil
 }
 
-func (a *APK) cachePackage(ctx context.Context, pkg *repository.RepositoryPackage, exp *APKExpanded, cacheDir string) (*APKExpanded, error) {
+func (a *APK) cachePackage(ctx context.Context, pkg *repository.RepositoryPackage, exp *expandapk.APKExpanded, cacheDir string) (*expandapk.APKExpanded, error) {
 	_, span := otel.Tracer("go-apk").Start(ctx, "cachePackage", trace.WithAttributes(attribute.String("package", pkg.Name)))
 	defer span.End()
 
@@ -672,15 +673,15 @@ func (a *APK) cachePackage(ctx context.Context, pkg *repository.RepositoryPackag
 	exp.PackageFile = datDst
 
 	tarDst := strings.TrimSuffix(exp.PackageFile, ".gz")
-	if err := os.Rename(exp.tarFile, tarDst); err != nil {
+	if err := os.Rename(exp.TarFile, tarDst); err != nil {
 		return nil, fmt.Errorf("renaming control file: %w", err)
 	}
-	exp.tarFile = tarDst
+	exp.TarFile = tarDst
 
 	return exp, nil
 }
 
-func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPackage, cacheDir string) (*APKExpanded, error) {
+func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPackage, cacheDir string) (*expandapk.APKExpanded, error) {
 	_, span := otel.Tracer("go-apk").Start(ctx, "cachedPackage", trace.WithAttributes(attribute.String("package", pkg.Name)))
 	defer span.End()
 
@@ -696,7 +697,7 @@ func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPacka
 
 	pkgHexSum := hex.EncodeToString(checksum)
 
-	exp := APKExpanded{}
+	exp := expandapk.APKExpanded{}
 
 	ctl := filepath.Join(cacheDir, pkgHexSum+".ctl.tar.gz")
 	cf, err := os.Stat(ctl)
@@ -739,16 +740,16 @@ func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPacka
 		return nil, err
 	}
 
-	exp.tarFile = strings.TrimSuffix(exp.PackageFile, ".gz")
-	exp.tarfs, err = tarfs.New(exp.PackageData)
+	exp.TarFile = strings.TrimSuffix(exp.PackageFile, ".gz")
+	exp.TarFS, err = tarfs.New(exp.PackageData)
 	if err != nil {
 		return nil, err
 	}
 
 	return &exp, nil
 }
 
-func (a *APK) expandPackage(ctx context.Context, pkg *repository.RepositoryPackage) (*APKExpanded, error) {
+func (a *APK) expandPackage(ctx context.Context, pkg *repository.RepositoryPackage) (*expandapk.APKExpanded, error) {
 	ctx, span := otel.Tracer("go-apk").Start(ctx, "expandPackage", trace.WithAttributes(attribute.String("package", pkg.Name)))
 	defer span.End()
 
@@ -779,7 +780,7 @@ func (a *APK) expandPackage(ctx context.Context, pkg *repository.RepositoryPacka
 	}
 	defer rc.Close()
 
-	exp, err := ExpandApk(ctx, rc, cacheDir)
+	exp, err := expandapk.ExpandApk(ctx, rc, cacheDir)
 	if err != nil {
 		return nil, fmt.Errorf("expanding %s: %w", pkg.Name, err)
 	}
@@ -868,7 +869,7 @@ type writeHeaderer interface {
 }
 
 // installPackage installs a single package and updates installed db.
-func (a *APK) installPackage(ctx context.Context, pkg *repository.RepositoryPackage, expanded *APKExpanded, sourceDateEpoch *time.Time) error {
+func (a *APK) installPackage(ctx context.Context, pkg *repository.RepositoryPackage, expanded *expandapk.APKExpanded, sourceDateEpoch *time.Time) error {
 	a.logger.Debugf("installing %s (%s)", pkg.Name, pkg.Version)
 
 	ctx, span := otel.Tracer("go-apk").Start(ctx, "installPackage", trace.WithAttributes(attribute.String("package", pkg.Name)))
@@ -882,7 +883,7 @@ func (a *APK) installPackage(ctx context.Context, pkg *repository.RepositoryPack
 	)
 
 	if wh, ok := a.fs.(writeHeaderer); ok {
-		installedFiles, err = a.lazilyInstallAPKFiles(ctx, wh, expanded.tarfs, pkg.Package)
+		installedFiles, err = a.lazilyInstallAPKFiles(ctx, wh, expanded.TarFS, pkg.Package)
 		if err != nil {
 			return fmt.Errorf("unable to install files for pkg %s: %w", pkg.Name, err)
 		}

pkg/expandapk/const.go

@@ -0,0 +1,5 @@
+package expandapk
+
+const (
+	paxRecordsChecksumKey = "APK-TOOLS.checksum.SHA1"
+)

pkg/apk/expandapk.go → pkg/expandapk/expandapk.go

@@ -4,7 +4,7 @@
 // this duplicate file goes away!
 
 //nolint:all
-package apk
+package expandapk
 
 import (
 	"archive/tar"
@@ -50,10 +50,10 @@ type APKExpanded struct {
 	PackageFile string
 
 	// The package data filename in .tar format.
-	tarFile string
+	TarFile string
 
-	// Exposes tarFile as an indexed FS implementation.
-	tarfs *tarfs.FS
+	// Exposes TarFile as an indexed FS implementation.
+	TarFS *tarfs.FS
 
 	ControlHash []byte
 	PackageHash []byte
@@ -62,7 +62,7 @@ type APKExpanded struct {
 const meg = 1 << 20
 
 func (a *APKExpanded) PackageData() (io.ReadSeekCloser, error) {
-	uf, err := os.Open(a.tarFile)
+	uf, err := os.Open(a.TarFile)
 	if err == nil {
 		return uf, nil
 	} else if !os.IsNotExist(err) {
@@ -87,9 +87,9 @@ func (a *APKExpanded) PackageData() (io.ReadSeekCloser, error) {
 		return nil, fmt.Errorf("parsing %q: %w", a.PackageFile, err)
 	}
 
-	uf, err = os.Create(a.tarFile)
+	uf, err = os.Create(a.TarFile)
 	if err != nil {
-		return nil, fmt.Errorf("opening tar file %q: %w", a.tarFile, err)
+		return nil, fmt.Errorf("opening tar file %q: %w", a.TarFile, err)
 	}
 
 	buf := make([]byte, bufSize)
@@ -98,10 +98,10 @@ func (a *APKExpanded) PackageData() (io.ReadSeekCloser, error) {
 	}
 
 	if err := uf.Close(); err != nil {
-		return nil, fmt.Errorf("closing %q: %w", a.tarFile, err)
+		return nil, fmt.Errorf("closing %q: %w", a.TarFile, err)
 	}
 
-	return os.Open(a.tarFile)
+	return os.Open(a.TarFile)
 }
 
 func (a *APKExpanded) APK() (io.ReadCloser, error) {
@@ -422,12 +422,12 @@ func ExpandApk(ctx context.Context, source io.Reader, cacheDir string) (*APKExpa
 		expanded.SignatureFile = gzipStreams[0]
 	}
 
-	expanded.tarFile = strings.TrimSuffix(expanded.PackageFile, ".gz")
+	expanded.TarFile = strings.TrimSuffix(expanded.PackageFile, ".gz")
 
 	// TODO: We could overlap this with checkSums.
-	expanded.tarfs, err = tarfs.New(expanded.PackageData)
+	expanded.TarFS, err = tarfs.New(expanded.PackageData)
 	if err != nil {
-		return nil, fmt.Errorf("indexing %q: %w", expanded.tarFile, err)
+		return nil, fmt.Errorf("indexing %q: %w", expanded.TarFile, err)
 	}
 
 	return &expanded, nil

pkg/expandapk/utility.go

@@ -0,0 +1,41 @@
+package expandapk
+
+import (
+	"archive/tar"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"strings"
+)
+
+func checksumFromHeader(header *tar.Header) ([]byte, error) {
+	pax := header.PAXRecords
+	if pax == nil {
+		return nil, nil
+	}
+
+	hexsum, ok := pax[paxRecordsChecksumKey]
+	if !ok {
+		return nil, nil
+	}
+
+	if strings.HasPrefix(hexsum, "Q1") {
+		// This is nonstandard but something we did at one point, handle it.
+		// In other contexts, this Q1 prefix means "this is sha1 not md5".
+		b64 := strings.TrimPrefix(hexsum, "Q1")
+
+		checksum, err := base64.StdEncoding.DecodeString(b64)
+		if err != nil {
+			return nil, fmt.Errorf("decoding base64 checksum from header for %q: %w", header.Name, err)
+		}
+
+		return checksum, nil
+	}
+
+	checksum, err := hex.DecodeString(hexsum)
+	if err != nil {
+		return nil, fmt.Errorf("decoding hex checksum from header for %q: %w", header.Name, err)
+	}
+
+	return checksum, nil
+}