diff --git a/pkg/apk/implementation.go b/pkg/apk/implementation.go index 106cd0a..f08b846 100644 --- a/pkg/apk/implementation.go +++ b/pkg/apk/implementation.go @@ -32,6 +32,8 @@ import ( "strings" "time" + "github.com/chainguard-dev/go-apk/pkg/expandapk" + "gitlab.alpinelinux.org/alpine/go/repository" "go.lsp.dev/uri" "go.opentelemetry.io/otel" @@ -261,12 +263,12 @@ func (a *APK) InitDB(ctx context.Context, alpineVersions ...string) error { // add scripts.tar with nothing in it scriptsTarPerms := 0o644 - tarfile, err := a.fs.OpenFile(scriptsFilePath, os.O_CREATE|os.O_WRONLY, fs.FileMode(scriptsTarPerms)) + TarFile, err := a.fs.OpenFile(scriptsFilePath, os.O_CREATE|os.O_WRONLY, fs.FileMode(scriptsTarPerms)) if err != nil { return fmt.Errorf("could not create tarball file '%s', got error '%w'", scriptsFilePath, err) } - defer tarfile.Close() - tarWriter := tar.NewWriter(tarfile) + defer TarFile.Close() + tarWriter := tar.NewWriter(TarFile) defer tarWriter.Close() // nothing to add to it; scripts.tar should be empty @@ -492,7 +494,7 @@ func (a *APK) FixateWorld(ctx context.Context, sourceDateEpoch *time.Time) error g, gctx := errgroup.WithContext(ctx) g.SetLimit(jobs + 1) - expanded := make([]*APKExpanded, len(allpkgs)) + expanded := make([]*expandapk.APKExpanded, len(allpkgs)) // A slice of pseudo-promises that get closed when expanded[i] is ready. done := make([]chan struct{}, len(allpkgs)) @@ -637,7 +639,7 @@ func (a *APK) fetchAlpineKeys(ctx context.Context, alpineVersions []string) erro return nil } -func (a *APK) cachePackage(ctx context.Context, pkg *repository.RepositoryPackage, exp *APKExpanded, cacheDir string) (*APKExpanded, error) { +func (a *APK) cachePackage(ctx context.Context, pkg *repository.RepositoryPackage, exp *expandapk.APKExpanded, cacheDir string) (*expandapk.APKExpanded, error) { _, span := otel.Tracer("go-apk").Start(ctx, "cachePackage", trace.WithAttributes(attribute.String("package", pkg.Name))) defer span.End() @@ -672,15 +674,15 @@ func (a *APK) cachePackage(ctx context.Context, pkg *repository.RepositoryPackag exp.PackageFile = datDst tarDst := strings.TrimSuffix(exp.PackageFile, ".gz") - if err := os.Rename(exp.tarFile, tarDst); err != nil { + if err := os.Rename(exp.TarFile, tarDst); err != nil { return nil, fmt.Errorf("renaming control file: %w", err) } - exp.tarFile = tarDst + exp.TarFile = tarDst return exp, nil } -func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPackage, cacheDir string) (*APKExpanded, error) { +func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPackage, cacheDir string) (*expandapk.APKExpanded, error) { _, span := otel.Tracer("go-apk").Start(ctx, "cachedPackage", trace.WithAttributes(attribute.String("package", pkg.Name))) defer span.End() @@ -696,7 +698,7 @@ func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPacka pkgHexSum := hex.EncodeToString(checksum) - exp := APKExpanded{} + exp := expandapk.APKExpanded{} ctl := filepath.Join(cacheDir, pkgHexSum+".ctl.tar.gz") cf, err := os.Stat(ctl) @@ -739,8 +741,8 @@ func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPacka return nil, err } - exp.tarFile = strings.TrimSuffix(exp.PackageFile, ".gz") - exp.tarfs, err = tarfs.New(exp.PackageData) + exp.TarFile = strings.TrimSuffix(exp.PackageFile, ".gz") + exp.TarFS, err = tarfs.New(exp.PackageData) if err != nil { return nil, err } @@ -748,7 +750,7 @@ func (a *APK) cachedPackage(ctx context.Context, pkg *repository.RepositoryPacka return &exp, nil } -func (a *APK) expandPackage(ctx context.Context, pkg *repository.RepositoryPackage) (*APKExpanded, error) { +func (a *APK) expandPackage(ctx context.Context, pkg *repository.RepositoryPackage) (*expandapk.APKExpanded, error) { ctx, span := otel.Tracer("go-apk").Start(ctx, "expandPackage", trace.WithAttributes(attribute.String("package", pkg.Name))) defer span.End() @@ -779,7 +781,7 @@ func (a *APK) expandPackage(ctx context.Context, pkg *repository.RepositoryPacka } defer rc.Close() - exp, err := ExpandApk(ctx, rc, cacheDir) + exp, err := expandapk.ExpandApk(ctx, rc, cacheDir) if err != nil { return nil, fmt.Errorf("expanding %s: %w", pkg.Name, err) } @@ -868,7 +870,7 @@ type writeHeaderer interface { } // installPackage installs a single package and updates installed db. -func (a *APK) installPackage(ctx context.Context, pkg *repository.RepositoryPackage, expanded *APKExpanded, sourceDateEpoch *time.Time) error { +func (a *APK) installPackage(ctx context.Context, pkg *repository.RepositoryPackage, expanded *expandapk.APKExpanded, sourceDateEpoch *time.Time) error { a.logger.Debugf("installing %s (%s)", pkg.Name, pkg.Version) ctx, span := otel.Tracer("go-apk").Start(ctx, "installPackage", trace.WithAttributes(attribute.String("package", pkg.Name))) @@ -882,7 +884,7 @@ func (a *APK) installPackage(ctx context.Context, pkg *repository.RepositoryPack ) if wh, ok := a.fs.(writeHeaderer); ok { - installedFiles, err = a.lazilyInstallAPKFiles(ctx, wh, expanded.tarfs, pkg.Package) + installedFiles, err = a.lazilyInstallAPKFiles(ctx, wh, expanded.TarFS, pkg.Package) if err != nil { return fmt.Errorf("unable to install files for pkg %s: %w", pkg.Name, err) } diff --git a/pkg/expandapk/const.go b/pkg/expandapk/const.go new file mode 100644 index 0000000..20c3f06 --- /dev/null +++ b/pkg/expandapk/const.go @@ -0,0 +1,5 @@ +package expandapk + +const ( + paxRecordsChecksumKey = "APK-TOOLS.checksum.SHA1" +) diff --git a/pkg/apk/expandapk.go b/pkg/expandapk/expandapk.go similarity index 95% rename from pkg/apk/expandapk.go rename to pkg/expandapk/expandapk.go index a08974d..c4739e6 100644 --- a/pkg/apk/expandapk.go +++ b/pkg/expandapk/expandapk.go @@ -4,7 +4,7 @@ // this duplicate file goes away! //nolint:all -package apk +package expandapk import ( "archive/tar" @@ -50,10 +50,10 @@ type APKExpanded struct { PackageFile string // The package data filename in .tar format. - tarFile string + TarFile string - // Exposes tarFile as an indexed FS implementation. - tarfs *tarfs.FS + // Exposes TarFile as an indexed FS implementation. + TarFS *tarfs.FS ControlHash []byte PackageHash []byte @@ -62,7 +62,7 @@ type APKExpanded struct { const meg = 1 << 20 func (a *APKExpanded) PackageData() (io.ReadSeekCloser, error) { - uf, err := os.Open(a.tarFile) + uf, err := os.Open(a.TarFile) if err == nil { return uf, nil } else if !os.IsNotExist(err) { @@ -87,9 +87,9 @@ func (a *APKExpanded) PackageData() (io.ReadSeekCloser, error) { return nil, fmt.Errorf("parsing %q: %w", a.PackageFile, err) } - uf, err = os.Create(a.tarFile) + uf, err = os.Create(a.TarFile) if err != nil { - return nil, fmt.Errorf("opening tar file %q: %w", a.tarFile, err) + return nil, fmt.Errorf("opening tar file %q: %w", a.TarFile, err) } buf := make([]byte, bufSize) @@ -98,10 +98,10 @@ func (a *APKExpanded) PackageData() (io.ReadSeekCloser, error) { } if err := uf.Close(); err != nil { - return nil, fmt.Errorf("closing %q: %w", a.tarFile, err) + return nil, fmt.Errorf("closing %q: %w", a.TarFile, err) } - return os.Open(a.tarFile) + return os.Open(a.TarFile) } func (a *APKExpanded) APK() (io.ReadCloser, error) { @@ -422,12 +422,12 @@ func ExpandApk(ctx context.Context, source io.Reader, cacheDir string) (*APKExpa expanded.SignatureFile = gzipStreams[0] } - expanded.tarFile = strings.TrimSuffix(expanded.PackageFile, ".gz") + expanded.TarFile = strings.TrimSuffix(expanded.PackageFile, ".gz") // TODO: We could overlap this with checkSums. - expanded.tarfs, err = tarfs.New(expanded.PackageData) + expanded.TarFS, err = tarfs.New(expanded.PackageData) if err != nil { - return nil, fmt.Errorf("indexing %q: %w", expanded.tarFile, err) + return nil, fmt.Errorf("indexing %q: %w", expanded.TarFile, err) } return &expanded, nil diff --git a/pkg/expandapk/utility.go b/pkg/expandapk/utility.go new file mode 100644 index 0000000..d37ce7c --- /dev/null +++ b/pkg/expandapk/utility.go @@ -0,0 +1,41 @@ +package expandapk + +import ( + "archive/tar" + "encoding/base64" + "encoding/hex" + "fmt" + "strings" +) + +func checksumFromHeader(header *tar.Header) ([]byte, error) { + pax := header.PAXRecords + if pax == nil { + return nil, nil + } + + hexsum, ok := pax[paxRecordsChecksumKey] + if !ok { + return nil, nil + } + + if strings.HasPrefix(hexsum, "Q1") { + // This is nonstandard but something we did at one point, handle it. + // In other contexts, this Q1 prefix means "this is sha1 not md5". + b64 := strings.TrimPrefix(hexsum, "Q1") + + checksum, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, fmt.Errorf("decoding base64 checksum from header for %q: %w", header.Name, err) + } + + return checksum, nil + } + + checksum, err := hex.DecodeString(hexsum) + if err != nil { + return nil, fmt.Errorf("decoding hex checksum from header for %q: %w", header.Name, err) + } + + return checksum, nil +} diff --git a/pkg/fs/apkfs.go b/pkg/fs/apkfs.go new file mode 100644 index 0000000..5747669 --- /dev/null +++ b/pkg/fs/apkfs.go @@ -0,0 +1,222 @@ +package fs + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "io/fs" + "os" + "strings" + "time" + + "github.com/chainguard-dev/go-apk/pkg/expandapk" +) + +type APKFS struct { + path string + files map[string]*apkFSFile + ctx context.Context + cache *expandapk.APKExpanded +} + +func (a *APKFS) acquireCache() (*expandapk.APKExpanded, error) { + if a.cache == nil { + file, err := os.Open(a.path) + if err != nil { + return nil, err + } + defer file.Close() + a.cache, err = expandapk.ExpandApk(a.ctx, file, "/tmp/") + if err != nil { + return nil, err + } + } + return a.cache, nil +} +func (a *APKFS) getTarReader() (*os.File, *tar.Reader, error) { + file, err := os.Open(a.cache.PackageFile) + + if err != nil { + return nil, nil, err + } + gzipStream, err := gzip.NewReader(file) + if err != nil { + return nil, nil, err + } + tr := tar.NewReader(gzipStream) + return file, tr, nil +} +func NewAPKFS(ctx context.Context, archive string) (*APKFS, error) { + result := APKFS{archive, make(map[string]*apkFSFile), ctx, nil} + + file, err := os.Open(archive) + if err != nil { + return nil, err + } + defer file.Close() + + apkExpanded, err := expandapk.ExpandApk(ctx, file, "") + if err != nil { + return nil, err + } + defer apkExpanded.Close() + gzipFile, err := os.Open(apkExpanded.PackageFile) + if err != nil { + return nil, err + } + defer gzipFile.Close() + gzipStream, err := gzip.NewReader(gzipFile) + if err != nil { + return nil, err + } + + reader := tar.NewReader(gzipStream) + for { + header, err := reader.Next() + + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + currentEntry := apkFSFile{mode: fs.FileMode(header.Mode), name: "/" + header.Name, + uid: header.Uid, gid: header.Gid, + size: uint64(header.Size), modTime: header.ModTime, + createTime: header.ChangeTime, + linkTarget: header.Linkname, isDir: header.Typeflag == tar.TypeDir, + xattrs: make(map[string][]byte)} + for k, v := range header.PAXRecords { + // If this trend continues then it would be wise to move the + // named constant for this into a place accessible from here + attrname := strings.TrimPrefix(k, "SCHILY.xattr.") + if len(attrname) != len(k) { + currentEntry.xattrs[attrname] = []byte(v) + } + } + result.files["/"+header.Name] = ¤tEntry + } + result.cache, err = result.acquireCache() + if err != nil { + return nil, err + } + return &result, nil +} +func (a *APKFS) Close() error { + if a.cache == nil { + return nil + } + return a.cache.Close() +} + +type apkFSFile struct { + mode fs.FileMode + uid, gid int + name string + size uint64 + modTime time.Time + createTime time.Time + linkTarget string + linkCount int + xattrs map[string][]byte + isDir bool + fs *APKFS + // The following fields are not initialized in the copies held + // by the apkfs object. + fileDescriptor io.Closer + tarReader *tar.Reader +} + +// Users of the api should not handle the copies referred to in the +// filesystem object. +func (a *apkFSFile) acquireCopy() *apkFSFile { + return &apkFSFile{mode: a.mode, uid: a.uid, gid: a.gid, size: a.size, + name: a.name, modTime: a.modTime, createTime: a.createTime, linkTarget: a.linkTarget, + linkCount: a.linkCount, xattrs: a.xattrs, isDir: a.isDir, fs: a.fs, + fileDescriptor: nil, tarReader: nil} +} +func (a *apkFSFile) seekTo(reader *tar.Reader) error { + for { + header, err := reader.Next() + if err == os.ErrNotExist { + break + } else if err != nil { + return err + } + if header.Name == a.name[1:] { + return nil + } + } + return os.ErrNotExist +} + +func (a *apkFSFile) Read(b []byte) (int, error) { + return a.tarReader.Read(b) +} +func (a *apkFSFile) Stat() (fs.FileInfo, error) { + return &apkFSFileInfo{file: a, name: a.name}, nil +} +func (a *apkFSFile) Close() error { + if a.fileDescriptor != nil { + err := a.fileDescriptor.Close() + if err != nil { + return err + } + } + + return nil +} + +func (a *APKFS) Stat(path string) (fs.FileInfo, error) { + file, ok := a.files[path] + if !ok { + return nil, os.ErrNotExist + } + return &apkFSFileInfo{file: file, name: file.name[strings.LastIndex(file.name, "/"):]}, nil +} + +func (a *APKFS) Open(path string) (fs.File, error) { + file, ok := a.files[path] + if !ok { + return nil, os.ErrNotExist + } + fileCopy := file.acquireCopy() + var err error + fileCopy.fileDescriptor, fileCopy.tarReader, err = a.getTarReader() + if err != nil { + return nil, err + } + err = fileCopy.seekTo(fileCopy.tarReader) + if err != nil { + return nil, err + } + return fileCopy, nil +} + +type apkFSFileInfo struct { + file *apkFSFile + name string +} + +func (a *apkFSFileInfo) Name() string { + return a.file.name[strings.LastIndex(a.name, "/")+1:] +} +func (a *apkFSFileInfo) Size() int64 { + return int64(a.file.size) +} +func (a *apkFSFileInfo) Mode() fs.FileMode { + return a.file.mode +} +func (a *apkFSFileInfo) ModTime() time.Time { + return a.file.modTime +} +func (a *apkFSFileInfo) IsDir() bool { + return a.file.isDir +} +func (a *apkFSFileInfo) Sys() any { + return &tar.Header{ + Mode: int64(a.file.mode), + Uid: a.file.uid, + Gid: a.file.gid, + } +} diff --git a/pkg/fs/apkfs_test.go b/pkg/fs/apkfs_test.go new file mode 100644 index 0000000..777b734 --- /dev/null +++ b/pkg/fs/apkfs_test.go @@ -0,0 +1,49 @@ +package fs + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadAPKFile(t *testing.T) { + t.Run("stat", func(t *testing.T) { + apkfs, err := NewAPKFS(context.Background(), "testdata/hello-2.12-r0.apk") + require.Nil(t, err) + defer apkfs.Close() + require.NotNil(t, apkfs) + file, err := apkfs.Open("/usr/bin/hello") + require.Nil(t, err) + defer file.Close() + info, err := file.Stat() + require.Nil(t, err) + require.Equal(t, info.Name(), "hello") + }) + t.Run("read", func(t *testing.T) { + apkfs, err := NewAPKFS(context.Background(), "testdata/hello-2.12-r0.apk") + require.Nil(t, err) + defer apkfs.Close() + require.NotNil(t, apkfs) + file, err := apkfs.Open("/usr/bin/hello") + require.Nil(t, err) + defer file.Close() + info, err := file.Stat() + require.Nil(t, err) + buffer := make([]byte, 4096) + var readSoFar int64 + for { + readThisTime, err := file.Read(buffer) + if err != io.EOF { + require.Nil(t, err) + } + readSoFar += int64(readThisTime) + if readThisTime == 0 { + break + } + } + require.Equal(t, info.Size(), readSoFar) + require.Equal(t, info.Name(), "hello") + }) +} diff --git a/pkg/fs/testdata/hello-2.12-r0.apk b/pkg/fs/testdata/hello-2.12-r0.apk new file mode 100644 index 0000000..55087f1 Binary files /dev/null and b/pkg/fs/testdata/hello-2.12-r0.apk differ