diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index c361adba..3c2cb540 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -5,13 +5,19 @@ import ( "io" "os" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/distrofactory" "github.com/osbuild/images/pkg/reporegistry" + "github.com/osbuild/images/pkg/rpmmd" ) var ( GetOneImage = getOneImage + Run = run ) +type ManifestGenerator = manifestGenerator + func MockOsArgs(new []string) (restore func()) { saved := os.Args os.Args = append([]string{"argv0"}, new...) @@ -49,6 +55,32 @@ func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore } } -var ( - Run = run -) +func MockDistrofactoryNew(f func() *distrofactory.Factory) (restore func()) { + saved := distrofactoryNew + distrofactoryNew = f + return func() { + distrofactoryNew = saved + } +} + +func MockDepsolve() (restore func()) { + saved := depsolve + // XXX: move to images + depsolve = func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSet := range packageSets { + depsolvedSets[name] = []rpmmd.PackageSpec{ + { + Name: pkgSet[0].Include[0], + Checksum: "sha256:01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + }, + } + //repoSets[name] = res.Repos + } + return depsolvedSets, repoSets, nil + } + return func() { + depsolve = saved + } +} diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 58a89aaf..ff96253b 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -10,12 +10,15 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) +var distrofactoryNew = distrofactory.NewDefault + func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) { - fac := distrofactory.NewDefault() + fac := distrofactoryNew() repos, err := newRepoRegistry(dataDir) if err != nil { return nil, err } + return imagefilter.New(fac, repos) } diff --git a/cmd/image-builder/list.go b/cmd/image-builder/list.go index 81e357ee..9bf06d94 100644 --- a/cmd/image-builder/list.go +++ b/cmd/image-builder/list.go @@ -4,8 +4,8 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) -func listImages(output string, filterExprs []string, opts *cmdlineOpts) error { - imageFilter, err := newImageFilterDefault(opts.dataDir) +func listImages(dataDir, output string, filterExprs []string) error { + imageFilter, err := newImageFilterDefault(dataDir) if err != nil { return err } @@ -19,7 +19,7 @@ func listImages(output string, filterExprs []string, opts *cmdlineOpts) error { if err != nil { return err } - if err := fmter.Output(opts.out, filteredResult); err != nil { + if err := fmter.Output(osStdout, filteredResult); err != nil { return err } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 96fe0f98..86f545e8 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -7,6 +7,8 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/osbuild/images/pkg/arch" ) var ( @@ -14,11 +16,6 @@ var ( osStderr io.Writer = os.Stderr ) -type cmdlineOpts struct { - dataDir string - out io.Writer -} - func cmdListImages(cmd *cobra.Command, args []string) error { filter, err := cmd.Flags().GetStringArray("filter") if err != nil { @@ -33,11 +30,25 @@ func cmdListImages(cmd *cobra.Command, args []string) error { return err } - opts := &cmdlineOpts{ - out: osStdout, - dataDir: dataDir, + return listImages(dataDir, output, filter) +} + +func cmdManifest(cmd *cobra.Command, args []string) error { + dataDir, err := cmd.Flags().GetString("datadir") + if err != nil { + return err + } + + mg := &manifestGenerator{dataDir, osStdout} + distro := args[0] + imgType := args[1] + var archStr string + if len(args) > 2 { + archStr = args[2] + } else { + archStr = arch.Current().String() } - return listImages(output, filter, opts) + return mg.Generate(distro, imgType, archStr) } func run() error { @@ -69,6 +80,17 @@ operating sytsems like centos and RHEL with easy customizations support.`, listImagesCmd.Flags().String("output", "", "Output in a specific format (text, json)") rootCmd.AddCommand(listImagesCmd) + manifestCmd := &cobra.Command{ + Use: "manifest []", + Short: "Build manifest for the given distro/image-type, e.g. centos-9 qcow2", + RunE: cmdManifest, + SilenceUsage: true, + Args: cobra.MinimumNArgs(2), + Hidden: true, + } + // XXX: add blueprint switch + rootCmd.AddCommand(manifestCmd) + return rootCmd.Execute() } diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index 2d949e4d..a16e01a0 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -3,6 +3,7 @@ package main_test import ( "bytes" "encoding/json" + "os" "testing" "github.com/sirupsen/logrus" @@ -108,3 +109,35 @@ func TestListImagesOverrideDatadir(t *testing.T) { err := main.Run() assert.EqualError(t, err, `no repositories found in the given paths: [/this/path/does/not/exist]`) } + +func hasDepsolveDnf() bool { + // XXX: expose images/pkg/depsolve:findDepsolveDnf() + _, err := os.Stat("/usr/libexec/osbuild-depsolve-dnf") + return err == nil +} + +// XXX: move to pytest like bib maybe? +func TestManifestIntegrationSmoke(t *testing.T) { + if testing.Short() { + t.Skip("manifest generation takes a while") + } + if !hasDepsolveDnf() { + t.Skip("no osbuild-depsolve-dnf binary found") + } + + restore := main.MockNewRepoRegistry(testrepos.New) + defer restore() + + restore = main.MockOsArgs([]string{"manifest", "centos-9", "qcow2"}) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + err := main.Run() + assert.NoError(t, err) + + pipelineNames := pipelineNamesFrom(t, fakeStdout.Bytes()) + assert.Contains(t, pipelineNames, "qcow2") +} diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go new file mode 100644 index 00000000..a0b701f0 --- /dev/null +++ b/cmd/image-builder/manifest.go @@ -0,0 +1,83 @@ +package main + +import ( + "fmt" + "io" + "strings" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/images/pkg/sbom" +) + +// XXX: duplicated from cmd/build/main.go:depsolve (and probably more places) +// should go into a common helper in "images" or images should do this on +// its own +var depsolve = func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSet := range packageSets { + res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone) + if err != nil { + return nil, nil, err + } + depsolvedSets[name] = res.Packages + repoSets[name] = res.Repos + } + return depsolvedSets, repoSets, nil +} + +type manifestGenerator struct { + DataDir string + Out io.Writer +} + +func (mg *manifestGenerator) Generate(distroStr, imgTypeStr, archStr string) error { + // Note that distroStr/imgTypeStr/archStr may contain prefixes like + // "distro:" so only the getOneImage() result should be used in the + // rest of the function. + res, err := getOneImage(mg.DataDir, distroStr, imgTypeStr, archStr) + if err != nil { + return err + } + + var bp blueprint.Blueprint + var options distro.ImageOptions + reporeg, err := newRepoRegistry(mg.DataDir) + if err != nil { + return err + } + repos, err := reporeg.ReposByImageTypeName(res.Distro.Name(), res.Arch.Name(), res.ImgType.Name()) + if err != nil { + return err + } + preManifest, warnings, err := res.ImgType.Manifest(&bp, options, repos, 0) + if err != nil { + return err + } + if len(warnings) > 0 { + // XXX: what can we do here? for things like json output? + // what are these warnings? + return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n")) + } + // XXX: add something like "--rpmmd" (like bib) + cacheDir := "" + packageSpecs, _, err := depsolve(cacheDir, preManifest.GetPackageSetChains(), res.Distro, res.Arch.Name()) + if err != nil { + return err + } + if packageSpecs == nil { + return fmt.Errorf("depsolve did not return any packages") + } + // XXX: support commit/container specs + mf, err := preManifest.Serialize(packageSpecs, nil, nil, nil) + if err != nil { + return err + } + fmt.Fprintf(mg.Out, "%s\n", mf) + + return nil +} diff --git a/cmd/image-builder/manifest_test.go b/cmd/image-builder/manifest_test.go new file mode 100644 index 00000000..d0ca300f --- /dev/null +++ b/cmd/image-builder/manifest_test.go @@ -0,0 +1,56 @@ +package main_test + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/distrofactory" + testrepos "github.com/osbuild/images/test/data/repositories" + + "github.com/osbuild/image-builder-cli/cmd/image-builder" +) + +func TestManifestGeneratorSad(t *testing.T) { + restore := main.MockNewRepoRegistry(testrepos.New) + defer restore() + + mg := &main.ManifestGenerator{} + assert.NotNil(t, mg) + err := mg.Generate("bad-distro", "bad-type", "bad-arch") + assert.EqualError(t, err, `cannot find image for: distro:"bad-distro" type:"bad-type" arch:"bad-arch"`) +} + +// XXX: move to images:testutil.go or something +func pipelineNamesFrom(t *testing.T, osbuildManifest []byte) []string { + var manifest map[string]interface{} + + err := json.Unmarshal(osbuildManifest, &manifest) + assert.NoError(t, err) + assert.NotNil(t, manifest["pipelines"]) + pipelines := manifest["pipelines"].([]interface{}) + pipelineNames := make([]string, len(pipelines)) + for idx, pi := range pipelines { + pipelineNames[idx] = pi.(map[string]interface{})["name"].(string) + } + return pipelineNames +} + +func TestManifestGenerator(t *testing.T) { + var osbuildManifest bytes.Buffer + + restore := main.MockDistrofactoryNew(distrofactory.NewTestDefault) + defer restore() + restore = main.MockDepsolve() + defer restore() + + mg := &main.ManifestGenerator{DataDir: "../../test/data/", Out: &osbuildManifest} + assert.NotNil(t, mg) + err := mg.Generate("test-distro-1", "test_type", "test_arch") + assert.NoError(t, err) + + pipelineNames := pipelineNamesFrom(t, osbuildManifest.Bytes()) + assert.Equal(t, []string{"build", "os"}, pipelineNames) +} diff --git a/go.mod b/go.mod index 044652c7..ad5e2b85 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ module github.com/osbuild/image-builder-cli go 1.21.0 require ( + github.com/gobwas/glob v0.2.3 github.com/osbuild/images v0.98.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 @@ -50,7 +51,6 @@ require ( github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-openapi/validate v0.24.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/test/data/repositories/test-distro-1.json b/test/data/repositories/test-distro-1.json new file mode 100644 index 00000000..f91f04f4 --- /dev/null +++ b/test/data/repositories/test-distro-1.json @@ -0,0 +1,6 @@ +{ + "test_arch": [ + { + } + ] +}