From b21dea4b02a3ef5a27a5df0a4e6923d5169099c1 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Sat, 10 Aug 2024 21:47:48 +0200 Subject: [PATCH 01/22] feat: git based ctf tracking repositories --- api/oci/extensions/repositories/git/README.md | 32 ++ api/oci/extensions/repositories/git/format.go | 44 +++ .../extensions/repositories/git/git_test.go | 133 ++++++++ .../extensions/repositories/git/namespace.go | 304 ++++++++++++++++++ .../extensions/repositories/git/repository.go | 103 ++++++ .../extensions/repositories/git/suite_test.go | 13 + .../git/testdata/repo/file_in_repo | 1 + api/oci/extensions/repositories/git/type.go | 94 ++++++ api/oci/extensions/repositories/init.go | 1 + .../artifactaccess/gitaccess/options.go | 58 ++++ .../artifactaccess/gitaccess/resource.go | 25 ++ .../extensions/accessmethods/git/README.md | 43 +++ api/ocm/extensions/accessmethods/git/cli.go | 43 +++ .../extensions/accessmethods/git/method.go | 176 ++++++++++ .../accessmethods/git/method_test.go | 13 + .../accessmethods/git/suite_test.go | 96 ++++++ .../git/testdata/repo/file_in_repo | 1 + api/ocm/extensions/accessmethods/init.go | 1 + api/tech/git/fs.go | 174 ++++++++++ api/tech/git/ref.go | 79 +++++ api/tech/git/resolver.go | 209 ++++++++++++ .../accessio/downloader/git/downloader.go | 111 +++++++ 22 files changed, 1754 insertions(+) create mode 100644 api/oci/extensions/repositories/git/README.md create mode 100644 api/oci/extensions/repositories/git/format.go create mode 100644 api/oci/extensions/repositories/git/git_test.go create mode 100644 api/oci/extensions/repositories/git/namespace.go create mode 100644 api/oci/extensions/repositories/git/repository.go create mode 100644 api/oci/extensions/repositories/git/suite_test.go create mode 100644 api/oci/extensions/repositories/git/testdata/repo/file_in_repo create mode 100644 api/oci/extensions/repositories/git/type.go create mode 100644 api/ocm/elements/artifactaccess/gitaccess/options.go create mode 100644 api/ocm/elements/artifactaccess/gitaccess/resource.go create mode 100644 api/ocm/extensions/accessmethods/git/README.md create mode 100644 api/ocm/extensions/accessmethods/git/cli.go create mode 100644 api/ocm/extensions/accessmethods/git/method.go create mode 100644 api/ocm/extensions/accessmethods/git/method_test.go create mode 100644 api/ocm/extensions/accessmethods/git/suite_test.go create mode 100644 api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo create mode 100644 api/tech/git/fs.go create mode 100644 api/tech/git/ref.go create mode 100644 api/tech/git/resolver.go create mode 100644 api/utils/accessio/downloader/git/downloader.go diff --git a/api/oci/extensions/repositories/git/README.md b/api/oci/extensions/repositories/git/README.md new file mode 100644 index 0000000000..cf81f06ee1 --- /dev/null +++ b/api/oci/extensions/repositories/git/README.md @@ -0,0 +1,32 @@ + +# Repository `GitRepository` - git based repository + + +### Synopsis + +``` +type: GitRepository/v1 +``` + +### Description + +Artifact namespaces/repositories of the API layer will be mapped to git repository paths. + +Supported specification version is `v1`. + +### Specification Versions + +#### Version `v1` + +The type specific specification fields are: + +- **`url`** *string* + + URL of the git repository in the form of @# ^([^@#]+)(@[^#\n]+)?(#[^@\n]+)? + - url is the URL of the git repository + - ref is the git reference to checkout, if not specified, defaults to "HEAD" + - path is the path to the file or directory to use as the source, if not specified,defaults to the root of the repository. + +### Go Bindings + +The Go binding can be found [here](type.go) diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go new file mode 100644 index 0000000000..2c63170d50 --- /dev/null +++ b/api/oci/extensions/repositories/git/format.go @@ -0,0 +1,44 @@ +package git + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/oci/extensions/repositories/ctf" + "ocm.software/ocm/api/oci/extensions/repositories/ctf/format" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" +) + +const ( + ArtifactIndexFileName = format.ArtifactIndexFileName + BlobsDirectoryName = format.BlobsDirectoryName +) + +var accessObjectInfo = &accessobj.DefaultAccessObjectInfo{ + DescriptorFileName: ArtifactIndexFileName, + ObjectTypeName: "repository", + ElementDirectoryName: BlobsDirectoryName, + ElementTypeName: "blob", + DescriptorHandlerFactory: ctf.NewStateHandler, +} + +type Object = Repository + +// ////////////////////////////////////////////////////////////////////////////// + +func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string) (Object, error) { + return New(cpi.FromProvider(ctx), &RepositorySpec{ + URL: url, + AccessMode: acc, + }) +} + +func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, mode vfs.FileMode, option ...accessio.Option) (Object, error) { + spec, err := NewRepositorySpec(acc, url, mode, option...) + if err != nil { + return nil, err + } + + return New(cpi.FromProvider(ctx), spec) +} diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go new file mode 100644 index 0000000000..1c2e4b9061 --- /dev/null +++ b/api/oci/extensions/repositories/git/git_test.go @@ -0,0 +1,133 @@ +package git_test + +import ( + "embed" + "fmt" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/opencontainers/go-digest" + + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/oci" + "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/cpi" + rgit "ocm.software/ocm/api/oci/extensions/repositories/git" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" + "ocm.software/ocm/api/utils/blobaccess" + ocmlog "ocm.software/ocm/api/utils/logging" + "ocm.software/ocm/api/utils/mime" + "ocm.software/ocm/api/utils/refmgmt" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("ctf management", func() { + var remoteRepo *git.Repository + + var tmp vfs.FileSystem + var workspace vfs.FileSystem + + var repoDir string + var repoURL string + + ocmlog.Context().AddRule(logging.NewConditionRule(logging.TraceLevel, refmgmt.ALLOC_REALM)) + + ctx := oci.New() + + BeforeEach(func() { + path := GinkgoT().TempDir() + tmp = Must(cwdfs.New(osfs.New(), path)) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tmp}) + vfsattr.Set(ctx, tmp) + + Expect(tmp.Mkdir("repo", 0o700)).To(Succeed()) + repoDir = path + filepath.PathSeparatorString + "repo" + repoURL = "file://" + repoDir + + Expect(tmp.Mkdir("workspace", 0o700)).To(Succeed()) + workspace = Must(cwdfs.New(tmp, "workspace")) + }) + + BeforeEach(func() { + remoteRepo = Must(git.PlainInit(repoDir, true)) + }) + + AfterEach(func() { + Expect(vfs.Cleanup(tmp)).To(Succeed()) + }) + + It("instantiate git based ctf", func() { + repo := Must(rgit.Create(ctx, accessobj.ACC_CREATE, repoURL, 0o700, + accessio.RepresentationFileSystem(workspace), + )) + ns := Must(repo.LookupNamespace("test")) + + testData := []byte("testdata") + + aa := NewArtifact(ns, testData) + + Expect(aa.Close()).To(Succeed()) + Expect(ns.Close()).To(Succeed()) + Expect(repo.Close()).To(Succeed()) + + commits := Must(remoteRepo.CommitObjects()) + validAdd := 0 + validSync := 0 + var messages []string + Expect(commits.ForEach(func(commit *object.Commit) error { + if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa); commit.Message == expected { + validAdd++ + } + if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationSync, aa); commit.Message == expected { + validSync++ + } + messages = append(messages, commit.Message) + return nil + })).To(Succeed()) + + Expect(validAdd).To(Equal(1), + fmt.Sprintf( + "expected exactly one commit with message %q, got %d commits with messages:\n%v", + rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa), + validAdd, + messages, + )) + Expect(validSync).To(Equal(1), + fmt.Sprintf( + "expected exactly one commit with message %q, got %d commits with messages:\n%v", + rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa), + validAdd, + messages, + )) + }) +}) + +func NewArtifact(n cpi.NamespaceAccess, data []byte) cpi.ArtifactAccess { + art := Must(n.NewArtifact()) + Expect(art.AddLayer(blobaccess.ForData(mime.MIME_OCTET, data), nil)).To(Equal(0)) + desc := Must(art.Manifest()) + Expect(desc).NotTo(BeNil()) + + Expect(desc.Layers[0].Digest).To(Equal(digest.FromBytes(data))) + Expect(desc.Layers[0].MediaType).To(Equal(mime.MIME_OCTET)) + Expect(desc.Layers[0].Size).To(Equal(int64(8))) + + config := blobaccess.ForData(mime.MIME_OCTET, []byte("{}")) + desc.Config = *artdesc.DefaultBlobDescriptor(config) + MustBeSuccessful(n.AddBlob(config)) + MustBeSuccessful(n.AddArtifact(desc)) + return art +} diff --git a/api/oci/extensions/repositories/git/namespace.go b/api/oci/extensions/repositories/git/namespace.go new file mode 100644 index 0000000000..aac57830f1 --- /dev/null +++ b/api/oci/extensions/repositories/git/namespace.go @@ -0,0 +1,304 @@ +package git + +import ( + "context" + "fmt" + + "github.com/opencontainers/go-digest" + + "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/oci/cpi/support" + "ocm.software/ocm/api/oci/internal" + "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/utils/blobaccess/blobaccess" +) + +func NewNamespace(repo *repository, name string) (cpi.NamespaceAccess, error) { + ctfNamespace, err := newNamespaceContainer(repo, name) + if err != nil { + return nil, err + } + return support.NewNamespaceAccess(name, ctfNamespace, repo, "Git repository Branch") +} + +type namespaceContainer struct { + impl support.NamespaceAccessImpl + name string + client git.Client + ctf cpi.NamespaceAccess +} + +var _ support.NamespaceContainer = (*namespaceContainer)(nil) + +func newNamespaceContainer(repo *repository, name string) (support.NamespaceContainer, error) { + ctfNamespace, err := repo.ctf.LookupNamespace(name) + if err != nil { + return nil, err + } + return &namespaceContainer{ + name: name, + client: repo.client, + ctf: ctfNamespace, + }, nil +} + +func (n *namespaceContainer) SetImplementation(impl support.NamespaceAccessImpl) { + n.impl = impl +} + +func (n *namespaceContainer) IsReadOnly() bool { + return false +} + +func (n *namespaceContainer) Close() error { + if err := n.ctf.Close(); err != nil { + return err + } + return n.client.Update(context.Background(), fmt.Sprintf("namespace update %q", n.name), true) +} + +func (n *namespaceContainer) ListTags() ([]string, error) { + if err := n.client.Refresh(context.Background()); err != nil { + return nil, err + } + return n.ctf.ListTags() +} + +func (n *namespaceContainer) GetBlobData(digest digest.Digest) (int64, cpi.DataAccess, error) { + if err := n.client.Refresh(context.Background()); err != nil { + return 0, nil, err + } + + return n.ctf.GetBlobData(digest) +} + +func (n *namespaceContainer) AddBlob(blob cpi.BlobAccess) error { + if err := n.ctf.AddBlob(blob); err != nil { + return err + } + + if err := n.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, blob), false); err != nil { + return err + } + return nil +} + +func (n *namespaceContainer) GetArtifact(i support.NamespaceAccessImpl, vers string) (cpi.ArtifactAccess, error) { + if err := n.client.Refresh(context.Background()); err != nil { + return nil, err + } + + return n.ctf.GetArtifact(vers) +} + +func (n *namespaceContainer) HasArtifact(vers string) (bool, error) { + if err := n.client.Refresh(context.Background()); err != nil { + return false, err + } + return n.ctf.HasArtifact(vers) +} + +func (n *namespaceContainer) AddArtifact(artifact cpi.Artifact, tags ...string) (access blobaccess.BlobAccess, err error) { + blobAccess, err := n.ctf.AddArtifact(artifact, tags...) + if err != nil { + return nil, err + } + msg := GenerateCommitMessageForArtifact(OperationAdd, artifact) + + if err := n.client.Update(context.Background(), msg, false); err != nil { + return nil, err + } + + return blobAccess, nil +} + +func (n *namespaceContainer) AddTags(digest digest.Digest, tags ...string) error { + if err := n.ctf.AddTags(digest, tags...); err != nil { + return err + } + + if err := n.client.Update(context.Background(), fmt.Sprintf("added tags %s to %s", tags, digest.String()), true); err != nil { + return err + } + + return nil +} + +func (n *namespaceContainer) NewArtifact(i support.NamespaceAccessImpl, art ...cpi.Artifact) (cpi.ArtifactAccess, error) { + artifactAccess, err := n.ctf.NewArtifact(art...) + if err != nil { + return nil, err + } + return &artifactContainer{ + client: n.client, + ArtifactAccess: artifactAccess, + }, nil +} + +type Operation string + +const ( + OperationAdd Operation = "add" + OperationMod Operation = "mod" + OperationSync Operation = "sync" +) + +func GenerateCommitMessageForArtifact(operation Operation, artifact cpi.Artifact) string { + a := artifact.Artifact() + + var msg string + if artifact.IsManifest() { + msg = fmt.Sprintf("update(ocm): %s manifest %s (%s)", operation, a.Digest(), a.MimeType()) + } else if artifact.IsIndex() { + msg = fmt.Sprintf("update(ocm): %s index %s (%s)", operation, a.Digest(), a.MimeType()) + } else { + msg = fmt.Sprintf("update(ocm): %s artifact %s (%s)", operation, a.Digest(), a.MimeType()) + } + return msg +} + +func GenerateCommitMessageForBlob(operation Operation, blob cpi.BlobAccess) string { + var msg string + if blob.DigestKnown() { + msg = fmt.Sprintf("update(ocm): %s blob %s of type %s", operation, blob.Digest(), blob.MimeType()) + } else { + msg = fmt.Sprintf("update(ocm): %s blob of type %s", operation, blob.MimeType()) + } + return msg +} + +type artifactContainer struct { + client git.Client + cpi.ArtifactAccess +} + +var _ cpi.ArtifactAccess = (*artifactContainer)(nil) + +func (a *artifactContainer) Close() error { + if err := a.ArtifactAccess.Close(); err != nil { + return err + } + return a.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationSync, a.ArtifactAccess), true) +} + +func (a *artifactContainer) Dup() (cpi.ArtifactAccess, error) { + access, err := a.ArtifactAccess.Dup() + if err != nil { + return nil, err + } + return &artifactContainer{ + client: a.client, + ArtifactAccess: access, + }, nil +} + +func (a *artifactContainer) AddBlob(access internal.BlobAccess) error { + if err := a.ArtifactAccess.AddBlob(access); err != nil { + return err + } + return a.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, access), false) +} + +func (a *artifactContainer) AddArtifact(artifact cpi.Artifact, platform *artdesc.Platform) (cpi.BlobAccess, error) { + b, err := a.ArtifactAccess.AddArtifact(artifact, platform) + if err != nil { + return nil, err + } + return b, a.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationAdd, artifact), true) +} + +func (a *artifactContainer) AddLayer(access cpi.BlobAccess, descriptor *artdesc.Descriptor) (int, error) { + n, err := a.ArtifactAccess.AddLayer(access, descriptor) + if err != nil { + return -1, err + } + return n, a.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationMod, access), false) +} + +func (a *artifactContainer) NewArtifact(artifact ...cpi.Artifact) (cpi.ArtifactAccess, error) { + access, err := a.ArtifactAccess.NewArtifact(artifact...) + if err != nil { + return nil, err + } + return &artifactContainer{ + client: a.client, + ArtifactAccess: access, + }, nil +} + +func (a *artifactContainer) ManifestAccess() cpi.ManifestAccess { + return &manifestContainer{ + client: a.client, + ManifestAccess: a.ArtifactAccess.ManifestAccess(), + } +} + +func (a *artifactContainer) IndexAccess() cpi.IndexAccess { + return &indexContainer{ + client: a.client, + IndexAccess: a.ArtifactAccess.IndexAccess(), + } +} + +type manifestContainer struct { + cpi.ManifestAccess + client git.Client +} + +var _ cpi.ManifestAccess = (*manifestContainer)(nil) + +func (m *manifestContainer) AddBlob(access internal.BlobAccess) error { + if err := m.ManifestAccess.AddBlob(access); err != nil { + return err + } + return m.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, access), false) +} + +func (m *manifestContainer) AddLayer(access internal.BlobAccess, descriptor *artdesc.Descriptor) (int, error) { + n, err := m.ManifestAccess.AddLayer(access, descriptor) + if err != nil { + return -1, err + } + return n, m.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationMod, access), false) +} + +func (m *manifestContainer) SetConfigBlob(blob internal.BlobAccess, d *artdesc.Descriptor) error { + if err := m.ManifestAccess.SetConfigBlob(blob, d); err != nil { + return err + } + return m.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationMod, blob), false) +} + +type indexContainer struct { + cpi.IndexAccess + client git.Client +} + +var _ cpi.IndexAccess = (*indexContainer)(nil) + +func (i *indexContainer) GetArtifact(digest digest.Digest) (internal.ArtifactAccess, error) { + a, err := i.IndexAccess.GetArtifact(digest) + if err != nil { + return nil, err + } + return &artifactContainer{ + client: i.client, + ArtifactAccess: a, + }, nil +} + +func (i *indexContainer) AddBlob(access internal.BlobAccess) error { + if err := i.IndexAccess.AddBlob(access); err != nil { + return err + } + return i.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, access), false) +} + +func (i *indexContainer) AddArtifact(artifact internal.Artifact, platform *artdesc.Platform) (internal.BlobAccess, error) { + b, err := i.IndexAccess.AddArtifact(artifact, platform) + if err != nil { + return nil, err + } + return b, i.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationAdd, artifact), false) +} diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go new file mode 100644 index 0000000000..0dd42bce44 --- /dev/null +++ b/api/oci/extensions/repositories/git/repository.go @@ -0,0 +1,103 @@ +package git + +import ( + "context" + + "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/credentials/builtin/oci/identity" + cpicredentials "ocm.software/ocm/api/credentials/cpi" + "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/oci/extensions/repositories/ctf" + "ocm.software/ocm/api/tech/git" + ocmlog "ocm.software/ocm/api/utils/logging" +) + +type Repository interface { + cpi.Repository +} + +type repository struct { + cpi.RepositoryImplBase + logger logging.UnboundLogger + spec *RepositorySpec + ctf *ctf.Repository + client git.Client +} + +var ( + _ cpi.RepositoryImpl = (*repository)(nil) + _ credentials.ConsumerIdentityProvider = &repository{} +) + +func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { + urs := spec.UniformRepositorySpec() + i := &repository{ + RepositoryImplBase: cpi.NewRepositoryImplBase(ctx), + logger: logging.DynamicLogger(ctx, logging.NewAttribute(ocmlog.ATTR_HOST, urs.Host)), + spec: spec, + } + + var err error + if i.client, err = git.NewClient(spec.URL); err != nil { + return nil, err + } + + repo, err := ctf.New(ctx, &ctf.RepositorySpec{ + StandardOptions: spec.StandardOptions, + AccessMode: spec.AccessMode, + }, i.client, i.client, vfs.FileMode(0o770)) + if err != nil { + return nil, err + } + i.ctf = repo + + return cpi.NewRepository(i), nil +} + +func (r *repository) GetSpecification() cpi.RepositorySpec { + return r.spec +} + +func (r *repository) Close() error { + return r.ctf.Close() +} + +func (r *repository) GetIdentityMatcher() string { + return identity.CONSUMER_TYPE +} + +func (r *repository) NamespaceLister() cpi.NamespaceLister { + return r.ctf.NamespaceLister() +} + +func (r *repository) IsReadOnly() bool { + return false +} + +func (r *repository) ExistsArtifact(name string, version string) (bool, error) { + if err := r.client.Refresh(context.Background()); err != nil { + return false, err + } + return r.ctf.ExistsArtifact(name, version) +} + +func (r *repository) LookupArtifact(name string, version string) (cpi.ArtifactAccess, error) { + if err := r.client.Refresh(context.Background()); err != nil { + return nil, err + } + return r.ctf.LookupArtifact(name, version) +} + +func (r *repository) LookupNamespace(name string) (cpi.NamespaceAccess, error) { + if err := r.client.Refresh(context.Background()); err != nil { + return nil, err + } + return NewNamespace(r, name) +} + +func (r *repository) GetConsumerId(ctx ...cpicredentials.UsageContext) cpicredentials.ConsumerIdentity { + return nil +} diff --git a/api/oci/extensions/repositories/git/suite_test.go b/api/oci/extensions/repositories/git/suite_test.go new file mode 100644 index 0000000000..597d5f9350 --- /dev/null +++ b/api/oci/extensions/repositories/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCI Git Test Suite") +} diff --git a/api/oci/extensions/repositories/git/testdata/repo/file_in_repo b/api/oci/extensions/repositories/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/oci/extensions/repositories/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go new file mode 100644 index 0000000000..fb2d2511ab --- /dev/null +++ b/api/oci/extensions/repositories/git/type.go @@ -0,0 +1,94 @@ +package git + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" + "ocm.software/ocm/api/utils/runtime" +) + +const ( + Type = "GitRepository" + TypeV1 = Type + runtime.VersionSeparator + "v1" + + ShortType = "Git" + ShortTypeV1 = ShortType + runtime.VersionSeparator + "v1" +) + +func init() { + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](Type)) + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](TypeV1)) + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](ShortType)) + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](ShortTypeV1)) +} + +// Is checks the kind. +func Is(spec cpi.RepositorySpec) bool { + return spec != nil && (spec.GetKind() == Type || spec.GetKind() == ShortType) +} + +func IsKind(k string) bool { + return k == Type || k == ShortType +} + +// RepositorySpec describes an CTF repository interface backed by a git repository. +type RepositorySpec struct { + runtime.ObjectVersionedType `json:",inline"` + accessio.StandardOptions `json:",inline"` + + // URL is the url of the repository to resolve artifacts. + URL string `json:"baseUrl"` + + // AccessMode can be set to request readonly access or creation + AccessMode accessobj.AccessMode `json:"accessMode,omitempty"` + + // FileMode is the file mode for the repository in the filesystem. + FileMode vfs.FileMode `json:"fileMode"` +} + +var _ cpi.RepositorySpec = (*RepositorySpec)(nil) + +var _ cpi.IntermediateRepositorySpecAspect = (*RepositorySpec)(nil) + +// NewRepositorySpec creates a new RepositorySpec. +func NewRepositorySpec(mode accessobj.AccessMode, url string, fileMode vfs.FileMode, opts ...accessio.Option) (*RepositorySpec, error) { + o, err := accessio.AccessOptions(nil, opts...) + if err != nil { + return nil, err + } + o.Default() + return &RepositorySpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + URL: url, + FileMode: fileMode, + StandardOptions: *o.(*accessio.StandardOptions), + AccessMode: mode, + }, nil +} + +func (s *RepositorySpec) IsIntermediate() bool { + return true +} + +func (s *RepositorySpec) GetType() string { + return Type +} + +func (s *RepositorySpec) Name() string { + return s.URL +} + +func (s *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec { + u := &cpi.UniformRepositorySpec{ + Type: Type, + Info: s.URL, + } + return u +} + +func (s *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) { + return New(ctx, s) +} diff --git a/api/oci/extensions/repositories/init.go b/api/oci/extensions/repositories/init.go index 2d9efaeb2c..6ddf95b0cf 100644 --- a/api/oci/extensions/repositories/init.go +++ b/api/oci/extensions/repositories/init.go @@ -5,5 +5,6 @@ import ( _ "ocm.software/ocm/api/oci/extensions/repositories/ctf" _ "ocm.software/ocm/api/oci/extensions/repositories/docker" _ "ocm.software/ocm/api/oci/extensions/repositories/empty" + _ "ocm.software/ocm/api/oci/extensions/repositories/git" _ "ocm.software/ocm/api/oci/extensions/repositories/ocireg" ) diff --git a/api/ocm/elements/artifactaccess/gitaccess/options.go b/api/ocm/elements/artifactaccess/gitaccess/options.go new file mode 100644 index 0000000000..0ced931627 --- /dev/null +++ b/api/ocm/elements/artifactaccess/gitaccess/options.go @@ -0,0 +1,58 @@ +package githubaccess + +import ( + "github.com/mandelsoft/goutils/optionutils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + URL string + Ref string + PathSpec string +} + +var _ Option = (*Options)(nil) + +func (o *Options) ApplyTo(opts *Options) { + if o.URL != "" { + opts.URL = o.URL + } +} + +func (o *Options) Apply(opts ...Option) { + optionutils.ApplyOptions(o, opts...) +} + +// ////////////////////////////////////////////////////////////////////////////// +// Local options + +type url string + +func (h url) ApplyTo(opts *Options) { + opts.URL = string(h) +} + +func WithURL(h string) Option { + return url(h) +} + +type ref string + +func (h ref) ApplyTo(opts *Options) { + opts.Ref = string(h) +} + +func WithRef(h string) Option { + return ref(h) +} + +type pathSpec string + +func (h pathSpec) ApplyTo(opts *Options) { + opts.PathSpec = string(h) +} + +func WithPathSpec(h string) Option { + return pathSpec(h) +} diff --git a/api/ocm/elements/artifactaccess/gitaccess/resource.go b/api/ocm/elements/artifactaccess/gitaccess/resource.go new file mode 100644 index 0000000000..6f9adcbf9b --- /dev/null +++ b/api/ocm/elements/artifactaccess/gitaccess/resource.go @@ -0,0 +1,25 @@ +package githubaccess + +import ( + "github.com/mandelsoft/goutils/optionutils" + + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/compdesc" + "ocm.software/ocm/api/ocm/cpi" + "ocm.software/ocm/api/ocm/elements/artifactaccess/genericaccess" + access "ocm.software/ocm/api/ocm/extensions/accessmethods/git" + resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" +) + +const TYPE = resourcetypes.BLOB + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + if meta.GetType() == "" { + meta.SetType(TYPE) + } + + spec := access.New(eff.URL, eff.Ref, eff.PathSpec) + // is global access, must work, otherwise there is an error in the lib. + return genericaccess.MustAccess(ctx, meta, spec) +} diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md new file mode 100644 index 0000000000..1a2b2fca3f --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -0,0 +1,43 @@ + +# Access Method `git` - Git Commit Access + + +### Synopsis + +``` +type: git/v1 +``` + +Provided blobs use the following media type for: `application/x-tgz` + +The artifact content is provided as gnu-zipped tar archive + +### Description + +This method implements the access of the content of a git commit stored in a +GitHub repository. + +Supported specification version is `v1` + +### Specification Versions + +#### Version `v1` + +The type specific specification fields are: + +- **`repoUrl`** *string* + + Repository URL with or without scheme. + +- **`ref`** (optional) *string* + + Original ref used to get the commit from + +- **`commit`** *string* + + The sha/id of the git commit + + +### Go Bindings + +The go binding can be found [here](method.go) diff --git a/api/ocm/extensions/accessmethods/git/cli.go b/api/ocm/extensions/accessmethods/git/cli.go new file mode 100644 index 0000000000..da01ffae4f --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/cli.go @@ -0,0 +1,43 @@ +package git + +import ( + "ocm.software/ocm/api/ocm/extensions/accessmethods/options" + "ocm.software/ocm/api/utils/cobrautils/flagsets" +) + +func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { + return flagsets.NewConfigOptionTypeSetHandler( + Type, AddConfig, + options.RepositoryOption, + options.ReferenceOption, + options.CommitOption, + ) +} + +func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repoUrl") + flagsets.AddFieldByOptionP(opts, options.CommitOption, config, "commit") + flagsets.AddFieldByOptionP(opts, options.ReferenceOption, config, "ref") + return nil +} + +var usage = ` +This method implements the access of the content of a git commit stored in a +Git repository. +` + +var formatV1 = ` +The type specific specification fields are: + +- **repoUrl** *string* + + Repository URL with or without scheme. + +- **ref** (optional) *string* + + Original ref used to get the commit from + +- **commit** *string* + + The sha/id of the git commit +` diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go new file mode 100644 index 0000000000..2ab8517637 --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -0,0 +1,176 @@ +package git + +import ( + "fmt" + "io" + "net/http" + "net/url" + "sync" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/mandelsoft/goutils/errors" + + "ocm.software/ocm/api/ocm/cpi/accspeccpi" + "ocm.software/ocm/api/ocm/internal" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessio/downloader" + "ocm.software/ocm/api/utils/accessio/downloader/git" + "ocm.software/ocm/api/utils/accessobj" + "ocm.software/ocm/api/utils/blobaccess/blobaccess" + "ocm.software/ocm/api/utils/mime" + "ocm.software/ocm/api/utils/runtime" +) + +const ( + Type = "git" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) + +func init() { + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage))) + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) +} + +// AccessSpec describes the access for a GitHub registry. +type AccessSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // RepoURL is the repository URL + RepoURL string `json:"repoUrl"` + + // Ref defines the hash of the commit + Ref string `json:"ref"` + + // PathSpec is a path in the repository to download, can be a file or a regex matching multiple files + PathSpec string `json:"pathSpec"` + + client *http.Client + downloader downloader.Downloader +} + +// AccessSpecOptions defines a set of options which can be applied to the access spec. +type AccessSpecOptions func(s *AccessSpec) + +// New creates a new git registry access spec version v1. +func New(url, ref string, pathSpec string, opts ...AccessSpecOptions) *AccessSpec { + s := &AccessSpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + RepoURL: url, + Ref: ref, + PathSpec: pathSpec, + } + for _, o := range opts { + o(s) + } + return s +} + +func (a *AccessSpec) Describe(internal.Context) string { + return fmt.Sprintf("git commit %s[%s]", a.RepoURL, a.Ref) +} + +func (*AccessSpec) IsLocal(internal.Context) bool { + return false +} + +func (a *AccessSpec) GlobalAccessSpec(accspeccpi.Context) accspeccpi.AccessSpec { + return a +} + +func (*AccessSpec) GetType() string { + return Type +} + +func (a *AccessSpec) AccessMethod(c internal.ComponentVersionAccess) (internal.AccessMethod, error) { + return accspeccpi.AccessMethodForImplementation(newMethod(c, a)) +} + +func newMethod(c internal.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { + u, err := url.Parse(a.RepoURL) + if err != nil { + return nil, errors.ErrInvalidWrap(err, "repository repoURL", a.RepoURL) + } + if err := plumbing.ReferenceName(a.Ref).Validate(); err != nil { + return nil, errors.ErrInvalidWrap(err, "commit hash", a.Ref) + } + + return &accessMethod{ + repoURL: u.String(), + compvers: c, + spec: a, + ref: a.Ref, + }, nil +} + +type accessMethod struct { + lock sync.Mutex + access blobaccess.BlobAccess + + compvers accspeccpi.ComponentVersionAccess + spec *AccessSpec + + repoURL string + path string + ref string +} + +var _ accspeccpi.AccessMethodImpl = &accessMethod{} + +func (m *accessMethod) Close() error { + if m.access == nil { + return nil + } + return m.access.Close() +} + +func (m *accessMethod) Get() ([]byte, error) { + if err := m.setup(); err != nil { + return nil, err + } + return m.access.Get() +} + +func (m *accessMethod) Reader() (io.ReadCloser, error) { + if err := m.setup(); err != nil { + return nil, err + } + return m.access.Reader() +} + +func (m *accessMethod) setup() error { + m.lock.Lock() + defer m.lock.Unlock() + + if m.access != nil { + return nil + } + + d := git.NewDownloader(m.repoURL, m.ref, m.path) + defer d.Close() + + cacheBlobAccess := accessobj.CachedBlobAccessForWriter( + m.compvers.GetContext(), + m.MimeType(), + accessio.NewWriteAtWriter(d.Download), + ) + + m.access = cacheBlobAccess + + return nil +} + +func (m *accessMethod) MimeType() string { + return mime.MIME_OCTET +} + +func (*accessMethod) IsLocal() bool { + return false +} + +func (m *accessMethod) GetKind() string { + return Type +} + +func (m *accessMethod) AccessSpec() internal.AccessSpec { + return m.spec +} diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go new file mode 100644 index 0000000000..59ce0ec1fc --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -0,0 +1,13 @@ +package git + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Github Test Suite") +} diff --git a/api/ocm/extensions/accessmethods/git/suite_test.go b/api/ocm/extensions/accessmethods/git/suite_test.go new file mode 100644 index 0000000000..8a6d992a8c --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/suite_test.go @@ -0,0 +1,96 @@ +package git_test + +import ( + "embed" + _ "embed" + "fmt" + "io" + "os" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/cpi" + me "ocm.software/ocm/api/ocm/extensions/accessmethods/git" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("Method", func() { + var ( + ctx ocm.Context + expectedBlobContent []byte + accessSpec *me.AccessSpec + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + BeforeEach(func() { + repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo, err := git.PlainInit(repoDir, false) + Expect(err).ToNot(HaveOccurred()) + + repoBase := filepath.Join("testdata", "repo") + repoTestData, err := testData.ReadDir(repoBase) + Expect(err).ToNot(HaveOccurred()) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file, err := testData.Open(path) + Expect(err).ToNot(HaveOccurred()) + + fileInRepo, err := os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0600, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = io.Copy(fileInRepo, file) + Expect(err).ToNot(HaveOccurred()) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt, err := repo.Worktree() + Expect(err).ToNot(HaveOccurred()) + Expect(wt.AddGlob("*")).To(Succeed()) + _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{}) + Expect(err).ToNot(HaveOccurred()) + + accessSpec = me.New( + fmt.Sprintf("file://%s", repoDir), + string(plumbing.Master), + ".", + ) + }) + + It("downloads artifacts", func() { + m, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) + Expect(err).ToNot(HaveOccurred()) + content, err := m.Get() + Expect(err).ToNot(HaveOccurred()) + Expect(content).To(Equal(expectedBlobContent)) + }) + +}) diff --git a/api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo b/api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/api/ocm/extensions/accessmethods/init.go b/api/ocm/extensions/accessmethods/init.go index 5f4094d3d5..08ce51bf32 100644 --- a/api/ocm/extensions/accessmethods/init.go +++ b/api/ocm/extensions/accessmethods/init.go @@ -1,6 +1,7 @@ package accessmethods import ( + _ "ocm.software/ocm/api/ocm/extensions/accessmethods/git" _ "ocm.software/ocm/api/ocm/extensions/accessmethods/github" _ "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" _ "ocm.software/ocm/api/ocm/extensions/accessmethods/localblob" diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go new file mode 100644 index 0000000000..8701371cda --- /dev/null +++ b/api/tech/git/fs.go @@ -0,0 +1,174 @@ +package git + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" + + "github.com/go-git/go-billy/v5" + "github.com/juju/fslock" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/vfs" +) + +func VFSBillyFS(fsToWrap vfs.VFS) billy.Filesystem { + if fsToWrap == nil { + fsToWrap = vfs.New(memoryfs.New()) + } + fi, err := fsToWrap.Stat(".") + if err != nil || !fi.IsDir() { + panic(fmt.Errorf("invalid vfs: %v", err)) + } + + return &fs{ + vfs: fsToWrap, + root: fi.Name(), + } +} + +type fs struct { + vfs vfs.VFS + root string +} + +type file struct { + lock *fslock.Lock + vfsFile vfs.File +} + +func (f *file) Name() string { + return f.vfsFile.Name() +} + +func (f *file) Write(p []byte) (n int, err error) { + return f.vfsFile.Write(p) +} + +func (f *file) Read(p []byte) (n int, err error) { + return f.vfsFile.Read(p) +} + +func (f *file) ReadAt(p []byte, off int64) (n int, err error) { + return f.vfsFile.ReadAt(p, off) +} + +func (f *file) Seek(offset int64, whence int) (int64, error) { + return f.vfsFile.Seek(offset, whence) +} + +func (f *file) Close() error { + return f.vfsFile.Close() +} + +func (f *file) Lock() error { + return f.lock.Lock() +} + +func (f *file) Unlock() error { + return f.lock.Unlock() +} + +func (f *file) Truncate(size int64) error { + return f.vfsFile.Truncate(size) +} + +var _ billy.File = &file{} + +func (f *fs) Create(filename string) (billy.File, error) { + vfsFile, err := f.vfs.Create(filename) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile), nil +} + +func (f *fs) vfsToBillyFileInfo(vf vfs.File) billy.File { + return &file{ + vfsFile: vf, + lock: fslock.New(fmt.Sprintf("%s.lock", vf.Name())), + } +} + +func (f *fs) Open(filename string) (billy.File, error) { + vfsFile, err := f.vfs.Open(filename) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile), nil +} + +func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { + vfsFile, err := f.vfs.OpenFile(filename, flag, perm) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile), nil +} + +func (f *fs) Stat(filename string) (os.FileInfo, error) { + fi, err := f.vfs.Stat(filename) + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } + return fi, nil +} + +func (f *fs) Rename(oldpath, newpath string) error { + return f.vfs.Rename(oldpath, newpath) +} + +func (f *fs) Remove(filename string) error { + return f.vfs.Remove(filename) +} + +func (f *fs) Join(elem ...string) string { + return filepath.Join(elem...) +} + +func (f *fs) TempFile(dir, prefix string) (billy.File, error) { + vfsFile, err := f.vfs.TempFile(dir, prefix) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile), nil +} + +func (f *fs) ReadDir(path string) ([]os.FileInfo, error) { + return f.vfs.ReadDir(path) +} + +func (f *fs) MkdirAll(filename string, perm os.FileMode) error { + return f.vfs.MkdirAll(filename, perm) +} + +func (f *fs) Lstat(filename string) (os.FileInfo, error) { + return f.vfs.Lstat(filename) +} + +func (f *fs) Symlink(target, link string) error { + return f.vfs.Symlink(target, link) +} + +func (f *fs) Readlink(link string) (string, error) { + return f.vfs.Readlink(link) +} + +func (f *fs) Chroot(path string) (billy.Filesystem, error) { + chfs, err := cwdfs.New(f.vfs, path) + if err != nil { + return nil, err + } + return &fs{ + root: path, + vfs: vfs.New(chfs), + }, nil +} + +func (f *fs) Root() string { + return f.root +} + +var _ billy.Filesystem = &fs{} diff --git a/api/tech/git/ref.go b/api/tech/git/ref.go new file mode 100644 index 0000000000..7a3a039309 --- /dev/null +++ b/api/tech/git/ref.go @@ -0,0 +1,79 @@ +package git + +import ( + "fmt" + "hash/fnv" + "net/url" + "regexp" + + "github.com/go-git/go-git/v5/plumbing" + + "ocm.software/ocm/api/utils" +) + +const urlToRefSeparator = "@" +const refToPathSeparator = "#" + +// refRegexp is a regular expression that matches a git ref string. +// The ref string is expected to be in the format of: +// @# +// where: +// - url is the URL of the git repository +// - ref is the git reference to checkout, if not specified, defaults to "HEAD" +// - path is the path to the file or directory to use as the source, if not specified, defaults to the root of the repository. +var refRegexp = regexp.MustCompile(`^([^@#]+)(@[^#\n]+)?(#[^@\n]+)?`) + +type gurl struct { + url *url.URL + ref plumbing.ReferenceName + path string +} + +// decodeGitURL decodes a git ref string into a gurl struct. +// The ref string is expected to be in the format of: +// @# +// see refRegexp for more details. +func decodeGitURL(rawRef string) (*gurl, error) { + matches := refRegexp.FindStringSubmatch(rawRef) + if matches == nil { + return nil, fmt.Errorf("failed to match ref: %s via %s", rawRef, refRegexp) + } + rawURL := matches[1] + matchedRef := matches[2] + path := matches[3] + + parsedURL, err := utils.ParseURL(rawURL) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %w", err) + } + + var ref plumbing.ReferenceName + if matchedRef == "" { + ref = plumbing.HEAD + } else { + ref = plumbing.ReferenceName(matchedRef) + if err := ref.Validate(); err != nil { + return nil, fmt.Errorf("failed to validate ref: %w", err) + } + } + + return &gurl{ + url: parsedURL, + ref: ref, + path: path, + }, nil +} + +func (ref *gurl) String() string { + return fmt.Sprintf("%s%s%s%s%s", ref.url.String(), urlToRefSeparator, ref.ref, refToPathSeparator, ref.path) +} + +func (ref *gurl) Hash() []byte { + hash := fnv.New64() + _, _ = hash.Write([]byte(ref.url.String())) + return hash.Sum(nil) +} + +func (ref *gurl) HashString() string { + return fmt.Sprintf("%x", ref.Hash()) +} diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go new file mode 100644 index 0000000000..55223e6915 --- /dev/null +++ b/api/tech/git/resolver.go @@ -0,0 +1,209 @@ +package git + +import ( + "context" + "errors" + "os" + + osfs2 "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/storage" + "github.com/go-git/go-git/v5/storage/filesystem" + "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/utils/accessobj" +) + +var worktreeBranch = plumbing.NewBranchReferenceName("ocm") + +type client struct { + vfs vfs.VFS + *gurl + + storage storage.Storer +} + +type Client interface { + Repository(ctx context.Context) (*git.Repository, error) + Refresh(ctx context.Context) error + Update(ctx context.Context, msg string, push bool) error + accessobj.Setup + accessobj.Closer +} + +var _ Client = &client{} + +func NewClient(url string) (Client, error) { + gitURL, err := decodeGitURL(url) + if err != nil { + return nil, err + } + + return &client{ + vfs: vfs.New(memoryfs.New()), + gurl: gitURL, + }, nil +} + +func (c *client) Repository(ctx context.Context) (*git.Repository, error) { + strg, err := getStorage(c.vfs) + if err != nil { + return nil, err + } + + wd, err := c.vfs.Getwd() + if err != nil { + return nil, err + } + billy := osfs2.New(wd, osfs2.WithBoundOS()) + + newRepo := false + repo, err := git.Open(strg, billy) + if errors.Is(err, git.ErrRepositoryNotExists) { + repo, err = git.CloneContext(ctx, strg, billy, &git.CloneOptions{ + URL: c.url.String(), + RemoteName: git.DefaultRemoteName, + ReferenceName: c.ref, + SingleBranch: true, + Depth: 0, + Tags: git.AllTags, + }) + newRepo = true + } + if errors.Is(err, transport.ErrEmptyRemoteRepository) { + return git.Open(strg, billy) + } + + if err != nil { + return nil, err + } + if newRepo { + if err := repo.FetchContext(ctx, &git.FetchOptions{ + RemoteName: git.DefaultRemoteName, + Depth: 0, + Tags: git.AllTags, + Force: false, + }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return nil, err + } + worktree, err := repo.Worktree() + if err != nil { + return nil, err + } + + if err := worktree.Checkout(&git.CheckoutOptions{ + Branch: worktreeBranch, + Create: true, + Keep: true, + }); err != nil { + return nil, err + } + + if err := worktree.AddGlob("*"); err != nil { + return nil, err + } + + if _, err := worktree.Commit("OCM Repository Setup", &git.CommitOptions{}); err != nil && !errors.Is(err, git.ErrEmptyCommit) { + return nil, err + } + } + + return repo, nil +} + +func getStorage(base vfs.VFS) (storage.Storer, error) { + wd, err := base.Getwd() + if err != nil { + return nil, err + } + + return filesystem.NewStorage( + osfs2.New(filepath.Join(wd, git.GitDirName), osfs2.WithBoundOS()), + cache.NewObjectLRUDefault(), + ), nil +} + +func (c *client) TopLevelDirs(ctx context.Context) ([]os.FileInfo, error) { + repo, err := c.Repository(ctx) + if err != nil { + return nil, err + } + + fs, err := repo.Worktree() + if err != nil { + return nil, err + } + + return fs.Filesystem.ReadDir(".") +} + +func (c *client) Refresh(ctx context.Context) error { + repo, err := c.Repository(ctx) + if err != nil { + return err + } + + worktree, err := repo.Worktree() + if err != nil { + return err + } + + if err := worktree.PullContext(ctx, &git.PullOptions{ + RemoteName: git.DefaultRemoteName, + }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) && !errors.Is(err, transport.ErrEmptyRemoteRepository) { + return err + } + + return nil +} + +func (c *client) Update(ctx context.Context, msg string, push bool) error { + repo, err := c.Repository(ctx) + if err != nil { + return err + } + + worktree, err := repo.Worktree() + if err != nil { + return err + } + + err = worktree.AddGlob("*") + + if err != nil { + return err + } + + _, err = worktree.Commit(msg, &git.CommitOptions{}) + + if err != nil { + return err + } + + if !push { + return nil + } + + if err := repo.PushContext(ctx, &git.PushOptions{ + RemoteName: git.DefaultRemoteName, + }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + + return nil +} + +func (c *client) Setup(system vfs.FileSystem) error { + c.vfs = vfs.New(system) + _, err := c.Repository(context.Background()) + return err +} + +func (c *client) Close(object *accessobj.AccessObject) error { + return c.Update(context.Background(), "OCM Repository Update", true) +} diff --git a/api/utils/accessio/downloader/git/downloader.go b/api/utils/accessio/downloader/git/downloader.go new file mode 100644 index 0000000000..d91081cb0c --- /dev/null +++ b/api/utils/accessio/downloader/git/downloader.go @@ -0,0 +1,111 @@ +package git + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "regexp" + "sync" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage" + "github.com/go-git/go-git/v5/storage/memory" + + "ocm.software/ocm/api/utils/accessio/downloader" +) + +const localRemoteName = "origin" + +type CloseableDownloader interface { + downloader.Downloader + Close() error +} + +// Downloader simply uses the default HTTP client to download the contents of a URL. +type Downloader struct { + cloneOpts *git.CloneOptions + grepOpts *git.GrepOptions + + matching *regexp.Regexp + + mu sync.Mutex + buf *bytes.Buffer + storage storage.Storer +} + +var _ downloader.Downloader = (*Downloader)(nil) + +func NewDownloader(url string, ref string, path string) CloseableDownloader { + refName := plumbing.ReferenceName(ref) + return &Downloader{ + cloneOpts: &git.CloneOptions{ + URL: url, + RemoteName: localRemoteName, + ReferenceName: refName, + SingleBranch: true, + Tags: git.NoTags, + Depth: 0, + }, + matching: regexp.MustCompile(fmt.Sprintf(`%s`, path)), + buf: bytes.NewBuffer(make([]byte, 0, 4096)), + storage: memory.NewStorage(), + } +} + +func (d *Downloader) Download(w io.WriterAt) error { + d.mu.Lock() + defer d.mu.Unlock() + + ctx := context.Background() + + // no support for git archive yet, so we need to clone the repository in bare mode + repo, err := git.CloneContext(ctx, d.storage, nil, d.cloneOpts) + if err != nil { + return fmt.Errorf("failed to clone repository %s: %w", d.cloneOpts.URL, err) + } + + trees, err := repo.TreeObjects() + if err != nil { + return fmt.Errorf("failed to get tree objects: %w", err) + } + + if err := trees.ForEach(func(t *object.Tree) error { + return t.Files().ForEach(d.copyFileToBuffer) + }); err != nil { + return fmt.Errorf("failed to iterate over trees: %w", err) + } + + defer d.buf.Reset() + if _, err := w.WriteAt(d.buf.Bytes(), 0); err != nil { + return fmt.Errorf("failed to write blobs: %w", err) + } + + return nil +} + +func (d *Downloader) copyFileToBuffer(file *object.File) error { + if !d.matching.MatchString(file.Name) { + return nil + } + + reader, err := file.Reader() + if err != nil { + return err + } + _, err = io.Copy(d.buf, reader) + return errors.Join(err, reader.Close()) +} + +func (d *Downloader) Close() error { + d.mu.Lock() + defer d.mu.Unlock() + + d.buf = nil + d.storage = nil + + return nil +} From 554390d123828216c0624ea72aed62cb21bfb73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Tue, 20 Aug 2024 16:56:01 +0200 Subject: [PATCH 02/22] feat: git authentication support --- .../builtin/git/identity/identity.go | 132 ++++++++++++++++ .../builtin/git/identity/identity_test.go | 143 ++++++++++++++++++ .../builtin/git/identity/suite_test.go | 13 ++ api/credentials/builtin/init.go | 1 + api/oci/extensions/repositories/git/format.go | 15 -- .../extensions/repositories/git/git_test.go | 4 - .../extensions/accessmethods/git/method.go | 111 ++++++++------ .../accessmethods/git/suite_test.go | 6 + api/tech/git/auth.go | 46 ++++++ api/tech/git/ref.go | 4 + api/tech/git/resolver.go | 6 +- .../accessio/downloader/git/downloader.go | 4 +- 12 files changed, 417 insertions(+), 68 deletions(-) create mode 100644 api/credentials/builtin/git/identity/identity.go create mode 100644 api/credentials/builtin/git/identity/identity_test.go create mode 100644 api/credentials/builtin/git/identity/suite_test.go create mode 100644 api/tech/git/auth.go diff --git a/api/credentials/builtin/git/identity/identity.go b/api/credentials/builtin/git/identity/identity.go new file mode 100644 index 0000000000..e1afecc4ef --- /dev/null +++ b/api/credentials/builtin/git/identity/identity.go @@ -0,0 +1,132 @@ +package identity + +import ( + "strings" + + giturls "github.com/whilp/git-urls" + "ocm.software/ocm/api/credentials/cpi" + "ocm.software/ocm/api/credentials/identity/hostpath" + "ocm.software/ocm/api/utils/listformat" +) + +const CONSUMER_TYPE = "Git" + +var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) + +func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { + return identityMatcher(pattern, cur, id) +} + +func init() { + attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + ATTR_TOKEN, "HTTP token authentication", + ATTR_PRIVATE_KEY, "Private Key authentication certificate", + }) + cpi.RegisterStandardIdentity(CONSUMER_TYPE, identityMatcher, + `Git credential matcher + +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +the `+hostpath.IDENTITY_TYPE+` type.`, + attrs) +} + +const ( + ID_HOSTNAME = hostpath.ID_HOSTNAME + ID_PATH = "path" + ID_PORT = hostpath.ID_PORT + ID_SCHEME = hostpath.ID_SCHEME +) + +const ( + ATTR_TOKEN = cpi.ATTR_TOKEN + ATTR_USERNAME = cpi.ATTR_USERNAME + ATTR_PASSWORD = cpi.ATTR_PASSWORD + ATTR_PRIVATE_KEY = cpi.ATTR_PRIVATE_KEY +) + +func GetConsumerId(repoURL string) (cpi.ConsumerIdentity, error) { + host := "" + port := "" + defaultPort := "" + scheme := "" + path := "" + + if repoURL != "" { + u, err := giturls.Parse(repoURL) + if err == nil { + host = u.Host + } else { + return nil, err + } + + scheme = u.Scheme + switch scheme { + case "http": + defaultPort = "80" + case "https": + defaultPort = "443" + case "git": + defaultPort = "9418" + case "ssh": + defaultPort = "22" + case "file": + host = "localhost" + path = u.Path + } + + } + + if idx := strings.Index(host, ":"); idx > 0 { + port = host[idx+1:] + host = host[:idx] + } + + id := cpi.ConsumerIdentity{ + cpi.ID_TYPE: CONSUMER_TYPE, + ID_HOSTNAME: host, + } + + if port != "" { + id[ID_PORT] = port + } else if defaultPort != "" { + id[ID_PORT] = defaultPort + } + + if path != "" { + id[ID_PATH] = path + } + + id[ID_SCHEME] = scheme + + return id, nil +} + +func TokenCredentials(token string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_TOKEN: token, + } +} + +func BasicAuthCredentials(username, password string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_USERNAME: username, + ATTR_PASSWORD: password, + } +} + +func PublicKeyCredentials(username, publicKey string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_USERNAME: username, + ATTR_PRIVATE_KEY: publicKey, + } +} + +func GetCredentials(ctx cpi.ContextProvider, repoURL string) (cpi.Credentials, error) { + id, err := GetConsumerId(repoURL) + if err != nil { + return nil, err + } + return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id, identityMatcher) +} diff --git a/api/credentials/builtin/git/identity/identity_test.go b/api/credentials/builtin/git/identity/identity_test.go new file mode 100644 index 0000000000..8fc82507b2 --- /dev/null +++ b/api/credentials/builtin/git/identity/identity_test.go @@ -0,0 +1,143 @@ +package identity_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "ocm.software/ocm/api/credentials/builtin/git/identity" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext" + "ocm.software/ocm/api/oci" + common "ocm.software/ocm/api/utils/misc" +) + +var _ = Describe("consumer id handling", func() { + repo := "https://github.com/torvalds/linux.git" + + Context("id determination", func() { + It("handles https repos", func() { + id, err := GetConsumerId(repo) + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "443", + "hostname", "github.com", + "scheme", "https", + ))) + }) + + It("handles http repos", func() { + id, err := GetConsumerId("http://github.com/torvalds/linux.git") + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "80", + "hostname", "github.com", + "scheme", "http", + ))) + }) + + It("handles ssh standard format repos", func() { + id, err := GetConsumerId("ssh://github.com/torvalds/linux.git") + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "22", + "hostname", "github.com", + "scheme", "ssh", + ))) + }) + + It("handles ssh git @ format repos", func() { + id, err := GetConsumerId("git@github.com:torvalds/linux.git") + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "22", + "hostname", "github.com", + "scheme", "ssh", + ))) + }) + + It("handles git format repos", func() { + id, err := GetConsumerId("git://github.com/torvalds/linux.git") + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "9418", + "hostname", "github.com", + "scheme", "git", + ))) + }) + + It("handles file format repos", func() { + id, err := GetConsumerId("file:///path/to/linux/repo") + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "scheme", "file", + "hostname", "localhost", + "path", "/path/to/linux/repo", + ))) + }) + }) + + Context("query credentials", func() { + var ctx oci.Context + var credctx credentials.Context + + BeforeEach(func() { + ctx = oci.New(datacontext.MODE_EXTENDED) + credctx = ctx.CredentialsContext() + }) + + It("Basic Auth", func() { + user, pass := "linus", "torvalds" + id, err := GetConsumerId(repo) + Expect(err).ToNot(HaveOccurred()) + credctx.SetCredentialsForConsumer(id, + credentials.CredentialsFromList( + ATTR_USERNAME, user, + ATTR_PASSWORD, pass, + ), + ) + + creds, err := GetCredentials(ctx, repo) + Expect(err).ToNot(HaveOccurred()) + Expect(creds).To(BeEquivalentTo(common.Properties{ + ATTR_USERNAME: user, + ATTR_PASSWORD: pass, + })) + }) + + It("Token Auth", func() { + token := "mytoken" + id, err := GetConsumerId(repo) + Expect(err).ToNot(HaveOccurred()) + credctx.SetCredentialsForConsumer(id, + credentials.CredentialsFromList( + ATTR_TOKEN, token, + ), + ) + + creds, err := GetCredentials(ctx, repo) + Expect(err).ToNot(HaveOccurred()) + Expect(creds).To(BeEquivalentTo(common.Properties{ + ATTR_TOKEN: token, + })) + }) + + It("Public Key Auth", func() { + user, key := "linus", "path/to/my/id_rsa" + id, err := GetConsumerId(repo) + Expect(err).ToNot(HaveOccurred()) + credctx.SetCredentialsForConsumer(id, + credentials.CredentialsFromList( + ATTR_USERNAME, user, + ATTR_PRIVATE_KEY, key, + ), + ) + + creds, err := GetCredentials(ctx, repo) + Expect(err).ToNot(HaveOccurred()) + Expect(creds).To(BeEquivalentTo(common.Properties{ + ATTR_USERNAME: user, + ATTR_PRIVATE_KEY: key, + })) + }) + }) +}) diff --git a/api/credentials/builtin/git/identity/suite_test.go b/api/credentials/builtin/git/identity/suite_test.go new file mode 100644 index 0000000000..d79c7330b1 --- /dev/null +++ b/api/credentials/builtin/git/identity/suite_test.go @@ -0,0 +1,13 @@ +package identity_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Git Identity Suite") +} diff --git a/api/credentials/builtin/init.go b/api/credentials/builtin/init.go index 7b7cc5062d..38a23142b9 100644 --- a/api/credentials/builtin/init.go +++ b/api/credentials/builtin/init.go @@ -1,5 +1,6 @@ package builtin import ( + _ "ocm.software/ocm/api/credentials/builtin/git/identity" _ "ocm.software/ocm/api/credentials/builtin/github" ) diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go index 2c63170d50..d0ddea381a 100644 --- a/api/oci/extensions/repositories/git/format.go +++ b/api/oci/extensions/repositories/git/format.go @@ -4,25 +4,10 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/oci/extensions/repositories/ctf" - "ocm.software/ocm/api/oci/extensions/repositories/ctf/format" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" ) -const ( - ArtifactIndexFileName = format.ArtifactIndexFileName - BlobsDirectoryName = format.BlobsDirectoryName -) - -var accessObjectInfo = &accessobj.DefaultAccessObjectInfo{ - DescriptorFileName: ArtifactIndexFileName, - ObjectTypeName: "repository", - ElementDirectoryName: BlobsDirectoryName, - ElementTypeName: "blob", - DescriptorHandlerFactory: ctf.NewStateHandler, -} - type Object = Repository // ////////////////////////////////////////////////////////////////////////////// diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go index 1c2e4b9061..413760fc75 100644 --- a/api/oci/extensions/repositories/git/git_test.go +++ b/api/oci/extensions/repositories/git/git_test.go @@ -1,7 +1,6 @@ package git_test import ( - "embed" "fmt" "github.com/go-git/go-git/v5" @@ -31,9 +30,6 @@ import ( "ocm.software/ocm/api/utils/refmgmt" ) -//go:embed testdata/repo -var testData embed.FS - var _ = Describe("ctf management", func() { var remoteRepo *git.Repository diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index 2ab8517637..aaaf1149d9 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -4,11 +4,13 @@ import ( "fmt" "io" "net/http" - "net/url" - "sync" "github.com/go-git/go-git/v5/plumbing" "github.com/mandelsoft/goutils/errors" + giturls "github.com/whilp/git-urls" + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/credentials/builtin/git/identity" + techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/internal" @@ -85,33 +87,49 @@ func (a *AccessSpec) AccessMethod(c internal.ComponentVersionAccess) (internal.A return accspeccpi.AccessMethodForImplementation(newMethod(c, a)) } -func newMethod(c internal.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { - u, err := url.Parse(a.RepoURL) +func newMethod(componentVersionAccess internal.ComponentVersionAccess, accessSpec *AccessSpec) (accspeccpi.AccessMethodImpl, error) { + u, err := giturls.Parse(accessSpec.RepoURL) if err != nil { - return nil, errors.ErrInvalidWrap(err, "repository repoURL", a.RepoURL) + return nil, errors.ErrInvalidWrap(err, "repository repoURL", accessSpec.RepoURL) } - if err := plumbing.ReferenceName(a.Ref).Validate(); err != nil { - return nil, errors.ErrInvalidWrap(err, "commit hash", a.Ref) + if err := plumbing.ReferenceName(accessSpec.Ref).Validate(); err != nil { + return nil, errors.ErrInvalidWrap(err, "commit hash", accessSpec.Ref) + } + + creds, cid, err := getCreds(accessSpec.RepoURL, componentVersionAccess.GetContext().CredentialsContext()) + if err != nil { + return nil, fmt.Errorf("failed to get credentials for repository %s: %w", accessSpec.RepoURL, err) + } + + auth, err := techgit.AuthFromCredentials(creds) + if err != nil && !errors.Is(err, techgit.ErrNoValidGitCredentials) { + return nil, fmt.Errorf("failed to get auth method for repository %s: %w", accessSpec.RepoURL, err) + } + + gitDownloader := git.NewDownloader(u.String(), accessSpec.Ref, accessSpec.PathSpec, auth) + cachedGitBlobAccessor := accessobj.CachedBlobAccessForWriter( + componentVersionAccess.GetContext(), + mime.MIME_OCTET, + accessio.NewWriteAtWriter(gitDownloader.Download), + ) + jointCloser := func() error { + return errors.Join(gitDownloader.Close(), cachedGitBlobAccessor.Close()) } return &accessMethod{ - repoURL: u.String(), - compvers: c, - spec: a, - ref: a.Ref, + spec: accessSpec, + access: cachedGitBlobAccessor, + close: jointCloser, + cid: cid, }, nil } type accessMethod struct { - lock sync.Mutex + spec *AccessSpec access blobaccess.BlobAccess + close func() error - compvers accspeccpi.ComponentVersionAccess - spec *AccessSpec - - repoURL string - path string - ref string + cid credentials.ConsumerIdentity } var _ accspeccpi.AccessMethodImpl = &accessMethod{} @@ -120,45 +138,24 @@ func (m *accessMethod) Close() error { if m.access == nil { return nil } - return m.access.Close() + + var err error + if m.close != nil { + err = m.close() + } + err = errors.Join(err, m.access.Close()) + + return err } func (m *accessMethod) Get() ([]byte, error) { - if err := m.setup(); err != nil { - return nil, err - } return m.access.Get() } func (m *accessMethod) Reader() (io.ReadCloser, error) { - if err := m.setup(); err != nil { - return nil, err - } return m.access.Reader() } -func (m *accessMethod) setup() error { - m.lock.Lock() - defer m.lock.Unlock() - - if m.access != nil { - return nil - } - - d := git.NewDownloader(m.repoURL, m.ref, m.path) - defer d.Close() - - cacheBlobAccess := accessobj.CachedBlobAccessForWriter( - m.compvers.GetContext(), - m.MimeType(), - accessio.NewWriteAtWriter(d.Download), - ) - - m.access = cacheBlobAccess - - return nil -} - func (m *accessMethod) MimeType() string { return mime.MIME_OCTET } @@ -174,3 +171,23 @@ func (m *accessMethod) GetKind() string { func (m *accessMethod) AccessSpec() internal.AccessSpec { return m.spec } + +func (m *accessMethod) GetConsumerId(_ ...credentials.UsageContext) credentials.ConsumerIdentity { + return m.cid +} + +func (m *accessMethod) GetIdentityMatcher() string { + return identity.CONSUMER_TYPE +} + +func getCreds(repoURL string, cctx credentials.Context) (credentials.Credentials, credentials.ConsumerIdentity, error) { + id, err := identity.GetConsumerId(repoURL) + if err != nil { + return nil, nil, err + } + creds, err := credentials.CredentialsForConsumer(cctx.CredentialsContext(), id, identity.IdentityMatcher) + if creds == nil || err != nil { + return nil, id, err + } + return creds, id, nil +} diff --git a/api/ocm/extensions/accessmethods/git/suite_test.go b/api/ocm/extensions/accessmethods/git/suite_test.go index 8a6d992a8c..30a5c5b531 100644 --- a/api/ocm/extensions/accessmethods/git/suite_test.go +++ b/api/ocm/extensions/accessmethods/git/suite_test.go @@ -85,6 +85,12 @@ var _ = Describe("Method", func() { ) }) + BeforeEach(func() { + var err error + expectedBlobContent, err = testData.ReadFile(filepath.Join("testdata", "repo", "file_in_repo")) + Expect(err).ToNot(HaveOccurred()) + }) + It("downloads artifacts", func() { m, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) Expect(err).ToNot(HaveOccurred()) diff --git a/api/tech/git/auth.go b/api/tech/git/auth.go new file mode 100644 index 0000000000..76347d91a0 --- /dev/null +++ b/api/tech/git/auth.go @@ -0,0 +1,46 @@ +package git + +import ( + "errors" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/credentials/builtin/git/identity" + + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + gssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" +) + +var ErrNoValidGitCredentials = errors.New("no valid credentials found for git authentication") + +type AuthMethod = transport.AuthMethod + +// AuthFromCredentials creates a git authentication method from the given credentials. +// If no valid credentials are found, ErrNoValidGitCredentials is returned. +// However, one can still perform anonymous operations with the git client if the repo allows it. +func AuthFromCredentials(creds credentials.Credentials) (AuthMethod, error) { + if creds == nil { + return nil, ErrNoValidGitCredentials + } + + if creds.ExistsProperty(identity.ATTR_PRIVATE_KEY) { + return gssh.NewPublicKeysFromFile( + creds.GetProperty(identity.ATTR_USERNAME), + creds.GetProperty(identity.ATTR_PRIVATE_KEY), + creds.GetProperty(identity.ATTR_PASSWORD), + ) + } + + if creds.ExistsProperty(identity.ATTR_TOKEN) { + return &http.TokenAuth{Token: creds.GetProperty(identity.ATTR_TOKEN)}, nil + } + + if creds.ExistsProperty(identity.ATTR_USERNAME) { + return &http.BasicAuth{ + Username: creds.GetProperty(identity.ATTR_USERNAME), + Password: creds.GetProperty(identity.ATTR_PASSWORD), + }, nil + } + + return nil, ErrNoValidGitCredentials +} diff --git a/api/tech/git/ref.go b/api/tech/git/ref.go index 7a3a039309..a3d0a89ccf 100644 --- a/api/tech/git/ref.go +++ b/api/tech/git/ref.go @@ -23,6 +23,10 @@ const refToPathSeparator = "#" // - path is the path to the file or directory to use as the source, if not specified, defaults to the root of the repository. var refRegexp = regexp.MustCompile(`^([^@#]+)(@[^#\n]+)?(#[^@\n]+)?`) +// gurl represents a git URL reference. +// It contains the URL of the git repository, +// the reference to check out, +// and the path to the file or directory to use as the source. type gurl struct { url *url.URL ref plumbing.ReferenceName diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index 55223e6915..b0ea1fb803 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -26,6 +26,7 @@ type client struct { *gurl storage storage.Storer + auth AuthMethod } type Client interface { @@ -66,6 +67,7 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { repo, err := git.Open(strg, billy) if errors.Is(err, git.ErrRepositoryNotExists) { repo, err = git.CloneContext(ctx, strg, billy, &git.CloneOptions{ + Auth: c.auth, URL: c.url.String(), RemoteName: git.DefaultRemoteName, ReferenceName: c.ref, @@ -84,6 +86,7 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { } if newRepo { if err := repo.FetchContext(ctx, &git.FetchOptions{ + Auth: c.auth, RemoteName: git.DefaultRemoteName, Depth: 0, Tags: git.AllTags, @@ -154,6 +157,7 @@ func (c *client) Refresh(ctx context.Context) error { } if err := worktree.PullContext(ctx, &git.PullOptions{ + Auth: c.auth, RemoteName: git.DefaultRemoteName, }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) && !errors.Is(err, transport.ErrEmptyRemoteRepository) { return err @@ -204,6 +208,6 @@ func (c *client) Setup(system vfs.FileSystem) error { return err } -func (c *client) Close(object *accessobj.AccessObject) error { +func (c *client) Close(_ *accessobj.AccessObject) error { return c.Update(context.Background(), "OCM Repository Update", true) } diff --git a/api/utils/accessio/downloader/git/downloader.go b/api/utils/accessio/downloader/git/downloader.go index d91081cb0c..39d0be1884 100644 --- a/api/utils/accessio/downloader/git/downloader.go +++ b/api/utils/accessio/downloader/git/downloader.go @@ -14,6 +14,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/storage/memory" + techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio/downloader" ) @@ -39,10 +40,11 @@ type Downloader struct { var _ downloader.Downloader = (*Downloader)(nil) -func NewDownloader(url string, ref string, path string) CloseableDownloader { +func NewDownloader(url string, ref string, path string, auth techgit.AuthMethod) CloseableDownloader { refName := plumbing.ReferenceName(ref) return &Downloader{ cloneOpts: &git.CloneOptions{ + Auth: auth, URL: url, RemoteName: localRemoteName, ReferenceName: refName, From 955c3775758b00335826ef79f1d0a48ff581db76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Tue, 20 Aug 2024 23:57:35 +0200 Subject: [PATCH 03/22] DRAFT: git fs support --- api/oci/extensions/repositories/git/format.go | 20 +- .../extensions/repositories/git/git_test.go | 4 +- .../extensions/repositories/git/namespace.go | 6 +- .../extensions/repositories/git/repository.go | 30 +- api/oci/extensions/repositories/git/type.go | 14 +- .../accessmethods/git/method_test.go | 101 ++++++- .../accessmethods/git/suite_test.go | 101 +------ api/ocm/extensions/repositories/git/format.go | 49 ++++ .../extensions/repositories/git/repo_test.go | 260 ++++++++++++++++++ .../extensions/repositories/git/suite_test.go | 13 + api/ocm/extensions/repositories/git/type.go | 18 ++ api/ocm/extensions/repositories/init.go | 1 + api/tech/git/fs.go | 111 ++++++-- api/tech/git/ref.go | 5 +- api/tech/git/resolver.go | 40 +-- 15 files changed, 584 insertions(+), 189 deletions(-) create mode 100644 api/ocm/extensions/repositories/git/format.go create mode 100644 api/ocm/extensions/repositories/git/repo_test.go create mode 100644 api/ocm/extensions/repositories/git/suite_test.go create mode 100644 api/ocm/extensions/repositories/git/type.go diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go index d0ddea381a..5c31d92aa3 100644 --- a/api/oci/extensions/repositories/git/format.go +++ b/api/oci/extensions/repositories/git/format.go @@ -1,29 +1,21 @@ package git import ( - "github.com/mandelsoft/vfs/pkg/vfs" - "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" ) -type Object = Repository - // ////////////////////////////////////////////////////////////////////////////// -func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string) (Object, error) { - return New(cpi.FromProvider(ctx), &RepositorySpec{ - URL: url, - AccessMode: acc, - }) -} - -func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, mode vfs.FileMode, option ...accessio.Option) (Object, error) { - spec, err := NewRepositorySpec(acc, url, mode, option...) +func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, option ...accessio.Option) (Repository, error) { + spec, err := NewRepositorySpec(acc, url, option...) if err != nil { return nil, err } - return New(cpi.FromProvider(ctx), spec) } + +func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, option ...accessio.Option) (Repository, error) { + return Open(ctx, acc, url, option...) +} diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go index 413760fc75..05fb4b6187 100644 --- a/api/oci/extensions/repositories/git/git_test.go +++ b/api/oci/extensions/repositories/git/git_test.go @@ -66,9 +66,7 @@ var _ = Describe("ctf management", func() { }) It("instantiate git based ctf", func() { - repo := Must(rgit.Create(ctx, accessobj.ACC_CREATE, repoURL, 0o700, - accessio.RepresentationFileSystem(workspace), - )) + repo := Must(rgit.Create(ctx, accessobj.ACC_CREATE, repoURL, accessio.RepresentationFileSystem(workspace))) ns := Must(repo.LookupNamespace("test")) testData := []byte("testdata") diff --git a/api/oci/extensions/repositories/git/namespace.go b/api/oci/extensions/repositories/git/namespace.go index aac57830f1..5b3b2755d0 100644 --- a/api/oci/extensions/repositories/git/namespace.go +++ b/api/oci/extensions/repositories/git/namespace.go @@ -14,12 +14,12 @@ import ( "ocm.software/ocm/api/utils/blobaccess/blobaccess" ) -func NewNamespace(repo *repository, name string) (cpi.NamespaceAccess, error) { +func NewNamespace(repo *RepositoryImpl, name string) (cpi.NamespaceAccess, error) { ctfNamespace, err := newNamespaceContainer(repo, name) if err != nil { return nil, err } - return support.NewNamespaceAccess(name, ctfNamespace, repo, "Git repository Branch") + return support.NewNamespaceAccess(name, ctfNamespace, repo, "Git RepositoryImpl Branch") } type namespaceContainer struct { @@ -31,7 +31,7 @@ type namespaceContainer struct { var _ support.NamespaceContainer = (*namespaceContainer)(nil) -func newNamespaceContainer(repo *repository, name string) (support.NamespaceContainer, error) { +func newNamespaceContainer(repo *RepositoryImpl, name string) (support.NamespaceContainer, error) { ctfNamespace, err := repo.ctf.LookupNamespace(name) if err != nil { return nil, err diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index 0dd42bce44..dfb873f976 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -19,7 +19,7 @@ type Repository interface { cpi.Repository } -type repository struct { +type RepositoryImpl struct { cpi.RepositoryImplBase logger logging.UnboundLogger spec *RepositorySpec @@ -28,20 +28,20 @@ type repository struct { } var ( - _ cpi.RepositoryImpl = (*repository)(nil) - _ credentials.ConsumerIdentityProvider = &repository{} + _ cpi.RepositoryImpl = (*RepositoryImpl)(nil) + _ credentials.ConsumerIdentityProvider = &RepositoryImpl{} ) func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { urs := spec.UniformRepositorySpec() - i := &repository{ + i := &RepositoryImpl{ RepositoryImplBase: cpi.NewRepositoryImplBase(ctx), logger: logging.DynamicLogger(ctx, logging.NewAttribute(ocmlog.ATTR_HOST, urs.Host)), spec: spec, } var err error - if i.client, err = git.NewClient(spec.URL); err != nil { + if i.client, err = git.NewClient(spec.URL, git.ClientOptions{}); err != nil { return nil, err } @@ -54,50 +54,50 @@ func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { } i.ctf = repo - return cpi.NewRepository(i), nil + return cpi.NewRepository(i, "git"), nil } -func (r *repository) GetSpecification() cpi.RepositorySpec { +func (r *RepositoryImpl) GetSpecification() cpi.RepositorySpec { return r.spec } -func (r *repository) Close() error { +func (r *RepositoryImpl) Close() error { return r.ctf.Close() } -func (r *repository) GetIdentityMatcher() string { +func (r *RepositoryImpl) GetIdentityMatcher() string { return identity.CONSUMER_TYPE } -func (r *repository) NamespaceLister() cpi.NamespaceLister { +func (r *RepositoryImpl) NamespaceLister() cpi.NamespaceLister { return r.ctf.NamespaceLister() } -func (r *repository) IsReadOnly() bool { +func (r *RepositoryImpl) IsReadOnly() bool { return false } -func (r *repository) ExistsArtifact(name string, version string) (bool, error) { +func (r *RepositoryImpl) ExistsArtifact(name string, version string) (bool, error) { if err := r.client.Refresh(context.Background()); err != nil { return false, err } return r.ctf.ExistsArtifact(name, version) } -func (r *repository) LookupArtifact(name string, version string) (cpi.ArtifactAccess, error) { +func (r *RepositoryImpl) LookupArtifact(name string, version string) (cpi.ArtifactAccess, error) { if err := r.client.Refresh(context.Background()); err != nil { return nil, err } return r.ctf.LookupArtifact(name, version) } -func (r *repository) LookupNamespace(name string) (cpi.NamespaceAccess, error) { +func (r *RepositoryImpl) LookupNamespace(name string) (cpi.NamespaceAccess, error) { if err := r.client.Refresh(context.Background()); err != nil { return nil, err } return NewNamespace(r, name) } -func (r *repository) GetConsumerId(ctx ...cpicredentials.UsageContext) cpicredentials.ConsumerIdentity { +func (r *RepositoryImpl) GetConsumerId(ctx ...cpicredentials.UsageContext) cpicredentials.ConsumerIdentity { return nil } diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go index fb2d2511ab..04052af285 100644 --- a/api/oci/extensions/repositories/git/type.go +++ b/api/oci/extensions/repositories/git/type.go @@ -1,8 +1,6 @@ package git import ( - "github.com/mandelsoft/vfs/pkg/vfs" - "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/utils/accessio" @@ -34,19 +32,16 @@ func IsKind(k string) bool { return k == Type || k == ShortType } -// RepositorySpec describes an CTF repository interface backed by a git repository. +// RepositorySpec describes an CTF RepositoryImpl interface backed by a git RepositoryImpl. type RepositorySpec struct { runtime.ObjectVersionedType `json:",inline"` accessio.StandardOptions `json:",inline"` - // URL is the url of the repository to resolve artifacts. - URL string `json:"baseUrl"` + // URL is the url of the RepositoryImpl to resolve artifacts. + URL string `json:"url"` // AccessMode can be set to request readonly access or creation AccessMode accessobj.AccessMode `json:"accessMode,omitempty"` - - // FileMode is the file mode for the repository in the filesystem. - FileMode vfs.FileMode `json:"fileMode"` } var _ cpi.RepositorySpec = (*RepositorySpec)(nil) @@ -54,7 +49,7 @@ var _ cpi.RepositorySpec = (*RepositorySpec)(nil) var _ cpi.IntermediateRepositorySpecAspect = (*RepositorySpec)(nil) // NewRepositorySpec creates a new RepositorySpec. -func NewRepositorySpec(mode accessobj.AccessMode, url string, fileMode vfs.FileMode, opts ...accessio.Option) (*RepositorySpec, error) { +func NewRepositorySpec(mode accessobj.AccessMode, url string, opts ...accessio.Option) (*RepositorySpec, error) { o, err := accessio.AccessOptions(nil, opts...) if err != nil { return nil, err @@ -63,7 +58,6 @@ func NewRepositorySpec(mode accessobj.AccessMode, url string, fileMode vfs.FileM return &RepositorySpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), URL: url, - FileMode: fileMode, StandardOptions: *o.(*accessio.StandardOptions), AccessMode: mode, }, nil diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go index 59ce0ec1fc..30a5c5b531 100644 --- a/api/ocm/extensions/accessmethods/git/method_test.go +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -1,13 +1,102 @@ -package git +package git_test import ( - "testing" + "embed" + _ "embed" + "fmt" + "io" + "os" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/cpi" + me "ocm.software/ocm/api/ocm/extensions/accessmethods/git" ) -func TestConfig(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Github Test Suite") -} +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("Method", func() { + var ( + ctx ocm.Context + expectedBlobContent []byte + accessSpec *me.AccessSpec + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + BeforeEach(func() { + repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo, err := git.PlainInit(repoDir, false) + Expect(err).ToNot(HaveOccurred()) + + repoBase := filepath.Join("testdata", "repo") + repoTestData, err := testData.ReadDir(repoBase) + Expect(err).ToNot(HaveOccurred()) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file, err := testData.Open(path) + Expect(err).ToNot(HaveOccurred()) + + fileInRepo, err := os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0600, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = io.Copy(fileInRepo, file) + Expect(err).ToNot(HaveOccurred()) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt, err := repo.Worktree() + Expect(err).ToNot(HaveOccurred()) + Expect(wt.AddGlob("*")).To(Succeed()) + _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{}) + Expect(err).ToNot(HaveOccurred()) + + accessSpec = me.New( + fmt.Sprintf("file://%s", repoDir), + string(plumbing.Master), + ".", + ) + }) + + BeforeEach(func() { + var err error + expectedBlobContent, err = testData.ReadFile(filepath.Join("testdata", "repo", "file_in_repo")) + Expect(err).ToNot(HaveOccurred()) + }) + + It("downloads artifacts", func() { + m, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) + Expect(err).ToNot(HaveOccurred()) + content, err := m.Get() + Expect(err).ToNot(HaveOccurred()) + Expect(content).To(Equal(expectedBlobContent)) + }) + +}) diff --git a/api/ocm/extensions/accessmethods/git/suite_test.go b/api/ocm/extensions/accessmethods/git/suite_test.go index 30a5c5b531..59ce0ec1fc 100644 --- a/api/ocm/extensions/accessmethods/git/suite_test.go +++ b/api/ocm/extensions/accessmethods/git/suite_test.go @@ -1,102 +1,13 @@ -package git_test +package git import ( - "embed" - _ "embed" - "fmt" - "io" - "os" + "testing" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mandelsoft/filepath/pkg/filepath" - "github.com/mandelsoft/vfs/pkg/cwdfs" - "github.com/mandelsoft/vfs/pkg/osfs" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "ocm.software/ocm/api/datacontext/attrs/tmpcache" - "ocm.software/ocm/api/datacontext/attrs/vfsattr" - "ocm.software/ocm/api/ocm" - "ocm.software/ocm/api/ocm/cpi" - me "ocm.software/ocm/api/ocm/extensions/accessmethods/git" ) -//go:embed testdata/repo -var testData embed.FS - -var _ = Describe("Method", func() { - var ( - ctx ocm.Context - expectedBlobContent []byte - accessSpec *me.AccessSpec - ) - - ctx = ocm.New() - - BeforeEach(func() { - tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) - Expect(err).ToNot(HaveOccurred()) - tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) - vfsattr.Set(ctx, tempVFS) - }) - - BeforeEach(func() { - repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" - - repo, err := git.PlainInit(repoDir, false) - Expect(err).ToNot(HaveOccurred()) - - repoBase := filepath.Join("testdata", "repo") - repoTestData, err := testData.ReadDir(repoBase) - Expect(err).ToNot(HaveOccurred()) - - for _, entry := range repoTestData { - path := filepath.Join(repoBase, entry.Name()) - repoPath := filepath.Join(repoDir, entry.Name()) - - file, err := testData.Open(path) - Expect(err).ToNot(HaveOccurred()) - - fileInRepo, err := os.OpenFile( - repoPath, - os.O_CREATE|os.O_RDWR|os.O_TRUNC, - 0600, - ) - Expect(err).ToNot(HaveOccurred()) - - _, err = io.Copy(fileInRepo, file) - Expect(err).ToNot(HaveOccurred()) - - Expect(fileInRepo.Close()).To(Succeed()) - Expect(file.Close()).To(Succeed()) - } - - wt, err := repo.Worktree() - Expect(err).ToNot(HaveOccurred()) - Expect(wt.AddGlob("*")).To(Succeed()) - _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{}) - Expect(err).ToNot(HaveOccurred()) - - accessSpec = me.New( - fmt.Sprintf("file://%s", repoDir), - string(plumbing.Master), - ".", - ) - }) - - BeforeEach(func() { - var err error - expectedBlobContent, err = testData.ReadFile(filepath.Join("testdata", "repo", "file_in_repo")) - Expect(err).ToNot(HaveOccurred()) - }) - - It("downloads artifacts", func() { - m, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) - Expect(err).ToNot(HaveOccurred()) - content, err := m.Get() - Expect(err).ToNot(HaveOccurred()) - Expect(content).To(Equal(expectedBlobContent)) - }) - -}) +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Github Test Suite") +} diff --git a/api/ocm/extensions/repositories/git/format.go b/api/ocm/extensions/repositories/git/format.go new file mode 100644 index 0000000000..bcc77bb607 --- /dev/null +++ b/api/ocm/extensions/repositories/git/format.go @@ -0,0 +1,49 @@ +package git + +import ( + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" + + "ocm.software/ocm/api/oci/extensions/repositories/git" + "ocm.software/ocm/api/ocm/cpi" + "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" +) + +var ( + FormatDirectory = ctf.FormatDirectory +) + +type Object = ctf.Object + +type FormatHandler = ctf.FormatHandler + +func GetFormats() []string { + return ctf.GetFormats() +} + +func GetFormat(name accessio.FileFormat) FormatHandler { + return ctf.GetFormat(name) +} + +const ( + ACC_CREATE = accessobj.ACC_CREATE + ACC_WRITABLE = accessobj.ACC_WRITABLE + ACC_READONLY = accessobj.ACC_READONLY +) + +func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts ...accessio.Option) (cpi.Repository, error) { + r, err := git.Open(cpi.FromProvider(ctx), acc, url, opts...) + if err != nil { + return nil, err + } + return genericocireg.NewRepository(cpi.FromProvider(ctx), nil, r), nil +} + +func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts ...accessio.Option) (cpi.Repository, error) { + r, err := git.Create(cpi.FromProvider(ctx), acc, url, opts...) + if err != nil { + return nil, err + } + return genericocireg.NewRepository(cpi.FromProvider(ctx), nil, r), nil +} diff --git a/api/ocm/extensions/repositories/git/repo_test.go b/api/ocm/extensions/repositories/git/repo_test.go new file mode 100644 index 0000000000..dfa67b4c0b --- /dev/null +++ b/api/ocm/extensions/repositories/git/repo_test.go @@ -0,0 +1,260 @@ +package git_test + +import ( + "bytes" + + gitgo "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport/client" + "github.com/go-git/go-git/v5/plumbing/transport/server" + . "github.com/mandelsoft/goutils/finalizer" + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "ocm.software/ocm/api/ocm/extensions/repositories/git" + . "ocm.software/ocm/api/ocm/testhelper" + techgit "ocm.software/ocm/api/tech/git" + + "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/tonglil/buflogr" + + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/compdesc" + metav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" + resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/blobaccess" + ocmlog "ocm.software/ocm/api/utils/logging" + "ocm.software/ocm/api/utils/mime" + "ocm.software/ocm/api/utils/refmgmt" +) + +const ( + COMPONENT = "ocm.software/ocm" + VERSION = "1.0.0" +) + +var _ = Describe("access method", func() { + + var pathFS vfs.FileSystem + // var repFS vfs.FileSystem + ctx := ocm.DefaultContext() + + BeforeEach(func() { + pathFS = memoryfs.New() + }) + + BeforeEach(func() { + remoteFS := memoryfs.New() + client.InstallProtocol("file", server.NewClient(server.NewFilesystemLoader(techgit.VFSBillyFS(remoteFS)))) + gitgo.InitWithOptions() + }) + + It("adds naked component version and later lookup", func() { + final := Finalizer{} + defer Defer(final.Finalize) + + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", + accessio.PathFileSystem(pathFS), + )) + final.Close(a, "repository") + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c, "component") + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv, "version") + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(final.Finalize()) + + refmgmt.AllocLog.Trace("opening ctf") + a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + + refmgmt.AllocLog.Trace("lookup component") + c = Must(a.LookupComponent(COMPONENT)) + final.Close(c) + + refmgmt.AllocLog.Trace("lookup version") + cv = Must(c.LookupVersion(VERSION)) + final.Close(cv) + + refmgmt.AllocLog.Trace("closing") + MustBeSuccessful(final.Finalize()) + }) + + It("adds naked component version and later shortcut lookup", func() { + final := Finalizer{} + defer Defer(final.Finalize) + + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a, "repository") + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c, "component") + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv, "version") + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(final.Finalize()) + + refmgmt.AllocLog.Trace("opening ctf") + a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + + refmgmt.AllocLog.Trace("lookup component version") + cv = Must(a.LookupComponentVersion(COMPONENT, VERSION)) + final.Close(cv) + + refmgmt.AllocLog.Trace("closing") + MustBeSuccessful(final.Finalize()) + }) + + It("adds component version", func() { + final := Finalizer{} + defer Defer(final.Finalize) + + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c) + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv) + + // add resource + MustBeSuccessful(cv.SetResourceBlob(compdesc.NewResourceMeta("text1", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) + Expect(Must(cv.GetResource(compdesc.NewIdentity("text1"))).Meta().Digest).To(Equal(DS_TESTDATA)) + + // add resource with digest + meta := compdesc.NewResourceMeta("text2", resourcetypes.PLAIN_TEXT, metav1.LocalRelation) + meta.SetDigest(DS_TESTDATA) + MustBeSuccessful(cv.SetResourceBlob(meta, blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) + Expect(Must(cv.GetResource(compdesc.NewIdentity("text2"))).Meta().Digest).To(Equal(DS_TESTDATA)) + + // reject resource with wrong digest + meta = compdesc.NewResourceMeta("text3", resourcetypes.PLAIN_TEXT, metav1.LocalRelation) + meta.SetDigest(TextResourceDigestSpec("fake")) + Expect(cv.SetResourceBlob(meta, blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)).To(MatchError("unable to set resource: digest mismatch: " + D_TESTDATA + " != fake")) + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(final.Finalize()) + + a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + + cv = Must(a.LookupComponentVersion(COMPONENT, VERSION)) + final.Close(cv) + }) + + It("adds omits unadded new component version", func() { + final := Finalizer{} + defer Defer(final.Finalize) + + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c) + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv) + + MustBeSuccessful(final.Finalize()) + + a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + + _, err := a.LookupComponentVersion(COMPONENT, VERSION) + + Expect(err).To(MatchError(ContainSubstring("component version \"github.com/mandelsoft/ocm:1.0.0\" not found: oci artifact \"1.0.0\" not found in component-descriptors/github.com/mandelsoft/ocm"))) + }) + + It("provides error for invalid bloc access", func() { + final := Finalizer{} + defer Defer(final.Finalize) + + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + final.Close(a) + c := Must(a.LookupComponent(COMPONENT)) + final.Close(c) + + cv := Must(c.NewVersion(VERSION)) + final.Close(cv) + + // add resource + Expect(ErrorFrom(cv.SetResourceBlob(compdesc.NewResourceMeta("text1", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blobaccess.ForFile(mime.MIME_TEXT, "non-existing-file"), "", nil))).To(MatchError(`file "non-existing-file" not found`)) + + MustBeSuccessful(final.Finalize()) + }) + + It("logs diff", func() { + r := Must(git.Open(ctx, git.ACC_CREATE, "test.ctf", accessio.FormatDirectory, accessio.PathFileSystem(pathFS))) + defer Close(r, "repo") + + c := Must(r.LookupComponent("acme.org/test")) + defer Close(c, "comp") + + cv := Must(c.NewVersion("v1")) + + ocmlog.PushContext(nil) + ocmlog.Context().AddRule(logging.NewConditionRule(logging.DebugLevel, genericocireg.TAG_CDDIFF)) + var buf bytes.Buffer + def := buflogr.NewWithBuffer(&buf) + ocmlog.Context().SetBaseLogger(def) + defer ocmlog.Context().ResetRules() + defer ocmlog.PopContext() + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(cv.Close()) + + cv = Must(c.LookupVersion("v1")) + cv.GetDescriptor().Provider.Name = "acme.org" + MustBeSuccessful(cv.Close()) + Expect("\n" + buf.String()).To(Equal(` +V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != acme.org] +V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != acme.org] +`)) + }) + + It("handles readonly mode", func() { + r := Must(git.Open(ctx, git.ACC_CREATE, "test.ctf", accessio.FormatDirectory, accessio.PathFileSystem(pathFS))) + defer Close(r, "repo") + + c := Must(r.LookupComponent("acme.org/test")) + defer Close(c, "comp") + + cv := Must(c.NewVersion("v1")) + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(cv.Close()) + + cv = Must(c.LookupVersion("v1")) + cv.SetReadOnly() + Expect(cv.IsReadOnly()).To(BeTrue()) + cv.GetDescriptor().Provider.Name = "acme.org" + ExpectError(cv.Close()).To(MatchError(accessio.ErrReadOnly)) + }) + + It("handles readonly mode on repo", func() { + r := Must(git.Open(ctx, git.ACC_CREATE, "test.ctf", accessio.FormatDirectory, accessio.PathFileSystem(pathFS))) + defer Close(r, "repo") + + c := Must(r.LookupComponent("acme.org/test")) + defer Close(c, "comp") + + cv := Must(c.NewVersion("v1")) + + MustBeSuccessful(c.AddVersion(cv)) + MustBeSuccessful(cv.Close()) + + r.SetReadOnly() + cv = Must(c.LookupVersion("v1")) + Expect(cv.IsReadOnly()).To(BeTrue()) + cv.GetDescriptor().Provider.Name = "acme.org" + ExpectError(cv.Close()).To(MatchError(accessio.ErrReadOnly)) + + ExpectError(c.NewVersion("v2")).To(MatchError(accessio.ErrReadOnly)) + }) +}) diff --git a/api/ocm/extensions/repositories/git/suite_test.go b/api/ocm/extensions/repositories/git/suite_test.go new file mode 100644 index 0000000000..0995130649 --- /dev/null +++ b/api/ocm/extensions/repositories/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Git Repository Test Suite") +} diff --git a/api/ocm/extensions/repositories/git/type.go b/api/ocm/extensions/repositories/git/type.go new file mode 100644 index 0000000000..5dbc01b74d --- /dev/null +++ b/api/ocm/extensions/repositories/git/type.go @@ -0,0 +1,18 @@ +package git + +import ( + "ocm.software/ocm/api/oci/extensions/repositories/git" + "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" +) + +const Type = git.Type + +func NewRepositorySpec(acc accessobj.AccessMode, url string, opts ...accessio.Option) (*genericocireg.RepositorySpec, error) { + spec, err := git.NewRepositorySpec(acc, url, opts...) + if err != nil { + return nil, err + } + return genericocireg.NewRepositorySpec(spec, nil), nil +} diff --git a/api/ocm/extensions/repositories/init.go b/api/ocm/extensions/repositories/init.go index 5471dab2e8..6a1530b56d 100644 --- a/api/ocm/extensions/repositories/init.go +++ b/api/ocm/extensions/repositories/init.go @@ -4,4 +4,5 @@ import ( _ "ocm.software/ocm/api/ocm/extensions/repositories/comparch" _ "ocm.software/ocm/api/ocm/extensions/repositories/ctf" _ "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" + _ "ocm.software/ocm/api/ocm/extensions/repositories/git" ) diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go index 8701371cda..51435f887a 100644 --- a/api/tech/git/fs.go +++ b/api/tech/git/fs.go @@ -3,18 +3,27 @@ package git import ( "errors" "fmt" + "hash/fnv" "os" "path/filepath" "syscall" "github.com/go-git/go-billy/v5" "github.com/juju/fslock" - "github.com/mandelsoft/vfs/pkg/cwdfs" "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" ) -func VFSBillyFS(fsToWrap vfs.VFS) billy.Filesystem { +type Base interface { + Base() vfs.FileSystem +} +type HasRoot interface { + Root() string +} + +func VFSBillyFS(fsToWrap vfs.FileSystem) billy.Filesystem { if fsToWrap == nil { fsToWrap = vfs.New(memoryfs.New()) } @@ -24,14 +33,12 @@ func VFSBillyFS(fsToWrap vfs.VFS) billy.Filesystem { } return &fs{ - vfs: fsToWrap, - root: fi.Name(), + vfs: fsToWrap, } } type fs struct { - vfs vfs.VFS - root string + vfs vfs.FileSystem } type file struct { @@ -82,14 +89,44 @@ func (f *fs) Create(filename string) (billy.File, error) { if err != nil { return nil, err } - return f.vfsToBillyFileInfo(vfsFile), nil -} + return f.vfsToBillyFileInfo(vfsFile) +} + +// vfsToBillyFileInfo converts a vfs.File to a billy.File +// It also creates a fslock.Lock for the file to ensure that the file is lockable +// If the vfs is an osfs.OsFs, the lock is created in the same directory as the file +// If the vfs is not an osfs.OsFs, a temporary directory is created to store the lock +// because its not trivial to store the lock for jujufs on a virtual filesystem because +// juju vfs only operates on syscalls directly and without interface abstraction its not easy to get the root. +func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { + var lock *fslock.Lock + if f.vfs == osfs.OsFs { + lock = fslock.New(fmt.Sprintf("%s.lock", vf.Name())) + } else { + hash := fnv.New32() + _, _ = hash.Write([]byte(vf.Name())) + temp, err := os.MkdirTemp("", fmt.Sprintf("git-vfs-locks-%x", hash.Sum32())) + if err != nil { + return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+ + "this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err) + } + _, components := vfs.Components(f.vfs, vf.Name()) + lockPath := filepath.Join( + temp, + filepath.Join(components[:len(components)-1]...), + fmt.Sprintf("%s.lock", components[len(components)-1]), + ) + if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { + return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+ + "this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err) + } + lock = fslock.New(lockPath) + } -func (f *fs) vfsToBillyFileInfo(vf vfs.File) billy.File { return &file{ vfsFile: vf, - lock: fslock.New(fmt.Sprintf("%s.lock", vf.Name())), - } + lock: lock, + }, nil } func (f *fs) Open(filename string) (billy.File, error) { @@ -97,7 +134,7 @@ func (f *fs) Open(filename string) (billy.File, error) { if err != nil { return nil, err } - return f.vfsToBillyFileInfo(vfsFile), nil + return f.vfsToBillyFileInfo(vfsFile) } func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { @@ -105,7 +142,7 @@ func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, if err != nil { return nil, err } - return f.vfsToBillyFileInfo(vfsFile), nil + return f.vfsToBillyFileInfo(vfsFile) } func (f *fs) Stat(filename string) (os.FileInfo, error) { @@ -117,6 +154,12 @@ func (f *fs) Stat(filename string) (os.FileInfo, error) { } func (f *fs) Rename(oldpath, newpath string) error { + dir := filepath.Dir(newpath) + if dir != "." { + if err := f.vfs.MkdirAll(dir, 0o755); err != nil { + return err + } + } return f.vfs.Rename(oldpath, newpath) } @@ -129,15 +172,15 @@ func (f *fs) Join(elem ...string) string { } func (f *fs) TempFile(dir, prefix string) (billy.File, error) { - vfsFile, err := f.vfs.TempFile(dir, prefix) + vfsFile, err := vfs.TempFile(f.vfs, dir, prefix) if err != nil { return nil, err } - return f.vfsToBillyFileInfo(vfsFile), nil + return f.vfsToBillyFileInfo(vfsFile) } func (f *fs) ReadDir(path string) ([]os.FileInfo, error) { - return f.vfs.ReadDir(path) + return vfs.ReadDir(f.vfs, path) } func (f *fs) MkdirAll(filename string, perm os.FileMode) error { @@ -145,7 +188,13 @@ func (f *fs) MkdirAll(filename string, perm os.FileMode) error { } func (f *fs) Lstat(filename string) (os.FileInfo, error) { - return f.vfs.Lstat(filename) + fi, err := f.vfs.Lstat(filename) + if err != nil { + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } + } + return fi, nil } func (f *fs) Symlink(target, link string) error { @@ -157,18 +206,38 @@ func (f *fs) Readlink(link string) (string, error) { } func (f *fs) Chroot(path string) (billy.Filesystem, error) { - chfs, err := cwdfs.New(f.vfs, path) + fi, err := f.vfs.Stat(path) + if os.IsNotExist(err) { + if err = f.vfs.MkdirAll(path, 0o755); err != nil { + return nil, err + } + fi, err = f.vfs.Stat(path) + } + + if err != nil { + return nil, err + } else if !fi.IsDir() { + return nil, fmt.Errorf("path %s is not a directory", path) + } + + chfs, err := projectionfs.New(f.vfs, path) if err != nil { return nil, err } + return &fs{ - root: path, - vfs: vfs.New(chfs), + vfs: chfs, }, nil } func (f *fs) Root() string { - return f.root + if root := projectionfs.Root(f.vfs); root != "" { + return root + } + if canonicalRoot, err := vfs.Canonical(f.vfs, "/", true); err == nil { + return canonicalRoot + } + return "/" } var _ billy.Filesystem = &fs{} diff --git a/api/tech/git/ref.go b/api/tech/git/ref.go index a3d0a89ccf..1512aa5ec2 100644 --- a/api/tech/git/ref.go +++ b/api/tech/git/ref.go @@ -7,8 +7,7 @@ import ( "regexp" "github.com/go-git/go-git/v5/plumbing" - - "ocm.software/ocm/api/utils" + giturls "github.com/whilp/git-urls" ) const urlToRefSeparator = "@" @@ -46,7 +45,7 @@ func decodeGitURL(rawRef string) (*gurl, error) { matchedRef := matches[2] path := matches[3] - parsedURL, err := utils.ParseURL(rawURL) + parsedURL, err := giturls.Parse(rawURL) if err != nil { return nil, fmt.Errorf("failed to parse URL: %w", err) } diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index b0ea1fb803..bfed5da652 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -3,16 +3,16 @@ package git import ( "context" "errors" + "fmt" "os" - osfs2 "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/storage/filesystem" - "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/vfs/pkg/memoryfs" "github.com/mandelsoft/vfs/pkg/vfs" @@ -22,11 +22,10 @@ import ( var worktreeBranch = plumbing.NewBranchReferenceName("ocm") type client struct { - vfs vfs.VFS + vfs vfs.FileSystem *gurl - storage storage.Storer - auth AuthMethod + auth AuthMethod } type Client interface { @@ -37,31 +36,30 @@ type Client interface { accessobj.Closer } +type ClientOptions struct { +} + var _ Client = &client{} -func NewClient(url string) (Client, error) { +func NewClient(url string, _ ClientOptions) (Client, error) { gitURL, err := decodeGitURL(url) if err != nil { return nil, err } return &client{ - vfs: vfs.New(memoryfs.New()), + vfs: memoryfs.New(), gurl: gitURL, }, nil } func (c *client) Repository(ctx context.Context) (*git.Repository, error) { - strg, err := getStorage(c.vfs) - if err != nil { - return nil, err - } + billy := VFSBillyFS(c.vfs) - wd, err := c.vfs.Getwd() + strg, err := getStorage(billy) if err != nil { return nil, err } - billy := osfs2.New(wd, osfs2.WithBoundOS()) newRepo := false repo, err := git.Open(strg, billy) @@ -119,14 +117,14 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { return repo, nil } -func getStorage(base vfs.VFS) (storage.Storer, error) { - wd, err := base.Getwd() +func getStorage(base billy.Filesystem) (storage.Storer, error) { + dotGit, err := base.Chroot(git.GitDirName) if err != nil { return nil, err } return filesystem.NewStorage( - osfs2.New(filepath.Join(wd, git.GitDirName), osfs2.WithBoundOS()), + dotGit, cache.NewObjectLRUDefault(), ), nil } @@ -203,9 +201,13 @@ func (c *client) Update(ctx context.Context, msg string, push bool) error { } func (c *client) Setup(system vfs.FileSystem) error { - c.vfs = vfs.New(system) - _, err := c.Repository(context.Background()) - return err + + c.vfs = system + + if _, err := c.Repository(context.Background()); err != nil { + return fmt.Errorf("failed to setup repository %q: %w", c.url.String(), err) + } + return nil } func (c *client) Close(_ *accessobj.AccessObject) error { From f2c76adb5d364c926e0f9a97d22dbcadc77dd126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Wed, 21 Aug 2024 00:27:37 +0200 Subject: [PATCH 04/22] feat: git repo support # Conflicts: # docs/reference/ocm_add_resource-configuration.md # docs/reference/ocm_add_resources.md # docs/reference/ocm_add_source-configuration.md # docs/reference/ocm_add_sources.md # docs/reference/ocm_ocm-accessmethods.md --- api/oci/cpi/support/base.go | 12 +- api/oci/extensions/repositories/git/format.go | 9 +- .../extensions/repositories/git/git_test.go | 16 +- .../extensions/repositories/git/namespace.go | 31 ++- .../extensions/repositories/git/repository.go | 2 +- api/oci/extensions/repositories/git/type.go | 41 ++++ api/ocm/extensions/repositories/git/format.go | 23 +- .../extensions/repositories/git/repo_test.go | 206 +++++++++++++++--- api/ocm/extensions/repositories/git/type.go | 9 +- api/tech/git/fs.go | 18 +- api/tech/git/resolver.go | 79 +++++-- docs/reference/ocm_add_routingslips.md | 2 + docs/reference/ocm_bootstrap_configuration.md | 2 + docs/reference/ocm_bootstrap_package.md | 2 + docs/reference/ocm_check_componentversions.md | 2 + docs/reference/ocm_describe_artifacts.md | 2 + docs/reference/ocm_describe_package.md | 2 + docs/reference/ocm_download_artifacts.md | 2 + docs/reference/ocm_download_cli.md | 2 + .../ocm_download_componentversions.md | 2 + docs/reference/ocm_download_resources.md | 2 + docs/reference/ocm_get_artifacts.md | 2 + docs/reference/ocm_get_componentversions.md | 2 + docs/reference/ocm_get_references.md | 2 + docs/reference/ocm_get_resources.md | 2 + docs/reference/ocm_get_routingslips.md | 2 + docs/reference/ocm_get_sources.md | 2 + docs/reference/ocm_hash_componentversions.md | 2 + docs/reference/ocm_install_plugins.md | 2 + docs/reference/ocm_list_componentversions.md | 2 + docs/reference/ocm_show_tags.md | 2 + docs/reference/ocm_show_versions.md | 2 + docs/reference/ocm_sign_componentversions.md | 2 + docs/reference/ocm_transfer_artifacts.md | 2 + .../ocm_transfer_componentversions.md | 2 + .../reference/ocm_verify_componentversions.md | 2 + 36 files changed, 385 insertions(+), 111 deletions(-) diff --git a/api/oci/cpi/support/base.go b/api/oci/cpi/support/base.go index 60cbf0a79c..5d3568bf33 100644 --- a/api/oci/cpi/support/base.go +++ b/api/oci/cpi/support/base.go @@ -31,18 +31,18 @@ func (a *artifactBase) IsReadOnly() bool { } func (a *artifactBase) IsIndex() bool { - d := a.state.GetState().(*artdesc.Artifact) - return d.IsIndex() + d, ok := a.state.GetState().(*artdesc.Artifact) + return ok && d.IsIndex() } func (a *artifactBase) IsManifest() bool { - d := a.state.GetState().(*artdesc.Artifact) - return d.IsManifest() + d, ok := a.state.GetState().(*artdesc.Artifact) + return ok && d.IsManifest() } func (a *artifactBase) IsValid() bool { - d := a.state.GetState().(*artdesc.Artifact) - return d.IsValid() + d, ok := a.state.GetState().(*artdesc.Artifact) + return ok && d.IsValid() } func (a *artifactBase) blob() (cpi.BlobAccess, error) { diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go index 5c31d92aa3..65665249b3 100644 --- a/api/oci/extensions/repositories/git/format.go +++ b/api/oci/extensions/repositories/git/format.go @@ -2,20 +2,19 @@ package git import ( "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" ) // ////////////////////////////////////////////////////////////////////////////// -func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, option ...accessio.Option) (Repository, error) { - spec, err := NewRepositorySpec(acc, url, option...) +func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { + spec, err := NewRepositorySpecFromOptions(acc, url, opts) if err != nil { return nil, err } return New(cpi.FromProvider(ctx), spec) } -func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, option ...accessio.Option) (Repository, error) { - return Open(ctx, acc, url, option...) +func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { + return Open(ctx, acc, url, opts) } diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go index 05fb4b6187..f95b97cf23 100644 --- a/api/oci/extensions/repositories/git/git_test.go +++ b/api/oci/extensions/repositories/git/git_test.go @@ -7,8 +7,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/logging" - "github.com/mandelsoft/vfs/pkg/cwdfs" "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/opencontainers/go-digest" @@ -45,7 +45,7 @@ var _ = Describe("ctf management", func() { BeforeEach(func() { path := GinkgoT().TempDir() - tmp = Must(cwdfs.New(osfs.New(), path)) + tmp = Must(projectionfs.New(osfs.New(), path)) tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tmp}) vfsattr.Set(ctx, tmp) @@ -54,7 +54,7 @@ var _ = Describe("ctf management", func() { repoURL = "file://" + repoDir Expect(tmp.Mkdir("workspace", 0o700)).To(Succeed()) - workspace = Must(cwdfs.New(tmp, "workspace")) + workspace = Must(projectionfs.New(tmp, "workspace")) }) BeforeEach(func() { @@ -66,7 +66,13 @@ var _ = Describe("ctf management", func() { }) It("instantiate git based ctf", func() { - repo := Must(rgit.Create(ctx, accessobj.ACC_CREATE, repoURL, accessio.RepresentationFileSystem(workspace))) + repo := Must(rgit.Create(ctx, accessobj.ACC_CREATE, repoURL, rgit.Options{ + Author: &rgit.Author{ + Name: fmt.Sprintf("OCM Test Case: %s", GinkgoT().Name()), + Email: "dummy@ocm.software", + }, + Options: Must(accessio.AccessOptions(nil, accessio.RepresentationFileSystem(workspace))), + })) ns := Must(repo.LookupNamespace("test")) testData := []byte("testdata") @@ -85,7 +91,7 @@ var _ = Describe("ctf management", func() { if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa); commit.Message == expected { validAdd++ } - if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationSync, aa); commit.Message == expected { + if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationUpdate, aa); commit.Message == expected { validSync++ } messages = append(messages, commit.Message) diff --git a/api/oci/extensions/repositories/git/namespace.go b/api/oci/extensions/repositories/git/namespace.go index 5b3b2755d0..0683a8ac4d 100644 --- a/api/oci/extensions/repositories/git/namespace.go +++ b/api/oci/extensions/repositories/git/namespace.go @@ -14,6 +14,8 @@ import ( "ocm.software/ocm/api/utils/blobaccess/blobaccess" ) +const CommitPrefix = "update(ocm)" + func NewNamespace(repo *RepositoryImpl, name string) (cpi.NamespaceAccess, error) { ctfNamespace, err := newNamespaceContainer(repo, name) if err != nil { @@ -55,7 +57,7 @@ func (n *namespaceContainer) Close() error { if err := n.ctf.Close(); err != nil { return err } - return n.client.Update(context.Background(), fmt.Sprintf("namespace update %q", n.name), true) + return n.client.Update(context.Background(), GenerateCommitMessageForNamespace(OperationUpdate, n.name), true) } func (n *namespaceContainer) ListTags() ([]string, error) { @@ -139,35 +141,40 @@ func (n *namespaceContainer) NewArtifact(i support.NamespaceAccessImpl, art ...c type Operation string const ( - OperationAdd Operation = "add" - OperationMod Operation = "mod" - OperationSync Operation = "sync" + OperationAdd Operation = "add" + OperationMod Operation = "mod" + OperationUpdate Operation = "update" ) func GenerateCommitMessageForArtifact(operation Operation, artifact cpi.Artifact) string { a := artifact.Artifact() - var msg string + var typ string if artifact.IsManifest() { - msg = fmt.Sprintf("update(ocm): %s manifest %s (%s)", operation, a.Digest(), a.MimeType()) + typ = "manifest" } else if artifact.IsIndex() { - msg = fmt.Sprintf("update(ocm): %s index %s (%s)", operation, a.Digest(), a.MimeType()) + typ = "index" } else { - msg = fmt.Sprintf("update(ocm): %s artifact %s (%s)", operation, a.Digest(), a.MimeType()) + typ = "artifact" } - return msg + + return fmt.Sprintf("%s: %s %s %s (%s)", CommitPrefix, operation, typ, a.Digest(), a.MimeType()) } func GenerateCommitMessageForBlob(operation Operation, blob cpi.BlobAccess) string { var msg string if blob.DigestKnown() { - msg = fmt.Sprintf("update(ocm): %s blob %s of type %s", operation, blob.Digest(), blob.MimeType()) + msg = fmt.Sprintf("%s: %s blob(%s) of type %s", CommitPrefix, operation, blob.Digest(), blob.MimeType()) } else { - msg = fmt.Sprintf("update(ocm): %s blob of type %s", operation, blob.MimeType()) + msg = fmt.Sprintf("%s: %s blob of type %s", CommitPrefix, operation, blob.MimeType()) } return msg } +func GenerateCommitMessageForNamespace(operation Operation, namespace string) string { + return fmt.Sprintf("update(ocm): %s namespace %q", operation, namespace) +} + type artifactContainer struct { client git.Client cpi.ArtifactAccess @@ -179,7 +186,7 @@ func (a *artifactContainer) Close() error { if err := a.ArtifactAccess.Close(); err != nil { return err } - return a.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationSync, a.ArtifactAccess), true) + return a.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationUpdate, a.ArtifactAccess), true) } func (a *artifactContainer) Dup() (cpi.ArtifactAccess, error) { diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index dfb873f976..852b36edaf 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -41,7 +41,7 @@ func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { } var err error - if i.client, err = git.NewClient(spec.URL, git.ClientOptions{}); err != nil { + if i.client, err = git.NewClient(spec.ToClientOptions()); err != nil { return nil, err } diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go index 04052af285..b1833fcf91 100644 --- a/api/oci/extensions/repositories/git/type.go +++ b/api/oci/extensions/repositories/git/type.go @@ -3,6 +3,7 @@ package git import ( "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" "ocm.software/ocm/api/utils/runtime" @@ -40,14 +41,39 @@ type RepositorySpec struct { // URL is the url of the RepositoryImpl to resolve artifacts. URL string `json:"url"` + // Author is the author of commits generated by the repository. If not set, is defaulted from environment and git + // configuration of the host system. + Author *Author `json:"author,omitempty"` + // AccessMode can be set to request readonly access or creation AccessMode accessobj.AccessMode `json:"accessMode,omitempty"` } +// Author describes the author of commits generated by the repository. +type Author struct { + Name string `json:"name"` + Email string `json:"email"` +} + var _ cpi.RepositorySpec = (*RepositorySpec)(nil) var _ cpi.IntermediateRepositorySpecAspect = (*RepositorySpec)(nil) +type Options struct { + *Author + accessio.Options +} + +// NewRepositorySpecFromOptions creates a new RepositorySpec from options. +func NewRepositorySpecFromOptions(mode accessobj.AccessMode, url string, opts Options) (*RepositorySpec, error) { + spec, err := NewRepositorySpec(mode, url, opts.Options) + if err != nil { + return nil, err + } + spec.Author = opts.Author + return spec, nil +} + // NewRepositorySpec creates a new RepositorySpec. func NewRepositorySpec(mode accessobj.AccessMode, url string, opts ...accessio.Option) (*RepositorySpec, error) { o, err := accessio.AccessOptions(nil, opts...) @@ -86,3 +112,18 @@ func (s *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec { func (s *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) { return New(ctx, s) } + +func (s *RepositorySpec) ToClientOptions() git.ClientOptions { + opts := git.ClientOptions{} + + if s.Author != nil { + opts.Author = git.Author{ + Name: s.Author.Name, + Email: s.Author.Email, + } + } + + opts.URL = s.URL + + return opts +} diff --git a/api/ocm/extensions/repositories/git/format.go b/api/ocm/extensions/repositories/git/format.go index bcc77bb607..e1a6b05488 100644 --- a/api/ocm/extensions/repositories/git/format.go +++ b/api/ocm/extensions/repositories/git/format.go @@ -6,42 +6,27 @@ import ( "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/ocm/cpi" "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" ) -var ( - FormatDirectory = ctf.FormatDirectory -) - type Object = ctf.Object -type FormatHandler = ctf.FormatHandler - -func GetFormats() []string { - return ctf.GetFormats() -} - -func GetFormat(name accessio.FileFormat) FormatHandler { - return ctf.GetFormat(name) -} - const ( ACC_CREATE = accessobj.ACC_CREATE ACC_WRITABLE = accessobj.ACC_WRITABLE ACC_READONLY = accessobj.ACC_READONLY ) -func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts ...accessio.Option) (cpi.Repository, error) { - r, err := git.Open(cpi.FromProvider(ctx), acc, url, opts...) +func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (cpi.Repository, error) { + r, err := git.Open(cpi.FromProvider(ctx), acc, url, opts) if err != nil { return nil, err } return genericocireg.NewRepository(cpi.FromProvider(ctx), nil, r), nil } -func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts ...accessio.Option) (cpi.Repository, error) { - r, err := git.Create(cpi.FromProvider(ctx), acc, url, opts...) +func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (cpi.Repository, error) { + r, err := git.Create(cpi.FromProvider(ctx), acc, url, opts) if err != nil { return nil, err } diff --git a/api/ocm/extensions/repositories/git/repo_test.go b/api/ocm/extensions/repositories/git/repo_test.go index dfa67b4c0b..1dc71b8e22 100644 --- a/api/ocm/extensions/repositories/git/repo_test.go +++ b/api/ocm/extensions/repositories/git/repo_test.go @@ -2,28 +2,39 @@ package git_test import ( "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "github.com/go-git/go-billy/v5" gitgo "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/client" "github.com/go-git/go-git/v5/plumbing/transport/server" + "github.com/go-git/go-git/v5/storage/filesystem" . "github.com/mandelsoft/goutils/finalizer" . "github.com/mandelsoft/goutils/testutils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "ocm.software/ocm/api/ocm/extensions/repositories/git" - . "ocm.software/ocm/api/ocm/testhelper" - techgit "ocm.software/ocm/api/tech/git" - "github.com/mandelsoft/logging" - "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/tonglil/buflogr" - + "ocm.software/ocm/api/oci/artdesc" + gitrepo "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/ocm" "ocm.software/ocm/api/ocm/compdesc" metav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" + "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg/componentmapping" + "ocm.software/ocm/api/ocm/extensions/repositories/git" + . "ocm.software/ocm/api/ocm/testhelper" + techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess" ocmlog "ocm.software/ocm/api/utils/logging" @@ -32,33 +43,76 @@ import ( ) const ( - COMPONENT = "ocm.software/ocm" - VERSION = "1.0.0" + COMPONENT = "ocm.software/ocm" + VERSION = "1.0.0" + REMOTE_REPO = "repo.git" ) var _ = Describe("access method", func() { - var pathFS vfs.FileSystem - // var repFS vfs.FileSystem + // remoteFS contains the remote repository on the filesystem + // pathFS contains the local PWD + // repFS contains the local representation of the repository, meaning the cloned repo to work on the Repository + var remoteFS, pathFS, repFS vfs.FileSystem + // access contains the access configuration to the above filesystems + var access accessio.Options + + // repoURL is the URL specification to access the remote repository in remoteFS + var repoURL string + + // remoteRepo is the remote repository that can be used for test assertions on pushed content + var remoteRepo *gitgo.Repository + + var opts git.Options + ctx := ocm.DefaultContext() BeforeEach(func() { - pathFS = memoryfs.New() + By("setting up test filesystems") + basePath := GinkgoT().TempDir() + baseFS := Must(cwdfs.New(osfs.New(), basePath)) + for _, dir := range []string{"remote", "path", "rep"} { + Expect(os.Mkdir(filepath.Join(basePath, dir), 0777)).To(Succeed()) + } + remoteFS = Must(projectionfs.New(baseFS, "remote")) + pathFS = Must(projectionfs.New(baseFS, "path")) + repFS = Must(projectionfs.New(baseFS, "rep")) + + access = &accessio.StandardOptions{ + PathFileSystem: pathFS, + Representation: repFS, + } + }) + + AfterEach(func() { + Expect(Must(vfs.ReadDir(pathFS, "."))).To(BeEmpty(), "nothing of the CTF should be stored in the path, "+ + "because everything should be handled in the representation which contains the local git repository") }) BeforeEach(func() { - remoteFS := memoryfs.New() - client.InstallProtocol("file", server.NewClient(server.NewFilesystemLoader(techgit.VFSBillyFS(remoteFS)))) - gitgo.InitWithOptions() + By("setting up local bare git repository to work against when pushing/updating") + billy := techgit.VFSBillyFS(remoteFS) + client.InstallProtocol("file", server.NewClient(server.NewFilesystemLoader(billy))) + remoteRepo = Must(newBareTestRepo(billy, REMOTE_REPO, gitgo.InitOptions{})) + // now that we have a bare repository, we can reference it via URL to access it like a remote repository + repoURL = fmt.Sprintf("file:///%s", REMOTE_REPO) + }) + + BeforeEach(func() { + opts = git.Options{ + Author: &git.Author{ + Name: fmt.Sprintf("OCM Test Case: %s", GinkgoT().Name()), + Email: "dummy@ocm.software", + }, + Options: access, + } }) It("adds naked component version and later lookup", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", - accessio.PathFileSystem(pathFS), - )) + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) final.Close(a, "repository") c := Must(a.LookupComponent(COMPONENT)) final.Close(c, "component") @@ -69,12 +123,56 @@ var _ = Describe("access method", func() { MustBeSuccessful(c.AddVersion(cv)) MustBeSuccessful(final.Finalize()) + componentCommitExpectation := gitrepo.GenerateCommitMessageForNamespace(gitrepo.OperationUpdate, fmt.Sprintf("component-descriptors/%s", COMPONENT)) + descriptorCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s blob.* of type %s", + regexp.QuoteMeta(gitrepo.CommitPrefix), + gitrepo.OperationAdd, + regexp.QuoteMeta(componentmapping.ComponentDescriptorTarMimeType)), + ) + descriptorConfigCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s blob.* of type %s", + regexp.QuoteMeta(gitrepo.CommitPrefix), + gitrepo.OperationAdd, + regexp.QuoteMeta(componentmapping.ComponentDescriptorConfigMimeType)), + ) + manifestAddCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s artifact .* %s", + regexp.QuoteMeta(gitrepo.CommitPrefix), + gitrepo.OperationAdd, + regexp.QuoteMeta(fmt.Sprintf("(%s)", artdesc.MediaTypeImageManifest))), + ) + manifestUpdateCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s manifest .* %s", + regexp.QuoteMeta(gitrepo.CommitPrefix), + gitrepo.OperationUpdate, + regexp.QuoteMeta(fmt.Sprintf("(%s)", artdesc.MediaTypeImageManifest))), + ) + + componentUpdate := 0 + descriptorCommits := 0 + manifestUpdateCommits := 0 + commits := Must(remoteRepo.CommitObjects()) + Expect(commits.ForEach(func(commit *object.Commit) error { + Expect(commit.Author.Name).To(Equal(opts.Author.Name)) + Expect(commit.Author.Email).To(Equal(opts.Author.Email)) + + if commit.Message == componentCommitExpectation { + componentUpdate++ + } else if descriptorCommitExpectation.MatchString(commit.Message) || descriptorConfigCommitExpectation.MatchString(commit.Message) { + descriptorCommits++ + } else if manifestUpdateCommitExpectation.MatchString(commit.Message) || manifestAddCommitExpectation.MatchString(commit.Message) { + manifestUpdateCommits++ + } + return nil + })).To(Succeed()) + Expect(componentUpdate).To(Equal(1)) + Expect(descriptorCommits).To(Equal(2)) + Expect(manifestUpdateCommits).To(Equal(2)) + refmgmt.AllocLog.Trace("opening ctf") - a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) final.Close(a) refmgmt.AllocLog.Trace("lookup component") - c = Must(a.LookupComponent(COMPONENT)) + c, err := a.LookupComponent(COMPONENT) + Expect(err).ToNot(HaveOccurred()) final.Close(c) refmgmt.AllocLog.Trace("lookup version") @@ -89,7 +187,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) final.Close(a, "repository") c := Must(a.LookupComponent(COMPONENT)) final.Close(c, "component") @@ -101,7 +199,7 @@ var _ = Describe("access method", func() { MustBeSuccessful(final.Finalize()) refmgmt.AllocLog.Trace("opening ctf") - a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) final.Close(a) refmgmt.AllocLog.Trace("lookup component version") @@ -116,7 +214,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) final.Close(a) c := Must(a.LookupComponent(COMPONENT)) final.Close(c) @@ -142,7 +240,7 @@ var _ = Describe("access method", func() { MustBeSuccessful(c.AddVersion(cv)) MustBeSuccessful(final.Finalize()) - a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) final.Close(a) cv = Must(a.LookupComponentVersion(COMPONENT, VERSION)) @@ -153,7 +251,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) final.Close(a) c := Must(a.LookupComponent(COMPONENT)) final.Close(c) @@ -163,19 +261,19 @@ var _ = Describe("access method", func() { MustBeSuccessful(final.Finalize()) - a = Must(git.Open(ctx, git.ACC_READONLY, "ctf", accessio.PathFileSystem(pathFS))) + a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) final.Close(a) _, err := a.LookupComponentVersion(COMPONENT, VERSION) - Expect(err).To(MatchError(ContainSubstring("component version \"github.com/mandelsoft/ocm:1.0.0\" not found: oci artifact \"1.0.0\" not found in component-descriptors/github.com/mandelsoft/ocm"))) + Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("component version \"%[1]s:%[2]s\" not found: oci artifact \"%[2]s\" not found in component-descriptors/%[1]s", COMPONENT, VERSION)))) }) It("provides error for invalid bloc access", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, "ctf", accessio.PathFileSystem(pathFS))) + a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) final.Close(a) c := Must(a.LookupComponent(COMPONENT)) final.Close(c) @@ -190,7 +288,8 @@ var _ = Describe("access method", func() { }) It("logs diff", func() { - r := Must(git.Open(ctx, git.ACC_CREATE, "test.ctf", accessio.FormatDirectory, accessio.PathFileSystem(pathFS))) + MustBeSuccessful(accessio.FormatDirectory.ApplyOption(opts.Options)) + r := Must(git.Open(ctx, git.ACC_CREATE, repoURL, opts)) defer Close(r, "repo") c := Must(r.LookupComponent("acme.org/test")) @@ -212,14 +311,15 @@ var _ = Describe("access method", func() { cv = Must(c.LookupVersion("v1")) cv.GetDescriptor().Provider.Name = "acme.org" MustBeSuccessful(cv.Close()) - Expect("\n" + buf.String()).To(Equal(` -V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != acme.org] -V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != acme.org] -`)) + Expect("\n" + buf.String()).To(Equal(fmt.Sprintf(` +V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != %[1]s] +V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != %[1]s] +`, cv.GetDescriptor().Provider.Name))) }) It("handles readonly mode", func() { - r := Must(git.Open(ctx, git.ACC_CREATE, "test.ctf", accessio.FormatDirectory, accessio.PathFileSystem(pathFS))) + MustBeSuccessful(accessio.FormatDirectory.ApplyOption(opts.Options)) + r := Must(git.Open(ctx, git.ACC_CREATE, repoURL, opts)) defer Close(r, "repo") c := Must(r.LookupComponent("acme.org/test")) @@ -238,7 +338,8 @@ V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff }) It("handles readonly mode on repo", func() { - r := Must(git.Open(ctx, git.ACC_CREATE, "test.ctf", accessio.FormatDirectory, accessio.PathFileSystem(pathFS))) + MustBeSuccessful(accessio.FormatDirectory.ApplyOption(opts.Options)) + r := Must(git.Open(ctx, git.ACC_CREATE, repoURL, opts)) defer Close(r, "repo") c := Must(r.LookupComponent("acme.org/test")) @@ -258,3 +359,36 @@ V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff ExpectError(c.NewVersion("v2")).To(MatchError(accessio.ErrReadOnly)) }) }) + +func newBareTestRepo(fs billy.Filesystem, path string, opts gitgo.InitOptions) (*gitgo.Repository, error) { + var wt, dot billy.Filesystem + + var err error + dot, err = fs.Chroot(path) + if err != nil { + return nil, err + } + wt, err = fs.Chroot(path) + if err != nil { + return nil, err + } + + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) + + r, err := gitgo.InitWithOptions(s, wt, opts) + if err != nil { + return nil, err + } + + cfg, err := r.Config() + if err != nil { + return nil, err + } + + err = r.Storer.SetConfig(cfg) + if err != nil { + return nil, err + } + + return r, err +} diff --git a/api/ocm/extensions/repositories/git/type.go b/api/ocm/extensions/repositories/git/type.go index 5dbc01b74d..6c85613a46 100644 --- a/api/ocm/extensions/repositories/git/type.go +++ b/api/ocm/extensions/repositories/git/type.go @@ -3,14 +3,17 @@ package git import ( "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" ) const Type = git.Type -func NewRepositorySpec(acc accessobj.AccessMode, url string, opts ...accessio.Option) (*genericocireg.RepositorySpec, error) { - spec, err := git.NewRepositorySpec(acc, url, opts...) +type Options = git.Options + +type Author = git.Author + +func NewRepositorySpec(acc accessobj.AccessMode, url string, opts Options) (*genericocireg.RepositorySpec, error) { + spec, err := git.NewRepositorySpecFromOptions(acc, url, opts) if err != nil { return nil, err } diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go index 51435f887a..0a74cd0e3a 100644 --- a/api/tech/git/fs.go +++ b/api/tech/git/fs.go @@ -16,13 +16,6 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" ) -type Base interface { - Base() vfs.FileSystem -} -type HasRoot interface { - Root() string -} - func VFSBillyFS(fsToWrap vfs.FileSystem) billy.Filesystem { if fsToWrap == nil { fsToWrap = vfs.New(memoryfs.New()) @@ -104,7 +97,7 @@ func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { lock = fslock.New(fmt.Sprintf("%s.lock", vf.Name())) } else { hash := fnv.New32() - _, _ = hash.Write([]byte(vf.Name())) + _, _ = hash.Write([]byte(f.vfs.Name())) temp, err := os.MkdirTemp("", fmt.Sprintf("git-vfs-locks-%x", hash.Sum32())) if err != nil { return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+ @@ -138,6 +131,11 @@ func (f *fs) Open(filename string) (billy.File, error) { } func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { + if flag&os.O_CREATE != 0 { + if err := f.vfs.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return nil, err + } + } vfsFile, err := f.vfs.OpenFile(filename, flag, perm) if err != nil { return nil, err @@ -150,7 +148,7 @@ func (f *fs) Stat(filename string) (os.FileInfo, error) { if errors.Is(err, syscall.ENOENT) { return nil, os.ErrNotExist } - return fi, nil + return fi, err } func (f *fs) Rename(oldpath, newpath string) error { @@ -194,7 +192,7 @@ func (f *fs) Lstat(filename string) (os.FileInfo, error) { return nil, os.ErrNotExist } } - return fi, nil + return fi, err } func (f *fs) Symlink(target, link string) error { diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index bfed5da652..6c73ed2bf9 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "sync" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" @@ -22,10 +23,20 @@ import ( var worktreeBranch = plumbing.NewBranchReferenceName("ocm") type client struct { + opts ClientOptions + + // vfs tracks the current filesystem where the repo will be stored (at the root) vfs vfs.FileSystem + + // url is the git URL of the repository *gurl + // auth is the authentication method to use when accessing the repository auth AuthMethod + + // repo is a reference to the git repository if it is already open + repo *git.Repository + repoMu sync.Mutex } type Client interface { @@ -37,12 +48,19 @@ type Client interface { } type ClientOptions struct { + URL string + Author +} + +type Author struct { + Name string + Email string } var _ Client = &client{} -func NewClient(url string, _ ClientOptions) (Client, error) { - gitURL, err := decodeGitURL(url) +func NewClient(opts ClientOptions) (Client, error) { + gitURL, err := decodeGitURL(opts.URL) if err != nil { return nil, err } @@ -50,21 +68,28 @@ func NewClient(url string, _ ClientOptions) (Client, error) { return &client{ vfs: memoryfs.New(), gurl: gitURL, + opts: opts, }, nil } func (c *client) Repository(ctx context.Context) (*git.Repository, error) { - billy := VFSBillyFS(c.vfs) + c.repoMu.Lock() + defer c.repoMu.Unlock() + if c.repo != nil { + return c.repo, nil + } - strg, err := getStorage(billy) + billyFS := VFSBillyFS(c.vfs) + + strg, err := GetStorage(billyFS) if err != nil { return nil, err } newRepo := false - repo, err := git.Open(strg, billy) + repo, err := git.Open(strg, billyFS) if errors.Is(err, git.ErrRepositoryNotExists) { - repo, err = git.CloneContext(ctx, strg, billy, &git.CloneOptions{ + repo, err = git.CloneContext(ctx, strg, billyFS, &git.CloneOptions{ Auth: c.auth, URL: c.url.String(), RemoteName: git.DefaultRemoteName, @@ -76,7 +101,7 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { newRepo = true } if errors.Is(err, transport.ErrEmptyRemoteRepository) { - return git.Open(strg, billy) + return git.Open(strg, billyFS) } if err != nil { @@ -114,10 +139,16 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { } } + if err := c.opts.applyToRepo(repo); err != nil { + return nil, err + } + + c.repo = repo + return repo, nil } -func getStorage(base billy.Filesystem) (storage.Storer, error) { +func GetStorage(base billy.Filesystem) (storage.Storer, error) { dotGit, err := base.Chroot(git.GitDirName) if err != nil { return nil, err @@ -175,14 +206,16 @@ func (c *client) Update(ctx context.Context, msg string, push bool) error { return err } - err = worktree.AddGlob("*") - - if err != nil { + if err = worktree.AddGlob("*"); err != nil { return err } _, err = worktree.Commit(msg, &git.CommitOptions{}) + if errors.Is(err, git.ErrEmptyCommit) { + return nil + } + if err != nil { return err } @@ -201,9 +234,7 @@ func (c *client) Update(ctx context.Context, msg string, push bool) error { } func (c *client) Setup(system vfs.FileSystem) error { - c.vfs = system - if _, err := c.Repository(context.Background()); err != nil { return fmt.Errorf("failed to setup repository %q: %w", c.url.String(), err) } @@ -211,5 +242,25 @@ func (c *client) Setup(system vfs.FileSystem) error { } func (c *client) Close(_ *accessobj.AccessObject) error { - return c.Update(context.Background(), "OCM Repository Update", true) + if err := c.Update(context.Background(), "OCM Repository Update", true); err != nil { + return fmt.Errorf("failed to close repository %q: %w", c.url.String(), err) + } + return nil +} + +func (o ClientOptions) applyToRepo(repo *git.Repository) error { + cfg, err := repo.Config() + if err != nil { + return err + } + + if o.Author.Name != "" { + cfg.User.Name = o.Author.Name + } + + if o.Author.Email != "" { + cfg.User.Email = o.Author.Email + } + + return repo.SetConfig(cfg) } diff --git a/docs/reference/ocm_add_routingslips.md b/docs/reference/ocm_add_routingslips.md index 4bebc50df7..5742f762f9 100644 --- a/docs/reference/ocm_add_routingslips.md +++ b/docs/reference/ocm_add_routingslips.md @@ -99,6 +99,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_bootstrap_configuration.md b/docs/reference/ocm_bootstrap_configuration.md index 92396ca672..03ecfcfce0 100644 --- a/docs/reference/ocm_bootstrap_configuration.md +++ b/docs/reference/ocm_bootstrap_configuration.md @@ -80,6 +80,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_bootstrap_package.md b/docs/reference/ocm_bootstrap_package.md index 19e343b811..b678f70594 100644 --- a/docs/reference/ocm_bootstrap_package.md +++ b/docs/reference/ocm_bootstrap_package.md @@ -161,6 +161,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_check_componentversions.md b/docs/reference/ocm_check_componentversions.md index 23f9849220..4379e553c4 100644 --- a/docs/reference/ocm_check_componentversions.md +++ b/docs/reference/ocm_check_componentversions.md @@ -68,6 +68,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_describe_artifacts.md b/docs/reference/ocm_describe_artifacts.md index a2bf2e3d07..9cf092f0a6 100644 --- a/docs/reference/ocm_describe_artifacts.md +++ b/docs/reference/ocm_describe_artifacts.md @@ -58,6 +58,8 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_describe_package.md b/docs/reference/ocm_describe_package.md index def83647ae..4ee5bf9727 100644 --- a/docs/reference/ocm_describe_package.md +++ b/docs/reference/ocm_describe_package.md @@ -71,6 +71,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_artifacts.md b/docs/reference/ocm_download_artifacts.md index a7a7928e3e..878ee7da76 100644 --- a/docs/reference/ocm_download_artifacts.md +++ b/docs/reference/ocm_download_artifacts.md @@ -60,6 +60,8 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_cli.md b/docs/reference/ocm_download_cli.md index ff71ec89dd..53e53b7d8d 100644 --- a/docs/reference/ocm_download_cli.md +++ b/docs/reference/ocm_download_cli.md @@ -81,6 +81,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_componentversions.md b/docs/reference/ocm_download_componentversions.md index 79dbd5c69e..36d51a1d78 100644 --- a/docs/reference/ocm_download_componentversions.md +++ b/docs/reference/ocm_download_componentversions.md @@ -67,6 +67,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_resources.md b/docs/reference/ocm_download_resources.md index fd14528178..e5ee73bd65 100644 --- a/docs/reference/ocm_download_resources.md +++ b/docs/reference/ocm_download_resources.md @@ -106,6 +106,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_artifacts.md b/docs/reference/ocm_get_artifacts.md index a4bfa73b68..db0af2b205 100644 --- a/docs/reference/ocm_get_artifacts.md +++ b/docs/reference/ocm_get_artifacts.md @@ -58,6 +58,8 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_componentversions.md b/docs/reference/ocm_get_componentversions.md index ceebb7e970..6c93c6ada5 100644 --- a/docs/reference/ocm_get_componentversions.md +++ b/docs/reference/ocm_get_componentversions.md @@ -77,6 +77,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_references.md b/docs/reference/ocm_get_references.md index 684f1b4787..751e378f96 100644 --- a/docs/reference/ocm_get_references.md +++ b/docs/reference/ocm_get_references.md @@ -78,6 +78,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_resources.md b/docs/reference/ocm_get_resources.md index 9790deeb88..9f88968abe 100644 --- a/docs/reference/ocm_get_resources.md +++ b/docs/reference/ocm_get_resources.md @@ -78,6 +78,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_routingslips.md b/docs/reference/ocm_get_routingslips.md index a5402ac441..b4c804a0f9 100644 --- a/docs/reference/ocm_get_routingslips.md +++ b/docs/reference/ocm_get_routingslips.md @@ -77,6 +77,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_sources.md b/docs/reference/ocm_get_sources.md index fc32e41373..ecb6fe7eac 100644 --- a/docs/reference/ocm_get_sources.md +++ b/docs/reference/ocm_get_sources.md @@ -78,6 +78,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_hash_componentversions.md b/docs/reference/ocm_hash_componentversions.md index 1e49ba07f4..7237ac1e35 100644 --- a/docs/reference/ocm_hash_componentversions.md +++ b/docs/reference/ocm_hash_componentversions.md @@ -111,6 +111,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_install_plugins.md b/docs/reference/ocm_install_plugins.md index 9c20720c97..140a88f083 100644 --- a/docs/reference/ocm_install_plugins.md +++ b/docs/reference/ocm_install_plugins.md @@ -70,6 +70,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_list_componentversions.md b/docs/reference/ocm_list_componentversions.md index 630d009e9b..e1c1b5703c 100644 --- a/docs/reference/ocm_list_componentversions.md +++ b/docs/reference/ocm_list_componentversions.md @@ -76,6 +76,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_show_tags.md b/docs/reference/ocm_show_tags.md index 92250cbf21..5a3a616843 100644 --- a/docs/reference/ocm_show_tags.md +++ b/docs/reference/ocm_show_tags.md @@ -50,6 +50,8 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_show_versions.md b/docs/reference/ocm_show_versions.md index 075203cf3c..da82b0fe66 100644 --- a/docs/reference/ocm_show_versions.md +++ b/docs/reference/ocm_show_versions.md @@ -64,6 +64,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_sign_componentversions.md b/docs/reference/ocm_sign_componentversions.md index b1d3f582bf..5a2d72c970 100644 --- a/docs/reference/ocm_sign_componentversions.md +++ b/docs/reference/ocm_sign_componentversions.md @@ -88,6 +88,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_transfer_artifacts.md b/docs/reference/ocm_transfer_artifacts.md index 7cbe20ee89..351fd6fb37 100644 --- a/docs/reference/ocm_transfer_artifacts.md +++ b/docs/reference/ocm_transfer_artifacts.md @@ -71,6 +71,8 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_transfer_componentversions.md b/docs/reference/ocm_transfer_componentversions.md index 1c235f0e82..e7fa5b7908 100644 --- a/docs/reference/ocm_transfer_componentversions.md +++ b/docs/reference/ocm_transfer_componentversions.md @@ -89,6 +89,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_verify_componentversions.md b/docs/reference/ocm_verify_componentversions.md index dabdd5d270..5f2211b99b 100644 --- a/docs/reference/ocm_verify_componentversions.md +++ b/docs/reference/ocm_verify_componentversions.md @@ -85,6 +85,8 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 + - Git: v1 + - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry From 70d59d183d51dccd1cc0292e28cfccbff8f7f9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Wed, 21 Aug 2024 16:43:25 +0200 Subject: [PATCH 05/22] feat: git repo support --- api/oci/extensions/repositories/git/namespace.go | 7 ++++--- api/ocm/extensions/accessmethods/git/method.go | 5 ----- api/ocm/extensions/repositories/git/repo_test.go | 2 +- api/tech/git/fs.go | 6 +++--- api/tech/git/ref.go | 14 +++++++++++--- api/tech/git/resolver.go | 5 ++++- api/utils/accessio/downloader/git/downloader.go | 3 +-- 7 files changed, 24 insertions(+), 18 deletions(-) diff --git a/api/oci/extensions/repositories/git/namespace.go b/api/oci/extensions/repositories/git/namespace.go index 0683a8ac4d..2db42a820a 100644 --- a/api/oci/extensions/repositories/git/namespace.go +++ b/api/oci/extensions/repositories/git/namespace.go @@ -150,11 +150,12 @@ func GenerateCommitMessageForArtifact(operation Operation, artifact cpi.Artifact a := artifact.Artifact() var typ string - if artifact.IsManifest() { + switch { + case artifact.IsManifest(): typ = "manifest" - } else if artifact.IsIndex() { + case artifact.IsIndex(): typ = "index" - } else { + default: typ = "artifact" } diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index aaaf1149d9..cab76f0da8 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -3,7 +3,6 @@ package git import ( "fmt" "io" - "net/http" "github.com/go-git/go-git/v5/plumbing" "github.com/mandelsoft/goutils/errors" @@ -15,7 +14,6 @@ import ( "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/internal" "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/accessio/downloader" "ocm.software/ocm/api/utils/accessio/downloader/git" "ocm.software/ocm/api/utils/accessobj" "ocm.software/ocm/api/utils/blobaccess/blobaccess" @@ -45,9 +43,6 @@ type AccessSpec struct { // PathSpec is a path in the repository to download, can be a file or a regex matching multiple files PathSpec string `json:"pathSpec"` - - client *http.Client - downloader downloader.Downloader } // AccessSpecOptions defines a set of options which can be applied to the access spec. diff --git a/api/ocm/extensions/repositories/git/repo_test.go b/api/ocm/extensions/repositories/git/repo_test.go index 1dc71b8e22..c4fa9ae0f1 100644 --- a/api/ocm/extensions/repositories/git/repo_test.go +++ b/api/ocm/extensions/repositories/git/repo_test.go @@ -91,7 +91,7 @@ var _ = Describe("access method", func() { BeforeEach(func() { By("setting up local bare git repository to work against when pushing/updating") - billy := techgit.VFSBillyFS(remoteFS) + billy := Must(techgit.VFSBillyFS(remoteFS)) client.InstallProtocol("file", server.NewClient(server.NewFilesystemLoader(billy))) remoteRepo = Must(newBareTestRepo(billy, REMOTE_REPO, gitgo.InitOptions{})) // now that we have a bare repository, we can reference it via URL to access it like a remote repository diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go index 0a74cd0e3a..1cbfe43608 100644 --- a/api/tech/git/fs.go +++ b/api/tech/git/fs.go @@ -16,18 +16,18 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" ) -func VFSBillyFS(fsToWrap vfs.FileSystem) billy.Filesystem { +func VFSBillyFS(fsToWrap vfs.FileSystem) (billy.Filesystem, error) { if fsToWrap == nil { fsToWrap = vfs.New(memoryfs.New()) } fi, err := fsToWrap.Stat(".") if err != nil || !fi.IsDir() { - panic(fmt.Errorf("invalid vfs: %v", err)) + return nil, fmt.Errorf("invalid vfs for billy conversion: %w", err) } return &fs{ vfs: fsToWrap, - } + }, nil } type fs struct { diff --git a/api/tech/git/ref.go b/api/tech/git/ref.go index 1512aa5ec2..df0161b52b 100644 --- a/api/tech/git/ref.go +++ b/api/tech/git/ref.go @@ -10,8 +10,10 @@ import ( giturls "github.com/whilp/git-urls" ) -const urlToRefSeparator = "@" -const refToPathSeparator = "#" +const ( + urlToRefSeparator = "@" + refToPathSeparator = "#" +) // refRegexp is a regular expression that matches a git ref string. // The ref string is expected to be in the format of: @@ -20,7 +22,13 @@ const refToPathSeparator = "#" // - url is the URL of the git repository // - ref is the git reference to checkout, if not specified, defaults to "HEAD" // - path is the path to the file or directory to use as the source, if not specified, defaults to the root of the repository. -var refRegexp = regexp.MustCompile(`^([^@#]+)(@[^#\n]+)?(#[^@\n]+)?`) +var refRegexp = regexp.MustCompile( + fmt.Sprintf( + `^([^%[1]s%[2]s]+)(%[1]s[^%[2]s\n]+)?(%[2]s[^%[1]s\n]+)?`, + urlToRefSeparator, + refToPathSeparator, + ), +) // gurl represents a git URL reference. // It contains the URL of the git repository, diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index 6c73ed2bf9..bc18bb83bc 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -79,7 +79,10 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { return c.repo, nil } - billyFS := VFSBillyFS(c.vfs) + billyFS, err := VFSBillyFS(c.vfs) + if err != nil { + return nil, err + } strg, err := GetStorage(billyFS) if err != nil { diff --git a/api/utils/accessio/downloader/git/downloader.go b/api/utils/accessio/downloader/git/downloader.go index 39d0be1884..a636c35951 100644 --- a/api/utils/accessio/downloader/git/downloader.go +++ b/api/utils/accessio/downloader/git/downloader.go @@ -29,7 +29,6 @@ type CloseableDownloader interface { // Downloader simply uses the default HTTP client to download the contents of a URL. type Downloader struct { cloneOpts *git.CloneOptions - grepOpts *git.GrepOptions matching *regexp.Regexp @@ -52,7 +51,7 @@ func NewDownloader(url string, ref string, path string, auth techgit.AuthMethod) Tags: git.NoTags, Depth: 0, }, - matching: regexp.MustCompile(fmt.Sprintf(`%s`, path)), + matching: regexp.MustCompile(path), buf: bytes.NewBuffer(make([]byte, 0, 4096)), storage: memory.NewStorage(), } From 0a844d50295442a4d5e8794f6d655fa174e27be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Wed, 21 Aug 2024 16:45:41 +0200 Subject: [PATCH 06/22] feat: git repo support --- .../builtin/git/identity/identity.go | 1 + api/oci/extensions/repositories/git/README.md | 17 ++-- api/oci/extensions/repositories/git/format.go | 2 +- .../extensions/repositories/git/git_test.go | 8 +- .../extensions/repositories/git/repository.go | 19 +++- api/oci/extensions/repositories/git/type.go | 30 +++++- .../extensions/accessmethods/git/README.md | 6 +- .../extensions/accessmethods/git/method.go | 4 +- .../accessmethods/git/method_test.go | 21 ++-- api/ocm/extensions/repositories/git/format.go | 3 +- .../extensions/repositories/git/repo_test.go | 15 +-- api/tech/git/auth.go | 6 +- api/tech/git/ref.go | 90 ----------------- api/tech/git/resolver.go | 96 ++++++++++--------- .../accessio/downloader/git/downloader.go | 2 +- 15 files changed, 142 insertions(+), 178 deletions(-) delete mode 100644 api/tech/git/ref.go diff --git a/api/credentials/builtin/git/identity/identity.go b/api/credentials/builtin/git/identity/identity.go index e1afecc4ef..d698a3a114 100644 --- a/api/credentials/builtin/git/identity/identity.go +++ b/api/credentials/builtin/git/identity/identity.go @@ -4,6 +4,7 @@ import ( "strings" giturls "github.com/whilp/git-urls" + "ocm.software/ocm/api/credentials/cpi" "ocm.software/ocm/api/credentials/identity/hostpath" "ocm.software/ocm/api/utils/listformat" diff --git a/api/oci/extensions/repositories/git/README.md b/api/oci/extensions/repositories/git/README.md index cf81f06ee1..35e1c46ccf 100644 --- a/api/oci/extensions/repositories/git/README.md +++ b/api/oci/extensions/repositories/git/README.md @@ -1,10 +1,9 @@ # Repository `GitRepository` - git based repository +## Synopsis -### Synopsis - -``` +```yaml type: GitRepository/v1 ``` @@ -21,11 +20,13 @@ Supported specification version is `v1`. The type specific specification fields are: - **`url`** *string* - - URL of the git repository in the form of @# ^([^@#]+)(@[^#\n]+)?(#[^@\n]+)? - - url is the URL of the git repository - - ref is the git reference to checkout, if not specified, defaults to "HEAD" - - path is the path to the file or directory to use as the source, if not specified,defaults to the root of the repository. + + URL of the git repository in any standard git URL format. + The schemes `http`, `https`, `git`, `ssh` and `file` are supported. + +- **`ref`** *string* + + The git reference to use. This can be a branch, tag, or commit hash. The default is `HEAD`, pointing to the default branch of a repository in most implementations. ### Go Bindings diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go index 65665249b3..39190493ec 100644 --- a/api/oci/extensions/repositories/git/format.go +++ b/api/oci/extensions/repositories/git/format.go @@ -12,7 +12,7 @@ func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Op if err != nil { return nil, err } - return New(cpi.FromProvider(ctx), spec) + return New(cpi.FromProvider(ctx), spec, nil) } func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go index f95b97cf23..40fd8186a3 100644 --- a/api/oci/extensions/repositories/git/git_test.go +++ b/api/oci/extensions/repositories/git/git_test.go @@ -3,6 +3,10 @@ package git_test import ( "fmt" + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/object" "github.com/mandelsoft/filepath/pkg/filepath" @@ -12,10 +16,6 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "github.com/opencontainers/go-digest" - . "github.com/mandelsoft/goutils/testutils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "ocm.software/ocm/api/datacontext/attrs/tmpcache" "ocm.software/ocm/api/datacontext/attrs/vfsattr" "ocm.software/ocm/api/oci" diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index 852b36edaf..5c40a7913d 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -2,6 +2,7 @@ package git import ( "context" + "fmt" "github.com/mandelsoft/logging" "github.com/mandelsoft/vfs/pkg/vfs" @@ -32,7 +33,7 @@ var ( _ credentials.ConsumerIdentityProvider = &RepositoryImpl{} ) -func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { +func New(ctx cpi.Context, spec *RepositorySpec, creds credentials.Credentials) (Repository, error) { urs := spec.UniformRepositorySpec() i := &RepositoryImpl{ RepositoryImplBase: cpi.NewRepositoryImplBase(ctx), @@ -40,9 +41,19 @@ func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { spec: spec, } + opts := spec.ToClientOptions() + + if creds != nil { + auth, err := git.AuthFromCredentials(creds) + if err != nil { + return nil, fmt.Errorf("failed to create git authentication from given credentials: %w", err) + } + opts.AuthMethod = auth + } + var err error - if i.client, err = git.NewClient(spec.ToClientOptions()); err != nil { - return nil, err + if i.client, err = git.NewClient(opts); err != nil { + return nil, fmt.Errorf("failed to create new git client for interacting with the repository: %w", err) } repo, err := ctf.New(ctx, &ctf.RepositorySpec{ @@ -50,7 +61,7 @@ func New(ctx cpi.Context, spec *RepositorySpec) (Repository, error) { AccessMode: spec.AccessMode, }, i.client, i.client, vfs.FileMode(0o770)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create new ctf repository within the git repository: %w", err) } i.ctf = repo diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go index b1833fcf91..fcf003c453 100644 --- a/api/oci/extensions/repositories/git/type.go +++ b/api/oci/extensions/repositories/git/type.go @@ -1,8 +1,11 @@ package git import ( + "fmt" + "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/oci/internal" "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessobj" @@ -38,9 +41,18 @@ type RepositorySpec struct { runtime.ObjectVersionedType `json:",inline"` accessio.StandardOptions `json:",inline"` - // URL is the url of the RepositoryImpl to resolve artifacts. + // URL is the git url of the RepositoryImpl to resolve artifacts. URL string `json:"url"` + // Ref is the git ref of the RepositoryImpl to resolve artifacts. + // Examples include + // - heads/master + // - tags/v1.0.0 + // - pull/123/head + // - remotes/origin/feature + // If empty, the default is set to HEAD. + Ref string `json:"ref,omitempty"` + // Author is the author of commits generated by the repository. If not set, is defaulted from environment and git // configuration of the host system. Author *Author `json:"author,omitempty"` @@ -110,7 +122,7 @@ func (s *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec { } func (s *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) { - return New(ctx, s) + return New(ctx, s, creds) } func (s *RepositorySpec) ToClientOptions() git.ClientOptions { @@ -124,6 +136,20 @@ func (s *RepositorySpec) ToClientOptions() git.ClientOptions { } opts.URL = s.URL + opts.Ref = s.Ref return opts } + +func (s *RepositorySpec) Validate(ctx internal.Context, c credentials.Credentials, context ...credentials.UsageContext) error { + repo, err := New(ctx, s, c) + if err != nil { + return fmt.Errorf("failed to initialize repository for validation: %w", err) + } + defer repo.Close() + + if _, err := repo.NamespaceLister().NumNamespaces(""); err != nil { + return fmt.Errorf("failed to list namespaces to validate repository: %w", err) + } + return nil +} diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md index 1a2b2fca3f..4521d90cd5 100644 --- a/api/ocm/extensions/accessmethods/git/README.md +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -1,10 +1,9 @@ # Access Method `git` - Git Commit Access +## Synopsis -### Synopsis - -``` +```yaml type: git/v1 ``` @@ -37,7 +36,6 @@ The type specific specification fields are: The sha/id of the git commit - ### Go Bindings The go binding can be found [here](method.go) diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index cab76f0da8..bfec8b3914 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -7,12 +7,12 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/mandelsoft/goutils/errors" giturls "github.com/whilp/git-urls" + "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/credentials/builtin/git/identity" - techgit "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/internal" + techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessio/downloader/git" "ocm.software/ocm/api/utils/accessobj" diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go index 30a5c5b531..a12c51c910 100644 --- a/api/ocm/extensions/accessmethods/git/method_test.go +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -2,18 +2,22 @@ package git_test import ( "embed" - _ "embed" "fmt" "io" "os" + "time" + + _ "embed" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/vfs/pkg/cwdfs" "github.com/mandelsoft/vfs/pkg/osfs" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "ocm.software/ocm/api/datacontext/attrs/tmpcache" "ocm.software/ocm/api/datacontext/attrs/vfsattr" @@ -61,7 +65,7 @@ var _ = Describe("Method", func() { fileInRepo, err := os.OpenFile( repoPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, - 0600, + 0o600, ) Expect(err).ToNot(HaveOccurred()) @@ -75,7 +79,13 @@ var _ = Describe("Method", func() { wt, err := repo.Worktree() Expect(err).ToNot(HaveOccurred()) Expect(wt.AddGlob("*")).To(Succeed()) - _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{}) + _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "OCM Test", + Email: "dummy@ocm.software", + When: time.Now(), + }, + }) Expect(err).ToNot(HaveOccurred()) accessSpec = me.New( @@ -98,5 +108,4 @@ var _ = Describe("Method", func() { Expect(err).ToNot(HaveOccurred()) Expect(content).To(Equal(expectedBlobContent)) }) - }) diff --git a/api/ocm/extensions/repositories/git/format.go b/api/ocm/extensions/repositories/git/format.go index e1a6b05488..75dc816345 100644 --- a/api/ocm/extensions/repositories/git/format.go +++ b/api/ocm/extensions/repositories/git/format.go @@ -1,10 +1,9 @@ package git import ( - "ocm.software/ocm/api/ocm/extensions/repositories/ctf" - "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/ocm/cpi" + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" "ocm.software/ocm/api/utils/accessobj" ) diff --git a/api/ocm/extensions/repositories/git/repo_test.go b/api/ocm/extensions/repositories/git/repo_test.go index c4fa9ae0f1..b1dc187565 100644 --- a/api/ocm/extensions/repositories/git/repo_test.go +++ b/api/ocm/extensions/repositories/git/repo_test.go @@ -7,6 +7,12 @@ import ( "path/filepath" "regexp" + . "github.com/mandelsoft/goutils/finalizer" + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "ocm.software/ocm/api/ocm/testhelper" + "github.com/go-git/go-billy/v5" gitgo "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/cache" @@ -14,16 +20,13 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport/client" "github.com/go-git/go-git/v5/plumbing/transport/server" "github.com/go-git/go-git/v5/storage/filesystem" - . "github.com/mandelsoft/goutils/finalizer" - . "github.com/mandelsoft/goutils/testutils" "github.com/mandelsoft/logging" "github.com/mandelsoft/vfs/pkg/cwdfs" "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/tonglil/buflogr" + "ocm.software/ocm/api/oci/artdesc" gitrepo "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/ocm" @@ -33,7 +36,6 @@ import ( "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg/componentmapping" "ocm.software/ocm/api/ocm/extensions/repositories/git" - . "ocm.software/ocm/api/ocm/testhelper" techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess" @@ -49,7 +51,6 @@ const ( ) var _ = Describe("access method", func() { - // remoteFS contains the remote repository on the filesystem // pathFS contains the local PWD // repFS contains the local representation of the repository, meaning the cloned repo to work on the Repository @@ -72,7 +73,7 @@ var _ = Describe("access method", func() { basePath := GinkgoT().TempDir() baseFS := Must(cwdfs.New(osfs.New(), basePath)) for _, dir := range []string{"remote", "path", "rep"} { - Expect(os.Mkdir(filepath.Join(basePath, dir), 0777)).To(Succeed()) + Expect(os.Mkdir(filepath.Join(basePath, dir), 0o777)).To(Succeed()) } remoteFS = Must(projectionfs.New(baseFS, "remote")) pathFS = Must(projectionfs.New(baseFS, "path")) diff --git a/api/tech/git/auth.go b/api/tech/git/auth.go index 76347d91a0..1f49551ccd 100644 --- a/api/tech/git/auth.go +++ b/api/tech/git/auth.go @@ -3,12 +3,12 @@ package git import ( "errors" - "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/credentials/builtin/git/identity" - "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" gssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/credentials/builtin/git/identity" ) var ErrNoValidGitCredentials = errors.New("no valid credentials found for git authentication") diff --git a/api/tech/git/ref.go b/api/tech/git/ref.go deleted file mode 100644 index df0161b52b..0000000000 --- a/api/tech/git/ref.go +++ /dev/null @@ -1,90 +0,0 @@ -package git - -import ( - "fmt" - "hash/fnv" - "net/url" - "regexp" - - "github.com/go-git/go-git/v5/plumbing" - giturls "github.com/whilp/git-urls" -) - -const ( - urlToRefSeparator = "@" - refToPathSeparator = "#" -) - -// refRegexp is a regular expression that matches a git ref string. -// The ref string is expected to be in the format of: -// @# -// where: -// - url is the URL of the git repository -// - ref is the git reference to checkout, if not specified, defaults to "HEAD" -// - path is the path to the file or directory to use as the source, if not specified, defaults to the root of the repository. -var refRegexp = regexp.MustCompile( - fmt.Sprintf( - `^([^%[1]s%[2]s]+)(%[1]s[^%[2]s\n]+)?(%[2]s[^%[1]s\n]+)?`, - urlToRefSeparator, - refToPathSeparator, - ), -) - -// gurl represents a git URL reference. -// It contains the URL of the git repository, -// the reference to check out, -// and the path to the file or directory to use as the source. -type gurl struct { - url *url.URL - ref plumbing.ReferenceName - path string -} - -// decodeGitURL decodes a git ref string into a gurl struct. -// The ref string is expected to be in the format of: -// @# -// see refRegexp for more details. -func decodeGitURL(rawRef string) (*gurl, error) { - matches := refRegexp.FindStringSubmatch(rawRef) - if matches == nil { - return nil, fmt.Errorf("failed to match ref: %s via %s", rawRef, refRegexp) - } - rawURL := matches[1] - matchedRef := matches[2] - path := matches[3] - - parsedURL, err := giturls.Parse(rawURL) - if err != nil { - return nil, fmt.Errorf("failed to parse URL: %w", err) - } - - var ref plumbing.ReferenceName - if matchedRef == "" { - ref = plumbing.HEAD - } else { - ref = plumbing.ReferenceName(matchedRef) - if err := ref.Validate(); err != nil { - return nil, fmt.Errorf("failed to validate ref: %w", err) - } - } - - return &gurl{ - url: parsedURL, - ref: ref, - path: path, - }, nil -} - -func (ref *gurl) String() string { - return fmt.Sprintf("%s%s%s%s%s", ref.url.String(), urlToRefSeparator, ref.ref, refToPathSeparator, ref.path) -} - -func (ref *gurl) Hash() []byte { - hash := fnv.New64() - _, _ = hash.Write([]byte(ref.url.String())) - return hash.Sum(nil) -} - -func (ref *gurl) HashString() string { - return fmt.Sprintf("%x", ref.Hash()) -} diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index bc18bb83bc..66d1bc9cd6 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -20,7 +20,7 @@ import ( "ocm.software/ocm/api/utils/accessobj" ) -var worktreeBranch = plumbing.NewBranchReferenceName("ocm") +var DefaultWorktreeBranch = plumbing.NewBranchReferenceName("ocm") type client struct { opts ClientOptions @@ -28,12 +28,6 @@ type client struct { // vfs tracks the current filesystem where the repo will be stored (at the root) vfs vfs.FileSystem - // url is the git URL of the repository - *gurl - - // auth is the authentication method to use when accessing the repository - auth AuthMethod - // repo is a reference to the git repository if it is already open repo *git.Repository repoMu sync.Mutex @@ -49,7 +43,9 @@ type Client interface { type ClientOptions struct { URL string + Ref string Author + AuthMethod AuthMethod } type Author struct { @@ -60,14 +56,18 @@ type Author struct { var _ Client = &client{} func NewClient(opts ClientOptions) (Client, error) { - gitURL, err := decodeGitURL(opts.URL) - if err != nil { - return nil, err + var pref plumbing.ReferenceName + if opts.Ref == "" { + pref = plumbing.HEAD + } else { + pref = plumbing.ReferenceName(opts.Ref) + if err := pref.Validate(); err != nil { + return nil, fmt.Errorf("invalid reference %q: %w", opts.Ref, err) + } } return &client{ vfs: memoryfs.New(), - gurl: gitURL, opts: opts, }, nil } @@ -93,10 +93,10 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { repo, err := git.Open(strg, billyFS) if errors.Is(err, git.ErrRepositoryNotExists) { repo, err = git.CloneContext(ctx, strg, billyFS, &git.CloneOptions{ - Auth: c.auth, - URL: c.url.String(), + Auth: c.opts.AuthMethod, + URL: c.opts.URL, RemoteName: git.DefaultRemoteName, - ReferenceName: c.ref, + ReferenceName: plumbing.ReferenceName(c.opts.Ref), SingleBranch: true, Depth: 0, Tags: git.AllTags, @@ -111,33 +111,7 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { return nil, err } if newRepo { - if err := repo.FetchContext(ctx, &git.FetchOptions{ - Auth: c.auth, - RemoteName: git.DefaultRemoteName, - Depth: 0, - Tags: git.AllTags, - Force: false, - }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { - return nil, err - } - worktree, err := repo.Worktree() - if err != nil { - return nil, err - } - - if err := worktree.Checkout(&git.CheckoutOptions{ - Branch: worktreeBranch, - Create: true, - Keep: true, - }); err != nil { - return nil, err - } - - if err := worktree.AddGlob("*"); err != nil { - return nil, err - } - - if _, err := worktree.Commit("OCM Repository Setup", &git.CommitOptions{}); err != nil && !errors.Is(err, git.ErrEmptyCommit) { + if err := c.newRepository(ctx, repo); err != nil { return nil, err } } @@ -151,6 +125,40 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { return repo, nil } +func (c *client) newRepository(ctx context.Context, repo *git.Repository) error { + if err := repo.FetchContext(ctx, &git.FetchOptions{ + Auth: c.opts.AuthMethod, + RemoteName: git.DefaultRemoteName, + Depth: 0, + Tags: git.AllTags, + Force: false, + }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + worktree, err := repo.Worktree() + if err != nil { + return err + } + + if err := worktree.Checkout(&git.CheckoutOptions{ + Branch: DefaultWorktreeBranch, + Create: true, + Keep: true, + }); err != nil { + return err + } + + if err := worktree.AddGlob("*"); err != nil { + return err + } + + if _, err := worktree.Commit("OCM Repository Setup", &git.CommitOptions{}); err != nil && !errors.Is(err, git.ErrEmptyCommit) { + return err + } + + return nil +} + func GetStorage(base billy.Filesystem) (storage.Storer, error) { dotGit, err := base.Chroot(git.GitDirName) if err != nil { @@ -189,7 +197,7 @@ func (c *client) Refresh(ctx context.Context) error { } if err := worktree.PullContext(ctx, &git.PullOptions{ - Auth: c.auth, + Auth: c.opts.AuthMethod, RemoteName: git.DefaultRemoteName, }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) && !errors.Is(err, transport.ErrEmptyRemoteRepository) { return err @@ -239,14 +247,14 @@ func (c *client) Update(ctx context.Context, msg string, push bool) error { func (c *client) Setup(system vfs.FileSystem) error { c.vfs = system if _, err := c.Repository(context.Background()); err != nil { - return fmt.Errorf("failed to setup repository %q: %w", c.url.String(), err) + return fmt.Errorf("failed to setup repository %q: %w", c.opts.URL, err) } return nil } func (c *client) Close(_ *accessobj.AccessObject) error { if err := c.Update(context.Background(), "OCM Repository Update", true); err != nil { - return fmt.Errorf("failed to close repository %q: %w", c.url.String(), err) + return fmt.Errorf("failed to close repository %q: %w", c.opts.URL, err) } return nil } diff --git a/api/utils/accessio/downloader/git/downloader.go b/api/utils/accessio/downloader/git/downloader.go index a636c35951..184805ba33 100644 --- a/api/utils/accessio/downloader/git/downloader.go +++ b/api/utils/accessio/downloader/git/downloader.go @@ -14,8 +14,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/storage/memory" - techgit "ocm.software/ocm/api/tech/git" + techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio/downloader" ) From ae82837aa459bb28a7ea029f322a4e828d2d89bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Wed, 25 Sep 2024 13:07:01 +0200 Subject: [PATCH 07/22] feat: git repo support --- api/oci/extensions/repositories/git/repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index 5c40a7913d..14ddd56973 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -8,11 +8,11 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/credentials/builtin/oci/identity" cpicredentials "ocm.software/ocm/api/credentials/cpi" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/extensions/repositories/ctf" "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/tech/oci/identity" ocmlog "ocm.software/ocm/api/utils/logging" ) From 8432e5f2b380e466547b169e40f903b400020eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Wed, 25 Sep 2024 13:28:32 +0200 Subject: [PATCH 08/22] feat: git repo support --- docs/reference/ocm_credential-handling.md | 13 +++++++++++++ docs/reference/ocm_get_credentials.md | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/docs/reference/ocm_credential-handling.md b/docs/reference/ocm_credential-handling.md index 71e3340e28..d0ba951423 100644 --- a/docs/reference/ocm_credential-handling.md +++ b/docs/reference/ocm_credential-handling.md @@ -110,6 +110,19 @@ The following credential consumer types are used/supported: - key: secret key use to access the credential server + - Git: Git credential matcher + + It matches the Git consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type Git evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - token: HTTP token authentication + - privateKey: Private Key authentication certificate + + - Github: GitHub credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_get_credentials.md b/docs/reference/ocm_get_credentials.md index 55ed158abc..4ef43b318d 100644 --- a/docs/reference/ocm_get_credentials.md +++ b/docs/reference/ocm_get_credentials.md @@ -36,6 +36,19 @@ Matchers exist for the following usage contexts or consumer types: - key: secret key use to access the credential server + - Git: Git credential matcher + + It matches the Git consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type Git evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - token: HTTP token authentication + - privateKey: Private Key authentication certificate + + - Github: GitHub credential matcher This matcher is a hostpath matcher. From 66fc603bee7378e2d46b434866323dc49f59f969 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Wed, 2 Oct 2024 17:08:36 +0200 Subject: [PATCH 09/22] chore: various pr review adjustments --- api/credentials/builtin/init.go | 2 +- api/oci/extensions/repositories/git/type.go | 25 +++-- api/oci/internal/uniform.go | 2 +- .../artifactaccess/gitaccess/options.go | 2 +- .../artifactaccess/gitaccess/resource.go | 2 +- .../extensions/accessmethods/git/README.md | 2 +- .../extensions/accessmethods/git/method.go | 2 +- api/tech/git/auth.go | 2 +- api/tech/git/fs.go | 98 ++++++------------- .../builtin => tech}/git/identity/identity.go | 14 +-- .../git/identity/identity_test.go | 41 +++----- .../git/identity/suite_test.go | 0 12 files changed, 73 insertions(+), 119 deletions(-) rename api/{credentials/builtin => tech}/git/identity/identity.go (90%) rename api/{credentials/builtin => tech}/git/identity/identity_test.go (70%) rename api/{credentials/builtin => tech}/git/identity/suite_test.go (100%) diff --git a/api/credentials/builtin/init.go b/api/credentials/builtin/init.go index 38a23142b9..d06d239ae2 100644 --- a/api/credentials/builtin/init.go +++ b/api/credentials/builtin/init.go @@ -1,6 +1,6 @@ package builtin import ( - _ "ocm.software/ocm/api/credentials/builtin/git/identity" _ "ocm.software/ocm/api/credentials/builtin/github" + _ "ocm.software/ocm/api/tech/git/identity" ) diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go index fcf003c453..92dbe0935d 100644 --- a/api/oci/extensions/repositories/git/type.go +++ b/api/oci/extensions/repositories/git/type.go @@ -2,7 +2,7 @@ package git import ( "fmt" - + giturls "github.com/whilp/git-urls" "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/internal" @@ -53,7 +53,7 @@ type RepositorySpec struct { // If empty, the default is set to HEAD. Ref string `json:"ref,omitempty"` - // Author is the author of commits generated by the repository. If not set, is defaulted from environment and git + // Author is the author of commits generated by the repository. If not set, it is defaulted from environment and git // configuration of the host system. Author *Author `json:"author,omitempty"` @@ -102,7 +102,7 @@ func NewRepositorySpec(mode accessobj.AccessMode, url string, opts ...accessio.O } func (s *RepositorySpec) IsIntermediate() bool { - return true + return false } func (s *RepositorySpec) GetType() string { @@ -118,6 +118,12 @@ func (s *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec { Type: Type, Info: s.URL, } + url, err := giturls.Parse(s.URL) + if err == nil { + u.Host = url.Host + u.Scheme = url.Scheme + } + return u } @@ -141,15 +147,14 @@ func (s *RepositorySpec) ToClientOptions() git.ClientOptions { return opts } -func (s *RepositorySpec) Validate(ctx internal.Context, c credentials.Credentials, context ...credentials.UsageContext) error { - repo, err := New(ctx, s, c) - if err != nil { - return fmt.Errorf("failed to initialize repository for validation: %w", err) +func (s *RepositorySpec) Validate(_ internal.Context, c credentials.Credentials, _ ...credentials.UsageContext) error { + if _, err := giturls.Parse(s.URL); err != nil { + return fmt.Errorf("failed to parse git url: %w", err) } - defer repo.Close() - if _, err := repo.NamespaceLister().NumNamespaces(""); err != nil { - return fmt.Errorf("failed to list namespaces to validate repository: %w", err) + if _, err := git.AuthFromCredentials(c); err != nil { + return fmt.Errorf("failed to create git authentication from given credentials: %w", err) } + return nil } diff --git a/api/oci/internal/uniform.go b/api/oci/internal/uniform.go index d35d102afb..32e3b0ef89 100644 --- a/api/oci/internal/uniform.go +++ b/api/oci/internal/uniform.go @@ -31,7 +31,7 @@ type UniformRepositorySpec struct { // Host is the hostname of an oci ref. Host string `json:"host,omitempty"` // Info is the file path used to host ctf component versions - Info string `json:"filePath,omitempty"` + Info string `json:"info,omitempty"` // CreateIfMissing indicates whether a file based or dynamic repo should be created if it does not exist CreateIfMissing bool `json:"createIfMissing,omitempty"` diff --git a/api/ocm/elements/artifactaccess/gitaccess/options.go b/api/ocm/elements/artifactaccess/gitaccess/options.go index 0ced931627..aad1e1911b 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/options.go +++ b/api/ocm/elements/artifactaccess/gitaccess/options.go @@ -1,4 +1,4 @@ -package githubaccess +package gitaccess import ( "github.com/mandelsoft/goutils/optionutils" diff --git a/api/ocm/elements/artifactaccess/gitaccess/resource.go b/api/ocm/elements/artifactaccess/gitaccess/resource.go index 6f9adcbf9b..46e5748046 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/resource.go +++ b/api/ocm/elements/artifactaccess/gitaccess/resource.go @@ -1,4 +1,4 @@ -package githubaccess +package gitaccess import ( "github.com/mandelsoft/goutils/optionutils" diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md index 4521d90cd5..9532b934f9 100644 --- a/api/ocm/extensions/accessmethods/git/README.md +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -14,7 +14,7 @@ The artifact content is provided as gnu-zipped tar archive ### Description This method implements the access of the content of a git commit stored in a -GitHub repository. +git repository. Supported specification version is `v1` diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index bfec8b3914..6752260cc0 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -3,13 +3,13 @@ package git import ( "fmt" "io" + "ocm.software/ocm/api/tech/git/identity" "github.com/go-git/go-git/v5/plumbing" "github.com/mandelsoft/goutils/errors" giturls "github.com/whilp/git-urls" "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/credentials/builtin/git/identity" "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/internal" techgit "ocm.software/ocm/api/tech/git" diff --git a/api/tech/git/auth.go b/api/tech/git/auth.go index 1f49551ccd..26b72d3554 100644 --- a/api/tech/git/auth.go +++ b/api/tech/git/auth.go @@ -2,13 +2,13 @@ package git import ( "errors" + "ocm.software/ocm/api/tech/git/identity" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" gssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/credentials/builtin/git/identity" ) var ErrNoValidGitCredentials = errors.New("no valid credentials found for git authentication") diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go index 1cbfe43608..3ccdc80108 100644 --- a/api/tech/git/fs.go +++ b/api/tech/git/fs.go @@ -26,42 +26,22 @@ func VFSBillyFS(fsToWrap vfs.FileSystem) (billy.Filesystem, error) { } return &fs{ - vfs: fsToWrap, + FileSystem: fsToWrap, }, nil } type fs struct { - vfs vfs.FileSystem + vfs.FileSystem } -type file struct { - lock *fslock.Lock - vfsFile vfs.File -} - -func (f *file) Name() string { - return f.vfsFile.Name() -} - -func (f *file) Write(p []byte) (n int, err error) { - return f.vfsFile.Write(p) -} - -func (f *file) Read(p []byte) (n int, err error) { - return f.vfsFile.Read(p) -} - -func (f *file) ReadAt(p []byte, off int64) (n int, err error) { - return f.vfsFile.ReadAt(p, off) -} +var _ billy.Filesystem = &fs{} -func (f *file) Seek(offset int64, whence int) (int64, error) { - return f.vfsFile.Seek(offset, whence) +type file struct { + lock *fslock.Lock + vfs.File } -func (f *file) Close() error { - return f.vfsFile.Close() -} +var _ billy.File = &file{} func (f *file) Lock() error { return f.lock.Lock() @@ -71,14 +51,10 @@ func (f *file) Unlock() error { return f.lock.Unlock() } -func (f *file) Truncate(size int64) error { - return f.vfsFile.Truncate(size) -} - var _ billy.File = &file{} func (f *fs) Create(filename string) (billy.File, error) { - vfsFile, err := f.vfs.Create(filename) + vfsFile, err := f.FileSystem.Create(filename) if err != nil { return nil, err } @@ -93,17 +69,17 @@ func (f *fs) Create(filename string) (billy.File, error) { // juju vfs only operates on syscalls directly and without interface abstraction its not easy to get the root. func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { var lock *fslock.Lock - if f.vfs == osfs.OsFs { + if f.FileSystem == osfs.OsFs { lock = fslock.New(fmt.Sprintf("%s.lock", vf.Name())) } else { hash := fnv.New32() - _, _ = hash.Write([]byte(f.vfs.Name())) + _, _ = hash.Write([]byte(f.FileSystem.Name())) temp, err := os.MkdirTemp("", fmt.Sprintf("git-vfs-locks-%x", hash.Sum32())) if err != nil { return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+ "this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err) } - _, components := vfs.Components(f.vfs, vf.Name()) + _, components := vfs.Components(f.FileSystem, vf.Name()) lockPath := filepath.Join( temp, filepath.Join(components[:len(components)-1]...), @@ -117,13 +93,13 @@ func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { } return &file{ - vfsFile: vf, - lock: lock, + File: vf, + lock: lock, }, nil } func (f *fs) Open(filename string) (billy.File, error) { - vfsFile, err := f.vfs.Open(filename) + vfsFile, err := f.FileSystem.Open(filename) if err != nil { return nil, err } @@ -132,11 +108,11 @@ func (f *fs) Open(filename string) (billy.File, error) { func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { if flag&os.O_CREATE != 0 { - if err := f.vfs.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + if err := f.FileSystem.MkdirAll(filepath.Dir(filename), 0o755); err != nil { return nil, err } } - vfsFile, err := f.vfs.OpenFile(filename, flag, perm) + vfsFile, err := f.FileSystem.OpenFile(filename, flag, perm) if err != nil { return nil, err } @@ -144,7 +120,7 @@ func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, } func (f *fs) Stat(filename string) (os.FileInfo, error) { - fi, err := f.vfs.Stat(filename) + fi, err := f.FileSystem.Stat(filename) if errors.Is(err, syscall.ENOENT) { return nil, os.ErrNotExist } @@ -154,15 +130,11 @@ func (f *fs) Stat(filename string) (os.FileInfo, error) { func (f *fs) Rename(oldpath, newpath string) error { dir := filepath.Dir(newpath) if dir != "." { - if err := f.vfs.MkdirAll(dir, 0o755); err != nil { + if err := f.FileSystem.MkdirAll(dir, 0o755); err != nil { return err } } - return f.vfs.Rename(oldpath, newpath) -} - -func (f *fs) Remove(filename string) error { - return f.vfs.Remove(filename) + return f.FileSystem.Rename(oldpath, newpath) } func (f *fs) Join(elem ...string) string { @@ -170,7 +142,7 @@ func (f *fs) Join(elem ...string) string { } func (f *fs) TempFile(dir, prefix string) (billy.File, error) { - vfsFile, err := vfs.TempFile(f.vfs, dir, prefix) + vfsFile, err := vfs.TempFile(f.FileSystem, dir, prefix) if err != nil { return nil, err } @@ -178,15 +150,11 @@ func (f *fs) TempFile(dir, prefix string) (billy.File, error) { } func (f *fs) ReadDir(path string) ([]os.FileInfo, error) { - return vfs.ReadDir(f.vfs, path) -} - -func (f *fs) MkdirAll(filename string, perm os.FileMode) error { - return f.vfs.MkdirAll(filename, perm) + return vfs.ReadDir(f.FileSystem, path) } func (f *fs) Lstat(filename string) (os.FileInfo, error) { - fi, err := f.vfs.Lstat(filename) + fi, err := f.FileSystem.Lstat(filename) if err != nil { if errors.Is(err, syscall.ENOENT) { return nil, os.ErrNotExist @@ -195,21 +163,13 @@ func (f *fs) Lstat(filename string) (os.FileInfo, error) { return fi, err } -func (f *fs) Symlink(target, link string) error { - return f.vfs.Symlink(target, link) -} - -func (f *fs) Readlink(link string) (string, error) { - return f.vfs.Readlink(link) -} - func (f *fs) Chroot(path string) (billy.Filesystem, error) { - fi, err := f.vfs.Stat(path) + fi, err := f.FileSystem.Stat(path) if os.IsNotExist(err) { - if err = f.vfs.MkdirAll(path, 0o755); err != nil { + if err = f.FileSystem.MkdirAll(path, 0o755); err != nil { return nil, err } - fi, err = f.vfs.Stat(path) + fi, err = f.FileSystem.Stat(path) } if err != nil { @@ -218,21 +178,21 @@ func (f *fs) Chroot(path string) (billy.Filesystem, error) { return nil, fmt.Errorf("path %s is not a directory", path) } - chfs, err := projectionfs.New(f.vfs, path) + chfs, err := projectionfs.New(f.FileSystem, path) if err != nil { return nil, err } return &fs{ - vfs: chfs, + FileSystem: chfs, }, nil } func (f *fs) Root() string { - if root := projectionfs.Root(f.vfs); root != "" { + if root := projectionfs.Root(f.FileSystem); root != "" { return root } - if canonicalRoot, err := vfs.Canonical(f.vfs, "/", true); err == nil { + if canonicalRoot, err := vfs.Canonical(f.FileSystem, "/", true); err == nil { return canonicalRoot } return "/" diff --git a/api/credentials/builtin/git/identity/identity.go b/api/tech/git/identity/identity.go similarity index 90% rename from api/credentials/builtin/git/identity/identity.go rename to api/tech/git/identity/identity.go index d698a3a114..61fe0dc67b 100644 --- a/api/credentials/builtin/git/identity/identity.go +++ b/api/tech/git/identity/identity.go @@ -34,10 +34,10 @@ the `+hostpath.IDENTITY_TYPE+` type.`, } const ( - ID_HOSTNAME = hostpath.ID_HOSTNAME - ID_PATH = "path" - ID_PORT = hostpath.ID_PORT - ID_SCHEME = hostpath.ID_SCHEME + ID_HOSTNAME = hostpath.ID_HOSTNAME + ID_PATHPREFIX = hostpath.ID_PATHPREFIX + ID_PORT = hostpath.ID_PORT + ID_SCHEME = hostpath.ID_SCHEME ) const ( @@ -96,7 +96,7 @@ func GetConsumerId(repoURL string) (cpi.ConsumerIdentity, error) { } if path != "" { - id[ID_PATH] = path + id[ID_PATHPREFIX] = path } id[ID_SCHEME] = scheme @@ -117,10 +117,10 @@ func BasicAuthCredentials(username, password string) cpi.Credentials { } } -func PublicKeyCredentials(username, publicKey string) cpi.Credentials { +func PrivateKeyCredentials(username, privateKey string) cpi.Credentials { return cpi.DirectCredentials{ ATTR_USERNAME: username, - ATTR_PRIVATE_KEY: publicKey, + ATTR_PRIVATE_KEY: privateKey, } } diff --git a/api/credentials/builtin/git/identity/identity_test.go b/api/tech/git/identity/identity_test.go similarity index 70% rename from api/credentials/builtin/git/identity/identity_test.go rename to api/tech/git/identity/identity_test.go index 8fc82507b2..13a95f4e2c 100644 --- a/api/credentials/builtin/git/identity/identity_test.go +++ b/api/tech/git/identity/identity_test.go @@ -1,9 +1,10 @@ package identity_test import ( + "github.com/mandelsoft/goutils/testutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "ocm.software/ocm/api/credentials/builtin/git/identity" + . "ocm.software/ocm/api/tech/git/identity" "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/datacontext" @@ -16,8 +17,7 @@ var _ = Describe("consumer id handling", func() { Context("id determination", func() { It("handles https repos", func() { - id, err := GetConsumerId(repo) - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId(repo)) Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, "port", "443", "hostname", "github.com", @@ -26,8 +26,7 @@ var _ = Describe("consumer id handling", func() { }) It("handles http repos", func() { - id, err := GetConsumerId("http://github.com/torvalds/linux.git") - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId("http://github.com/torvalds/linux.git")) Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, "port", "80", "hostname", "github.com", @@ -36,8 +35,7 @@ var _ = Describe("consumer id handling", func() { }) It("handles ssh standard format repos", func() { - id, err := GetConsumerId("ssh://github.com/torvalds/linux.git") - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId("ssh://github.com/torvalds/linux.git")) Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, "port", "22", "hostname", "github.com", @@ -46,8 +44,7 @@ var _ = Describe("consumer id handling", func() { }) It("handles ssh git @ format repos", func() { - id, err := GetConsumerId("git@github.com:torvalds/linux.git") - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId("git@github.com:torvalds/linux.git")) Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, "port", "22", "hostname", "github.com", @@ -56,8 +53,7 @@ var _ = Describe("consumer id handling", func() { }) It("handles git format repos", func() { - id, err := GetConsumerId("git://github.com/torvalds/linux.git") - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId("git://github.com/torvalds/linux.git")) Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, "port", "9418", "hostname", "github.com", @@ -66,12 +62,11 @@ var _ = Describe("consumer id handling", func() { }) It("handles file format repos", func() { - id, err := GetConsumerId("file:///path/to/linux/repo") - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId("file:///path/to/linux/repo")) Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, "scheme", "file", "hostname", "localhost", - "path", "/path/to/linux/repo", + "pathprefix", "/path/to/linux/repo", ))) }) }) @@ -87,8 +82,7 @@ var _ = Describe("consumer id handling", func() { It("Basic Auth", func() { user, pass := "linus", "torvalds" - id, err := GetConsumerId(repo) - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId(repo)) credctx.SetCredentialsForConsumer(id, credentials.CredentialsFromList( ATTR_USERNAME, user, @@ -96,8 +90,7 @@ var _ = Describe("consumer id handling", func() { ), ) - creds, err := GetCredentials(ctx, repo) - Expect(err).ToNot(HaveOccurred()) + creds := testutils.Must(GetCredentials(ctx, repo)) Expect(creds).To(BeEquivalentTo(common.Properties{ ATTR_USERNAME: user, ATTR_PASSWORD: pass, @@ -106,16 +99,14 @@ var _ = Describe("consumer id handling", func() { It("Token Auth", func() { token := "mytoken" - id, err := GetConsumerId(repo) - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId(repo)) credctx.SetCredentialsForConsumer(id, credentials.CredentialsFromList( ATTR_TOKEN, token, ), ) - creds, err := GetCredentials(ctx, repo) - Expect(err).ToNot(HaveOccurred()) + creds := testutils.Must(GetCredentials(ctx, repo)) Expect(creds).To(BeEquivalentTo(common.Properties{ ATTR_TOKEN: token, })) @@ -123,8 +114,7 @@ var _ = Describe("consumer id handling", func() { It("Public Key Auth", func() { user, key := "linus", "path/to/my/id_rsa" - id, err := GetConsumerId(repo) - Expect(err).ToNot(HaveOccurred()) + id := testutils.Must(GetConsumerId(repo)) credctx.SetCredentialsForConsumer(id, credentials.CredentialsFromList( ATTR_USERNAME, user, @@ -132,8 +122,7 @@ var _ = Describe("consumer id handling", func() { ), ) - creds, err := GetCredentials(ctx, repo) - Expect(err).ToNot(HaveOccurred()) + creds := testutils.Must(GetCredentials(ctx, repo)) Expect(creds).To(BeEquivalentTo(common.Properties{ ATTR_USERNAME: user, ATTR_PRIVATE_KEY: key, diff --git a/api/credentials/builtin/git/identity/suite_test.go b/api/tech/git/identity/suite_test.go similarity index 100% rename from api/credentials/builtin/git/identity/suite_test.go rename to api/tech/git/identity/suite_test.go From 7f68c55ce8efccd70172855ecfa55725e26b52bc Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Wed, 2 Oct 2024 18:17:21 +0200 Subject: [PATCH 10/22] chore: imports fix --- api/oci/extensions/repositories/git/type.go | 2 ++ api/ocm/extensions/accessmethods/git/method.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go index 92dbe0935d..1b59ded114 100644 --- a/api/oci/extensions/repositories/git/type.go +++ b/api/oci/extensions/repositories/git/type.go @@ -2,7 +2,9 @@ package git import ( "fmt" + giturls "github.com/whilp/git-urls" + "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/internal" diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index 6752260cc0..919cfb3b98 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -3,7 +3,6 @@ package git import ( "fmt" "io" - "ocm.software/ocm/api/tech/git/identity" "github.com/go-git/go-git/v5/plumbing" "github.com/mandelsoft/goutils/errors" @@ -13,6 +12,7 @@ import ( "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/internal" techgit "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/tech/git/identity" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/accessio/downloader/git" "ocm.software/ocm/api/utils/accessobj" From 6f9fc7d50c7810ac4a97a39bb2ee2d68f2051168 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Fri, 4 Oct 2024 17:48:39 +0200 Subject: [PATCH 11/22] feat: allow generic blob accessmethod for git --- .../extensions/accessmethods/git/method.go | 115 ++++---------- .../accessmethods/git/method_test.go | 52 +++---- api/tech/git/auth.go | 2 +- api/tech/git/fs.go | 3 + api/tech/git/identity/identity.go | 1 - api/tech/git/logging.go | 7 + api/utils/blobaccess/git/access.go | 92 +++++++++++ api/utils/blobaccess/git/access_test.go | 144 ++++++++++++++++++ api/utils/blobaccess/git/options.go | 119 +++++++++++++++ api/utils/blobaccess/git/suite_test.go | 13 ++ .../blobaccess/git/testdata/repo/file_in_repo | 1 + docs/reference/ocm_logging.md | 1 + 12 files changed, 434 insertions(+), 116 deletions(-) create mode 100644 api/tech/git/logging.go create mode 100644 api/utils/blobaccess/git/access.go create mode 100644 api/utils/blobaccess/git/access_test.go create mode 100644 api/utils/blobaccess/git/options.go create mode 100644 api/utils/blobaccess/git/suite_test.go create mode 100644 api/utils/blobaccess/git/testdata/repo/file_in_repo diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index 919cfb3b98..060ed139e2 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -2,21 +2,18 @@ package git import ( "fmt" - "io" "github.com/go-git/go-git/v5/plumbing" "github.com/mandelsoft/goutils/errors" giturls "github.com/whilp/git-urls" "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/internal" - techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/tech/git/identity" - "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/accessio/downloader/git" - "ocm.software/ocm/api/utils/accessobj" "ocm.software/ocm/api/utils/blobaccess/blobaccess" + gitblob "ocm.software/ocm/api/utils/blobaccess/git" "ocm.software/ocm/api/utils/mime" "ocm.software/ocm/api/utils/runtime" ) @@ -78,101 +75,43 @@ func (*AccessSpec) GetType() string { return Type } -func (a *AccessSpec) AccessMethod(c internal.ComponentVersionAccess) (internal.AccessMethod, error) { - return accspeccpi.AccessMethodForImplementation(newMethod(c, a)) -} - -func newMethod(componentVersionAccess internal.ComponentVersionAccess, accessSpec *AccessSpec) (accspeccpi.AccessMethodImpl, error) { - u, err := giturls.Parse(accessSpec.RepoURL) +func (a *AccessSpec) AccessMethod(cva internal.ComponentVersionAccess) (internal.AccessMethod, error) { + _, err := giturls.Parse(a.RepoURL) if err != nil { - return nil, errors.ErrInvalidWrap(err, "repository repoURL", accessSpec.RepoURL) + return nil, errors.ErrInvalidWrap(err, "repository repoURL", a.RepoURL) } - if err := plumbing.ReferenceName(accessSpec.Ref).Validate(); err != nil { - return nil, errors.ErrInvalidWrap(err, "commit hash", accessSpec.Ref) + if err := plumbing.ReferenceName(a.Ref).Validate(); err != nil { + return nil, errors.ErrInvalidWrap(err, "commit hash", a.Ref) } - - creds, cid, err := getCreds(accessSpec.RepoURL, componentVersionAccess.GetContext().CredentialsContext()) + creds, _, err := getCreds(a.RepoURL, cva.GetContext().CredentialsContext()) if err != nil { - return nil, fmt.Errorf("failed to get credentials for repository %s: %w", accessSpec.RepoURL, err) + return nil, fmt.Errorf("failed to get credentials for repository %s: %w", a.RepoURL, err) } - auth, err := techgit.AuthFromCredentials(creds) - if err != nil && !errors.Is(err, techgit.ErrNoValidGitCredentials) { - return nil, fmt.Errorf("failed to get auth method for repository %s: %w", accessSpec.RepoURL, err) - } + octx := cva.GetContext() - gitDownloader := git.NewDownloader(u.String(), accessSpec.Ref, accessSpec.PathSpec, auth) - cachedGitBlobAccessor := accessobj.CachedBlobAccessForWriter( - componentVersionAccess.GetContext(), - mime.MIME_OCTET, - accessio.NewWriteAtWriter(gitDownloader.Download), - ) - jointCloser := func() error { - return errors.Join(gitDownloader.Close(), cachedGitBlobAccessor.Close()) + opts := []gitblob.Option{ + gitblob.WithLoggingContext(octx), + gitblob.WithCredentialContext(octx), + gitblob.WithURL(a.RepoURL), + gitblob.WithRef(a.Ref), + gitblob.WithCachingFileSystem(vfsattr.Get(octx)), } - - return &accessMethod{ - spec: accessSpec, - access: cachedGitBlobAccessor, - close: jointCloser, - cid: cid, - }, nil -} - -type accessMethod struct { - spec *AccessSpec - access blobaccess.BlobAccess - close func() error - - cid credentials.ConsumerIdentity -} - -var _ accspeccpi.AccessMethodImpl = &accessMethod{} - -func (m *accessMethod) Close() error { - if m.access == nil { - return nil + if creds != nil { + opts = append(opts, gitblob.WithCredentials(creds)) } - var err error - if m.close != nil { - err = m.close() + factory := func() (blobaccess.BlobAccess, error) { + return gitblob.BlobAccess(opts...) } - err = errors.Join(err, m.access.Close()) - - return err -} - -func (m *accessMethod) Get() ([]byte, error) { - return m.access.Get() -} - -func (m *accessMethod) Reader() (io.ReadCloser, error) { - return m.access.Reader() -} - -func (m *accessMethod) MimeType() string { - return mime.MIME_OCTET -} - -func (*accessMethod) IsLocal() bool { - return false -} - -func (m *accessMethod) GetKind() string { - return Type -} - -func (m *accessMethod) AccessSpec() internal.AccessSpec { - return m.spec -} - -func (m *accessMethod) GetConsumerId(_ ...credentials.UsageContext) credentials.ConsumerIdentity { - return m.cid -} -func (m *accessMethod) GetIdentityMatcher() string { - return identity.CONSUMER_TYPE + return accspeccpi.AccessMethodForImplementation(accspeccpi.NewDefaultMethodImpl( + cva, + a, + "", + mime.MIME_TGZ, + factory, + ), nil) } func getCreds(repoURL string, cctx credentials.Context) (credentials.Credentials, credentials.ConsumerIdentity, error) { diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go index a12c51c910..1f8697f2ee 100644 --- a/api/ocm/extensions/accessmethods/git/method_test.go +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -1,6 +1,9 @@ package git_test import ( + "archive/tar" + "bytes" + "compress/gzip" "embed" "fmt" "io" @@ -9,6 +12,7 @@ import ( _ "embed" + . "github.com/mandelsoft/goutils/testutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -48,64 +52,60 @@ var _ = Describe("Method", func() { BeforeEach(func() { repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" - repo, err := git.PlainInit(repoDir, false) - Expect(err).ToNot(HaveOccurred()) + repo := Must(git.PlainInit(repoDir, false)) repoBase := filepath.Join("testdata", "repo") - repoTestData, err := testData.ReadDir(repoBase) - Expect(err).ToNot(HaveOccurred()) + repoTestData := Must(testData.ReadDir(repoBase)) for _, entry := range repoTestData { path := filepath.Join(repoBase, entry.Name()) repoPath := filepath.Join(repoDir, entry.Name()) - file, err := testData.Open(path) - Expect(err).ToNot(HaveOccurred()) + file := Must(testData.Open(path)) - fileInRepo, err := os.OpenFile( + fileInRepo := Must(os.OpenFile( repoPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600, - ) - Expect(err).ToNot(HaveOccurred()) + )) - _, err = io.Copy(fileInRepo, file) - Expect(err).ToNot(HaveOccurred()) + Must(io.Copy(fileInRepo, file)) Expect(fileInRepo.Close()).To(Succeed()) Expect(file.Close()).To(Succeed()) } - wt, err := repo.Worktree() - Expect(err).ToNot(HaveOccurred()) + wt := Must(repo.Worktree()) Expect(wt.AddGlob("*")).To(Succeed()) - _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{ + Must(wt.Commit("OCM Test Commit", &git.CommitOptions{ Author: &object.Signature{ Name: "OCM Test", Email: "dummy@ocm.software", When: time.Now(), }, - }) - Expect(err).ToNot(HaveOccurred()) + })) accessSpec = me.New( fmt.Sprintf("file://%s", repoDir), string(plumbing.Master), ".", ) - }) - BeforeEach(func() { - var err error - expectedBlobContent, err = testData.ReadFile(filepath.Join("testdata", "repo", "file_in_repo")) - Expect(err).ToNot(HaveOccurred()) + expectedBlobContent = Must(testData.ReadFile(filepath.Join("testdata", "repo", "file_in_repo"))) }) It("downloads artifacts", func() { - m, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) - Expect(err).ToNot(HaveOccurred()) - content, err := m.Get() - Expect(err).ToNot(HaveOccurred()) - Expect(content).To(Equal(expectedBlobContent)) + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) + content := Must(m.Get()) + unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) + + r := tar.NewReader(unzippedContent) + + file := Must(r.Next()) + Expect(file.Name).To(Equal("file_in_repo")) + Expect(file.Size).To(Equal(int64(len(expectedBlobContent)))) + + data := Must(io.ReadAll(r)) + Expect(data).To(Equal(expectedBlobContent)) }) }) diff --git a/api/tech/git/auth.go b/api/tech/git/auth.go index 26b72d3554..276d12a962 100644 --- a/api/tech/git/auth.go +++ b/api/tech/git/auth.go @@ -2,13 +2,13 @@ package git import ( "errors" - "ocm.software/ocm/api/tech/git/identity" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" gssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/tech/git/identity" ) var ErrNoValidGitCredentials = errors.New("no valid credentials found for git authentication") diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go index 3ccdc80108..c3bc5a84c0 100644 --- a/api/tech/git/fs.go +++ b/api/tech/git/fs.go @@ -100,6 +100,9 @@ func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { func (f *fs) Open(filename string) (billy.File, error) { vfsFile, err := f.FileSystem.Open(filename) + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } if err != nil { return nil, err } diff --git a/api/tech/git/identity/identity.go b/api/tech/git/identity/identity.go index 61fe0dc67b..a9ae244705 100644 --- a/api/tech/git/identity/identity.go +++ b/api/tech/git/identity/identity.go @@ -76,7 +76,6 @@ func GetConsumerId(repoURL string) (cpi.ConsumerIdentity, error) { host = "localhost" path = u.Path } - } if idx := strings.Index(host, ":"); idx > 0 { diff --git a/api/tech/git/logging.go b/api/tech/git/logging.go new file mode 100644 index 0000000000..d5f6dfd3d0 --- /dev/null +++ b/api/tech/git/logging.go @@ -0,0 +1,7 @@ +package git + +import "ocm.software/ocm/api/utils/logging" + +var REALM = logging.DefineSubRealm("git repository", "git") + +var Log = logging.DynamicLogger(REALM) diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go new file mode 100644 index 0000000000..27ec35806d --- /dev/null +++ b/api/utils/blobaccess/git/access.go @@ -0,0 +1,92 @@ +package git + +import ( + "context" + + gogit "github.com/go-git/go-git/v5" + "github.com/mandelsoft/goutils/errors" + "github.com/mandelsoft/goutils/finalizer" + "github.com/mandelsoft/goutils/optionutils" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/opencontainers/go-digest" + + "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/utils/blobaccess/bpi" + "ocm.software/ocm/api/utils/blobaccess/file" + "ocm.software/ocm/api/utils/iotools" + "ocm.software/ocm/api/utils/mime" + "ocm.software/ocm/api/utils/tarutils" +) + +func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagation(&rerr) + + options := optionutils.EvalOptions(opt...) + if options.URL == "" { + return nil, errors.New("no URL specified") + } + log := options.Logger("RepoUrl", options.URL) + + if options.AuthMethod == nil && options.Credentials.Value != nil { + authMethod, err := git.AuthFromCredentials(options.Credentials.Value) + if err != nil && !errors.Is(err, git.ErrNoValidGitCredentials) { + return nil, err + } else { + options.AuthMethod = authMethod + } + } + + c, err := git.NewClient(options.ClientOptions) + if err != nil { + return nil, err + } + + tmpfs, err := osfs.NewTempFileSystem() + if err != nil { + return nil, err + } + finalize.With(func() error { + return vfs.Cleanup(tmpfs) + }) + + // redirect the client to the temporary filesystem for storage of the repo, otherwise it would use memory + if err := c.Setup(tmpfs); err != nil { + return nil, err + } + + // get the repository, triggering a clone if not present into the filesystem provided by setup + if _, err := c.Repository(context.Background()); err != nil { + return nil, err + } + + // remove the .git directory as it shouldn't be part of the tarball + if err := tmpfs.RemoveAll(gogit.GitDirName); err != nil { + return nil, err + } + + // pack all downloaded files into a tar.gz file + fs := options.GetCachingFileSystem() + tgz, err := vfs.TempFile(fs, "", "git-*.tar.gz") + if err != nil { + return nil, err + } + + dw := iotools.NewDigestWriterWith(digest.SHA256, tgz) + finalize.Close(dw) + + if err := tarutils.TgzFs(tmpfs, dw); err != nil { + return nil, err + } + + log.Debug("created", "file", tgz.Name()) + + return file.BlobAccessForTemporaryFilePath( + mime.MIME_TGZ, + tgz.Name(), + file.WithFileSystem(fs), + file.WithDigest(dw.Digest()), + file.WithSize(dw.Size()), + ), nil +} diff --git a/api/utils/blobaccess/git/access_test.go b/api/utils/blobaccess/git/access_test.go new file mode 100644 index 0000000000..da1b7c2717 --- /dev/null +++ b/api/utils/blobaccess/git/access_test.go @@ -0,0 +1,144 @@ +package git_test + +import ( + "embed" + _ "embed" + "fmt" + "io" + "os" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/mandelsoft/filepath/pkg/filepath" + . "github.com/mandelsoft/goutils/testutils" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + gitblob "ocm.software/ocm/api/utils/blobaccess/git" + "ocm.software/ocm/api/utils/tarutils" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("git Blob Access", func() { + var ( + ctx ocm.Context + url string + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + Context("git filesystem repository", func() { + BeforeEach(func() { + repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo, err := git.PlainInit(repoDir, false) + Expect(err).ToNot(HaveOccurred()) + + repoBase := filepath.Join("testdata", "repo") + repoTestData, err := testData.ReadDir(repoBase) + Expect(err).ToNot(HaveOccurred()) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file, err := testData.Open(path) + Expect(err).ToNot(HaveOccurred()) + + fileInRepo, err := os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0o600, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = io.Copy(fileInRepo, file) + Expect(err).ToNot(HaveOccurred()) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt, err := repo.Worktree() + Expect(err).ToNot(HaveOccurred()) + Expect(wt.AddGlob("*")).To(Succeed()) + _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "OCM Test", + Email: "dummy@ocm.software", + When: time.Now(), + }, + }) + Expect(err).ToNot(HaveOccurred()) + + url = fmt.Sprintf("file://%s", repoDir) + }) + + It("blobaccess for simple repository", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithCachingContext(ctx), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingPath(GinkgoT().TempDir()), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("file_in_repo")) + }) + + }) + + Context("git http repository", func() { + BeforeEach(func() { + host := "github.com:443" + if PingTCPServer(host, time.Second) != nil { + Skip(fmt.Sprintf("no connection to %s, skipping test connection to remote", url)) + } + // This repo is a public repo owned by the Github Kraken Bot, so its as good of a public available + // example as any. + url = fmt.Sprintf("https://%s/octocat/Hello-World.git", host) + }) + + It("hello world from github with master branch", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithCachingContext(ctx), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingPath(GinkgoT().TempDir()), + gitblob.WithRef(plumbing.Master.String()), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("README")) + }) + + It("hello world from github with custom ref", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithCachingContext(ctx), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingPath(GinkgoT().TempDir()), + gitblob.WithRef("refs/heads/test"), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("README", "CONTRIBUTING.md")) + }) + }) +}) diff --git a/api/utils/blobaccess/git/options.go b/api/utils/blobaccess/git/options.go new file mode 100644 index 0000000000..d0450ed791 --- /dev/null +++ b/api/utils/blobaccess/git/options.go @@ -0,0 +1,119 @@ +package git + +import ( + "github.com/mandelsoft/goutils/optionutils" + "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext" + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/tech/git" + ocmlog "ocm.software/ocm/api/utils/logging" + "ocm.software/ocm/api/utils/stdopts" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + git.ClientOptions + + stdopts.StandardContexts +} + +func (o *Options) Logger(keyValuePairs ...interface{}) logging.Logger { + return ocmlog.LogContext(o.LoggingContext.Value, o.CredentialContext.Value).Logger(git.REALM).WithValues(keyValuePairs...) +} + +func (o *Options) Cache() *tmpcache.Attribute { + if o.CachingPath.Value != "" { + return tmpcache.New(o.CachingPath.Value, o.CachingFileSystem.Value) + } + if o.CachingContext.Value != nil { + return tmpcache.Get(o.CachingContext.Value) + } + return tmpcache.Get(o.CredentialContext.Value) +} + +func (o *Options) ApplyTo(opts *Options) { + if opts == nil { + return + } + if o.CredentialContext.Value != nil { + opts.CredentialContext = o.CredentialContext + } + if o.Credentials.Value != nil { + opts.Credentials = o.Credentials + } + if o.LoggingContext.Value != nil { + opts.LoggingContext = o.LoggingContext + } + if o.CachingFileSystem.Value != nil { + opts.CachingFileSystem = o.CachingFileSystem + } + if o.URL != "" { + opts.URL = o.URL + } + if o.Author.Name != "" && o.Author.Email != "" { + opts.Author = o.Author + } + if o.Ref != "" { + opts.Ref = o.Ref + } +} + +func option[S any, T any](v T) optionutils.Option[*Options] { + return optionutils.WithGenericOption[S, *Options](v) +} + +func WithCredentialContext(ctx credentials.ContextProvider) Option { + return option[stdopts.CredentialContextOptionBag](ctx) +} + +func WithLoggingContext(ctx logging.ContextProvider) Option { + return option[stdopts.LoggingContextOptionBag](ctx) +} + +func WithCachingContext(ctx datacontext.Context) Option { + return option[stdopts.CachingContextOptionBag](ctx) +} + +func WithCachingFileSystem(fs vfs.FileSystem) Option { + return option[stdopts.CachingFileSystemOptionBag](fs) +} + +func WithCachingPath(p string) Option { + return option[stdopts.CachingPathOptionBag](p) +} + +func WithCredentials(c credentials.Credentials) Option { + return option[stdopts.CredentialsOptionBag](c) +} + +//////////////////////////////////////////////////////////////////////////////// + +type URLOptionBag interface { + SetURL(v string) +} + +func (o *Options) SetURL(v string) { + o.URL = v +} + +func WithURL(url string) Option { + return option[URLOptionBag](url) +} + +//////////////////////////////////////////////////////////////////////////////// + +type RefOptionBag interface { + SetRef(v string) +} + +func (o *Options) SetRef(v string) { + o.Ref = v +} + +func WithRef(ref string) Option { + return option[RefOptionBag](ref) +} diff --git a/api/utils/blobaccess/git/suite_test.go b/api/utils/blobaccess/git/suite_test.go new file mode 100644 index 0000000000..1e348d8652 --- /dev/null +++ b/api/utils/blobaccess/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "git Blob Access Test Suite") +} diff --git a/api/utils/blobaccess/git/testdata/repo/file_in_repo b/api/utils/blobaccess/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/utils/blobaccess/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/docs/reference/ocm_logging.md b/docs/reference/ocm_logging.md index 9a1c45800b..c73dc6cc3e 100644 --- a/docs/reference/ocm_logging.md +++ b/docs/reference/ocm_logging.md @@ -27,6 +27,7 @@ The following *realms* are used by the command line tool: - ocm/credentials/dockerconfig: docker config handling as credential repository - ocm/credentials/vault: HashiCorp Vault Access - ocm/downloader: Downloaders + - ocm/git: git repository - ocm/maven: Maven repository - ocm/npm: NPM registry - ocm/oci/docker: Docker repository handling From 8b34ef841426c7921c34e7b789aaa18d407a0fa2 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Mon, 7 Oct 2024 12:40:27 +0200 Subject: [PATCH 12/22] chore: make sure creds get read from repo and enforce create in format --- api/oci/extensions/repositories/git/format.go | 9 +++++---- api/oci/extensions/repositories/git/git_test.go | 3 +-- api/oci/extensions/repositories/git/repository.go | 8 +++++++- api/tech/git/identity/identity.go | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go index 39190493ec..8408cd21c6 100644 --- a/api/oci/extensions/repositories/git/format.go +++ b/api/oci/extensions/repositories/git/format.go @@ -7,14 +7,15 @@ import ( // ////////////////////////////////////////////////////////////////////////////// -func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { +func Open(ctxp cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { + ctx := cpi.FromProvider(ctxp) spec, err := NewRepositorySpecFromOptions(acc, url, opts) if err != nil { return nil, err } - return New(cpi.FromProvider(ctx), spec, nil) + return New(ctx, spec, nil) } -func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { - return Open(ctx, acc, url, opts) +func Create(ctx cpi.ContextProvider, url string, opts Options) (Repository, error) { + return Open(ctx, accessobj.ACC_CREATE, url, opts) } diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go index 40fd8186a3..987552d2b0 100644 --- a/api/oci/extensions/repositories/git/git_test.go +++ b/api/oci/extensions/repositories/git/git_test.go @@ -23,7 +23,6 @@ import ( "ocm.software/ocm/api/oci/cpi" rgit "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/accessobj" "ocm.software/ocm/api/utils/blobaccess" ocmlog "ocm.software/ocm/api/utils/logging" "ocm.software/ocm/api/utils/mime" @@ -66,7 +65,7 @@ var _ = Describe("ctf management", func() { }) It("instantiate git based ctf", func() { - repo := Must(rgit.Create(ctx, accessobj.ACC_CREATE, repoURL, rgit.Options{ + repo := Must(rgit.Create(ctx, repoURL, rgit.Options{ Author: &rgit.Author{ Name: fmt.Sprintf("OCM Test Case: %s", GinkgoT().Name()), Email: "dummy@ocm.software", diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index 14ddd56973..dbdf9917e4 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -12,7 +12,7 @@ import ( "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/extensions/repositories/ctf" "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/tech/oci/identity" + "ocm.software/ocm/api/tech/git/identity" ocmlog "ocm.software/ocm/api/utils/logging" ) @@ -43,6 +43,12 @@ func New(ctx cpi.Context, spec *RepositorySpec, creds credentials.Credentials) ( opts := spec.ToClientOptions() + if creds == nil { + // if no credentials are provided, try to get them from the context, + // if the credential is not provided, the client will try to use unauthenticated access, so allow error + creds, _ = identity.GetCredentials(ctx, spec.URL) + } + if creds != nil { auth, err := git.AuthFromCredentials(creds) if err != nil { diff --git a/api/tech/git/identity/identity.go b/api/tech/git/identity/identity.go index a9ae244705..5f924dd0e8 100644 --- a/api/tech/git/identity/identity.go +++ b/api/tech/git/identity/identity.go @@ -128,5 +128,5 @@ func GetCredentials(ctx cpi.ContextProvider, repoURL string) (cpi.Credentials, e if err != nil { return nil, err } - return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id, identityMatcher) + return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id, IdentityMatcher) } From 607c3abd473769a272846af9c74e1c160ea7b020 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Mon, 7 Oct 2024 15:42:35 +0200 Subject: [PATCH 13/22] chore: make repo simpler --- api/oci/extensions/repositories/git/format.go | 2 +- .../extensions/repositories/git/git_test.go | 15 +- .../extensions/repositories/git/namespace.go | 288 +----------------- .../extensions/repositories/git/repository.go | 48 +-- api/ocm/extensions/repositories/git/format.go | 4 +- .../extensions/repositories/git/repo_test.go | 44 +-- api/tech/git/resolver.go | 31 +- 7 files changed, 75 insertions(+), 357 deletions(-) diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go index 8408cd21c6..6b5b84d30f 100644 --- a/api/oci/extensions/repositories/git/format.go +++ b/api/oci/extensions/repositories/git/format.go @@ -17,5 +17,5 @@ func Open(ctxp cpi.ContextProvider, acc accessobj.AccessMode, url string, opts O } func Create(ctx cpi.ContextProvider, url string, opts Options) (Repository, error) { - return Open(ctx, accessobj.ACC_CREATE, url, opts) + return Open(ctx, accessobj.ACC_CREATE|accessobj.ACC_WRITABLE, url, opts) } diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go index 987552d2b0..a27620b973 100644 --- a/api/oci/extensions/repositories/git/git_test.go +++ b/api/oci/extensions/repositories/git/git_test.go @@ -84,15 +84,11 @@ var _ = Describe("ctf management", func() { commits := Must(remoteRepo.CommitObjects()) validAdd := 0 - validSync := 0 var messages []string Expect(commits.ForEach(func(commit *object.Commit) error { - if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa); commit.Message == expected { + if expected := rgit.GenerateCommitMessage(); commit.Message == expected { validAdd++ } - if expected := rgit.GenerateCommitMessageForArtifact(rgit.OperationUpdate, aa); commit.Message == expected { - validSync++ - } messages = append(messages, commit.Message) return nil })).To(Succeed()) @@ -100,14 +96,7 @@ var _ = Describe("ctf management", func() { Expect(validAdd).To(Equal(1), fmt.Sprintf( "expected exactly one commit with message %q, got %d commits with messages:\n%v", - rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa), - validAdd, - messages, - )) - Expect(validSync).To(Equal(1), - fmt.Sprintf( - "expected exactly one commit with message %q, got %d commits with messages:\n%v", - rgit.GenerateCommitMessageForArtifact(rgit.OperationAdd, aa), + rgit.GenerateCommitMessage(), validAdd, messages, )) diff --git a/api/oci/extensions/repositories/git/namespace.go b/api/oci/extensions/repositories/git/namespace.go index 2db42a820a..87546925b7 100644 --- a/api/oci/extensions/repositories/git/namespace.go +++ b/api/oci/extensions/repositories/git/namespace.go @@ -2,311 +2,55 @@ package git import ( "context" - "fmt" "github.com/opencontainers/go-digest" - "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/oci/cpi/support" - "ocm.software/ocm/api/oci/internal" "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/utils/blobaccess/blobaccess" ) -const CommitPrefix = "update(ocm)" - func NewNamespace(repo *RepositoryImpl, name string) (cpi.NamespaceAccess, error) { - ctfNamespace, err := newNamespaceContainer(repo, name) - if err != nil { - return nil, err - } - return support.NewNamespaceAccess(name, ctfNamespace, repo, "Git RepositoryImpl Branch") -} - -type namespaceContainer struct { - impl support.NamespaceAccessImpl - name string - client git.Client - ctf cpi.NamespaceAccess -} - -var _ support.NamespaceContainer = (*namespaceContainer)(nil) - -func newNamespaceContainer(repo *RepositoryImpl, name string) (support.NamespaceContainer, error) { - ctfNamespace, err := repo.ctf.LookupNamespace(name) + ctfNamespace, err := repo.Repository.LookupNamespace(name) if err != nil { return nil, err } - return &namespaceContainer{ - name: name, - client: repo.client, - ctf: ctfNamespace, + return &namespace{ + client: repo.client, + NamespaceAccess: ctfNamespace, }, nil } -func (n *namespaceContainer) SetImplementation(impl support.NamespaceAccessImpl) { - n.impl = impl -} - -func (n *namespaceContainer) IsReadOnly() bool { - return false +type namespace struct { + client git.Client + cpi.NamespaceAccess } -func (n *namespaceContainer) Close() error { - if err := n.ctf.Close(); err != nil { - return err - } - return n.client.Update(context.Background(), GenerateCommitMessageForNamespace(OperationUpdate, n.name), true) -} +var _ cpi.NamespaceAccess = (*namespace)(nil) -func (n *namespaceContainer) ListTags() ([]string, error) { +func (n *namespace) ListTags() ([]string, error) { if err := n.client.Refresh(context.Background()); err != nil { return nil, err } - return n.ctf.ListTags() + return n.NamespaceAccess.ListTags() } -func (n *namespaceContainer) GetBlobData(digest digest.Digest) (int64, cpi.DataAccess, error) { +func (n *namespace) GetBlobData(digest digest.Digest) (int64, cpi.DataAccess, error) { if err := n.client.Refresh(context.Background()); err != nil { return 0, nil, err } - - return n.ctf.GetBlobData(digest) -} - -func (n *namespaceContainer) AddBlob(blob cpi.BlobAccess) error { - if err := n.ctf.AddBlob(blob); err != nil { - return err - } - - if err := n.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, blob), false); err != nil { - return err - } - return nil + return n.NamespaceAccess.GetBlobData(digest) } -func (n *namespaceContainer) GetArtifact(i support.NamespaceAccessImpl, vers string) (cpi.ArtifactAccess, error) { +func (n *namespace) GetArtifact(version string) (cpi.ArtifactAccess, error) { if err := n.client.Refresh(context.Background()); err != nil { return nil, err } - - return n.ctf.GetArtifact(vers) + return n.NamespaceAccess.GetArtifact(version) } -func (n *namespaceContainer) HasArtifact(vers string) (bool, error) { +func (n *namespace) HasArtifact(vers string) (bool, error) { if err := n.client.Refresh(context.Background()); err != nil { return false, err } - return n.ctf.HasArtifact(vers) -} - -func (n *namespaceContainer) AddArtifact(artifact cpi.Artifact, tags ...string) (access blobaccess.BlobAccess, err error) { - blobAccess, err := n.ctf.AddArtifact(artifact, tags...) - if err != nil { - return nil, err - } - msg := GenerateCommitMessageForArtifact(OperationAdd, artifact) - - if err := n.client.Update(context.Background(), msg, false); err != nil { - return nil, err - } - - return blobAccess, nil -} - -func (n *namespaceContainer) AddTags(digest digest.Digest, tags ...string) error { - if err := n.ctf.AddTags(digest, tags...); err != nil { - return err - } - - if err := n.client.Update(context.Background(), fmt.Sprintf("added tags %s to %s", tags, digest.String()), true); err != nil { - return err - } - - return nil -} - -func (n *namespaceContainer) NewArtifact(i support.NamespaceAccessImpl, art ...cpi.Artifact) (cpi.ArtifactAccess, error) { - artifactAccess, err := n.ctf.NewArtifact(art...) - if err != nil { - return nil, err - } - return &artifactContainer{ - client: n.client, - ArtifactAccess: artifactAccess, - }, nil -} - -type Operation string - -const ( - OperationAdd Operation = "add" - OperationMod Operation = "mod" - OperationUpdate Operation = "update" -) - -func GenerateCommitMessageForArtifact(operation Operation, artifact cpi.Artifact) string { - a := artifact.Artifact() - - var typ string - switch { - case artifact.IsManifest(): - typ = "manifest" - case artifact.IsIndex(): - typ = "index" - default: - typ = "artifact" - } - - return fmt.Sprintf("%s: %s %s %s (%s)", CommitPrefix, operation, typ, a.Digest(), a.MimeType()) -} - -func GenerateCommitMessageForBlob(operation Operation, blob cpi.BlobAccess) string { - var msg string - if blob.DigestKnown() { - msg = fmt.Sprintf("%s: %s blob(%s) of type %s", CommitPrefix, operation, blob.Digest(), blob.MimeType()) - } else { - msg = fmt.Sprintf("%s: %s blob of type %s", CommitPrefix, operation, blob.MimeType()) - } - return msg -} - -func GenerateCommitMessageForNamespace(operation Operation, namespace string) string { - return fmt.Sprintf("update(ocm): %s namespace %q", operation, namespace) -} - -type artifactContainer struct { - client git.Client - cpi.ArtifactAccess -} - -var _ cpi.ArtifactAccess = (*artifactContainer)(nil) - -func (a *artifactContainer) Close() error { - if err := a.ArtifactAccess.Close(); err != nil { - return err - } - return a.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationUpdate, a.ArtifactAccess), true) -} - -func (a *artifactContainer) Dup() (cpi.ArtifactAccess, error) { - access, err := a.ArtifactAccess.Dup() - if err != nil { - return nil, err - } - return &artifactContainer{ - client: a.client, - ArtifactAccess: access, - }, nil -} - -func (a *artifactContainer) AddBlob(access internal.BlobAccess) error { - if err := a.ArtifactAccess.AddBlob(access); err != nil { - return err - } - return a.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, access), false) -} - -func (a *artifactContainer) AddArtifact(artifact cpi.Artifact, platform *artdesc.Platform) (cpi.BlobAccess, error) { - b, err := a.ArtifactAccess.AddArtifact(artifact, platform) - if err != nil { - return nil, err - } - return b, a.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationAdd, artifact), true) -} - -func (a *artifactContainer) AddLayer(access cpi.BlobAccess, descriptor *artdesc.Descriptor) (int, error) { - n, err := a.ArtifactAccess.AddLayer(access, descriptor) - if err != nil { - return -1, err - } - return n, a.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationMod, access), false) -} - -func (a *artifactContainer) NewArtifact(artifact ...cpi.Artifact) (cpi.ArtifactAccess, error) { - access, err := a.ArtifactAccess.NewArtifact(artifact...) - if err != nil { - return nil, err - } - return &artifactContainer{ - client: a.client, - ArtifactAccess: access, - }, nil -} - -func (a *artifactContainer) ManifestAccess() cpi.ManifestAccess { - return &manifestContainer{ - client: a.client, - ManifestAccess: a.ArtifactAccess.ManifestAccess(), - } -} - -func (a *artifactContainer) IndexAccess() cpi.IndexAccess { - return &indexContainer{ - client: a.client, - IndexAccess: a.ArtifactAccess.IndexAccess(), - } -} - -type manifestContainer struct { - cpi.ManifestAccess - client git.Client -} - -var _ cpi.ManifestAccess = (*manifestContainer)(nil) - -func (m *manifestContainer) AddBlob(access internal.BlobAccess) error { - if err := m.ManifestAccess.AddBlob(access); err != nil { - return err - } - return m.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, access), false) -} - -func (m *manifestContainer) AddLayer(access internal.BlobAccess, descriptor *artdesc.Descriptor) (int, error) { - n, err := m.ManifestAccess.AddLayer(access, descriptor) - if err != nil { - return -1, err - } - return n, m.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationMod, access), false) -} - -func (m *manifestContainer) SetConfigBlob(blob internal.BlobAccess, d *artdesc.Descriptor) error { - if err := m.ManifestAccess.SetConfigBlob(blob, d); err != nil { - return err - } - return m.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationMod, blob), false) -} - -type indexContainer struct { - cpi.IndexAccess - client git.Client -} - -var _ cpi.IndexAccess = (*indexContainer)(nil) - -func (i *indexContainer) GetArtifact(digest digest.Digest) (internal.ArtifactAccess, error) { - a, err := i.IndexAccess.GetArtifact(digest) - if err != nil { - return nil, err - } - return &artifactContainer{ - client: i.client, - ArtifactAccess: a, - }, nil -} - -func (i *indexContainer) AddBlob(access internal.BlobAccess) error { - if err := i.IndexAccess.AddBlob(access); err != nil { - return err - } - return i.client.Update(context.Background(), GenerateCommitMessageForBlob(OperationAdd, access), false) -} - -func (i *indexContainer) AddArtifact(artifact internal.Artifact, platform *artdesc.Platform) (internal.BlobAccess, error) { - b, err := i.IndexAccess.AddArtifact(artifact, platform) - if err != nil { - return nil, err - } - return b, i.client.Update(context.Background(), GenerateCommitMessageForArtifact(OperationAdd, artifact), false) + return n.NamespaceAccess.HasArtifact(vers) } diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index dbdf9917e4..e9fbd27707 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -8,37 +8,36 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "ocm.software/ocm/api/credentials" - cpicredentials "ocm.software/ocm/api/credentials/cpi" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/extensions/repositories/ctf" "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/tech/git/identity" + "ocm.software/ocm/api/utils/accessobj" ocmlog "ocm.software/ocm/api/utils/logging" ) +const CommitPrefix = "update(ocm)" + type Repository interface { cpi.Repository } type RepositoryImpl struct { - cpi.RepositoryImplBase logger logging.UnboundLogger spec *RepositorySpec - ctf *ctf.Repository + *ctf.Repository client git.Client } var ( - _ cpi.RepositoryImpl = (*RepositoryImpl)(nil) - _ credentials.ConsumerIdentityProvider = &RepositoryImpl{} + _ cpi.Repository = (*RepositoryImpl)(nil) ) func New(ctx cpi.Context, spec *RepositorySpec, creds credentials.Credentials) (Repository, error) { urs := spec.UniformRepositorySpec() i := &RepositoryImpl{ - RepositoryImplBase: cpi.NewRepositoryImplBase(ctx), - logger: logging.DynamicLogger(ctx, logging.NewAttribute(ocmlog.ATTR_HOST, urs.Host)), - spec: spec, + logger: logging.DynamicLogger(ctx, logging.NewAttribute(ocmlog.ATTR_HOST, urs.Host)), + spec: spec, } opts := spec.ToClientOptions() @@ -65,31 +64,33 @@ func New(ctx cpi.Context, spec *RepositorySpec, creds credentials.Credentials) ( repo, err := ctf.New(ctx, &ctf.RepositorySpec{ StandardOptions: spec.StandardOptions, AccessMode: spec.AccessMode, - }, i.client, i.client, vfs.FileMode(0o770)) + }, i.client, &repoCloseUpdater{func() error { + if i.IsReadOnly() { + return nil + } + // on close make sure that we update and push the latest changes + return i.client.Update(context.Background(), GenerateCommitMessage(), true) + }}, vfs.FileMode(0o770)) if err != nil { return nil, fmt.Errorf("failed to create new ctf repository within the git repository: %w", err) } - i.ctf = repo + i.Repository = repo - return cpi.NewRepository(i, "git"), nil + return i, nil } func (r *RepositoryImpl) GetSpecification() cpi.RepositorySpec { return r.spec } -func (r *RepositoryImpl) Close() error { - return r.ctf.Close() +func GenerateCommitMessage() string { + return fmt.Sprintf("%s: update repository", CommitPrefix) } func (r *RepositoryImpl) GetIdentityMatcher() string { return identity.CONSUMER_TYPE } -func (r *RepositoryImpl) NamespaceLister() cpi.NamespaceLister { - return r.ctf.NamespaceLister() -} - func (r *RepositoryImpl) IsReadOnly() bool { return false } @@ -98,14 +99,14 @@ func (r *RepositoryImpl) ExistsArtifact(name string, version string) (bool, erro if err := r.client.Refresh(context.Background()); err != nil { return false, err } - return r.ctf.ExistsArtifact(name, version) + return r.Repository.ExistsArtifact(name, version) } func (r *RepositoryImpl) LookupArtifact(name string, version string) (cpi.ArtifactAccess, error) { if err := r.client.Refresh(context.Background()); err != nil { return nil, err } - return r.ctf.LookupArtifact(name, version) + return r.Repository.LookupArtifact(name, version) } func (r *RepositoryImpl) LookupNamespace(name string) (cpi.NamespaceAccess, error) { @@ -115,6 +116,11 @@ func (r *RepositoryImpl) LookupNamespace(name string) (cpi.NamespaceAccess, erro return NewNamespace(r, name) } -func (r *RepositoryImpl) GetConsumerId(ctx ...cpicredentials.UsageContext) cpicredentials.ConsumerIdentity { - return nil +// small helper to wrap accessio.Closer to allow calling an arbitrary closing logic +type repoCloseUpdater struct { + close func() error +} + +func (r *repoCloseUpdater) Close(*accessobj.AccessObject) error { + return r.close() } diff --git a/api/ocm/extensions/repositories/git/format.go b/api/ocm/extensions/repositories/git/format.go index 75dc816345..e85542da5c 100644 --- a/api/ocm/extensions/repositories/git/format.go +++ b/api/ocm/extensions/repositories/git/format.go @@ -24,8 +24,8 @@ func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Op return genericocireg.NewRepository(cpi.FromProvider(ctx), nil, r), nil } -func Create(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (cpi.Repository, error) { - r, err := git.Create(cpi.FromProvider(ctx), acc, url, opts) +func Create(ctx cpi.ContextProvider, url string, opts Options) (cpi.Repository, error) { + r, err := git.Create(cpi.FromProvider(ctx), url, opts) if err != nil { return nil, err } diff --git a/api/ocm/extensions/repositories/git/repo_test.go b/api/ocm/extensions/repositories/git/repo_test.go index b1dc187565..91de46cde6 100644 --- a/api/ocm/extensions/repositories/git/repo_test.go +++ b/api/ocm/extensions/repositories/git/repo_test.go @@ -5,12 +5,12 @@ import ( "fmt" "os" "path/filepath" - "regexp" . "github.com/mandelsoft/goutils/finalizer" . "github.com/mandelsoft/goutils/testutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "ocm.software/ocm/api/ocm/testhelper" "github.com/go-git/go-billy/v5" @@ -27,14 +27,12 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "github.com/tonglil/buflogr" - "ocm.software/ocm/api/oci/artdesc" gitrepo "ocm.software/ocm/api/oci/extensions/repositories/git" "ocm.software/ocm/api/ocm" "ocm.software/ocm/api/ocm/compdesc" metav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg/componentmapping" "ocm.software/ocm/api/ocm/extensions/repositories/git" techgit "ocm.software/ocm/api/tech/git" "ocm.software/ocm/api/utils/accessio" @@ -113,7 +111,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) + a := Must(git.Create(ctx, repoURL, opts)) final.Close(a, "repository") c := Must(a.LookupComponent(COMPONENT)) final.Close(c, "component") @@ -124,31 +122,9 @@ var _ = Describe("access method", func() { MustBeSuccessful(c.AddVersion(cv)) MustBeSuccessful(final.Finalize()) - componentCommitExpectation := gitrepo.GenerateCommitMessageForNamespace(gitrepo.OperationUpdate, fmt.Sprintf("component-descriptors/%s", COMPONENT)) - descriptorCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s blob.* of type %s", - regexp.QuoteMeta(gitrepo.CommitPrefix), - gitrepo.OperationAdd, - regexp.QuoteMeta(componentmapping.ComponentDescriptorTarMimeType)), - ) - descriptorConfigCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s blob.* of type %s", - regexp.QuoteMeta(gitrepo.CommitPrefix), - gitrepo.OperationAdd, - regexp.QuoteMeta(componentmapping.ComponentDescriptorConfigMimeType)), - ) - manifestAddCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s artifact .* %s", - regexp.QuoteMeta(gitrepo.CommitPrefix), - gitrepo.OperationAdd, - regexp.QuoteMeta(fmt.Sprintf("(%s)", artdesc.MediaTypeImageManifest))), - ) - manifestUpdateCommitExpectation := regexp.MustCompile(fmt.Sprintf("%s: %s manifest .* %s", - regexp.QuoteMeta(gitrepo.CommitPrefix), - gitrepo.OperationUpdate, - regexp.QuoteMeta(fmt.Sprintf("(%s)", artdesc.MediaTypeImageManifest))), - ) + componentCommitExpectation := gitrepo.GenerateCommitMessage() componentUpdate := 0 - descriptorCommits := 0 - manifestUpdateCommits := 0 commits := Must(remoteRepo.CommitObjects()) Expect(commits.ForEach(func(commit *object.Commit) error { Expect(commit.Author.Name).To(Equal(opts.Author.Name)) @@ -156,16 +132,10 @@ var _ = Describe("access method", func() { if commit.Message == componentCommitExpectation { componentUpdate++ - } else if descriptorCommitExpectation.MatchString(commit.Message) || descriptorConfigCommitExpectation.MatchString(commit.Message) { - descriptorCommits++ - } else if manifestUpdateCommitExpectation.MatchString(commit.Message) || manifestAddCommitExpectation.MatchString(commit.Message) { - manifestUpdateCommits++ } return nil })).To(Succeed()) Expect(componentUpdate).To(Equal(1)) - Expect(descriptorCommits).To(Equal(2)) - Expect(manifestUpdateCommits).To(Equal(2)) refmgmt.AllocLog.Trace("opening ctf") a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) @@ -188,7 +158,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) + a := Must(git.Create(ctx, repoURL, opts)) final.Close(a, "repository") c := Must(a.LookupComponent(COMPONENT)) final.Close(c, "component") @@ -215,7 +185,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) + a := Must(git.Create(ctx, repoURL, opts)) final.Close(a) c := Must(a.LookupComponent(COMPONENT)) final.Close(c) @@ -252,7 +222,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) + a := Must(git.Create(ctx, repoURL, opts)) final.Close(a) c := Must(a.LookupComponent(COMPONENT)) final.Close(c) @@ -274,7 +244,7 @@ var _ = Describe("access method", func() { final := Finalizer{} defer Defer(final.Finalize) - a := Must(git.Create(ctx, git.ACC_WRITABLE|git.ACC_CREATE, repoURL, opts)) + a := Must(git.Create(ctx, repoURL, opts)) final.Close(a) c := Must(a.LookupComponent(COMPONENT)) final.Close(c) diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index 66d1bc9cd6..8e563bd1c9 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -16,8 +16,6 @@ import ( "github.com/go-git/go-git/v5/storage/filesystem" "github.com/mandelsoft/vfs/pkg/memoryfs" "github.com/mandelsoft/vfs/pkg/vfs" - - "ocm.software/ocm/api/utils/accessobj" ) var DefaultWorktreeBranch = plumbing.NewBranchReferenceName("ocm") @@ -33,12 +31,30 @@ type client struct { repoMu sync.Mutex } +// Client is a heavy abstraction over the go git Client that opinionates the remote as git.DefaultRemoteName +// as well as access to it via high level functions that are usually required for operation within OCM CTFs that are stored +// within Git. It is not general-purpose. type Client interface { + // Repository returns the git repository for the client initialized in the Filesystem given to Setup. + // If Setup is not called before Repository, it will an in-memory filesystem. + // Repository will attempt to initially clone the repository if it does not exist. + // If the repository is already open or cloned in the filesystem, it will attempt to open & return the existing repository. + // If the remote repository does not exist, a new repository will be created with a dummy commit and the remote + // configured to the given URL. At that point it is up to the remote to accept an initial push to the repository or not with the + // given AuthMethod. Repository(ctx context.Context) (*git.Repository, error) + + // Refresh will attempt to fetch & pull the latest changes from the remote repository. + // In case there are no changes, it will do a no-op after having realized that no changes are in the remote. Refresh(ctx context.Context) error + + // Update will stage all changes in the repository, commit them with the given message and push them to the remote repository. Update(ctx context.Context, msg string, push bool) error - accessobj.Setup - accessobj.Closer + + // Setup will override the current filesystem with the given filesystem. This will be the filesystem where the repository will be stored. + // There can be only one filesystem per client. + // If the filesystem contains a repository already, it can be consumed by a subsequent call to Repository. + Setup(vfs.FileSystem) error } type ClientOptions struct { @@ -252,13 +268,6 @@ func (c *client) Setup(system vfs.FileSystem) error { return nil } -func (c *client) Close(_ *accessobj.AccessObject) error { - if err := c.Update(context.Background(), "OCM Repository Update", true); err != nil { - return fmt.Errorf("failed to close repository %q: %w", c.opts.URL, err) - } - return nil -} - func (o ClientOptions) applyToRepo(repo *git.Repository) error { cfg, err := repo.Config() if err != nil { From 9e017dee14c887b62639cf6445f30db63f052e58 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Mon, 7 Oct 2024 16:21:57 +0200 Subject: [PATCH 14/22] chore: allow no author --- api/oci/extensions/repositories/git/repository.go | 6 ++---- api/tech/git/resolver.go | 8 -------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go index e9fbd27707..3c51db31de 100644 --- a/api/oci/extensions/repositories/git/repository.go +++ b/api/oci/extensions/repositories/git/repository.go @@ -29,9 +29,7 @@ type RepositoryImpl struct { client git.Client } -var ( - _ cpi.Repository = (*RepositoryImpl)(nil) -) +var _ cpi.Repository = (*RepositoryImpl)(nil) func New(ctx cpi.Context, spec *RepositorySpec, creds credentials.Credentials) (Repository, error) { urs := spec.UniformRepositorySpec() @@ -116,7 +114,7 @@ func (r *RepositoryImpl) LookupNamespace(name string) (cpi.NamespaceAccess, erro return NewNamespace(r, name) } -// small helper to wrap accessio.Closer to allow calling an arbitrary closing logic +// small helper to wrap accessio.Closer to allow calling an arbitrary closing logic. type repoCloseUpdater struct { close func() error } diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index 8e563bd1c9..9488f12eac 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -164,14 +164,6 @@ func (c *client) newRepository(ctx context.Context, repo *git.Repository) error return err } - if err := worktree.AddGlob("*"); err != nil { - return err - } - - if _, err := worktree.Commit("OCM Repository Setup", &git.CommitOptions{}); err != nil && !errors.Is(err, git.ErrEmptyCommit) { - return err - } - return nil } From 648e800b78e8c191186867839dc096ddc437624d Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Mon, 7 Oct 2024 19:59:23 +0200 Subject: [PATCH 15/22] chore: implement cli input type --- .../artifactaccess/gitaccess/resource.go | 2 +- api/utils/blobaccess/git/access.go | 34 +++++---- api/utils/blobaccess/git/access_test.go | 29 +++++--- api/utils/blobaccess/git/options.go | 60 ++++++++++++++- .../ocmcmds/common/inputs/types/git/cli.go | 20 +++++ .../ocmcmds/common/inputs/types/git/spec.go | 74 +++++++++++++++++++ .../ocmcmds/common/inputs/types/git/type.go | 28 +++++++ 7 files changed, 220 insertions(+), 27 deletions(-) create mode 100644 cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go create mode 100644 cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go create mode 100644 cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go diff --git a/api/ocm/elements/artifactaccess/gitaccess/resource.go b/api/ocm/elements/artifactaccess/gitaccess/resource.go index 46e5748046..4d5076f01b 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/resource.go +++ b/api/ocm/elements/artifactaccess/gitaccess/resource.go @@ -11,7 +11,7 @@ import ( resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" ) -const TYPE = resourcetypes.BLOB +const TYPE = resourcetypes.DIRECTORY_TREE func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, opts ...Option) cpi.ArtifactAccess[M] { eff := optionutils.EvalOptions(opts...) diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go index 27ec35806d..25e2c13524 100644 --- a/api/utils/blobaccess/git/access.go +++ b/api/utils/blobaccess/git/access.go @@ -7,7 +7,7 @@ import ( "github.com/mandelsoft/goutils/errors" "github.com/mandelsoft/goutils/finalizer" "github.com/mandelsoft/goutils/optionutils" - "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/opencontainers/go-digest" @@ -29,13 +29,8 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { } log := options.Logger("RepoUrl", options.URL) - if options.AuthMethod == nil && options.Credentials.Value != nil { - authMethod, err := git.AuthFromCredentials(options.Credentials.Value) - if err != nil && !errors.Is(err, git.ErrNoValidGitCredentials) { - return nil, err - } else { - options.AuthMethod = authMethod - } + if err := options.ConfigureAuthMethod(); err != nil { + return nil, err } c, err := git.NewClient(options.ClientOptions) @@ -43,16 +38,27 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { return nil, err } - tmpfs, err := osfs.NewTempFileSystem() + tmpFS, cleanup, err := options.CachingFilesystem() + if err != nil { + return nil, err + } else if cleanup != nil { + finalize.With(cleanup) + } + + // store the repo in a temporary filesystem subfolder, so the tgz can go in the root without issues. + if err := tmpFS.MkdirAll("repository", 0700); err != nil { + return nil, err + } + repositoryFS, err := projectionfs.New(tmpFS, "repository") if err != nil { return nil, err } finalize.With(func() error { - return vfs.Cleanup(tmpfs) + return tmpFS.RemoveAll("repository") }) // redirect the client to the temporary filesystem for storage of the repo, otherwise it would use memory - if err := c.Setup(tmpfs); err != nil { + if err := c.Setup(repositoryFS); err != nil { return nil, err } @@ -62,12 +68,12 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { } // remove the .git directory as it shouldn't be part of the tarball - if err := tmpfs.RemoveAll(gogit.GitDirName); err != nil { + if err := repositoryFS.RemoveAll(gogit.GitDirName); err != nil { return nil, err } // pack all downloaded files into a tar.gz file - fs := options.GetCachingFileSystem() + fs := tmpFS tgz, err := vfs.TempFile(fs, "", "git-*.tar.gz") if err != nil { return nil, err @@ -76,7 +82,7 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { dw := iotools.NewDigestWriterWith(digest.SHA256, tgz) finalize.Close(dw) - if err := tarutils.TgzFs(tmpfs, dw); err != nil { + if err := tarutils.TgzFs(repositoryFS, dw); err != nil { return nil, err } diff --git a/api/utils/blobaccess/git/access_test.go b/api/utils/blobaccess/git/access_test.go index da1b7c2717..a35495a7aa 100644 --- a/api/utils/blobaccess/git/access_test.go +++ b/api/utils/blobaccess/git/access_test.go @@ -13,8 +13,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/mandelsoft/filepath/pkg/filepath" . "github.com/mandelsoft/goutils/testutils" - "github.com/mandelsoft/vfs/pkg/cwdfs" "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,7 +37,7 @@ var _ = Describe("git Blob Access", func() { ctx = ocm.New() BeforeEach(func() { - tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + tempVFS, err := projectionfs.New(osfs.New(), GinkgoT().TempDir()) Expect(err).ToNot(HaveOccurred()) tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) vfsattr.Set(ctx, tempVFS) @@ -93,9 +93,8 @@ var _ = Describe("git Blob Access", func() { It("blobaccess for simple repository", func() { b := Must(gitblob.BlobAccess( gitblob.WithURL(url), - gitblob.WithCachingContext(ctx), gitblob.WithLoggingContext(ctx), - gitblob.WithCachingPath(GinkgoT().TempDir()), + gitblob.WithCachingContext(ctx), )) defer Close(b) files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) @@ -105,9 +104,10 @@ var _ = Describe("git Blob Access", func() { }) Context("git http repository", func() { + host := "github.com:443" + reachable := PingTCPServer(host, time.Second) == nil BeforeEach(func() { - host := "github.com:443" - if PingTCPServer(host, time.Second) != nil { + if !reachable { Skip(fmt.Sprintf("no connection to %s, skipping test connection to remote", url)) } // This repo is a public repo owned by the Github Kraken Bot, so its as good of a public available @@ -115,12 +115,22 @@ var _ = Describe("git Blob Access", func() { url = fmt.Sprintf("https://%s/octocat/Hello-World.git", host) }) - It("hello world from github with master branch", func() { + It("hello world from github without explicit branch", func() { b := Must(gitblob.BlobAccess( gitblob.WithURL(url), + gitblob.WithLoggingContext(ctx), gitblob.WithCachingContext(ctx), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("README")) + }) + + It("hello world from github with master branch", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), gitblob.WithLoggingContext(ctx), - gitblob.WithCachingPath(GinkgoT().TempDir()), + gitblob.WithCachingContext(ctx), gitblob.WithRef(plumbing.Master.String()), )) defer Close(b) @@ -131,9 +141,8 @@ var _ = Describe("git Blob Access", func() { It("hello world from github with custom ref", func() { b := Must(gitblob.BlobAccess( gitblob.WithURL(url), - gitblob.WithCachingContext(ctx), gitblob.WithLoggingContext(ctx), - gitblob.WithCachingPath(GinkgoT().TempDir()), + gitblob.WithCachingContext(ctx), gitblob.WithRef("refs/heads/test"), )) defer Close(b) diff --git a/api/utils/blobaccess/git/options.go b/api/utils/blobaccess/git/options.go index d0450ed791..b654114934 100644 --- a/api/utils/blobaccess/git/options.go +++ b/api/utils/blobaccess/git/options.go @@ -3,12 +3,16 @@ package git import ( "github.com/mandelsoft/goutils/optionutils" "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/datacontext" "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/tech/git/identity" ocmlog "ocm.software/ocm/api/utils/logging" "ocm.software/ocm/api/utils/stdopts" ) @@ -19,6 +23,7 @@ type Options struct { git.ClientOptions stdopts.StandardContexts + stdopts.PathFileSystem } func (o *Options) Logger(keyValuePairs ...interface{}) logging.Logger { @@ -62,6 +67,57 @@ func (o *Options) ApplyTo(opts *Options) { } } +func (o *Options) ConfigureAuthMethod() error { + if o.ClientOptions.AuthMethod != nil { + return nil + } + + var err error + + if o.Credentials.Value != nil { + if o.ClientOptions.AuthMethod, err = git.AuthFromCredentials(o.Credentials.Value); err != nil { + return err + } + } + + if o.CredentialContext.Value != nil { + creds, err := identity.GetCredentials(o.CredentialContext.Value, o.URL) + if err != nil { + return err + } + if o.ClientOptions.AuthMethod, err = git.AuthFromCredentials(creds); err != nil { + return err + } + } + + return nil +} + +func (o *Options) CachingFilesystem() (vfs.FileSystem, func() error, error) { + if o.PathFileSystem.Value != nil { + return o.PathFileSystem.Value, nil, nil + } + if o.CachingFileSystem.Value != nil { + return o.CachingFileSystem.Value, nil, nil + } + + if o.CachingContext.Value != nil { + if fs := vfsattr.Get(o.CachingContext.Value); fs != nil { + return fs, nil, nil + } + + if fromtmp := tmpcache.Get(o.CachingContext.Value); fromtmp != nil { + fs, err := projectionfs.New(fromtmp.Filesystem, fromtmp.Path) + if err != nil { + return nil, nil, err + } + return fs, nil, nil + } + } + tmpfs, err := osfs.NewTempFileSystem() + return tmpfs, func() error { return vfs.Cleanup(tmpfs) }, err +} + func option[S any, T any](v T) optionutils.Option[*Options] { return optionutils.WithGenericOption[S, *Options](v) } @@ -82,8 +138,8 @@ func WithCachingFileSystem(fs vfs.FileSystem) Option { return option[stdopts.CachingFileSystemOptionBag](fs) } -func WithCachingPath(p string) Option { - return option[stdopts.CachingPathOptionBag](p) +func WithPathFileSystem(fs vfs.FileSystem) Option { + return option[stdopts.PathFileSystemOptionBag](fs) } func WithCredentials(c credentials.Credentials) Option { diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go new file mode 100644 index 0000000000..a5699723ee --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go @@ -0,0 +1,20 @@ +package docker + +import ( + "ocm.software/ocm/api/utils/cobrautils/flagsets" + "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/options" +) + +func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { + return flagsets.NewConfigOptionTypeSetHandler( + TYPE, AddConfig, + options.RepositoryOption, + options.VersionOption, + ) +} + +func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository") + flagsets.AddFieldByOptionP(opts, options.VersionOption, config, "ref") + return nil +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go new file mode 100644 index 0000000000..5825e807e6 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go @@ -0,0 +1,74 @@ +package docker + +import ( + "github.com/go-git/go-git/v5/plumbing" + giturls "github.com/whilp/git-urls" + "k8s.io/apimachinery/pkg/util/validation/field" + + "ocm.software/ocm/api/utils/blobaccess" + "ocm.software/ocm/api/utils/blobaccess/git" + "ocm.software/ocm/api/utils/runtime" + "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs" +) + +type Spec struct { + inputs.InputSpecBase `json:",inline"` + + // Repository is the Git Repository URL + Repository string `json:"repository"` + + // Ref is the Git Ref to check out. + // If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + Ref string `json:"ref,omitempty"` +} + +var _ inputs.InputSpec = (*Spec)(nil) + +func New(repository string) *Spec { + return &Spec{ + InputSpecBase: inputs.InputSpecBase{ + ObjectVersionedType: runtime.ObjectVersionedType{ + Type: TYPE, + }, + }, + Repository: repository, + } +} + +func (s *Spec) Validate(fldPath *field.Path, _ inputs.Context, _ string) field.ErrorList { + var allErrs field.ErrorList + + if path := fldPath.Child("repository"); s.Repository == "" { + allErrs = append(allErrs, field.Invalid(path, s.Repository, "no repository")) + } else { + if _, err := giturls.Parse(s.Repository); err != nil { + allErrs = append(allErrs, field.Invalid(path, s.Repository, err.Error())) + } + } + + if ref := fldPath.Child("ref"); s.Ref != "" { + if plumbing.ReferenceName(s.Ref).Validate() != nil { + allErrs = append(allErrs, field.Invalid(ref, s.Ref, "invalid ref")) + } + } + + return allErrs +} + +func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blobaccess.BlobAccess, string, error) { + if _, err := giturls.Parse(s.Repository); err != nil { + return nil, "", err + } + + blob, err := git.BlobAccess( + git.WithURL(s.Repository), + git.WithRef(s.Ref), + git.WithCredentialContext(ctx), + git.WithLoggingContext(ctx), + git.WithCachingContext(ctx), + ) + if err != nil { + return nil, "", err + } + return blob, "", nil +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go new file mode 100644 index 0000000000..b62aba2abb --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go @@ -0,0 +1,28 @@ +package docker + +import ( + "ocm.software/ocm/api/oci/annotations" + "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs" +) + +const TYPE = "git" + +func init() { + inputs.DefaultInputTypeScheme.Register(inputs.NewInputType(TYPE, &Spec{}, usage, ConfigHandler())) +} + +const usage = ` +The repository type allows accessing an arbitrary git repository +using the manifest annotation ` + annotations.COMPVERS_ANNOTATION + `. +The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + +This blob type specification supports the following fields: +- **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + +- **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. +` From 90f82f517cbf0d27dd50b21a3b3d103390d58f07 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Tue, 8 Oct 2024 10:39:10 +0200 Subject: [PATCH 16/22] chore: implement cli input type --- api/utils/blobaccess/git/access.go | 3 ++- api/utils/blobaccess/git/options.go | 15 ++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go index 25e2c13524..904032b04f 100644 --- a/api/utils/blobaccess/git/access.go +++ b/api/utils/blobaccess/git/access.go @@ -46,9 +46,10 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { } // store the repo in a temporary filesystem subfolder, so the tgz can go in the root without issues. - if err := tmpFS.MkdirAll("repository", 0700); err != nil { + if err := tmpFS.MkdirAll("repository", 0o700); err != nil { return nil, err } + repositoryFS, err := projectionfs.New(tmpFS, "repository") if err != nil { return nil, err diff --git a/api/utils/blobaccess/git/options.go b/api/utils/blobaccess/git/options.go index b654114934..fddc4ca97c 100644 --- a/api/utils/blobaccess/git/options.go +++ b/api/utils/blobaccess/git/options.go @@ -80,11 +80,16 @@ func (o *Options) ConfigureAuthMethod() error { } } - if o.CredentialContext.Value != nil { - creds, err := identity.GetCredentials(o.CredentialContext.Value, o.URL) - if err != nil { - return err - } + if o.CredentialContext.Value == nil { + return nil + } + + creds, err := identity.GetCredentials(o.CredentialContext.Value, o.URL) + if err != nil { + return err + } + + if creds != nil { if o.ClientOptions.AuthMethod, err = git.AuthFromCredentials(creds); err != nil { return err } From 38b3e41acceeb0da6d2a35911069ce658b65b407 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Wed, 9 Oct 2024 16:40:56 +0200 Subject: [PATCH 17/22] feat: allow access by commit --- api/oci/extensions/repositories/git/type.go | 12 +- .../artifactaccess/gitaccess/options.go | 11 ++ .../artifactaccess/gitaccess/resource.go | 2 +- .../extensions/accessmethods/git/method.go | 7 +- .../accessmethods/git/method_test.go | 1 + api/tech/git/resolver.go | 17 +++ api/utils/blobaccess/git/access.go | 46 +++++- api/utils/blobaccess/git/digest.go | 7 + api/utils/blobaccess/git/options.go | 17 +++ .../ocmcmds/common/inputs/types/git/cli.go | 2 +- .../common/inputs/types/git/input_test.go | 139 ++++++++++++++++++ .../ocmcmds/common/inputs/types/git/spec.go | 19 ++- .../common/inputs/types/git/suite_test.go | 13 ++ .../inputs/types/git/testdata/resources1.yaml | 6 + .../ocmcmds/common/inputs/types/git/type.go | 7 +- .../ocmcmds/common/inputs/types/init.go | 1 + 16 files changed, 289 insertions(+), 18 deletions(-) create mode 100644 api/utils/blobaccess/git/digest.go create mode 100644 cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go create mode 100644 cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go create mode 100644 cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go index 1b59ded114..7493da5a2b 100644 --- a/api/oci/extensions/repositories/git/type.go +++ b/api/oci/extensions/repositories/git/type.go @@ -48,13 +48,17 @@ type RepositorySpec struct { // Ref is the git ref of the RepositoryImpl to resolve artifacts. // Examples include - // - heads/master - // - tags/v1.0.0 + // - refs/heads/master + // - refs/tags/v1.0.0 // - pull/123/head // - remotes/origin/feature // If empty, the default is set to HEAD. Ref string `json:"ref,omitempty"` + // Commit is the commit hash of the RepositoryImpl inside the Ref to resolve artifacts. + // If empty, the default is set to the latest commit (the HEAD) of the Ref. + Commit string `json:"commit,omitempty"` + // Author is the author of commits generated by the repository. If not set, it is defaulted from environment and git // configuration of the host system. Author *Author `json:"author,omitempty"` @@ -75,6 +79,8 @@ var _ cpi.IntermediateRepositorySpecAspect = (*RepositorySpec)(nil) type Options struct { *Author + Ref string + Commit string accessio.Options } @@ -85,6 +91,8 @@ func NewRepositorySpecFromOptions(mode accessobj.AccessMode, url string, opts Op return nil, err } spec.Author = opts.Author + spec.Ref = opts.Ref + spec.Commit = opts.Commit return spec, nil } diff --git a/api/ocm/elements/artifactaccess/gitaccess/options.go b/api/ocm/elements/artifactaccess/gitaccess/options.go index aad1e1911b..9d89df6e72 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/options.go +++ b/api/ocm/elements/artifactaccess/gitaccess/options.go @@ -10,6 +10,7 @@ type Options struct { URL string Ref string PathSpec string + Commit string } var _ Option = (*Options)(nil) @@ -56,3 +57,13 @@ func (h pathSpec) ApplyTo(opts *Options) { func WithPathSpec(h string) Option { return pathSpec(h) } + +type commitSpec string + +func (h commitSpec) ApplyTo(opts *Options) { + opts.Commit = string(h) +} + +func WithCommit(c string) Option { + return commitSpec(c) +} diff --git a/api/ocm/elements/artifactaccess/gitaccess/resource.go b/api/ocm/elements/artifactaccess/gitaccess/resource.go index 4d5076f01b..b223198d0a 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/resource.go +++ b/api/ocm/elements/artifactaccess/gitaccess/resource.go @@ -19,7 +19,7 @@ func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, o meta.SetType(TYPE) } - spec := access.New(eff.URL, eff.Ref, eff.PathSpec) + spec := access.New(eff.URL, eff.Ref, eff.Commit, eff.PathSpec) // is global access, must work, otherwise there is an error in the lib. return genericaccess.MustAccess(ctx, meta, spec) } diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index 060ed139e2..601aae0ee1 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -38,6 +38,9 @@ type AccessSpec struct { // Ref defines the hash of the commit Ref string `json:"ref"` + // Commit defines the hash of the commit in string format to checkout from the Ref + Commit string `json:"commit"` + // PathSpec is a path in the repository to download, can be a file or a regex matching multiple files PathSpec string `json:"pathSpec"` } @@ -46,11 +49,12 @@ type AccessSpec struct { type AccessSpecOptions func(s *AccessSpec) // New creates a new git registry access spec version v1. -func New(url, ref string, pathSpec string, opts ...AccessSpecOptions) *AccessSpec { +func New(url, ref, commit, pathSpec string, opts ...AccessSpecOptions) *AccessSpec { s := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), RepoURL: url, Ref: ref, + Commit: commit, PathSpec: pathSpec, } for _, o := range opts { @@ -95,6 +99,7 @@ func (a *AccessSpec) AccessMethod(cva internal.ComponentVersionAccess) (internal gitblob.WithCredentialContext(octx), gitblob.WithURL(a.RepoURL), gitblob.WithRef(a.Ref), + gitblob.WithCommit(a.Commit), gitblob.WithCachingFileSystem(vfsattr.Get(octx)), } if creds != nil { diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go index 1f8697f2ee..cc37cb3a30 100644 --- a/api/ocm/extensions/accessmethods/git/method_test.go +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -88,6 +88,7 @@ var _ = Describe("Method", func() { accessSpec = me.New( fmt.Sprintf("file://%s", repoDir), string(plumbing.Master), + "", ".", ) diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index 9488f12eac..f952f479f9 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -58,9 +58,20 @@ type Client interface { } type ClientOptions struct { + // URL is the URL of the git repository to clone or open. URL string + // Ref is the reference to the repository to clone or open. + // If empty, it will default to plumbing.HEAD of the remote repository. + // If the remote does not exist, it will attempt to push to the remote with DefaultWorktreeBranch on Client.Update. + // To point to a remote branch, use refs/heads/. + // To point to a tag, use refs/tags/. Ref string + // Commit is the commit hash to checkout after cloning the repository. + // If empty, it will default to the plumbing.HEAD of the Ref. + Commit string + // Author is the author to use for commits. If empty, it will default to the git config of the user running the process. Author + // AuthMethod is the authentication method to use for the repository. AuthMethod AuthMethod } @@ -156,7 +167,13 @@ func (c *client) newRepository(ctx context.Context, repo *git.Repository) error return err } + var hash plumbing.Hash + if c.opts.Commit != "" { + hash = plumbing.NewHash(c.opts.Commit) + } + if err := worktree.Checkout(&git.CheckoutOptions{ + Hash: hash, Branch: DefaultWorktreeBranch, Create: true, Keep: true, diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go index 904032b04f..6c8f3ac524 100644 --- a/api/utils/blobaccess/git/access.go +++ b/api/utils/blobaccess/git/access.go @@ -68,9 +68,14 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { return nil, err } - // remove the .git directory as it shouldn't be part of the tarball - if err := repositoryFS.RemoveAll(gogit.GitDirName); err != nil { - return nil, err + filteredRepositoryFS := &filteredVFS{ + FileSystem: repositoryFS, + filter: func(s string) bool { + if s == gogit.GitDirName { + return false + } + return true + }, } // pack all downloaded files into a tar.gz file @@ -83,7 +88,7 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { dw := iotools.NewDigestWriterWith(digest.SHA256, tgz) finalize.Close(dw) - if err := tarutils.TgzFs(repositoryFS, dw); err != nil { + if err := tarutils.TgzFs(filteredRepositoryFS, dw); err != nil { return nil, err } @@ -97,3 +102,36 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { file.WithSize(dw.Size()), ), nil } + +type filteredVFS struct { + vfs.FileSystem + filter func(string) bool +} + +func (f *filteredVFS) Open(name string) (vfs.File, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.Open(name) +} + +func (f *filteredVFS) OpenFile(name string, flags int, perm vfs.FileMode) (vfs.File, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.OpenFile(name, flags, perm) +} + +func (f *filteredVFS) Stat(name string) (vfs.FileInfo, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.Stat(name) +} + +func (f *filteredVFS) Lstat(name string) (vfs.FileInfo, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.Lstat(name) +} diff --git a/api/utils/blobaccess/git/digest.go b/api/utils/blobaccess/git/digest.go new file mode 100644 index 0000000000..4d61d15310 --- /dev/null +++ b/api/utils/blobaccess/git/digest.go @@ -0,0 +1,7 @@ +package git + +import "github.com/opencontainers/go-digest" + +type Digest interface { + digest.Digest +} diff --git a/api/utils/blobaccess/git/options.go b/api/utils/blobaccess/git/options.go index fddc4ca97c..fa43a1ed60 100644 --- a/api/utils/blobaccess/git/options.go +++ b/api/utils/blobaccess/git/options.go @@ -65,6 +65,9 @@ func (o *Options) ApplyTo(opts *Options) { if o.Ref != "" { opts.Ref = o.Ref } + if o.Commit != "" { + opts.Commit = o.Commit + } } func (o *Options) ConfigureAuthMethod() error { @@ -178,3 +181,17 @@ func (o *Options) SetRef(v string) { func WithRef(ref string) Option { return option[RefOptionBag](ref) } + +//////////////////////////////////////////////////////////////////////////////// + +type CommitOptionBag interface { + SetCommit(v string) +} + +func (o *Options) SetCommit(v string) { + o.Commit = v +} + +func WithCommit(ref string) Option { + return option[CommitOptionBag](ref) +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go index a5699723ee..4d812d5c8f 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go @@ -1,4 +1,4 @@ -package docker +package git import ( "ocm.software/ocm/api/utils/cobrautils/flagsets" diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go new file mode 100644 index 0000000000..1e12ae5da2 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go @@ -0,0 +1,139 @@ +package git_test + +import ( + "io" + "os" + + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/utils/tarutils" + . "ocm.software/ocm/cmds/ocm/testhelper" + + "ocm.software/ocm/api/ocm/compdesc" + "ocm.software/ocm/api/ocm/extensions/accessmethods/localblob" + "ocm.software/ocm/api/ocm/extensions/repositories/comparch" + "ocm.software/ocm/api/utils/mime" +) + +const ( + ARCH = "test.ca" + VERSION = "v1" +) + +var _ = Describe("Test Environment", func() { + var env *TestEnv + + BeforeEach(func() { + env = NewTestEnv(TestData()) + + Expect(env.Execute( + "create", + "ca", + "-ft", + "directory", + "test.de/x", + VERSION, + "--provider", + "ocm", + "--file", + ARCH, + "--scheme", + "ocm.software/v3alpha1", + )).To(Succeed()) + }) + + AfterEach(func() { + Expect(env.Cleanup()).To(Succeed()) + }) + + It("add git repo described by access type specification", func() { + meta := ` +name: hello-world +type: git +` + Expect(env.Execute( + "add", "resources", + "--file", ARCH, + "--resource", meta, + "--accessType", "git", + "--accessRepository", "https://github.com/octocat/Hello-World.git", + "--reference", "refs/heads/master", + "--commit", "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + "--version", "0.0.1", + )).To(Succeed()) + + data, err := env.ReadFile(env.Join(ARCH, comparch.ComponentDescriptorFileName)) + Expect(err).To(Succeed()) + cd, err := compdesc.Decode(data) + Expect(err).To(Succeed()) + Expect(len(cd.Resources)).To(Equal(1)) + }) + + It("add git repo described by cli options through blob access via input described in file", func() { + meta := ` +name: hello-world +type: git +` + Expect(env.Execute( + "add", "resources", + "--file", ARCH, + "--resource", meta, + "--inputType", "git", + "--inputVersion", "refs/heads/master", + "--inputRepository", "https://github.com/octocat/Hello-World.git", + )).To(Succeed()) + data, err := env.ReadFile(env.Join(ARCH, comparch.ComponentDescriptorFileName)) + Expect(err).To(Succeed()) + cd, err := compdesc.Decode(data) + Expect(err).To(Succeed()) + Expect(len(cd.Resources)).To(Equal(1)) + + access := Must(env.Context.OCMContext().AccessSpecForSpec(cd.Resources[0].Access)).(*localblob.AccessSpec) + Expect(access.MediaType).To(Equal(mime.MIME_TGZ)) + fi := Must(env.FileSystem().Stat(env.Join(ARCH, "blobs", access.LocalReference))) + Expect(fi.Size()).To(Equal(int64(106))) + + Expect(tarutils.ExtractArchiveToFs(env.FileSystem(), env.Join(ARCH, "blobs", access.LocalReference), env.FileSystem())).To(Succeed()) + + readMeFi := Must(env.FileSystem().Stat("README")) + Expect(readMeFi.Size()).To(Equal(int64(13))) + readMe := Must(env.FileSystem().OpenFile("README", os.O_RDONLY, 0o400)) + defer readMe.Close() + Expect(string(Must(io.ReadAll(readMe)))).To(Equal("Hello World!\n")) + }) + + It("add git repo described by cli options through blob access via input", func() { + meta := ` +name: hello-world +type: git +` + Expect(env.Execute( + "add", "resources", + "--file", ARCH, + "--resource", meta, + "--inputType", "git", + "--inputVersion", "refs/heads/master", + "--inputRepository", "https://github.com/octocat/Hello-World.git", + )).To(Succeed()) + data, err := env.ReadFile(env.Join(ARCH, comparch.ComponentDescriptorFileName)) + Expect(err).To(Succeed()) + cd, err := compdesc.Decode(data) + Expect(err).To(Succeed()) + Expect(len(cd.Resources)).To(Equal(1)) + + access := Must(env.Context.OCMContext().AccessSpecForSpec(cd.Resources[0].Access)).(*localblob.AccessSpec) + Expect(access.MediaType).To(Equal(mime.MIME_TGZ)) + fi := Must(env.FileSystem().Stat(env.Join(ARCH, "blobs", access.LocalReference))) + Expect(fi.Size()).To(Equal(int64(106))) + + Expect(tarutils.ExtractArchiveToFs(env.FileSystem(), env.Join(ARCH, "blobs", access.LocalReference), env.FileSystem())).To(Succeed()) + + readMeFi := Must(env.FileSystem().Stat("README")) + Expect(readMeFi.Size()).To(Equal(int64(13))) + readMe := Must(env.FileSystem().OpenFile("README", os.O_RDONLY, 0o400)) + defer readMe.Close() + Expect(string(Must(io.ReadAll(readMe)))).To(Equal("Hello World!\n")) + }) +}) diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go index 5825e807e6..60b1178cc1 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go @@ -1,4 +1,4 @@ -package docker +package git import ( "github.com/go-git/go-git/v5/plumbing" @@ -18,13 +18,17 @@ type Spec struct { Repository string `json:"repository"` // Ref is the Git Ref to check out. - // If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + // If empty, the default HEAD (remotes/origin/HEAD) of the remote is used. Ref string `json:"ref,omitempty"` + + // Commit is the Git Commit to check out. + // If empty, the default HEAD of the Ref is used. + Commit string `json:"commit,omitempty"` } var _ inputs.InputSpec = (*Spec)(nil) -func New(repository string) *Spec { +func New(repository, ref, commit string) *Spec { return &Spec{ InputSpecBase: inputs.InputSpecBase{ ObjectVersionedType: runtime.ObjectVersionedType{ @@ -32,6 +36,8 @@ func New(repository string) *Spec { }, }, Repository: repository, + Ref: ref, + Commit: commit, } } @@ -47,7 +53,7 @@ func (s *Spec) Validate(fldPath *field.Path, _ inputs.Context, _ string) field.E } if ref := fldPath.Child("ref"); s.Ref != "" { - if plumbing.ReferenceName(s.Ref).Validate() != nil { + if err := plumbing.ReferenceName(s.Ref).Validate(); err != nil { allErrs = append(allErrs, field.Invalid(ref, s.Ref, "invalid ref")) } } @@ -56,13 +62,10 @@ func (s *Spec) Validate(fldPath *field.Path, _ inputs.Context, _ string) field.E } func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blobaccess.BlobAccess, string, error) { - if _, err := giturls.Parse(s.Repository); err != nil { - return nil, "", err - } - blob, err := git.BlobAccess( git.WithURL(s.Repository), git.WithRef(s.Ref), + git.WithCommit(s.Commit), git.WithCredentialContext(ctx), git.WithLoggingContext(ctx), git.WithCachingContext(ctx), diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go new file mode 100644 index 0000000000..b2afdea537 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Input Type git") +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml new file mode 100644 index 0000000000..1ebad10285 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml @@ -0,0 +1,6 @@ +name: hello-world +type: git +input: + type: git + repository: https://github.com/octocat/Hello-World.git + version: refs/heads/master \ No newline at end of file diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go index b62aba2abb..f4040197ae 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go @@ -1,4 +1,4 @@ -package docker +package git import ( "ocm.software/ocm/api/oci/annotations" @@ -25,4 +25,9 @@ This blob type specification supports the following fields: This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + +- **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. ` diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go index a03de2fceb..a73a166973 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go @@ -6,6 +6,7 @@ import ( _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/docker" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/file" + _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/git" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/helm" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/maven" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/npm" From 067d39751810839367f2641b0e5cbbb758b9723b Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Thu, 24 Oct 2024 10:19:10 +0200 Subject: [PATCH 18/22] chore: kill the git repository type because its unlikely to be used --- api/oci/extensions/repositories/git/README.md | 33 -- api/oci/extensions/repositories/git/format.go | 21 - .../extensions/repositories/git/git_test.go | 121 ------ .../extensions/repositories/git/namespace.go | 56 --- .../extensions/repositories/git/repository.go | 124 ------ .../extensions/repositories/git/suite_test.go | 13 - .../git/testdata/repo/file_in_repo | 1 - api/oci/extensions/repositories/git/type.go | 170 -------- api/oci/extensions/repositories/init.go | 1 - api/ocm/extensions/repositories/git/format.go | 33 -- .../extensions/repositories/git/repo_test.go | 365 ------------------ .../extensions/repositories/git/suite_test.go | 13 - api/ocm/extensions/repositories/git/type.go | 21 - api/ocm/extensions/repositories/init.go | 1 - api/utils/blobaccess/git/access.go | 5 +- docs/reference/ocm_add_routingslips.md | 2 - docs/reference/ocm_bootstrap_configuration.md | 2 - docs/reference/ocm_bootstrap_package.md | 2 - docs/reference/ocm_check_componentversions.md | 2 - docs/reference/ocm_describe_artifacts.md | 2 - docs/reference/ocm_describe_package.md | 2 - docs/reference/ocm_download_artifacts.md | 2 - docs/reference/ocm_download_cli.md | 2 - .../ocm_download_componentversions.md | 2 - docs/reference/ocm_download_resources.md | 2 - docs/reference/ocm_get_artifacts.md | 2 - docs/reference/ocm_get_componentversions.md | 2 - docs/reference/ocm_get_references.md | 2 - docs/reference/ocm_get_resources.md | 2 - docs/reference/ocm_get_routingslips.md | 2 - docs/reference/ocm_get_sources.md | 2 - docs/reference/ocm_hash_componentversions.md | 2 - docs/reference/ocm_install_plugins.md | 2 - docs/reference/ocm_list_componentversions.md | 2 - docs/reference/ocm_show_tags.md | 2 - docs/reference/ocm_show_versions.md | 2 - docs/reference/ocm_sign_componentversions.md | 2 - docs/reference/ocm_transfer_artifacts.md | 2 - .../ocm_transfer_componentversions.md | 2 - .../reference/ocm_verify_componentversions.md | 2 - 40 files changed, 1 insertion(+), 1027 deletions(-) delete mode 100644 api/oci/extensions/repositories/git/README.md delete mode 100644 api/oci/extensions/repositories/git/format.go delete mode 100644 api/oci/extensions/repositories/git/git_test.go delete mode 100644 api/oci/extensions/repositories/git/namespace.go delete mode 100644 api/oci/extensions/repositories/git/repository.go delete mode 100644 api/oci/extensions/repositories/git/suite_test.go delete mode 100644 api/oci/extensions/repositories/git/testdata/repo/file_in_repo delete mode 100644 api/oci/extensions/repositories/git/type.go delete mode 100644 api/ocm/extensions/repositories/git/format.go delete mode 100644 api/ocm/extensions/repositories/git/repo_test.go delete mode 100644 api/ocm/extensions/repositories/git/suite_test.go delete mode 100644 api/ocm/extensions/repositories/git/type.go diff --git a/api/oci/extensions/repositories/git/README.md b/api/oci/extensions/repositories/git/README.md deleted file mode 100644 index 35e1c46ccf..0000000000 --- a/api/oci/extensions/repositories/git/README.md +++ /dev/null @@ -1,33 +0,0 @@ - -# Repository `GitRepository` - git based repository - -## Synopsis - -```yaml -type: GitRepository/v1 -``` - -### Description - -Artifact namespaces/repositories of the API layer will be mapped to git repository paths. - -Supported specification version is `v1`. - -### Specification Versions - -#### Version `v1` - -The type specific specification fields are: - -- **`url`** *string* - - URL of the git repository in any standard git URL format. - The schemes `http`, `https`, `git`, `ssh` and `file` are supported. - -- **`ref`** *string* - - The git reference to use. This can be a branch, tag, or commit hash. The default is `HEAD`, pointing to the default branch of a repository in most implementations. - -### Go Bindings - -The Go binding can be found [here](type.go) diff --git a/api/oci/extensions/repositories/git/format.go b/api/oci/extensions/repositories/git/format.go deleted file mode 100644 index 6b5b84d30f..0000000000 --- a/api/oci/extensions/repositories/git/format.go +++ /dev/null @@ -1,21 +0,0 @@ -package git - -import ( - "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/utils/accessobj" -) - -// ////////////////////////////////////////////////////////////////////////////// - -func Open(ctxp cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (Repository, error) { - ctx := cpi.FromProvider(ctxp) - spec, err := NewRepositorySpecFromOptions(acc, url, opts) - if err != nil { - return nil, err - } - return New(ctx, spec, nil) -} - -func Create(ctx cpi.ContextProvider, url string, opts Options) (Repository, error) { - return Open(ctx, accessobj.ACC_CREATE|accessobj.ACC_WRITABLE, url, opts) -} diff --git a/api/oci/extensions/repositories/git/git_test.go b/api/oci/extensions/repositories/git/git_test.go deleted file mode 100644 index a27620b973..0000000000 --- a/api/oci/extensions/repositories/git/git_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package git_test - -import ( - "fmt" - - . "github.com/mandelsoft/goutils/testutils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/mandelsoft/filepath/pkg/filepath" - "github.com/mandelsoft/logging" - "github.com/mandelsoft/vfs/pkg/osfs" - "github.com/mandelsoft/vfs/pkg/projectionfs" - "github.com/mandelsoft/vfs/pkg/vfs" - "github.com/opencontainers/go-digest" - - "ocm.software/ocm/api/datacontext/attrs/tmpcache" - "ocm.software/ocm/api/datacontext/attrs/vfsattr" - "ocm.software/ocm/api/oci" - "ocm.software/ocm/api/oci/artdesc" - "ocm.software/ocm/api/oci/cpi" - rgit "ocm.software/ocm/api/oci/extensions/repositories/git" - "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/blobaccess" - ocmlog "ocm.software/ocm/api/utils/logging" - "ocm.software/ocm/api/utils/mime" - "ocm.software/ocm/api/utils/refmgmt" -) - -var _ = Describe("ctf management", func() { - var remoteRepo *git.Repository - - var tmp vfs.FileSystem - var workspace vfs.FileSystem - - var repoDir string - var repoURL string - - ocmlog.Context().AddRule(logging.NewConditionRule(logging.TraceLevel, refmgmt.ALLOC_REALM)) - - ctx := oci.New() - - BeforeEach(func() { - path := GinkgoT().TempDir() - tmp = Must(projectionfs.New(osfs.New(), path)) - tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tmp}) - vfsattr.Set(ctx, tmp) - - Expect(tmp.Mkdir("repo", 0o700)).To(Succeed()) - repoDir = path + filepath.PathSeparatorString + "repo" - repoURL = "file://" + repoDir - - Expect(tmp.Mkdir("workspace", 0o700)).To(Succeed()) - workspace = Must(projectionfs.New(tmp, "workspace")) - }) - - BeforeEach(func() { - remoteRepo = Must(git.PlainInit(repoDir, true)) - }) - - AfterEach(func() { - Expect(vfs.Cleanup(tmp)).To(Succeed()) - }) - - It("instantiate git based ctf", func() { - repo := Must(rgit.Create(ctx, repoURL, rgit.Options{ - Author: &rgit.Author{ - Name: fmt.Sprintf("OCM Test Case: %s", GinkgoT().Name()), - Email: "dummy@ocm.software", - }, - Options: Must(accessio.AccessOptions(nil, accessio.RepresentationFileSystem(workspace))), - })) - ns := Must(repo.LookupNamespace("test")) - - testData := []byte("testdata") - - aa := NewArtifact(ns, testData) - - Expect(aa.Close()).To(Succeed()) - Expect(ns.Close()).To(Succeed()) - Expect(repo.Close()).To(Succeed()) - - commits := Must(remoteRepo.CommitObjects()) - validAdd := 0 - var messages []string - Expect(commits.ForEach(func(commit *object.Commit) error { - if expected := rgit.GenerateCommitMessage(); commit.Message == expected { - validAdd++ - } - messages = append(messages, commit.Message) - return nil - })).To(Succeed()) - - Expect(validAdd).To(Equal(1), - fmt.Sprintf( - "expected exactly one commit with message %q, got %d commits with messages:\n%v", - rgit.GenerateCommitMessage(), - validAdd, - messages, - )) - }) -}) - -func NewArtifact(n cpi.NamespaceAccess, data []byte) cpi.ArtifactAccess { - art := Must(n.NewArtifact()) - Expect(art.AddLayer(blobaccess.ForData(mime.MIME_OCTET, data), nil)).To(Equal(0)) - desc := Must(art.Manifest()) - Expect(desc).NotTo(BeNil()) - - Expect(desc.Layers[0].Digest).To(Equal(digest.FromBytes(data))) - Expect(desc.Layers[0].MediaType).To(Equal(mime.MIME_OCTET)) - Expect(desc.Layers[0].Size).To(Equal(int64(8))) - - config := blobaccess.ForData(mime.MIME_OCTET, []byte("{}")) - desc.Config = *artdesc.DefaultBlobDescriptor(config) - MustBeSuccessful(n.AddBlob(config)) - MustBeSuccessful(n.AddArtifact(desc)) - return art -} diff --git a/api/oci/extensions/repositories/git/namespace.go b/api/oci/extensions/repositories/git/namespace.go deleted file mode 100644 index 87546925b7..0000000000 --- a/api/oci/extensions/repositories/git/namespace.go +++ /dev/null @@ -1,56 +0,0 @@ -package git - -import ( - "context" - - "github.com/opencontainers/go-digest" - - "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/tech/git" -) - -func NewNamespace(repo *RepositoryImpl, name string) (cpi.NamespaceAccess, error) { - ctfNamespace, err := repo.Repository.LookupNamespace(name) - if err != nil { - return nil, err - } - return &namespace{ - client: repo.client, - NamespaceAccess: ctfNamespace, - }, nil -} - -type namespace struct { - client git.Client - cpi.NamespaceAccess -} - -var _ cpi.NamespaceAccess = (*namespace)(nil) - -func (n *namespace) ListTags() ([]string, error) { - if err := n.client.Refresh(context.Background()); err != nil { - return nil, err - } - return n.NamespaceAccess.ListTags() -} - -func (n *namespace) GetBlobData(digest digest.Digest) (int64, cpi.DataAccess, error) { - if err := n.client.Refresh(context.Background()); err != nil { - return 0, nil, err - } - return n.NamespaceAccess.GetBlobData(digest) -} - -func (n *namespace) GetArtifact(version string) (cpi.ArtifactAccess, error) { - if err := n.client.Refresh(context.Background()); err != nil { - return nil, err - } - return n.NamespaceAccess.GetArtifact(version) -} - -func (n *namespace) HasArtifact(vers string) (bool, error) { - if err := n.client.Refresh(context.Background()); err != nil { - return false, err - } - return n.NamespaceAccess.HasArtifact(vers) -} diff --git a/api/oci/extensions/repositories/git/repository.go b/api/oci/extensions/repositories/git/repository.go deleted file mode 100644 index 3c51db31de..0000000000 --- a/api/oci/extensions/repositories/git/repository.go +++ /dev/null @@ -1,124 +0,0 @@ -package git - -import ( - "context" - "fmt" - - "github.com/mandelsoft/logging" - "github.com/mandelsoft/vfs/pkg/vfs" - - "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/oci/extensions/repositories/ctf" - "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/tech/git/identity" - "ocm.software/ocm/api/utils/accessobj" - ocmlog "ocm.software/ocm/api/utils/logging" -) - -const CommitPrefix = "update(ocm)" - -type Repository interface { - cpi.Repository -} - -type RepositoryImpl struct { - logger logging.UnboundLogger - spec *RepositorySpec - *ctf.Repository - client git.Client -} - -var _ cpi.Repository = (*RepositoryImpl)(nil) - -func New(ctx cpi.Context, spec *RepositorySpec, creds credentials.Credentials) (Repository, error) { - urs := spec.UniformRepositorySpec() - i := &RepositoryImpl{ - logger: logging.DynamicLogger(ctx, logging.NewAttribute(ocmlog.ATTR_HOST, urs.Host)), - spec: spec, - } - - opts := spec.ToClientOptions() - - if creds == nil { - // if no credentials are provided, try to get them from the context, - // if the credential is not provided, the client will try to use unauthenticated access, so allow error - creds, _ = identity.GetCredentials(ctx, spec.URL) - } - - if creds != nil { - auth, err := git.AuthFromCredentials(creds) - if err != nil { - return nil, fmt.Errorf("failed to create git authentication from given credentials: %w", err) - } - opts.AuthMethod = auth - } - - var err error - if i.client, err = git.NewClient(opts); err != nil { - return nil, fmt.Errorf("failed to create new git client for interacting with the repository: %w", err) - } - - repo, err := ctf.New(ctx, &ctf.RepositorySpec{ - StandardOptions: spec.StandardOptions, - AccessMode: spec.AccessMode, - }, i.client, &repoCloseUpdater{func() error { - if i.IsReadOnly() { - return nil - } - // on close make sure that we update and push the latest changes - return i.client.Update(context.Background(), GenerateCommitMessage(), true) - }}, vfs.FileMode(0o770)) - if err != nil { - return nil, fmt.Errorf("failed to create new ctf repository within the git repository: %w", err) - } - i.Repository = repo - - return i, nil -} - -func (r *RepositoryImpl) GetSpecification() cpi.RepositorySpec { - return r.spec -} - -func GenerateCommitMessage() string { - return fmt.Sprintf("%s: update repository", CommitPrefix) -} - -func (r *RepositoryImpl) GetIdentityMatcher() string { - return identity.CONSUMER_TYPE -} - -func (r *RepositoryImpl) IsReadOnly() bool { - return false -} - -func (r *RepositoryImpl) ExistsArtifact(name string, version string) (bool, error) { - if err := r.client.Refresh(context.Background()); err != nil { - return false, err - } - return r.Repository.ExistsArtifact(name, version) -} - -func (r *RepositoryImpl) LookupArtifact(name string, version string) (cpi.ArtifactAccess, error) { - if err := r.client.Refresh(context.Background()); err != nil { - return nil, err - } - return r.Repository.LookupArtifact(name, version) -} - -func (r *RepositoryImpl) LookupNamespace(name string) (cpi.NamespaceAccess, error) { - if err := r.client.Refresh(context.Background()); err != nil { - return nil, err - } - return NewNamespace(r, name) -} - -// small helper to wrap accessio.Closer to allow calling an arbitrary closing logic. -type repoCloseUpdater struct { - close func() error -} - -func (r *repoCloseUpdater) Close(*accessobj.AccessObject) error { - return r.close() -} diff --git a/api/oci/extensions/repositories/git/suite_test.go b/api/oci/extensions/repositories/git/suite_test.go deleted file mode 100644 index 597d5f9350..0000000000 --- a/api/oci/extensions/repositories/git/suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package git_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestConfig(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "OCI Git Test Suite") -} diff --git a/api/oci/extensions/repositories/git/testdata/repo/file_in_repo b/api/oci/extensions/repositories/git/testdata/repo/file_in_repo deleted file mode 100644 index 5eced95754..0000000000 --- a/api/oci/extensions/repositories/git/testdata/repo/file_in_repo +++ /dev/null @@ -1 +0,0 @@ -Foobar \ No newline at end of file diff --git a/api/oci/extensions/repositories/git/type.go b/api/oci/extensions/repositories/git/type.go deleted file mode 100644 index 7493da5a2b..0000000000 --- a/api/oci/extensions/repositories/git/type.go +++ /dev/null @@ -1,170 +0,0 @@ -package git - -import ( - "fmt" - - giturls "github.com/whilp/git-urls" - - "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/oci/internal" - "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/accessobj" - "ocm.software/ocm/api/utils/runtime" -) - -const ( - Type = "GitRepository" - TypeV1 = Type + runtime.VersionSeparator + "v1" - - ShortType = "Git" - ShortTypeV1 = ShortType + runtime.VersionSeparator + "v1" -) - -func init() { - cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](Type)) - cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](TypeV1)) - cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](ShortType)) - cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](ShortTypeV1)) -} - -// Is checks the kind. -func Is(spec cpi.RepositorySpec) bool { - return spec != nil && (spec.GetKind() == Type || spec.GetKind() == ShortType) -} - -func IsKind(k string) bool { - return k == Type || k == ShortType -} - -// RepositorySpec describes an CTF RepositoryImpl interface backed by a git RepositoryImpl. -type RepositorySpec struct { - runtime.ObjectVersionedType `json:",inline"` - accessio.StandardOptions `json:",inline"` - - // URL is the git url of the RepositoryImpl to resolve artifacts. - URL string `json:"url"` - - // Ref is the git ref of the RepositoryImpl to resolve artifacts. - // Examples include - // - refs/heads/master - // - refs/tags/v1.0.0 - // - pull/123/head - // - remotes/origin/feature - // If empty, the default is set to HEAD. - Ref string `json:"ref,omitempty"` - - // Commit is the commit hash of the RepositoryImpl inside the Ref to resolve artifacts. - // If empty, the default is set to the latest commit (the HEAD) of the Ref. - Commit string `json:"commit,omitempty"` - - // Author is the author of commits generated by the repository. If not set, it is defaulted from environment and git - // configuration of the host system. - Author *Author `json:"author,omitempty"` - - // AccessMode can be set to request readonly access or creation - AccessMode accessobj.AccessMode `json:"accessMode,omitempty"` -} - -// Author describes the author of commits generated by the repository. -type Author struct { - Name string `json:"name"` - Email string `json:"email"` -} - -var _ cpi.RepositorySpec = (*RepositorySpec)(nil) - -var _ cpi.IntermediateRepositorySpecAspect = (*RepositorySpec)(nil) - -type Options struct { - *Author - Ref string - Commit string - accessio.Options -} - -// NewRepositorySpecFromOptions creates a new RepositorySpec from options. -func NewRepositorySpecFromOptions(mode accessobj.AccessMode, url string, opts Options) (*RepositorySpec, error) { - spec, err := NewRepositorySpec(mode, url, opts.Options) - if err != nil { - return nil, err - } - spec.Author = opts.Author - spec.Ref = opts.Ref - spec.Commit = opts.Commit - return spec, nil -} - -// NewRepositorySpec creates a new RepositorySpec. -func NewRepositorySpec(mode accessobj.AccessMode, url string, opts ...accessio.Option) (*RepositorySpec, error) { - o, err := accessio.AccessOptions(nil, opts...) - if err != nil { - return nil, err - } - o.Default() - return &RepositorySpec{ - ObjectVersionedType: runtime.NewVersionedTypedObject(Type), - URL: url, - StandardOptions: *o.(*accessio.StandardOptions), - AccessMode: mode, - }, nil -} - -func (s *RepositorySpec) IsIntermediate() bool { - return false -} - -func (s *RepositorySpec) GetType() string { - return Type -} - -func (s *RepositorySpec) Name() string { - return s.URL -} - -func (s *RepositorySpec) UniformRepositorySpec() *cpi.UniformRepositorySpec { - u := &cpi.UniformRepositorySpec{ - Type: Type, - Info: s.URL, - } - url, err := giturls.Parse(s.URL) - if err == nil { - u.Host = url.Host - u.Scheme = url.Scheme - } - - return u -} - -func (s *RepositorySpec) Repository(ctx cpi.Context, creds credentials.Credentials) (cpi.Repository, error) { - return New(ctx, s, creds) -} - -func (s *RepositorySpec) ToClientOptions() git.ClientOptions { - opts := git.ClientOptions{} - - if s.Author != nil { - opts.Author = git.Author{ - Name: s.Author.Name, - Email: s.Author.Email, - } - } - - opts.URL = s.URL - opts.Ref = s.Ref - - return opts -} - -func (s *RepositorySpec) Validate(_ internal.Context, c credentials.Credentials, _ ...credentials.UsageContext) error { - if _, err := giturls.Parse(s.URL); err != nil { - return fmt.Errorf("failed to parse git url: %w", err) - } - - if _, err := git.AuthFromCredentials(c); err != nil { - return fmt.Errorf("failed to create git authentication from given credentials: %w", err) - } - - return nil -} diff --git a/api/oci/extensions/repositories/init.go b/api/oci/extensions/repositories/init.go index 6ddf95b0cf..2d9efaeb2c 100644 --- a/api/oci/extensions/repositories/init.go +++ b/api/oci/extensions/repositories/init.go @@ -5,6 +5,5 @@ import ( _ "ocm.software/ocm/api/oci/extensions/repositories/ctf" _ "ocm.software/ocm/api/oci/extensions/repositories/docker" _ "ocm.software/ocm/api/oci/extensions/repositories/empty" - _ "ocm.software/ocm/api/oci/extensions/repositories/git" _ "ocm.software/ocm/api/oci/extensions/repositories/ocireg" ) diff --git a/api/ocm/extensions/repositories/git/format.go b/api/ocm/extensions/repositories/git/format.go deleted file mode 100644 index e85542da5c..0000000000 --- a/api/ocm/extensions/repositories/git/format.go +++ /dev/null @@ -1,33 +0,0 @@ -package git - -import ( - "ocm.software/ocm/api/oci/extensions/repositories/git" - "ocm.software/ocm/api/ocm/cpi" - "ocm.software/ocm/api/ocm/extensions/repositories/ctf" - "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - "ocm.software/ocm/api/utils/accessobj" -) - -type Object = ctf.Object - -const ( - ACC_CREATE = accessobj.ACC_CREATE - ACC_WRITABLE = accessobj.ACC_WRITABLE - ACC_READONLY = accessobj.ACC_READONLY -) - -func Open(ctx cpi.ContextProvider, acc accessobj.AccessMode, url string, opts Options) (cpi.Repository, error) { - r, err := git.Open(cpi.FromProvider(ctx), acc, url, opts) - if err != nil { - return nil, err - } - return genericocireg.NewRepository(cpi.FromProvider(ctx), nil, r), nil -} - -func Create(ctx cpi.ContextProvider, url string, opts Options) (cpi.Repository, error) { - r, err := git.Create(cpi.FromProvider(ctx), url, opts) - if err != nil { - return nil, err - } - return genericocireg.NewRepository(cpi.FromProvider(ctx), nil, r), nil -} diff --git a/api/ocm/extensions/repositories/git/repo_test.go b/api/ocm/extensions/repositories/git/repo_test.go deleted file mode 100644 index 91de46cde6..0000000000 --- a/api/ocm/extensions/repositories/git/repo_test.go +++ /dev/null @@ -1,365 +0,0 @@ -package git_test - -import ( - "bytes" - "fmt" - "os" - "path/filepath" - - . "github.com/mandelsoft/goutils/finalizer" - . "github.com/mandelsoft/goutils/testutils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "ocm.software/ocm/api/ocm/testhelper" - - "github.com/go-git/go-billy/v5" - gitgo "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/server" - "github.com/go-git/go-git/v5/storage/filesystem" - "github.com/mandelsoft/logging" - "github.com/mandelsoft/vfs/pkg/cwdfs" - "github.com/mandelsoft/vfs/pkg/osfs" - "github.com/mandelsoft/vfs/pkg/projectionfs" - "github.com/mandelsoft/vfs/pkg/vfs" - "github.com/tonglil/buflogr" - - gitrepo "ocm.software/ocm/api/oci/extensions/repositories/git" - "ocm.software/ocm/api/ocm" - "ocm.software/ocm/api/ocm/compdesc" - metav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" - resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" - "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - "ocm.software/ocm/api/ocm/extensions/repositories/git" - techgit "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/blobaccess" - ocmlog "ocm.software/ocm/api/utils/logging" - "ocm.software/ocm/api/utils/mime" - "ocm.software/ocm/api/utils/refmgmt" -) - -const ( - COMPONENT = "ocm.software/ocm" - VERSION = "1.0.0" - REMOTE_REPO = "repo.git" -) - -var _ = Describe("access method", func() { - // remoteFS contains the remote repository on the filesystem - // pathFS contains the local PWD - // repFS contains the local representation of the repository, meaning the cloned repo to work on the Repository - var remoteFS, pathFS, repFS vfs.FileSystem - // access contains the access configuration to the above filesystems - var access accessio.Options - - // repoURL is the URL specification to access the remote repository in remoteFS - var repoURL string - - // remoteRepo is the remote repository that can be used for test assertions on pushed content - var remoteRepo *gitgo.Repository - - var opts git.Options - - ctx := ocm.DefaultContext() - - BeforeEach(func() { - By("setting up test filesystems") - basePath := GinkgoT().TempDir() - baseFS := Must(cwdfs.New(osfs.New(), basePath)) - for _, dir := range []string{"remote", "path", "rep"} { - Expect(os.Mkdir(filepath.Join(basePath, dir), 0o777)).To(Succeed()) - } - remoteFS = Must(projectionfs.New(baseFS, "remote")) - pathFS = Must(projectionfs.New(baseFS, "path")) - repFS = Must(projectionfs.New(baseFS, "rep")) - - access = &accessio.StandardOptions{ - PathFileSystem: pathFS, - Representation: repFS, - } - }) - - AfterEach(func() { - Expect(Must(vfs.ReadDir(pathFS, "."))).To(BeEmpty(), "nothing of the CTF should be stored in the path, "+ - "because everything should be handled in the representation which contains the local git repository") - }) - - BeforeEach(func() { - By("setting up local bare git repository to work against when pushing/updating") - billy := Must(techgit.VFSBillyFS(remoteFS)) - client.InstallProtocol("file", server.NewClient(server.NewFilesystemLoader(billy))) - remoteRepo = Must(newBareTestRepo(billy, REMOTE_REPO, gitgo.InitOptions{})) - // now that we have a bare repository, we can reference it via URL to access it like a remote repository - repoURL = fmt.Sprintf("file:///%s", REMOTE_REPO) - }) - - BeforeEach(func() { - opts = git.Options{ - Author: &git.Author{ - Name: fmt.Sprintf("OCM Test Case: %s", GinkgoT().Name()), - Email: "dummy@ocm.software", - }, - Options: access, - } - }) - - It("adds naked component version and later lookup", func() { - final := Finalizer{} - defer Defer(final.Finalize) - - a := Must(git.Create(ctx, repoURL, opts)) - final.Close(a, "repository") - c := Must(a.LookupComponent(COMPONENT)) - final.Close(c, "component") - - cv := Must(c.NewVersion(VERSION)) - final.Close(cv, "version") - - MustBeSuccessful(c.AddVersion(cv)) - MustBeSuccessful(final.Finalize()) - - componentCommitExpectation := gitrepo.GenerateCommitMessage() - - componentUpdate := 0 - commits := Must(remoteRepo.CommitObjects()) - Expect(commits.ForEach(func(commit *object.Commit) error { - Expect(commit.Author.Name).To(Equal(opts.Author.Name)) - Expect(commit.Author.Email).To(Equal(opts.Author.Email)) - - if commit.Message == componentCommitExpectation { - componentUpdate++ - } - return nil - })).To(Succeed()) - Expect(componentUpdate).To(Equal(1)) - - refmgmt.AllocLog.Trace("opening ctf") - a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) - final.Close(a) - - refmgmt.AllocLog.Trace("lookup component") - c, err := a.LookupComponent(COMPONENT) - Expect(err).ToNot(HaveOccurred()) - final.Close(c) - - refmgmt.AllocLog.Trace("lookup version") - cv = Must(c.LookupVersion(VERSION)) - final.Close(cv) - - refmgmt.AllocLog.Trace("closing") - MustBeSuccessful(final.Finalize()) - }) - - It("adds naked component version and later shortcut lookup", func() { - final := Finalizer{} - defer Defer(final.Finalize) - - a := Must(git.Create(ctx, repoURL, opts)) - final.Close(a, "repository") - c := Must(a.LookupComponent(COMPONENT)) - final.Close(c, "component") - - cv := Must(c.NewVersion(VERSION)) - final.Close(cv, "version") - - MustBeSuccessful(c.AddVersion(cv)) - MustBeSuccessful(final.Finalize()) - - refmgmt.AllocLog.Trace("opening ctf") - a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) - final.Close(a) - - refmgmt.AllocLog.Trace("lookup component version") - cv = Must(a.LookupComponentVersion(COMPONENT, VERSION)) - final.Close(cv) - - refmgmt.AllocLog.Trace("closing") - MustBeSuccessful(final.Finalize()) - }) - - It("adds component version", func() { - final := Finalizer{} - defer Defer(final.Finalize) - - a := Must(git.Create(ctx, repoURL, opts)) - final.Close(a) - c := Must(a.LookupComponent(COMPONENT)) - final.Close(c) - - cv := Must(c.NewVersion(VERSION)) - final.Close(cv) - - // add resource - MustBeSuccessful(cv.SetResourceBlob(compdesc.NewResourceMeta("text1", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) - Expect(Must(cv.GetResource(compdesc.NewIdentity("text1"))).Meta().Digest).To(Equal(DS_TESTDATA)) - - // add resource with digest - meta := compdesc.NewResourceMeta("text2", resourcetypes.PLAIN_TEXT, metav1.LocalRelation) - meta.SetDigest(DS_TESTDATA) - MustBeSuccessful(cv.SetResourceBlob(meta, blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)) - Expect(Must(cv.GetResource(compdesc.NewIdentity("text2"))).Meta().Digest).To(Equal(DS_TESTDATA)) - - // reject resource with wrong digest - meta = compdesc.NewResourceMeta("text3", resourcetypes.PLAIN_TEXT, metav1.LocalRelation) - meta.SetDigest(TextResourceDigestSpec("fake")) - Expect(cv.SetResourceBlob(meta, blobaccess.ForString(mime.MIME_TEXT, S_TESTDATA), "", nil)).To(MatchError("unable to set resource: digest mismatch: " + D_TESTDATA + " != fake")) - - MustBeSuccessful(c.AddVersion(cv)) - MustBeSuccessful(final.Finalize()) - - a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) - final.Close(a) - - cv = Must(a.LookupComponentVersion(COMPONENT, VERSION)) - final.Close(cv) - }) - - It("adds omits unadded new component version", func() { - final := Finalizer{} - defer Defer(final.Finalize) - - a := Must(git.Create(ctx, repoURL, opts)) - final.Close(a) - c := Must(a.LookupComponent(COMPONENT)) - final.Close(c) - - cv := Must(c.NewVersion(VERSION)) - final.Close(cv) - - MustBeSuccessful(final.Finalize()) - - a = Must(git.Open(ctx, git.ACC_READONLY, repoURL, opts)) - final.Close(a) - - _, err := a.LookupComponentVersion(COMPONENT, VERSION) - - Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("component version \"%[1]s:%[2]s\" not found: oci artifact \"%[2]s\" not found in component-descriptors/%[1]s", COMPONENT, VERSION)))) - }) - - It("provides error for invalid bloc access", func() { - final := Finalizer{} - defer Defer(final.Finalize) - - a := Must(git.Create(ctx, repoURL, opts)) - final.Close(a) - c := Must(a.LookupComponent(COMPONENT)) - final.Close(c) - - cv := Must(c.NewVersion(VERSION)) - final.Close(cv) - - // add resource - Expect(ErrorFrom(cv.SetResourceBlob(compdesc.NewResourceMeta("text1", resourcetypes.PLAIN_TEXT, metav1.LocalRelation), blobaccess.ForFile(mime.MIME_TEXT, "non-existing-file"), "", nil))).To(MatchError(`file "non-existing-file" not found`)) - - MustBeSuccessful(final.Finalize()) - }) - - It("logs diff", func() { - MustBeSuccessful(accessio.FormatDirectory.ApplyOption(opts.Options)) - r := Must(git.Open(ctx, git.ACC_CREATE, repoURL, opts)) - defer Close(r, "repo") - - c := Must(r.LookupComponent("acme.org/test")) - defer Close(c, "comp") - - cv := Must(c.NewVersion("v1")) - - ocmlog.PushContext(nil) - ocmlog.Context().AddRule(logging.NewConditionRule(logging.DebugLevel, genericocireg.TAG_CDDIFF)) - var buf bytes.Buffer - def := buflogr.NewWithBuffer(&buf) - ocmlog.Context().SetBaseLogger(def) - defer ocmlog.Context().ResetRules() - defer ocmlog.PopContext() - - MustBeSuccessful(c.AddVersion(cv)) - MustBeSuccessful(cv.Close()) - - cv = Must(c.LookupVersion("v1")) - cv.GetDescriptor().Provider.Name = "acme.org" - MustBeSuccessful(cv.Close()) - Expect("\n" + buf.String()).To(Equal(fmt.Sprintf(` -V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != %[1]s] -V[4] component descriptor has been changed realm ocm realm ocm/oci/mapping diff [ComponentSpec.ObjectMeta.Provider.Name: acme != %[1]s] -`, cv.GetDescriptor().Provider.Name))) - }) - - It("handles readonly mode", func() { - MustBeSuccessful(accessio.FormatDirectory.ApplyOption(opts.Options)) - r := Must(git.Open(ctx, git.ACC_CREATE, repoURL, opts)) - defer Close(r, "repo") - - c := Must(r.LookupComponent("acme.org/test")) - defer Close(c, "comp") - - cv := Must(c.NewVersion("v1")) - - MustBeSuccessful(c.AddVersion(cv)) - MustBeSuccessful(cv.Close()) - - cv = Must(c.LookupVersion("v1")) - cv.SetReadOnly() - Expect(cv.IsReadOnly()).To(BeTrue()) - cv.GetDescriptor().Provider.Name = "acme.org" - ExpectError(cv.Close()).To(MatchError(accessio.ErrReadOnly)) - }) - - It("handles readonly mode on repo", func() { - MustBeSuccessful(accessio.FormatDirectory.ApplyOption(opts.Options)) - r := Must(git.Open(ctx, git.ACC_CREATE, repoURL, opts)) - defer Close(r, "repo") - - c := Must(r.LookupComponent("acme.org/test")) - defer Close(c, "comp") - - cv := Must(c.NewVersion("v1")) - - MustBeSuccessful(c.AddVersion(cv)) - MustBeSuccessful(cv.Close()) - - r.SetReadOnly() - cv = Must(c.LookupVersion("v1")) - Expect(cv.IsReadOnly()).To(BeTrue()) - cv.GetDescriptor().Provider.Name = "acme.org" - ExpectError(cv.Close()).To(MatchError(accessio.ErrReadOnly)) - - ExpectError(c.NewVersion("v2")).To(MatchError(accessio.ErrReadOnly)) - }) -}) - -func newBareTestRepo(fs billy.Filesystem, path string, opts gitgo.InitOptions) (*gitgo.Repository, error) { - var wt, dot billy.Filesystem - - var err error - dot, err = fs.Chroot(path) - if err != nil { - return nil, err - } - wt, err = fs.Chroot(path) - if err != nil { - return nil, err - } - - s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - - r, err := gitgo.InitWithOptions(s, wt, opts) - if err != nil { - return nil, err - } - - cfg, err := r.Config() - if err != nil { - return nil, err - } - - err = r.Storer.SetConfig(cfg) - if err != nil { - return nil, err - } - - return r, err -} diff --git a/api/ocm/extensions/repositories/git/suite_test.go b/api/ocm/extensions/repositories/git/suite_test.go deleted file mode 100644 index 0995130649..0000000000 --- a/api/ocm/extensions/repositories/git/suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package git_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestConfig(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Git Repository Test Suite") -} diff --git a/api/ocm/extensions/repositories/git/type.go b/api/ocm/extensions/repositories/git/type.go deleted file mode 100644 index 6c85613a46..0000000000 --- a/api/ocm/extensions/repositories/git/type.go +++ /dev/null @@ -1,21 +0,0 @@ -package git - -import ( - "ocm.software/ocm/api/oci/extensions/repositories/git" - "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - "ocm.software/ocm/api/utils/accessobj" -) - -const Type = git.Type - -type Options = git.Options - -type Author = git.Author - -func NewRepositorySpec(acc accessobj.AccessMode, url string, opts Options) (*genericocireg.RepositorySpec, error) { - spec, err := git.NewRepositorySpecFromOptions(acc, url, opts) - if err != nil { - return nil, err - } - return genericocireg.NewRepositorySpec(spec, nil), nil -} diff --git a/api/ocm/extensions/repositories/init.go b/api/ocm/extensions/repositories/init.go index 6a1530b56d..5471dab2e8 100644 --- a/api/ocm/extensions/repositories/init.go +++ b/api/ocm/extensions/repositories/init.go @@ -4,5 +4,4 @@ import ( _ "ocm.software/ocm/api/ocm/extensions/repositories/comparch" _ "ocm.software/ocm/api/ocm/extensions/repositories/ctf" _ "ocm.software/ocm/api/ocm/extensions/repositories/genericocireg" - _ "ocm.software/ocm/api/ocm/extensions/repositories/git" ) diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go index 6c8f3ac524..988cfe705a 100644 --- a/api/utils/blobaccess/git/access.go +++ b/api/utils/blobaccess/git/access.go @@ -71,10 +71,7 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { filteredRepositoryFS := &filteredVFS{ FileSystem: repositoryFS, filter: func(s string) bool { - if s == gogit.GitDirName { - return false - } - return true + return s != gogit.GitDirName }, } diff --git a/docs/reference/ocm_add_routingslips.md b/docs/reference/ocm_add_routingslips.md index 5742f762f9..4bebc50df7 100644 --- a/docs/reference/ocm_add_routingslips.md +++ b/docs/reference/ocm_add_routingslips.md @@ -99,8 +99,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_bootstrap_configuration.md b/docs/reference/ocm_bootstrap_configuration.md index 03ecfcfce0..92396ca672 100644 --- a/docs/reference/ocm_bootstrap_configuration.md +++ b/docs/reference/ocm_bootstrap_configuration.md @@ -80,8 +80,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_bootstrap_package.md b/docs/reference/ocm_bootstrap_package.md index b678f70594..19e343b811 100644 --- a/docs/reference/ocm_bootstrap_package.md +++ b/docs/reference/ocm_bootstrap_package.md @@ -161,8 +161,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_check_componentversions.md b/docs/reference/ocm_check_componentversions.md index 4379e553c4..23f9849220 100644 --- a/docs/reference/ocm_check_componentversions.md +++ b/docs/reference/ocm_check_componentversions.md @@ -68,8 +68,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_describe_artifacts.md b/docs/reference/ocm_describe_artifacts.md index 9cf092f0a6..a2bf2e3d07 100644 --- a/docs/reference/ocm_describe_artifacts.md +++ b/docs/reference/ocm_describe_artifacts.md @@ -58,8 +58,6 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_describe_package.md b/docs/reference/ocm_describe_package.md index 4ee5bf9727..def83647ae 100644 --- a/docs/reference/ocm_describe_package.md +++ b/docs/reference/ocm_describe_package.md @@ -71,8 +71,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_artifacts.md b/docs/reference/ocm_download_artifacts.md index 878ee7da76..a7a7928e3e 100644 --- a/docs/reference/ocm_download_artifacts.md +++ b/docs/reference/ocm_download_artifacts.md @@ -60,8 +60,6 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_cli.md b/docs/reference/ocm_download_cli.md index 53e53b7d8d..ff71ec89dd 100644 --- a/docs/reference/ocm_download_cli.md +++ b/docs/reference/ocm_download_cli.md @@ -81,8 +81,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_componentversions.md b/docs/reference/ocm_download_componentversions.md index 36d51a1d78..79dbd5c69e 100644 --- a/docs/reference/ocm_download_componentversions.md +++ b/docs/reference/ocm_download_componentversions.md @@ -67,8 +67,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_download_resources.md b/docs/reference/ocm_download_resources.md index e5ee73bd65..fd14528178 100644 --- a/docs/reference/ocm_download_resources.md +++ b/docs/reference/ocm_download_resources.md @@ -106,8 +106,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_artifacts.md b/docs/reference/ocm_get_artifacts.md index db0af2b205..a4bfa73b68 100644 --- a/docs/reference/ocm_get_artifacts.md +++ b/docs/reference/ocm_get_artifacts.md @@ -58,8 +58,6 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_componentversions.md b/docs/reference/ocm_get_componentversions.md index 6c93c6ada5..ceebb7e970 100644 --- a/docs/reference/ocm_get_componentversions.md +++ b/docs/reference/ocm_get_componentversions.md @@ -77,8 +77,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_references.md b/docs/reference/ocm_get_references.md index 751e378f96..684f1b4787 100644 --- a/docs/reference/ocm_get_references.md +++ b/docs/reference/ocm_get_references.md @@ -78,8 +78,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_resources.md b/docs/reference/ocm_get_resources.md index 9f88968abe..9790deeb88 100644 --- a/docs/reference/ocm_get_resources.md +++ b/docs/reference/ocm_get_resources.md @@ -78,8 +78,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_routingslips.md b/docs/reference/ocm_get_routingslips.md index b4c804a0f9..a5402ac441 100644 --- a/docs/reference/ocm_get_routingslips.md +++ b/docs/reference/ocm_get_routingslips.md @@ -77,8 +77,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_get_sources.md b/docs/reference/ocm_get_sources.md index ecb6fe7eac..fc32e41373 100644 --- a/docs/reference/ocm_get_sources.md +++ b/docs/reference/ocm_get_sources.md @@ -78,8 +78,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_hash_componentversions.md b/docs/reference/ocm_hash_componentversions.md index 7237ac1e35..1e49ba07f4 100644 --- a/docs/reference/ocm_hash_componentversions.md +++ b/docs/reference/ocm_hash_componentversions.md @@ -111,8 +111,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_install_plugins.md b/docs/reference/ocm_install_plugins.md index 140a88f083..9c20720c97 100644 --- a/docs/reference/ocm_install_plugins.md +++ b/docs/reference/ocm_install_plugins.md @@ -70,8 +70,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_list_componentversions.md b/docs/reference/ocm_list_componentversions.md index e1c1b5703c..630d009e9b 100644 --- a/docs/reference/ocm_list_componentversions.md +++ b/docs/reference/ocm_list_componentversions.md @@ -76,8 +76,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_show_tags.md b/docs/reference/ocm_show_tags.md index 5a3a616843..92250cbf21 100644 --- a/docs/reference/ocm_show_tags.md +++ b/docs/reference/ocm_show_tags.md @@ -50,8 +50,6 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_show_versions.md b/docs/reference/ocm_show_versions.md index da82b0fe66..075203cf3c 100644 --- a/docs/reference/ocm_show_versions.md +++ b/docs/reference/ocm_show_versions.md @@ -64,8 +64,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_sign_componentversions.md b/docs/reference/ocm_sign_componentversions.md index 5a2d72c970..b1d3f582bf 100644 --- a/docs/reference/ocm_sign_componentversions.md +++ b/docs/reference/ocm_sign_componentversions.md @@ -88,8 +88,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_transfer_artifacts.md b/docs/reference/ocm_transfer_artifacts.md index 351fd6fb37..7cbe20ee89 100644 --- a/docs/reference/ocm_transfer_artifacts.md +++ b/docs/reference/ocm_transfer_artifacts.md @@ -71,8 +71,6 @@ linked library can be used: - CommonTransportFormat: v1 - DockerDaemon: v1 - Empty: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_transfer_componentversions.md b/docs/reference/ocm_transfer_componentversions.md index e7fa5b7908..1c235f0e82 100644 --- a/docs/reference/ocm_transfer_componentversions.md +++ b/docs/reference/ocm_transfer_componentversions.md @@ -89,8 +89,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry diff --git a/docs/reference/ocm_verify_componentversions.md b/docs/reference/ocm_verify_componentversions.md index 5f2211b99b..dabdd5d270 100644 --- a/docs/reference/ocm_verify_componentversions.md +++ b/docs/reference/ocm_verify_componentversions.md @@ -85,8 +85,6 @@ Dedicated OCM repository types: OCI Repository types (using standard component repository to OCI mapping): - CommonTransportFormat: v1 - - Git: v1 - - GitRepository: v1 - OCIRegistry: v1 - oci: v1 - ociRegistry From 99e94bd7f9771ebeabe48dbcad9ad448779474a9 Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Thu, 28 Nov 2024 09:47:02 +0100 Subject: [PATCH 19/22] chore: keep up to date --- .../extensions/accessmethods/git/method.go | 6 +-- api/tech/git/resolver.go | 19 ++++++++ api/utils/blobaccess/git/access.go | 2 + .../ocm_add_resource-configuration.md | 47 +++++++++++++++++++ docs/reference/ocm_add_resources.md | 47 +++++++++++++++++++ .../reference/ocm_add_source-configuration.md | 47 +++++++++++++++++++ docs/reference/ocm_add_sources.md | 47 +++++++++++++++++++ docs/reference/ocm_ocm-accessmethods.md | 24 ++++++++++ go.mod | 12 +++++ go.sum | 39 ++++++++++++++- 10 files changed, 285 insertions(+), 5 deletions(-) diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index 601aae0ee1..aace11f154 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -19,13 +19,13 @@ import ( ) const ( - Type = "git" - TypeV1 = Type + runtime.VersionSeparator + "v1" + Type = "git" + TypeV1Alpha1 = Type + runtime.VersionSeparator + "v1alpha1" ) func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage))) - accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1Alpha1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } // AccessSpec describes the access for a GitHub registry. diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index f952f479f9..2ddc996be0 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "os" "sync" @@ -12,12 +13,30 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/transport" + gitclient "github.com/go-git/go-git/v5/plumbing/transport/client" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/storage/filesystem" + mlog "github.com/mandelsoft/logging" "github.com/mandelsoft/vfs/pkg/memoryfs" "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/utils/logging" ) +const LogAttrProtocol = "protocol" + +func init() { + // override the logging realm for http based git clients + gitclient.InstallProtocol("http", githttp.NewClient(&http.Client{ + Transport: logging.NewRoundTripper(http.DefaultTransport, logging.DynamicLogger(REALM, mlog.NewAttribute(LogAttrProtocol, "http"))), + })) + gitclient.InstallProtocol("https", githttp.NewClient(&http.Client{ + Transport: logging.NewRoundTripper(http.DefaultTransport, logging.DynamicLogger(REALM, mlog.NewAttribute(LogAttrProtocol, "https"))), + })) + //TODO Determine how we ideally log for ssh+git protocol +} + var DefaultWorktreeBranch = plumbing.NewBranchReferenceName("ocm") type client struct { diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go index 988cfe705a..96aa173f68 100644 --- a/api/utils/blobaccess/git/access.go +++ b/api/utils/blobaccess/git/access.go @@ -19,6 +19,8 @@ import ( "ocm.software/ocm/api/utils/tarutils" ) +// BlobAccess clones the repository into a temporary filesystem, packs it into a tar.gz file, +// and returns a BlobAccess object for the tar.gz file. func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { var finalize finalizer.Finalizer defer finalize.FinalizeWithErrorPropagation(&rerr) diff --git a/docs/reference/ocm_add_resource-configuration.md b/docs/reference/ocm_add_resource-configuration.md index 2e541891bd..8d8c28441d 100644 --- a/docs/reference/ocm_add_resource-configuration.md +++ b/docs/reference/ocm_add_resource-configuration.md @@ -327,6 +327,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -607,6 +630,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_add_resources.md b/docs/reference/ocm_add_resources.md index 99d746c1ad..3e9754bd01 100644 --- a/docs/reference/ocm_add_resources.md +++ b/docs/reference/ocm_add_resources.md @@ -339,6 +339,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -619,6 +642,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_add_source-configuration.md b/docs/reference/ocm_add_source-configuration.md index 05153898b8..4090cbb55a 100644 --- a/docs/reference/ocm_add_source-configuration.md +++ b/docs/reference/ocm_add_source-configuration.md @@ -327,6 +327,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -607,6 +630,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_add_sources.md b/docs/reference/ocm_add_sources.md index 54f7d17b70..e6af2717b9 100644 --- a/docs/reference/ocm_add_sources.md +++ b/docs/reference/ocm_add_sources.md @@ -337,6 +337,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -617,6 +640,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_ocm-accessmethods.md b/docs/reference/ocm_ocm-accessmethods.md index 63d1ad8b44..2cd7c29fa2 100644 --- a/docs/reference/ocm_ocm-accessmethods.md +++ b/docs/reference/ocm_ocm-accessmethods.md @@ -15,6 +15,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/go.mod b/go.mod index 4623b27dde..24921294a3 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,8 @@ require ( github.com/fluxcd/pkg/ssa v0.41.1 github.com/gertd/go-pluralize v0.2.1 github.com/ghodss/yaml v1.0.0 + github.com/go-git/go-billy/v5 v5.6.0 + github.com/go-git/go-git/v5 v5.12.0 github.com/go-logr/logr v1.4.2 github.com/go-openapi/strfmt v0.23.0 github.com/go-openapi/swag v0.23.0 @@ -67,6 +69,7 @@ require ( github.com/texttheater/golang-levenshtein v1.0.1 github.com/tonglil/buflogr v1.1.1 github.com/ulikunitz/xz v0.5.12 + github.com/whilp/git-urls v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 @@ -176,6 +179,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/orderedmap v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect @@ -186,6 +190,7 @@ require ( github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-errors/errors v1.5.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect @@ -236,12 +241,14 @@ require ( github.com/huandu/xstrings v1.5.0 // indirect github.com/in-toto/in-toto-golang v0.9.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/letsencrypt/boulder v0.0.0-20241010192615-6692160cedfa // indirect @@ -280,6 +287,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -294,11 +302,13 @@ require ( github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect github.com/segmentio/ksuid v1.0.4 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sigstore/fulcio v1.6.5 // indirect github.com/sigstore/protobuf-specs v0.3.2 // indirect github.com/sigstore/timestamp-authority v1.2.3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect @@ -318,6 +328,7 @@ require ( github.com/vbatts/tar-split v0.11.6 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/go-gitlab v0.112.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xlab/treeprint v1.2.0 // indirect @@ -353,6 +364,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiserver v0.31.3 // indirect k8s.io/component-base v0.31.3 // indirect diff --git a/go.sum b/go.sum index e7febab052..ce78c1e499 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,7 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.12.9 h1:2zJy5KA+l0loz1HzEGqyNnjd3fyZA31ZBCGKacp6lLg= @@ -159,6 +160,8 @@ github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6q github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM= github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCoPipOEQA= github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -365,12 +368,16 @@ github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elliotchance/orderedmap v1.6.0 h1:xjn+kbbKXeDq6v9RVE+WYwRbYfAZKvlWfcJNxM8pvEw= github.com/elliotchance/orderedmap v1.6.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/proto v1.12.1 h1:6n/Z2pZAnBwuhU66Gs8160B8rrrYKo7h2F2sCOnNceE= github.com/emicklei/proto v1.12.1/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -409,10 +416,20 @@ github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlL github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8= +github.com/go-git/go-billy/v5 v5.6.0/go.mod h1:sFDq7xD3fn3E0GOwUSZqHo9lrkmx8xJhA0ZrfvjBRGM= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= @@ -630,6 +647,8 @@ github.com/in-toto/in-toto-golang v0.9.0/go.mod h1:xsBVrVsHNsB61++S6Dy2vWosKhuA3 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E= github.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc= @@ -661,6 +680,8 @@ github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b h1:FQ7+9fxhyp82ks9vAuy github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b/go.mod h1:HMcgvsgd0Fjj4XXDkbjdmlbI505rUPBs6WBMYg2pXks= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= @@ -830,6 +851,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= @@ -899,8 +922,8 @@ github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbm github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= @@ -930,8 +953,11 @@ github.com/sigstore/timestamp-authority v1.2.3/go.mod h1:q2tJKJzP34hLIbVu3Y1A9bB github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY= @@ -1010,10 +1036,14 @@ github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23env github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= github.com/weppos/publicsuffix-go v0.40.3-0.20240815124645-a8ed110559c9 h1:4pH9wXOWQdW8kVMJ8P/kxbuxJKR+iNvDeC8zEVLy7eM= github.com/weppos/publicsuffix-go v0.40.3-0.20240815124645-a8ed110559c9/go.mod h1:o4XOb/pL91sSlesP+I2Xcp38P4/emRvDF6N6xUWvwzg= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/go-gitlab v0.112.0 h1:6Z0cqEooCvBMfBIHw+CgO4AKGRV8na/9781xOb0+DKw= github.com/xanzy/go-gitlab v0.112.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1107,6 +1137,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -1194,6 +1225,7 @@ golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1303,6 +1335,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII= gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -1321,6 +1354,8 @@ gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From e5d7e85a3edcd6671ee553cacfb3039811f35dfa Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Mon, 16 Dec 2024 17:50:31 +0100 Subject: [PATCH 20/22] chore: pr review Co-authored-by: Gergely Brautigam <182850+skarlso@users.noreply.github.com> --- .../artifactaccess/gitaccess/options.go | 17 +- .../artifactaccess/gitaccess/resource.go | 2 +- .../extensions/accessmethods/git/README.md | 53 ++++++ api/ocm/extensions/accessmethods/git/cli.go | 2 +- .../extensions/accessmethods/git/method.go | 22 ++- .../accessmethods/git/method_test.go | 77 ++++++-- api/tech/git/identity/identity.go | 7 +- api/tech/git/resolver.go | 20 +- .../common/inputs/types/git/input_test.go | 180 ++++++++---------- .../ocm_add_resource-configuration.md | 2 +- docs/reference/ocm_add_resources.md | 2 +- .../reference/ocm_add_source-configuration.md | 2 +- docs/reference/ocm_add_sources.md | 2 +- docs/reference/ocm_ocm-accessmethods.md | 2 +- 14 files changed, 242 insertions(+), 148 deletions(-) diff --git a/api/ocm/elements/artifactaccess/gitaccess/options.go b/api/ocm/elements/artifactaccess/gitaccess/options.go index 9d89df6e72..5b569f1f0f 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/options.go +++ b/api/ocm/elements/artifactaccess/gitaccess/options.go @@ -7,10 +7,9 @@ import ( type Option = optionutils.Option[*Options] type Options struct { - URL string - Ref string - PathSpec string - Commit string + URL string + Ref string + Commit string } var _ Option = (*Options)(nil) @@ -48,16 +47,6 @@ func WithRef(h string) Option { return ref(h) } -type pathSpec string - -func (h pathSpec) ApplyTo(opts *Options) { - opts.PathSpec = string(h) -} - -func WithPathSpec(h string) Option { - return pathSpec(h) -} - type commitSpec string func (h commitSpec) ApplyTo(opts *Options) { diff --git a/api/ocm/elements/artifactaccess/gitaccess/resource.go b/api/ocm/elements/artifactaccess/gitaccess/resource.go index b223198d0a..24ade26b33 100644 --- a/api/ocm/elements/artifactaccess/gitaccess/resource.go +++ b/api/ocm/elements/artifactaccess/gitaccess/resource.go @@ -19,7 +19,7 @@ func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, o meta.SetType(TYPE) } - spec := access.New(eff.URL, eff.Ref, eff.Commit, eff.PathSpec) + spec := access.New(eff.URL, access.WithRef(eff.Ref), access.WithCommit(eff.Commit)) // is global access, must work, otherwise there is an error in the lib. return genericaccess.MustAccess(ctx, meta, spec) } diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md index 9532b934f9..0a4f404d1d 100644 --- a/api/ocm/extensions/accessmethods/git/README.md +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -39,3 +39,56 @@ The type specific specification fields are: ### Go Bindings The go binding can be found [here](method.go) + + +#### Example + +```go +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "io" + + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/cpi" + me "ocm.software/ocm/api/ocm/extensions/accessmethods/git" +) + +func main() { + ctx := ocm.New() + accessSpec := me.New( + "https://github.com/octocat/Hello-World.git", + me.WithRef("refs/heads/master"), + ) + method, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) + if err != nil { + panic(err) + } + content, err := method.GetContent() + if err != nil { + panic(err) + } + unzippedContent, err := gzip.NewReader(bytes.NewReader(content)) + + r := tar.NewReader(unzippedContent) + + file, err := r.Next() + if err != nil { + panic(err) + } + + if file.Name != "README.md" { + panic("Expected README.md") + } + + data, err := io.ReadAll(r) + if err != nil { + panic(err) + } + fmt.Println(string(data)) +} +``` \ No newline at end of file diff --git a/api/ocm/extensions/accessmethods/git/cli.go b/api/ocm/extensions/accessmethods/git/cli.go index da01ffae4f..fc0dce7d90 100644 --- a/api/ocm/extensions/accessmethods/git/cli.go +++ b/api/ocm/extensions/accessmethods/git/cli.go @@ -15,7 +15,7 @@ func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { } func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { - flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repoUrl") + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository", "repo", "repoUrl", "repoURL") flagsets.AddFieldByOptionP(opts, options.CommitOption, config, "commit") flagsets.AddFieldByOptionP(opts, options.ReferenceOption, config, "ref") return nil diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index aace11f154..e8c5d927ac 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -33,29 +33,35 @@ type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` // RepoURL is the repository URL - RepoURL string `json:"repoUrl"` + RepoURL string `json:"repository"` // Ref defines the hash of the commit Ref string `json:"ref"` // Commit defines the hash of the commit in string format to checkout from the Ref Commit string `json:"commit"` - - // PathSpec is a path in the repository to download, can be a file or a regex matching multiple files - PathSpec string `json:"pathSpec"` } // AccessSpecOptions defines a set of options which can be applied to the access spec. type AccessSpecOptions func(s *AccessSpec) +func WithCommit(commit string) AccessSpecOptions { + return func(s *AccessSpec) { + s.Commit = commit + } +} + +func WithRef(ref string) AccessSpecOptions { + return func(s *AccessSpec) { + s.Ref = ref + } +} + // New creates a new git registry access spec version v1. -func New(url, ref, commit, pathSpec string, opts ...AccessSpecOptions) *AccessSpec { +func New(url string, opts ...AccessSpecOptions) *AccessSpec { s := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), RepoURL: url, - Ref: ref, - Commit: commit, - PathSpec: pathSpec, } for _, o := range opts { o(s) diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go index cc37cb3a30..3a3ac663a4 100644 --- a/api/ocm/extensions/accessmethods/git/method_test.go +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -12,12 +12,12 @@ import ( _ "embed" + "github.com/go-git/go-git/v5/plumbing" . "github.com/mandelsoft/goutils/testutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/vfs/pkg/cwdfs" @@ -33,7 +33,7 @@ import ( //go:embed testdata/repo var testData embed.FS -var _ = Describe("Method", func() { +var _ = Describe("Method based on Filesystem", func() { var ( ctx ocm.Context expectedBlobContent []byte @@ -49,8 +49,10 @@ var _ = Describe("Method", func() { vfsattr.Set(ctx, tempVFS) }) + var repoDir string + BeforeEach(func() { - repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + repoDir = GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" repo := Must(git.PlainInit(repoDir, false)) @@ -77,7 +79,7 @@ var _ = Describe("Method", func() { wt := Must(repo.Worktree()) Expect(wt.AddGlob("*")).To(Succeed()) - Must(wt.Commit("OCM Test Commit", &git.CommitOptions{ + commit := Must(wt.Commit("OCM Test Commit", &git.CommitOptions{ Author: &object.Signature{ Name: "OCM Test", Email: "dummy@ocm.software", @@ -85,17 +87,34 @@ var _ = Describe("Method", func() { }, })) - accessSpec = me.New( - fmt.Sprintf("file://%s", repoDir), - string(plumbing.Master), - "", - ".", + path := filepath.Join("testdata", "repo", "file_in_repo") + + accessSpec = me.New(fmt.Sprintf("file://%s", repoDir), + me.WithRef(plumbing.Master.String()), + me.WithCommit(commit.String()), ) - expectedBlobContent = Must(testData.ReadFile(filepath.Join("testdata", "repo", "file_in_repo"))) + expectedBlobContent = Must(testData.ReadFile(path)) }) - It("downloads artifacts", func() { + It("downloads artifacts with full ref", func() { + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) + content := Must(m.Get()) + unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) + + r := tar.NewReader(unzippedContent) + + file := Must(r.Next()) + Expect(file.Name).To(Equal("file_in_repo")) + Expect(file.Size).To(Equal(int64(len(expectedBlobContent)))) + + data := Must(io.ReadAll(r)) + Expect(data).To(Equal(expectedBlobContent)) + }) + + It("downloads artifacts without commit because the url reference is enough", func() { + accessSpec = me.New(fmt.Sprintf("file://%s", repoDir), me.WithRef(plumbing.Master.String())) + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) content := Must(m.Get()) unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) @@ -109,4 +128,40 @@ var _ = Describe("Method", func() { data := Must(io.ReadAll(r)) Expect(data).To(Equal(expectedBlobContent)) }) + + It("cannot download artifacts ref without a reference", func() { + accessSpec = me.New(fmt.Sprintf("file://%s", repoDir), me.WithCommit(accessSpec.Commit)) + + _, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid reference name")) + }) +}) + +var _ = Describe("Method based on Real Repository", func() { + host := "github.com:443" + reachable := PingTCPServer(host, time.Second) == nil + var url string + BeforeEach(func() { + if !reachable { + Skip(fmt.Sprintf("no connection to %s, skipping test connection to remote", url)) + } + // This repo is a public repo owned by the Github Kraken Bot, so its as good of a public available + // example as any. + url = fmt.Sprintf("https://%s/octocat/Hello-World.git", host) + }) + + It("can download remote artifacts", func() { + ctx := ocm.New() + accessSpec := me.New(url, me.WithRef(plumbing.Master.String())) + + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) + content := Must(m.Get()) + unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) + + r := tar.NewReader(unzippedContent) + + file := Must(r.Next()) + Expect(file.Name).To(Equal("README")) + }) }) diff --git a/api/tech/git/identity/identity.go b/api/tech/git/identity/identity.go index 5f924dd0e8..20ae3fe257 100644 --- a/api/tech/git/identity/identity.go +++ b/api/tech/git/identity/identity.go @@ -1,7 +1,7 @@ package identity import ( - "strings" + "net" giturls "github.com/whilp/git-urls" @@ -78,9 +78,8 @@ func GetConsumerId(repoURL string) (cpi.ConsumerIdentity, error) { } } - if idx := strings.Index(host, ":"); idx > 0 { - port = host[idx+1:] - host = host[:idx] + if h, p, err := net.SplitHostPort(host); err == nil { + host, port = h, p } id := cpi.ConsumerIdentity{ diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index 2ddc996be0..ea1016eea2 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -135,17 +135,23 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { return nil, err } + depth := 0 + if c.opts.Commit == "" { + depth = 1 // if we have no dedicated commit we can checkout HEAD, and thus a shallow clone is ok + } + newRepo := false repo, err := git.Open(strg, billyFS) if errors.Is(err, git.ErrRepositoryNotExists) { repo, err = git.CloneContext(ctx, strg, billyFS, &git.CloneOptions{ - Auth: c.opts.AuthMethod, - URL: c.opts.URL, - RemoteName: git.DefaultRemoteName, - ReferenceName: plumbing.ReferenceName(c.opts.Ref), - SingleBranch: true, - Depth: 0, - Tags: git.AllTags, + Auth: c.opts.AuthMethod, + URL: c.opts.URL, + RemoteName: git.DefaultRemoteName, + ReferenceName: plumbing.ReferenceName(c.opts.Ref), + SingleBranch: true, + Depth: depth, + ShallowSubmodules: depth == 1, + Tags: git.AllTags, }) newRepo = true } diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go index 1e12ae5da2..b664f261eb 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go @@ -1,25 +1,24 @@ package git_test import ( - "io" + "fmt" "os" . "github.com/mandelsoft/goutils/testutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "ocm.software/ocm/api/utils/tarutils" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" + "ocm.software/ocm/api/utils/accessio" . "ocm.software/ocm/cmds/ocm/testhelper" - - "ocm.software/ocm/api/ocm/compdesc" - "ocm.software/ocm/api/ocm/extensions/accessmethods/localblob" - "ocm.software/ocm/api/ocm/extensions/repositories/comparch" - "ocm.software/ocm/api/utils/mime" ) const ( - ARCH = "test.ca" - VERSION = "v1" + ARCH = "test.ctf" + CONSTRUCTOR = "component-constructor.yaml" + VERSION = "v1" ) var _ = Describe("Test Environment", func() { @@ -27,21 +26,6 @@ var _ = Describe("Test Environment", func() { BeforeEach(func() { env = NewTestEnv(TestData()) - - Expect(env.Execute( - "create", - "ca", - "-ft", - "directory", - "test.de/x", - VERSION, - "--provider", - "ocm", - "--file", - ARCH, - "--scheme", - "ocm.software/v3alpha1", - )).To(Succeed()) }) AfterEach(func() { @@ -49,91 +33,93 @@ var _ = Describe("Test Environment", func() { }) It("add git repo described by access type specification", func() { - meta := ` -name: hello-world -type: git -` + constructor := fmt.Sprintf(`--- +name: test.de/x +version: %s +provider: + name: ocm +resources: +- name: hello-world + type: git + version: 0.0.1 + access: + type: git + commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d" + ref: refs/heads/master + repository: https://github.com/octocat/Hello-World.git +`, VERSION) + Expect( + env.WriteFile(CONSTRUCTOR, []byte(constructor), os.ModePerm), + ).To(Succeed()) + Expect(env.Execute( - "add", "resources", - "--file", ARCH, - "--resource", meta, - "--accessType", "git", - "--accessRepository", "https://github.com/octocat/Hello-World.git", - "--reference", "refs/heads/master", - "--commit", "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", - "--version", "0.0.1", + "add", + "cv", + "--create", + "--file", + ARCH, + "--force", + "--type", + "directory", + CONSTRUCTOR, )).To(Succeed()) - data, err := env.ReadFile(env.Join(ARCH, comparch.ComponentDescriptorFileName)) - Expect(err).To(Succeed()) - cd, err := compdesc.Decode(data) - Expect(err).To(Succeed()) + ctx := ocm.New() + vfsattr.Set(ctx, env.FileSystem()) + r := Must(ctf.Open(ctx, ctf.ACC_READONLY, ARCH, 0o400, accessio.FormatDirectory, accessio.PathFileSystem(env.FileSystem()))) + DeferCleanup(r.Close) + + c := Must(r.LookupComponent("test.de/x")) + DeferCleanup(c.Close) + cv := Must(c.LookupVersion(VERSION)) + DeferCleanup(cv.Close) + cd := cv.GetDescriptor() Expect(len(cd.Resources)).To(Equal(1)) }) It("add git repo described by cli options through blob access via input described in file", func() { - meta := ` -name: hello-world -type: git -` - Expect(env.Execute( - "add", "resources", - "--file", ARCH, - "--resource", meta, - "--inputType", "git", - "--inputVersion", "refs/heads/master", - "--inputRepository", "https://github.com/octocat/Hello-World.git", - )).To(Succeed()) - data, err := env.ReadFile(env.Join(ARCH, comparch.ComponentDescriptorFileName)) - Expect(err).To(Succeed()) - cd, err := compdesc.Decode(data) - Expect(err).To(Succeed()) - Expect(len(cd.Resources)).To(Equal(1)) - access := Must(env.Context.OCMContext().AccessSpecForSpec(cd.Resources[0].Access)).(*localblob.AccessSpec) - Expect(access.MediaType).To(Equal(mime.MIME_TGZ)) - fi := Must(env.FileSystem().Stat(env.Join(ARCH, "blobs", access.LocalReference))) - Expect(fi.Size()).To(Equal(int64(106))) - - Expect(tarutils.ExtractArchiveToFs(env.FileSystem(), env.Join(ARCH, "blobs", access.LocalReference), env.FileSystem())).To(Succeed()) - - readMeFi := Must(env.FileSystem().Stat("README")) - Expect(readMeFi.Size()).To(Equal(int64(13))) - readMe := Must(env.FileSystem().OpenFile("README", os.O_RDONLY, 0o400)) - defer readMe.Close() - Expect(string(Must(io.ReadAll(readMe)))).To(Equal("Hello World!\n")) - }) + constructor := fmt.Sprintf(`--- +name: test.de/x +version: %s +provider: + name: ocm +resources: +- name: hello-world + type: git + version: 0.0.1 + input: + type: git + ref: refs/heads/master + commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d" + repository: https://github.com/octocat/Hello-World.git +`, VERSION) + Expect( + env.WriteFile(CONSTRUCTOR, []byte(constructor), os.ModePerm), + ).To(Succeed()) - It("add git repo described by cli options through blob access via input", func() { - meta := ` -name: hello-world -type: git -` Expect(env.Execute( - "add", "resources", - "--file", ARCH, - "--resource", meta, - "--inputType", "git", - "--inputVersion", "refs/heads/master", - "--inputRepository", "https://github.com/octocat/Hello-World.git", + "add", + "cv", + "--file", + ARCH, + "--create", + "--force", + "--type", + "directory", + CONSTRUCTOR, )).To(Succeed()) - data, err := env.ReadFile(env.Join(ARCH, comparch.ComponentDescriptorFileName)) - Expect(err).To(Succeed()) - cd, err := compdesc.Decode(data) - Expect(err).To(Succeed()) - Expect(len(cd.Resources)).To(Equal(1)) - - access := Must(env.Context.OCMContext().AccessSpecForSpec(cd.Resources[0].Access)).(*localblob.AccessSpec) - Expect(access.MediaType).To(Equal(mime.MIME_TGZ)) - fi := Must(env.FileSystem().Stat(env.Join(ARCH, "blobs", access.LocalReference))) - Expect(fi.Size()).To(Equal(int64(106))) - Expect(tarutils.ExtractArchiveToFs(env.FileSystem(), env.Join(ARCH, "blobs", access.LocalReference), env.FileSystem())).To(Succeed()) + ctx := ocm.New() + vfsattr.Set(ctx, env.FileSystem()) + r := Must(ctf.Open(ctx, ctf.ACC_READONLY, ARCH, 0o400, accessio.FormatDirectory, accessio.PathFileSystem(env.FileSystem()))) + DeferCleanup(r.Close) - readMeFi := Must(env.FileSystem().Stat("README")) - Expect(readMeFi.Size()).To(Equal(int64(13))) - readMe := Must(env.FileSystem().OpenFile("README", os.O_RDONLY, 0o400)) - defer readMe.Close() - Expect(string(Must(io.ReadAll(readMe)))).To(Equal("Hello World!\n")) + c := Must(r.LookupComponent("test.de/x")) + DeferCleanup(c.Close) + cv := Must(c.LookupVersion(VERSION)) + DeferCleanup(cv.Close) + cd := cv.GetDescriptor() + Expect(len(cd.Resources)).To(Equal(1)) }) }) diff --git a/docs/reference/ocm_add_resource-configuration.md b/docs/reference/ocm_add_resource-configuration.md index 8d8c28441d..2d84437b48 100644 --- a/docs/reference/ocm_add_resource-configuration.md +++ b/docs/reference/ocm_add_resource-configuration.md @@ -636,7 +636,7 @@ shown below. Git repository. The following versions are supported: - - Version v1 + - Version v1alpha1 The type specific specification fields are: diff --git a/docs/reference/ocm_add_resources.md b/docs/reference/ocm_add_resources.md index 3e9754bd01..f350c6d481 100644 --- a/docs/reference/ocm_add_resources.md +++ b/docs/reference/ocm_add_resources.md @@ -648,7 +648,7 @@ shown below. Git repository. The following versions are supported: - - Version v1 + - Version v1alpha1 The type specific specification fields are: diff --git a/docs/reference/ocm_add_source-configuration.md b/docs/reference/ocm_add_source-configuration.md index 4090cbb55a..ecec431c58 100644 --- a/docs/reference/ocm_add_source-configuration.md +++ b/docs/reference/ocm_add_source-configuration.md @@ -636,7 +636,7 @@ shown below. Git repository. The following versions are supported: - - Version v1 + - Version v1alpha1 The type specific specification fields are: diff --git a/docs/reference/ocm_add_sources.md b/docs/reference/ocm_add_sources.md index e6af2717b9..4434cecd57 100644 --- a/docs/reference/ocm_add_sources.md +++ b/docs/reference/ocm_add_sources.md @@ -646,7 +646,7 @@ shown below. Git repository. The following versions are supported: - - Version v1 + - Version v1alpha1 The type specific specification fields are: diff --git a/docs/reference/ocm_ocm-accessmethods.md b/docs/reference/ocm_ocm-accessmethods.md index 2cd7c29fa2..ba86b327a5 100644 --- a/docs/reference/ocm_ocm-accessmethods.md +++ b/docs/reference/ocm_ocm-accessmethods.md @@ -21,7 +21,7 @@ shown below. Git repository. The following versions are supported: - - Version v1 + - Version v1alpha1 The type specific specification fields are: From 46a9e2c15cc24d14647025c959547b2ed069b7ca Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Tue, 17 Dec 2024 12:19:48 +0100 Subject: [PATCH 21/22] chore: cleanup resolver/downloader because no update is needed --- .../extensions/accessmethods/git/README.md | 3 +- .../extensions/accessmethods/git/method.go | 2 + api/tech/git/fs.go | 45 ++---- api/tech/git/logging.go | 2 - api/tech/git/resolver.go | 128 ++---------------- api/tech/git/resolver_test.go | 112 +++++++++++++++ api/tech/git/suite_test.go | 13 ++ api/tech/git/testdata/repo/file_in_repo | 1 + .../accessio/downloader/git/downloader.go | 112 --------------- api/utils/blobaccess/git/access.go | 2 +- api/utils/blobaccess/git/options.go | 3 - go.mod | 1 - go.sum | 2 - 13 files changed, 149 insertions(+), 277 deletions(-) create mode 100644 api/tech/git/resolver_test.go create mode 100644 api/tech/git/suite_test.go create mode 100644 api/tech/git/testdata/repo/file_in_repo delete mode 100644 api/utils/accessio/downloader/git/downloader.go diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md index 0a4f404d1d..6be153ac10 100644 --- a/api/ocm/extensions/accessmethods/git/README.md +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -40,7 +40,6 @@ The type specific specification fields are: The go binding can be found [here](method.go) - #### Example ```go @@ -91,4 +90,4 @@ func main() { } fmt.Println(string(data)) } -``` \ No newline at end of file +``` diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index e8c5d927ac..59fc39e64e 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -24,6 +24,8 @@ const ( ) func init() { + // If we remove the default registration, also the docs are gone. + // so we leave the default registration in. accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage))) accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1Alpha1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go index c3bc5a84c0..bfbb4c5386 100644 --- a/api/tech/git/fs.go +++ b/api/tech/git/fs.go @@ -3,15 +3,13 @@ package git import ( "errors" "fmt" - "hash/fnv" "os" "path/filepath" + "sync" "syscall" "github.com/go-git/go-billy/v5" - "github.com/juju/fslock" "github.com/mandelsoft/vfs/pkg/memoryfs" - "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/projectionfs" "github.com/mandelsoft/vfs/pkg/vfs" ) @@ -36,19 +34,24 @@ type fs struct { var _ billy.Filesystem = &fs{} +// file is a wrapper around a vfs.File that implements billy.File. +// it uses a mutex to lock the file, so it can be used concurrently from the same process, but +// not across processes (like a flock). type file struct { - lock *fslock.Lock vfs.File + lockMu sync.Mutex } var _ billy.File = &file{} func (f *file) Lock() error { - return f.lock.Lock() + f.lockMu.Lock() + return nil } func (f *file) Unlock() error { - return f.lock.Unlock() + f.lockMu.Unlock() + return nil } var _ billy.File = &file{} @@ -62,39 +65,9 @@ func (f *fs) Create(filename string) (billy.File, error) { } // vfsToBillyFileInfo converts a vfs.File to a billy.File -// It also creates a fslock.Lock for the file to ensure that the file is lockable -// If the vfs is an osfs.OsFs, the lock is created in the same directory as the file -// If the vfs is not an osfs.OsFs, a temporary directory is created to store the lock -// because its not trivial to store the lock for jujufs on a virtual filesystem because -// juju vfs only operates on syscalls directly and without interface abstraction its not easy to get the root. func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { - var lock *fslock.Lock - if f.FileSystem == osfs.OsFs { - lock = fslock.New(fmt.Sprintf("%s.lock", vf.Name())) - } else { - hash := fnv.New32() - _, _ = hash.Write([]byte(f.FileSystem.Name())) - temp, err := os.MkdirTemp("", fmt.Sprintf("git-vfs-locks-%x", hash.Sum32())) - if err != nil { - return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+ - "this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err) - } - _, components := vfs.Components(f.FileSystem, vf.Name()) - lockPath := filepath.Join( - temp, - filepath.Join(components[:len(components)-1]...), - fmt.Sprintf("%s.lock", components[len(components)-1]), - ) - if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { - return nil, fmt.Errorf("failed to create temp dir to allow mapping vfs to git (billy) filesystem; "+ - "this temporary directory is mandatory because a virtual filesystem cannot be used to accurately depict os syslocks: %w", err) - } - lock = fslock.New(lockPath) - } - return &file{ File: vf, - lock: lock, }, nil } diff --git a/api/tech/git/logging.go b/api/tech/git/logging.go index d5f6dfd3d0..1fce027755 100644 --- a/api/tech/git/logging.go +++ b/api/tech/git/logging.go @@ -3,5 +3,3 @@ package git import "ocm.software/ocm/api/utils/logging" var REALM = logging.DefineSubRealm("git repository", "git") - -var Log = logging.DynamicLogger(REALM) diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go index ea1016eea2..ef8098ee0b 100644 --- a/api/tech/git/resolver.go +++ b/api/tech/git/resolver.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/http" - "os" "sync" "github.com/go-git/go-billy/v5" @@ -34,7 +33,7 @@ func init() { gitclient.InstallProtocol("https", githttp.NewClient(&http.Client{ Transport: logging.NewRoundTripper(http.DefaultTransport, logging.DynamicLogger(REALM, mlog.NewAttribute(LogAttrProtocol, "https"))), })) - //TODO Determine how we ideally log for ssh+git protocol + // TODO Determine how we ideally log for ssh+git protocol } var DefaultWorktreeBranch = plumbing.NewBranchReferenceName("ocm") @@ -63,17 +62,10 @@ type Client interface { // given AuthMethod. Repository(ctx context.Context) (*git.Repository, error) - // Refresh will attempt to fetch & pull the latest changes from the remote repository. - // In case there are no changes, it will do a no-op after having realized that no changes are in the remote. - Refresh(ctx context.Context) error - - // Update will stage all changes in the repository, commit them with the given message and push them to the remote repository. - Update(ctx context.Context, msg string, push bool) error - // Setup will override the current filesystem with the given filesystem. This will be the filesystem where the repository will be stored. // There can be only one filesystem per client. // If the filesystem contains a repository already, it can be consumed by a subsequent call to Repository. - Setup(vfs.FileSystem) error + Setup(context.Context, vfs.FileSystem) error } type ClientOptions struct { @@ -88,24 +80,15 @@ type ClientOptions struct { // Commit is the commit hash to checkout after cloning the repository. // If empty, it will default to the plumbing.HEAD of the Ref. Commit string - // Author is the author to use for commits. If empty, it will default to the git config of the user running the process. - Author // AuthMethod is the authentication method to use for the repository. AuthMethod AuthMethod } -type Author struct { - Name string - Email string -} - var _ Client = &client{} func NewClient(opts ClientOptions) (Client, error) { - var pref plumbing.ReferenceName - if opts.Ref == "" { - pref = plumbing.HEAD - } else { + pref := plumbing.HEAD + if opts.Ref != "" { pref = plumbing.ReferenceName(opts.Ref) if err := pref.Validate(); err != nil { return nil, fmt.Errorf("invalid reference %q: %w", opts.Ref, err) @@ -156,7 +139,10 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { newRepo = true } if errors.Is(err, transport.ErrEmptyRemoteRepository) { - return git.Open(strg, billyFS) + repo, err = git.Open(strg, billyFS) + if err != nil { + return nil, fmt.Errorf("failed to open repository based on URL %q after it was determined to be an empty clone: %w", c.opts.URL, err) + } } if err != nil { @@ -168,10 +154,6 @@ func (c *client) Repository(ctx context.Context) (*git.Repository, error) { } } - if err := c.opts.applyToRepo(repo); err != nil { - return nil, err - } - c.repo = repo return repo, nil @@ -221,100 +203,10 @@ func GetStorage(base billy.Filesystem) (storage.Storer, error) { ), nil } -func (c *client) TopLevelDirs(ctx context.Context) ([]os.FileInfo, error) { - repo, err := c.Repository(ctx) - if err != nil { - return nil, err - } - - fs, err := repo.Worktree() - if err != nil { - return nil, err - } - - return fs.Filesystem.ReadDir(".") -} - -func (c *client) Refresh(ctx context.Context) error { - repo, err := c.Repository(ctx) - if err != nil { - return err - } - - worktree, err := repo.Worktree() - if err != nil { - return err - } - - if err := worktree.PullContext(ctx, &git.PullOptions{ - Auth: c.opts.AuthMethod, - RemoteName: git.DefaultRemoteName, - }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) && !errors.Is(err, transport.ErrEmptyRemoteRepository) { - return err - } - - return nil -} - -func (c *client) Update(ctx context.Context, msg string, push bool) error { - repo, err := c.Repository(ctx) - if err != nil { - return err - } - - worktree, err := repo.Worktree() - if err != nil { - return err - } - - if err = worktree.AddGlob("*"); err != nil { - return err - } - - _, err = worktree.Commit(msg, &git.CommitOptions{}) - - if errors.Is(err, git.ErrEmptyCommit) { - return nil - } - - if err != nil { - return err - } - - if !push { - return nil - } - - if err := repo.PushContext(ctx, &git.PushOptions{ - RemoteName: git.DefaultRemoteName, - }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { - return err - } - - return nil -} - -func (c *client) Setup(system vfs.FileSystem) error { +func (c *client) Setup(ctx context.Context, system vfs.FileSystem) error { c.vfs = system - if _, err := c.Repository(context.Background()); err != nil { + if _, err := c.Repository(ctx); err != nil { return fmt.Errorf("failed to setup repository %q: %w", c.opts.URL, err) } return nil } - -func (o ClientOptions) applyToRepo(repo *git.Repository) error { - cfg, err := repo.Config() - if err != nil { - return err - } - - if o.Author.Name != "" { - cfg.User.Name = o.Author.Name - } - - if o.Author.Email != "" { - cfg.User.Email = o.Author.Email - } - - return repo.SetConfig(cfg) -} diff --git a/api/tech/git/resolver_test.go b/api/tech/git/resolver_test.go new file mode 100644 index 0000000000..d128d07808 --- /dev/null +++ b/api/tech/git/resolver_test.go @@ -0,0 +1,112 @@ +package git_test + +import ( + "embed" + "fmt" + "io" + "os" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/mandelsoft/filepath/pkg/filepath" + . "github.com/mandelsoft/goutils/testutils" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + self "ocm.software/ocm/api/tech/git" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("standard tests with local file repo", func() { + var ( + ctx ocm.Context + expectedBlobContent []byte + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + var repoDir string + var repoURL string + var ref string + var commit string + + BeforeEach(func() { + repoDir = GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo := Must(git.PlainInit(repoDir, false)) + + repoBase := filepath.Join("testdata", "repo") + repoTestData := Must(testData.ReadDir(repoBase)) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file := Must(testData.Open(path)) + + fileInRepo := Must(os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0o600, + )) + + Must(io.Copy(fileInRepo, file)) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt := Must(repo.Worktree()) + Expect(wt.AddGlob("*")).To(Succeed()) + commit = Must(wt.Commit("OCM Test Commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "OCM Test", + Email: "dummy@ocm.software", + When: time.Now(), + }, + })).String() + + path := filepath.Join("testdata", "repo", "file_in_repo") + repoURL = fmt.Sprintf("file://%s", repoDir) + ref = plumbing.Master.String() + + expectedBlobContent = Must(testData.ReadFile(path)) + }) + + It("Resolver client can setup repository", func(ctx SpecContext) { + client := Must(self.NewClient(self.ClientOptions{ + URL: repoURL, + Ref: ref, + Commit: commit, + })) + + tempVFS, err := projectionfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + + Expect(client.Setup(ctx, tempVFS)).To(Succeed()) + + repo := Must(client.Repository(ctx)) + Expect(repo).ToNot(BeNil()) + + file := Must(tempVFS.Stat("file_in_repo")) + Expect(file.Size()).To(Equal(int64(len(expectedBlobContent)))) + + }) +}) diff --git a/api/tech/git/suite_test.go b/api/tech/git/suite_test.go new file mode 100644 index 0000000000..73e11ee635 --- /dev/null +++ b/api/tech/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCM Git Tech Test Suite") +} diff --git a/api/tech/git/testdata/repo/file_in_repo b/api/tech/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/tech/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/api/utils/accessio/downloader/git/downloader.go b/api/utils/accessio/downloader/git/downloader.go deleted file mode 100644 index 184805ba33..0000000000 --- a/api/utils/accessio/downloader/git/downloader.go +++ /dev/null @@ -1,112 +0,0 @@ -package git - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "regexp" - "sync" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/storage" - "github.com/go-git/go-git/v5/storage/memory" - - techgit "ocm.software/ocm/api/tech/git" - "ocm.software/ocm/api/utils/accessio/downloader" -) - -const localRemoteName = "origin" - -type CloseableDownloader interface { - downloader.Downloader - Close() error -} - -// Downloader simply uses the default HTTP client to download the contents of a URL. -type Downloader struct { - cloneOpts *git.CloneOptions - - matching *regexp.Regexp - - mu sync.Mutex - buf *bytes.Buffer - storage storage.Storer -} - -var _ downloader.Downloader = (*Downloader)(nil) - -func NewDownloader(url string, ref string, path string, auth techgit.AuthMethod) CloseableDownloader { - refName := plumbing.ReferenceName(ref) - return &Downloader{ - cloneOpts: &git.CloneOptions{ - Auth: auth, - URL: url, - RemoteName: localRemoteName, - ReferenceName: refName, - SingleBranch: true, - Tags: git.NoTags, - Depth: 0, - }, - matching: regexp.MustCompile(path), - buf: bytes.NewBuffer(make([]byte, 0, 4096)), - storage: memory.NewStorage(), - } -} - -func (d *Downloader) Download(w io.WriterAt) error { - d.mu.Lock() - defer d.mu.Unlock() - - ctx := context.Background() - - // no support for git archive yet, so we need to clone the repository in bare mode - repo, err := git.CloneContext(ctx, d.storage, nil, d.cloneOpts) - if err != nil { - return fmt.Errorf("failed to clone repository %s: %w", d.cloneOpts.URL, err) - } - - trees, err := repo.TreeObjects() - if err != nil { - return fmt.Errorf("failed to get tree objects: %w", err) - } - - if err := trees.ForEach(func(t *object.Tree) error { - return t.Files().ForEach(d.copyFileToBuffer) - }); err != nil { - return fmt.Errorf("failed to iterate over trees: %w", err) - } - - defer d.buf.Reset() - if _, err := w.WriteAt(d.buf.Bytes(), 0); err != nil { - return fmt.Errorf("failed to write blobs: %w", err) - } - - return nil -} - -func (d *Downloader) copyFileToBuffer(file *object.File) error { - if !d.matching.MatchString(file.Name) { - return nil - } - - reader, err := file.Reader() - if err != nil { - return err - } - _, err = io.Copy(d.buf, reader) - return errors.Join(err, reader.Close()) -} - -func (d *Downloader) Close() error { - d.mu.Lock() - defer d.mu.Unlock() - - d.buf = nil - d.storage = nil - - return nil -} diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go index 96aa173f68..d9aba5b631 100644 --- a/api/utils/blobaccess/git/access.go +++ b/api/utils/blobaccess/git/access.go @@ -61,7 +61,7 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { }) // redirect the client to the temporary filesystem for storage of the repo, otherwise it would use memory - if err := c.Setup(repositoryFS); err != nil { + if err := c.Setup(context.Background(), repositoryFS); err != nil { return nil, err } diff --git a/api/utils/blobaccess/git/options.go b/api/utils/blobaccess/git/options.go index fa43a1ed60..44ebd7cd14 100644 --- a/api/utils/blobaccess/git/options.go +++ b/api/utils/blobaccess/git/options.go @@ -59,9 +59,6 @@ func (o *Options) ApplyTo(opts *Options) { if o.URL != "" { opts.URL = o.URL } - if o.Author.Name != "" && o.Author.Email != "" { - opts.Author = o.Author - } if o.Ref != "" { opts.Ref = o.Ref } diff --git a/go.mod b/go.mod index 9c31445818..849c503d7e 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-github/v45 v45.2.0 github.com/hashicorp/vault-client-go v0.4.3 - github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b github.com/klauspost/compress v1.17.11 github.com/klauspost/pgzip v1.2.6 github.com/mandelsoft/filepath v0.0.0-20240223090642-3e2777258aa3 diff --git a/go.sum b/go.sum index a023338284..d75f2b78ba 100644 --- a/go.sum +++ b/go.sum @@ -674,8 +674,6 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b h1:FQ7+9fxhyp82ks9vAuyPzG0/vVbWwMwLJ+P6yJI5FN8= -github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b/go.mod h1:HMcgvsgd0Fjj4XXDkbjdmlbI505rUPBs6WBMYg2pXks= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= From 407988a06dd13a781be6eb8be5d0e5b4d471842d Mon Sep 17 00:00:00 2001 From: jakobmoellerdev Date: Wed, 18 Dec 2024 10:00:41 +0100 Subject: [PATCH 22/22] chore: fixup inconsistent access method --- .../extensions/accessmethods/git/README.md | 6 ++--- .../extensions/accessmethods/git/method.go | 22 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md index 6be153ac10..53adeffbd9 100644 --- a/api/ocm/extensions/accessmethods/git/README.md +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -16,15 +16,15 @@ The artifact content is provided as gnu-zipped tar archive This method implements the access of the content of a git commit stored in a git repository. -Supported specification version is `v1` +Supported specification version is `v1alpha1` ### Specification Versions -#### Version `v1` +#### Version `v1alpha1` The type specific specification fields are: -- **`repoUrl`** *string* +- **`repository`** *string* Repository URL with or without scheme. diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go index 59fc39e64e..b43b96841e 100644 --- a/api/ocm/extensions/accessmethods/git/method.go +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -34,14 +34,14 @@ func init() { type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` - // RepoURL is the repository URL - RepoURL string `json:"repository"` + // Repository is the repository URL + Repository string `json:"repository"` // Ref defines the hash of the commit - Ref string `json:"ref"` + Ref string `json:"ref,omitempty"` // Commit defines the hash of the commit in string format to checkout from the Ref - Commit string `json:"commit"` + Commit string `json:"commit,omitempty"` } // AccessSpecOptions defines a set of options which can be applied to the access spec. @@ -63,7 +63,7 @@ func WithRef(ref string) AccessSpecOptions { func New(url string, opts ...AccessSpecOptions) *AccessSpec { s := &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), - RepoURL: url, + Repository: url, } for _, o := range opts { o(s) @@ -72,7 +72,7 @@ func New(url string, opts ...AccessSpecOptions) *AccessSpec { } func (a *AccessSpec) Describe(internal.Context) string { - return fmt.Sprintf("git commit %s[%s]", a.RepoURL, a.Ref) + return fmt.Sprintf("git commit %s[%s]", a.Repository, a.Ref) } func (*AccessSpec) IsLocal(internal.Context) bool { @@ -88,16 +88,16 @@ func (*AccessSpec) GetType() string { } func (a *AccessSpec) AccessMethod(cva internal.ComponentVersionAccess) (internal.AccessMethod, error) { - _, err := giturls.Parse(a.RepoURL) + _, err := giturls.Parse(a.Repository) if err != nil { - return nil, errors.ErrInvalidWrap(err, "repository repoURL", a.RepoURL) + return nil, errors.ErrInvalidWrap(err, "repository repoURL", a.Repository) } if err := plumbing.ReferenceName(a.Ref).Validate(); err != nil { return nil, errors.ErrInvalidWrap(err, "commit hash", a.Ref) } - creds, _, err := getCreds(a.RepoURL, cva.GetContext().CredentialsContext()) + creds, _, err := getCreds(a.Repository, cva.GetContext().CredentialsContext()) if err != nil { - return nil, fmt.Errorf("failed to get credentials for repository %s: %w", a.RepoURL, err) + return nil, fmt.Errorf("failed to get credentials for repository %s: %w", a.Repository, err) } octx := cva.GetContext() @@ -105,7 +105,7 @@ func (a *AccessSpec) AccessMethod(cva internal.ComponentVersionAccess) (internal opts := []gitblob.Option{ gitblob.WithLoggingContext(octx), gitblob.WithCredentialContext(octx), - gitblob.WithURL(a.RepoURL), + gitblob.WithURL(a.Repository), gitblob.WithRef(a.Ref), gitblob.WithCommit(a.Commit), gitblob.WithCachingFileSystem(vfsattr.Get(octx)),