This commit adds pull request support to SCM generator so the generator

can create ArgoCD apps for PRs as well.

Fixes #466

Signed-off-by: Fardin Khanjani <[email protected]>

api/v1alpha1/applicationset_types.go

@@ -314,6 +314,8 @@ type SCMProviderGeneratorGithub struct {
 	TokenRef *SecretRef `json:"tokenRef,omitempty"`
 	// Scan all branches instead of just the default branch.
 	AllBranches bool `json:"allBranches,omitempty"`
+	// Scan all pull requests
+	AllPullRequests bool `json:"allPullRequests,omitempty"`
 }
 
 // SCMProviderGeneratorGitlab defines a connection info specific to Gitlab.
@@ -342,6 +344,10 @@ type SCMProviderGeneratorFilter struct {
 	LabelMatch *string `json:"labelMatch,omitempty"`
 	// A regex which must match the branch name.
 	BranchMatch *string `json:"branchMatch,omitempty"`
+	// A regex which must match the pull request tile.
+	PullRequestTitleMatch *string `json:"pullRequestTitleMatch,omitempty"`
+	// A regex which must match at least one pull request label.
+	PullRequestLabelMatch *string `json:"pullRequestLabelMatch,omitempty"`
 }
 
 // PullRequestGenerator defines a generator that scrapes a PullRequest API to find candidate pull requests.

manifests/crds/argoproj.io_applicationsets.yaml

@@ -2714,6 +2714,8 @@ spec:
                                     properties:
                                       allBranches:
                                         type: boolean
+                                      allPullRequests:
+                                        type: boolean
                                       api:
                                         type: string
                                       organization:
@@ -4800,6 +4802,8 @@ spec:
                                     properties:
                                       allBranches:
                                         type: boolean
+                                      allPullRequests:
+                                        type: boolean
                                       api:
                                         type: string
                                       organization:
@@ -5707,6 +5711,8 @@ spec:
                           properties:
                             allBranches:
                               type: boolean
+                            allPullRequests:
+                              type: boolean
                             api:
                               type: string
                             organization:

manifests/install-with-argo-cd.yaml

@@ -4531,6 +4531,8 @@ spec:
                                     properties:
                                       allBranches:
                                         type: boolean
+                                      allPullRequests:
+                                        type: boolean
                                       api:
                                         type: string
                                       organization:
@@ -6617,6 +6619,8 @@ spec:
                                     properties:
                                       allBranches:
                                         type: boolean
+                                      allPullRequests:
+                                        type: boolean
                                       api:
                                         type: string
                                       organization:
@@ -7524,6 +7528,8 @@ spec:
                           properties:
                             allBranches:
                               type: boolean
+                            allPullRequests:
+                              type: boolean
                             api:
                               type: string
                             organization:

manifests/install.yaml

@@ -2713,6 +2713,8 @@ spec:
                                     properties:
                                       allBranches:
                                         type: boolean
+                                      allPullRequests:
+                                        type: boolean
                                       api:
                                         type: string
                                       organization:
@@ -4799,6 +4801,8 @@ spec:
                                     properties:
                                       allBranches:
                                         type: boolean
+                                      allPullRequests:
+                                        type: boolean
                                       api:
                                         type: string
                                       organization:
@@ -5706,6 +5710,8 @@ spec:
                           properties:
                             allBranches:
                               type: boolean
+                            allPullRequests:
+                              type: boolean
                             api:
                               type: string
                             organization:

pkg/generators/scm_provider.go

@@ -64,7 +64,7 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha
 		if err != nil {
 			return nil, fmt.Errorf("error fetching Github token: %v", err)
 		}
-		provider, err = scm_provider.NewGithubProvider(ctx, providerConfig.Github.Organization, token, providerConfig.Github.API, providerConfig.Github.AllBranches)
+		provider, err = scm_provider.NewGithubProvider(ctx, providerConfig.Github.Organization, token, providerConfig.Github.API, providerConfig.Github.AllBranches, providerConfig.Github.AllPullRequests)
 		if err != nil {
 			return nil, fmt.Errorf("error initializing Github service: %v", err)
 		}

pkg/services/scm_provider/github.go

@@ -4,21 +4,21 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
-
 	"github.com/google/go-github/v35/github"
 	"golang.org/x/oauth2"
+	"os"
 )
 
 type GithubProvider struct {
-	client       *github.Client
-	organization string
-	allBranches  bool
+	client          *github.Client
+	organization    string
+	allBranches     bool
+	allPullRequests bool
 }
 
 var _ SCMProviderService = &GithubProvider{}
 
-func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool) (*GithubProvider, error) {
+func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool, allPullRequests bool) (*GithubProvider, error) {
 	var ts oauth2.TokenSource
 	// Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits.
 	if token == "" {
@@ -40,7 +40,7 @@ func NewGithubProvider(ctx context.Context, organization string, token string, u
 			return nil, err
 		}
 	}
-	return &GithubProvider{client: client, organization: organization, allBranches: allBranches}, nil
+	return &GithubProvider{client: client, organization: organization, allBranches: allBranches, allPullRequests: allPullRequests}, nil
 }
 
 func (g *GithubProvider) GetBranches(ctx context.Context, repo *Repository) ([]*Repository, error) {
@@ -64,6 +64,32 @@ func (g *GithubProvider) GetBranches(ctx context.Context, repo *Repository) ([]*
 	return repos, nil
 }
 
+func (g *GithubProvider) GetPullRequests(ctx context.Context, repo *Repository) ([]*Repository, error) {
+	repos := []*Repository{}
+	pullRequests, err := g.listPullRequests(ctx, repo)
+	if err != nil {
+		return nil, fmt.Errorf("error listing pull requests for %s/%s: %v", repo.Organization, repo.Repository, err)
+	}
+
+	// go-github's PullRequest type does not have a GetLabel() function.
+	var labels []string
+	for _, pullRequest := range pullRequests {
+		for _, label := range pullRequest.Labels {
+			labels = append(labels, label.GetName())
+		}
+		repos = append(repos, &Repository{
+			Organization: repo.Organization,
+			Repository:   repo.Repository,
+			URL:          repo.URL,
+			Branch:       pullRequest.GetTitle(),
+			SHA:          pullRequest.GetHead().GetSHA(),
+			Labels:       labels,
+			RepositoryId: repo.RepositoryId,
+		})
+	}
+	return repos, nil
+}
+
 func (g *GithubProvider) ListRepos(ctx context.Context, cloneProtocol string) ([]*Repository, error) {
 	opt := &github.RepositoryListByOrgOptions{
 		ListOptions: github.ListOptions{PerPage: 100},
@@ -104,7 +130,7 @@ func (g *GithubProvider) ListRepos(ctx context.Context, cloneProtocol string) ([
 
 func (g *GithubProvider) RepoHasPath(ctx context.Context, repo *Repository, path string) (bool, error) {
 	_, _, resp, err := g.client.Repositories.GetContents(ctx, repo.Organization, repo.Repository, path, &github.RepositoryContentGetOptions{
-		Ref: repo.Branch,
+		Ref: repo.SHA,
 	})
 	// 404s are not an error here, just a normal false.
 	if resp != nil && resp.StatusCode == 404 {
@@ -153,3 +179,28 @@ func (g *GithubProvider) listBranches(ctx context.Context, repo *Repository) ([]
 	}
 	return branches, nil
 }
+
+func (g *GithubProvider) listPullRequests(ctx context.Context, repo *Repository) ([]github.PullRequest, error) {
+	opt := &github.PullRequestListOptions{
+		ListOptions: github.ListOptions{PerPage: 100},
+	}
+
+	githubPullRequests := []github.PullRequest{}
+
+	for {
+		allPullRequests, resp, err := g.client.PullRequests.List(ctx, repo.Organization, repo.Repository, opt)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, pr := range allPullRequests {
+			githubPullRequests = append(githubPullRequests, *pr)
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	return githubPullRequests, nil
+}

pkg/services/scm_provider/gitlab.go

@@ -60,6 +60,28 @@ func (g *GitlabProvider) GetBranches(ctx context.Context, repo *Repository) ([]*
 	return repos, nil
 }
 
+func (g *GitlabProvider) GetPullRequests(ctx context.Context, repo *Repository) ([]*Repository, error) {
+	repos := []*Repository{}
+
+	pullRequests, err := g.listPullRequests(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, pullRequest := range pullRequests {
+		repos = append(repos, &Repository{
+			Organization: repo.Organization,
+			Repository:   repo.Repository,
+			URL:          repo.URL,
+			Branch:       pullRequest.Title,
+			SHA:          pullRequest.SHA,
+			Labels:       pullRequest.Labels,
+			RepositoryId: repo.RepositoryId,
+		})
+	}
+	return repos, nil
+}
+
 func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([]*Repository, error) {
 	opt := &gitlab.ListGroupProjectsOptions{
 		ListOptions:      gitlab.ListOptions{PerPage: 100},
@@ -149,3 +171,26 @@ func (g *GitlabProvider) listBranches(_ context.Context, repo *Repository) ([]gi
 	}
 	return branches, nil
 }
+
+func (g *GitlabProvider) listPullRequests(_ context.Context, repo *Repository) ([]gitlab.MergeRequest, error) {
+	opt := &gitlab.ListProjectMergeRequestsOptions{
+		ListOptions: gitlab.ListOptions{PerPage: 100},
+	}
+
+	pullRequests := []gitlab.MergeRequest{}
+	for {
+		gitlabPullRequests, resp, err := g.client.MergeRequests.ListProjectMergeRequests(repo.RepositoryId, opt)
+		if err != nil {
+			return nil, err
+		}
+		for _, gitlabPullRequest := range gitlabPullRequests {
+			pullRequests = append(pullRequests, *gitlabPullRequest)
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	return pullRequests, nil
+}

pkg/services/scm_provider/mock.go

@@ -48,3 +48,22 @@ func (m *MockProvider) GetBranches(_ context.Context, repo *Repository) ([]*Repo
 	}
 	return branchRepos, nil
 }
+
+func (m *MockProvider) GetPullRequests(_ context.Context, repo *Repository) ([]*Repository, error) {
+	pullRequestrepos := []*Repository{}
+	for _, candidateRepo := range m.Repos {
+		if candidateRepo.Repository == repo.Repository {
+			found := false
+			for _, alreadySetRepo := range pullRequestrepos {
+				if alreadySetRepo.Branch == candidateRepo.Branch {
+					found = true
+					break
+				}
+			}
+			if !found {
+				pullRequestrepos = append(pullRequestrepos, candidateRepo)
+			}
+		}
+	}
+	return pullRequestrepos, nil
+}

pkg/services/scm_provider/types.go

@@ -20,15 +20,18 @@ type SCMProviderService interface {
 	ListRepos(context.Context, string) ([]*Repository, error)
 	RepoHasPath(context.Context, *Repository, string) (bool, error)
 	GetBranches(context.Context, *Repository) ([]*Repository, error)
+	GetPullRequests(context.Context, *Repository) ([]*Repository, error)
 }
 
 // A compiled version of SCMProviderGeneratorFilter for performance.
 type Filter struct {
-	RepositoryMatch *regexp.Regexp
-	PathsExist      []string
-	LabelMatch      *regexp.Regexp
-	BranchMatch     *regexp.Regexp
-	FilterType      FilterType
+	RepositoryMatch       *regexp.Regexp
+	PathsExist            []string
+	LabelMatch            *regexp.Regexp
+	BranchMatch           *regexp.Regexp
+	PullRequestTitleMatch *regexp.Regexp
+	PullRequestLabelMatch *regexp.Regexp
+	FilterType            FilterType
 }
 
 // A convenience type for indicating where to apply a filter
@@ -39,4 +42,5 @@ const (
 	FilterTypeUndefined FilterType = iota
 	FilterTypeBranch
 	FilterTypeRepo
+	FilterTypePullRequest
 )