Merge pull request #43 from philips-labs/feature/fix-paging-rest

Add paging to github fetch private repos

lib/github/client.go

@@ -0,0 +1,34 @@
+package github
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/google/go-github/v32/github"
+	"github.com/shurcooL/githubv4"
+	"golang.org/x/oauth2"
+
+	"github.com/philips-labs/tabia/lib/transport"
+)
+
+type Client struct {
+	httpClient *http.Client
+	restClient *github.Client
+	*githubv4.Client
+}
+
+func NewClientWithTokenAuth(token string, writer io.Writer) *Client {
+	src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+	httpClient := oauth2.NewClient(context.Background(), src)
+	if writer != nil {
+		httpClient.Transport = transport.TeeRoundTripper{
+			RoundTripper: httpClient.Transport,
+			Writer:       writer,
+		}
+	}
+	client := githubv4.NewClient(httpClient)
+	restClient := github.NewClient(httpClient)
+
+	return &Client{httpClient, restClient, client}
+}

lib/github/client_test.go

@@ -0,0 +1,24 @@
+package github_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/philips-labs/tabia/lib/github"
+)
+
+func TestClient(t *testing.T) {
+	assert := assert.New(t)
+
+	var buf strings.Builder
+
+	client := github.NewClientWithTokenAuth("token", &buf)
+	var q struct{}
+	client.Client.Query(context.Background(), q, nil)
+
+	assert.NotEmpty(buf)
+	assert.Equal("POST: https://api.github.com/graphql {\"query\":\"{}\"}\n", buf.String())
+}

lib/github/repositories.go

@@ -3,41 +3,15 @@ package github
 import (
 	"context"
 	"fmt"
-	"io"
-	"net/http"
 	"strings"
 	"time"
 
-	"golang.org/x/oauth2"
-
 	"github.com/google/go-github/v32/github"
 	"github.com/shurcooL/githubv4"
 
 	"github.com/philips-labs/tabia/lib/github/graphql"
-	"github.com/philips-labs/tabia/lib/transport"
 )
 
-type Client struct {
-	httpClient *http.Client
-	restClient *github.Client
-	*githubv4.Client
-}
-
-func NewClientWithTokenAuth(token string, writer io.Writer) *Client {
-	src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
-	httpClient := oauth2.NewClient(context.Background(), src)
-	if writer != nil {
-		httpClient.Transport = transport.TeeRoundTripper{
-			RoundTripper: httpClient.Transport,
-			Writer:       writer,
-		}
-	}
-	client := githubv4.NewClient(httpClient)
-	restClient := github.NewClient(httpClient)
-
-	return &Client{httpClient, restClient, client}
-}
-
 //go:generate stringer -type=Visibility
 
 // Visibility indicates repository visibility
@@ -128,6 +102,7 @@ func (c *Client) FetchOrganziationRepositories(ctx context.Context, owner string
 		if !q.Repositories.PageInfo.HasNextPage {
 			break
 		}
+
 		variables["repoCursor"] = githubv4.NewString(q.Repositories.PageInfo.EndCursor)
 	}
 
@@ -137,16 +112,28 @@ func (c *Client) FetchOrganziationRepositories(ctx context.Context, owner string
 	if err != nil {
 		return nil, err
 	}
+
 	return Map(repositories, privateRepos)
 }
 
 func (c *Client) FetchRestRepositories(ctx context.Context, owner, repoType string) ([]*github.Repository, error) {
-	repos, resp, err := c.restClient.Repositories.ListByOrg(ctx, owner, &github.RepositoryListByOrgOptions{Type: repoType})
-	if err != nil {
-		return nil, err
+	var allRepos []*github.Repository
+
+	opt := &github.RepositoryListByOrgOptions{Type: repoType}
+	for {
+		repos, resp, err := c.restClient.Repositories.ListByOrg(ctx, owner, opt)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		allRepos = append(allRepos, repos...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
 	}
-	defer resp.Body.Close()
-	return repos, nil
+
+	return allRepos, nil
 }
 
 func Map(repositories []graphql.Repository, privateRepositories []*github.Repository) ([]Repository, error) {

lib/github/repositories_test.go

@@ -37,6 +37,12 @@ func TestRepositoryVisibilityToJSON(t *testing.T) {
 	jsonEnc.Encode(privRepo)
 	assert.Equal(fmt.Sprintf(expectedTemplate, "private-repo", "Private"), result.String())
 
+	var unmarshalledRepo github.Repository
+	err := json.Unmarshal([]byte(result.String()), &unmarshalledRepo)
+	if assert.NoError(err) {
+		assert.Equal(github.Private, unmarshalledRepo.Visibility)
+	}
+
 	internalRepo := github.Repository{
 		Name:       "internal-repo",
 		Visibility: github.Internal,
@@ -45,13 +51,23 @@ func TestRepositoryVisibilityToJSON(t *testing.T) {
 	jsonEnc.Encode(internalRepo)
 	assert.Equal(fmt.Sprintf(expectedTemplate, "internal-repo", "Internal"), result.String())
 
+	err = json.Unmarshal([]byte(result.String()), &unmarshalledRepo)
+	if assert.NoError(err) {
+		assert.Equal(github.Internal, unmarshalledRepo.Visibility)
+	}
+
 	publicRepo := github.Repository{
 		Name:       "public-repo",
 		Visibility: github.Public,
 	}
 	result.Reset()
 	jsonEnc.Encode(publicRepo)
 	assert.Equal(fmt.Sprintf(expectedTemplate, "public-repo", "Public"), result.String())
+
+	err = json.Unmarshal([]byte(result.String()), &unmarshalledRepo)
+	if assert.NoError(err) {
+		assert.Equal(github.Public, unmarshalledRepo.Visibility)
+	}
 }
 
 func TestMap(t *testing.T) {