Skip to content

Commit

Permalink
feat: cache GitHub queries (#57)
Browse files Browse the repository at this point in the history
  • Loading branch information
@freak12techno
freak12techno authored Aug 21, 2024
1 parent 0a14f61 commit 2e17036
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 53 deletions.
70 changes: 41 additions & 29 deletions pkg/clients/git/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"main/pkg/constants"
"main/pkg/query_info"
"net/http"
"strconv"
"time"

"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
Expand All @@ -19,14 +20,14 @@ import (
const API_BASE_URL = "https://api.github.com"

type Github struct {
ApiBaseUrl string
Organization string
Repository string
Token string
Logger zerolog.Logger
LastModified time.Time
LastResult string
Tracer trace.Tracer
ApiBaseUrl string
Organization string
Repository string
Token string
Logger zerolog.Logger
LastResult string
LastResultTime time.Time
Tracer trace.Tracer
}

type GithubReleaseInfo struct {
Expand All @@ -44,25 +45,31 @@ func NewGithub(config config.GitConfig, logger zerolog.Logger, tracer trace.Trac
Repository: value[2],
Token: config.Token,
Logger: logger.With().Str("component", "github").Logger(),
LastModified: time.Now(),
LastResult: "",
Tracer: tracer,
}
}

func (g *Github) UseCache() bool {
// If the last result is not present - do not use cache, for the first query.
func (g *Github) HasCachedResult() bool {
if g.LastResult == "" {
return false
}

// We need to make uncached requests once in a while, to make sure everything is ok
// (for example, if we messed up caching itself).
diff := time.Since(g.LastModified)
return diff < constants.UncachedGithubQueryTime
return g.LastResultTime.Add(constants.UncachedGithubQueryTime).Sub(time.Now()) > 0
}

func (g *Github) GetLatestRelease(ctx context.Context) (string, query_info.QueryInfo, error) {
if g.HasCachedResult() {
g.Logger.Trace().
Str("time-since-latest", time.Since(g.LastResultTime).String()).
Msg("Use Github response from cache")
return g.LastResult, query_info.QueryInfo{
Module: constants.ModuleGit,
Action: constants.ActionGitGetLatestRelease,
Success: true,
}, nil
}

childCtx, span := g.Tracer.Start(ctx, "HTTP request")
defer span.End()

Expand All @@ -89,35 +96,40 @@ func (g *Github) GetLatestRelease(ctx context.Context) (string, query_info.Query
return "", queryInfo, err
}

useCache := g.UseCache()

g.Logger.Trace().
Str("url", latestReleaseUrl).
Bool("cached", useCache).
Str("time-since-latest", time.Since(g.LastModified).String()).
Msg("Querying GitHub")

if useCache {
req.Header.Set("If-Modified-Since", g.LastModified.Format(http.TimeFormat))
}

if g.Token != "" {
g.Logger.Trace().Msg("Using personal token for Github requests")
req.Header.Set("Authorization", "Bearer "+g.Token)
}

res, err := client.Do(req)
if res != nil && res.Body != nil {
defer res.Body.Close()
}

if err != nil {
return "", queryInfo, err
}
defer res.Body.Close()

if res.StatusCode == http.StatusNotModified && g.LastResult != "" {
queryInfo.Success = true
g.Logger.Trace().Msg("Github returned cached response")
return g.LastResult, queryInfo, nil
// rate limiting
rateLimitTimeHeader := res.Header.Get("x-ratelimit-reset") //nolint:canonicalheader
rateLimitHeaderInt, err := strconv.ParseInt(rateLimitTimeHeader, 10, 64)
if err != nil {
return "", queryInfo, err
}

rateLimitTime := time.Unix(rateLimitHeaderInt, 0)

g.Logger.Trace().
Str("url", latestReleaseUrl).
Str("ratelimit-limit", res.Header.Get("x-ratelimit-limit")). //nolint:canonicalheader
Str("ratelimit-remaining", res.Header.Get("x-ratelimit-remaining")). //nolint:canonicalheader
Time("ratelimit-reset", rateLimitTime).
Msg("GitHub query finished")

releaseInfo := GithubReleaseInfo{}
err = json.NewDecoder(res.Body).Decode(&releaseInfo)

Expand All @@ -130,7 +142,7 @@ func (g *Github) GetLatestRelease(ctx context.Context) (string, query_info.Query
return "", queryInfo, fmt.Errorf("got error from Github: %s", releaseInfo.Message)
}

g.LastModified = time.Now()
g.LastResultTime = time.Now()
g.LastResult = releaseInfo.TagName

queryInfo.Success = true
Expand Down
47 changes: 28 additions & 19 deletions pkg/clients/git/github_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import (
configPkg "main/pkg/config"
loggerPkg "main/pkg/logger"
"main/pkg/tracing"
"net/http"
"testing"
"time"

"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -62,7 +64,9 @@ func TestGetGithubClientInvalidResponse(t *testing.T) {
httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("invalid.toml")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("invalid.toml")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand All @@ -79,38 +83,42 @@ func TestGetGithubClientInvalidResponse(t *testing.T) {
}

//nolint:paralleltest // disabled due to httpmock usage
func TestGetGithubClientRateLimit(t *testing.T) {
func TestGetGithubClientInvalidRatelimitHeader(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()

httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("github-error.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("invalid.toml")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"asd"}}),
)

config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia", Token: "aaa"}
logger := loggerPkg.GetNopLogger()
tracer := tracing.InitNoopTracer()
client := NewGithub(config, *logger, tracer)

release, queryInfo, err := client.GetLatestRelease(context.Background())

require.Error(t, err)
require.ErrorContains(t, err, "got error from Github: API rate limit exceeded")
require.ErrorContains(t, err, "invalid syntax")
assert.False(t, queryInfo.Success)
require.Empty(t, release)
}

//nolint:paralleltest // disabled due to httpmock usage
func TestGetGithubClientValidQueried(t *testing.T) {
func TestGetGithubClientRateLimit(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()

httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("github-error.json")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand All @@ -120,20 +128,23 @@ func TestGetGithubClientValidQueried(t *testing.T) {

release, queryInfo, err := client.GetLatestRelease(context.Background())

require.NoError(t, err)
assert.True(t, queryInfo.Success)
require.Equal(t, "v17.2.0", release)
require.Error(t, err)
require.ErrorContains(t, err, "got error from Github: API rate limit exceeded")
assert.False(t, queryInfo.Success)
require.Empty(t, release)
}

//nolint:paralleltest // disabled due to httpmock usage
func TestGetGithubClientCachedNoPreviousResponse(t *testing.T) {
func TestGetGithubClientValidQueried(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()

httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(304, assets.GetBytesOrPanic("github-valid.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand All @@ -153,17 +164,15 @@ func TestGetGithubClientCachedWithPreviousResponse(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()

httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(304, assets.GetBytesOrPanic("github-valid.json")),
)

config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia", Token: "aaa"}
config := configPkg.GitConfig{
Repository: "https://github.com/cosmos/gaia",
Token: "aaa",
}
logger := loggerPkg.GetNopLogger()
tracer := tracing.InitNoopTracer()
client := NewGithub(config, *logger, tracer)
client.LastResult = "v1.2.3"
client.LastResultTime = time.Now()

release, queryInfo, err := client.GetLatestRelease(context.Background())

Expand Down
2 changes: 1 addition & 1 deletion pkg/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ type Action string

const (
MetricsPrefix = "cosmos_node_exporter_"
UncachedGithubQueryTime = 30 * time.Second
UncachedGithubQueryTime = 120 * time.Second
ModuleCosmovisor Module = "cosmovisor"
ModuleTendermint Module = "tendermint"
ModuleGit Module = "git"
Expand Down
17 changes: 13 additions & 4 deletions pkg/queriers/versions/querier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"main/pkg/exec"
loggerPkg "main/pkg/logger"
"main/pkg/tracing"
"net/http"
"testing"

"github.com/jarcoal/httpmock"
Expand Down Expand Up @@ -58,7 +59,9 @@ func TestVersionsQuerierGitOk(t *testing.T) {
httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

config := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand Down Expand Up @@ -137,7 +140,9 @@ func TestVersionsQuerierLocalSemverInvalid(t *testing.T) {
httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

gitConfig := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand Down Expand Up @@ -183,7 +188,9 @@ func TestVersionsQuerierRemoteSemverInvalid(t *testing.T) {
httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("github-invalid.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("github-invalid.json")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

gitConfig := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand Down Expand Up @@ -229,7 +236,9 @@ func TestVersionsQuerierAllOk(t *testing.T) {
httpmock.RegisterResponder(
"GET",
"https://api.github.com/repos/cosmos/gaia/releases/latest",
httpmock.NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")),
httpmock.
NewBytesResponder(200, assets.GetBytesOrPanic("github-valid.json")).
HeaderAdd(http.Header{"x-ratelimit-reset": []string{"12345"}}),
)

gitConfig := configPkg.GitConfig{Repository: "https://github.com/cosmos/gaia"}
Expand Down

0 comments on commit 2e17036

Please sign in to comment.