Caching repodata file

Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.15 as build
 WORKDIR /void/repo-exporter
 COPY . .
 RUN go mod vendor && \
-        CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o /exporter .
+        CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o /exporter *.go
 
 FROM alpine:latest as certs
 RUN apk --update add ca-certificates

cache/groupcache.go

@@ -0,0 +1,42 @@
+package cache
+
+import (
+	"context"
+	"regexp"
+	"strings"
+
+	"github.com/golang/groupcache"
+)
+
+// RepoDataStorage store and search and store data through groupcache
+type RepoDataStorage struct {
+	group *groupcache.Group
+}
+
+// NewRepoDataStorage build a new repodata storage
+func NewRepoDataStorage(name string, cacheBytes int64, peers string, getter func(string) ([]byte, error)) RepoDataStorage {
+	p := strings.Split(peers, ",")
+	pool := groupcache.NewHTTPPool(p[0])
+	pool.Set(p...)
+
+	group := groupcache.NewGroup(name, cacheBytes, groupcache.GetterFunc(
+		func(ctx groupcache.Context, key string, dst groupcache.Sink) error {
+			var re = regexp.MustCompile(`{{\d*}}`)
+			s := re.ReplaceAllString(key, "")
+			b, err := getter(s)
+			if err != nil {
+				return err
+			}
+			dst.SetBytes(b)
+			return nil
+		},
+	))
+	return RepoDataStorage{
+		group: group,
+	}
+}
+
+// Get searches by repodata storage and store in dst
+func (r RepoDataStorage) Get(ctx context.Context, key string, dst *[]byte) error {
+	return r.group.Get(ctx, key, groupcache.AllocatingByteSliceSink(dst))
+}

cache/groupcache_test.go

@@ -0,0 +1,60 @@
+package cache
+
+import (
+	"bytes"
+	"context"
+	"testing"
+	"time"
+)
+
+func TestGet(t *testing.T) {
+	storage := NewRepoDataStorage("repodata", 64<<20, "http://localhost:8080", getBytes)
+	var tests = []struct {
+		name                string
+		givenKey            string
+		expectedData        []byte
+		expectedTimeElapsed time.Duration
+		expectedErr         error
+	}{
+		{
+			"Storing slow data for the first time with success",
+			"test",
+			[]byte("success"),
+			101 * time.Millisecond,
+			nil,
+		},
+		{
+			"Getting cached data with success",
+			"test",
+			[]byte("success"),
+			10 * time.Millisecond,
+			nil,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			var response []byte
+			start := time.Now()
+			err := storage.Get(context.Background(), tt.givenKey, &response)
+			finished := time.Since(start)
+			if err != tt.expectedErr {
+				t.Errorf("(%s): expected error %s, actual %s", tt.givenKey, tt.expectedErr, err)
+			}
+
+			if finished > tt.expectedTimeElapsed {
+				t.Errorf("(%s): expected %s, actual %s", tt.givenKey, tt.expectedTimeElapsed, finished)
+			}
+
+			if !bytes.Equal(response, tt.expectedData) {
+				t.Errorf("(%s): expected %s, actual %s", tt.givenKey, tt.expectedData, response)
+			}
+
+		})
+	}
+}
+
+func getBytes(_ string) ([]byte, error) {
+	time.Sleep(100 * time.Millisecond)
+	return []byte("success"), nil
+}

go.mod

@@ -2,4 +2,7 @@ module github.com/void-linux/repo-exporter
 
 go 1.15
 
-require github.com/prometheus/client_golang v1.9.0
+require (
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/prometheus/client_golang v1.9.0
+)

go.sum

@@ -69,6 +69,7 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -88,6 +89,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -347,6 +349,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

main.go

@@ -1,13 +1,17 @@
 package main
 
 import (
+	"flag"
+	"fmt"
 	"hash/crc32"
-	"io/ioutil"
+	"log"
 	"net/http"
 	"strconv"
-	"time"
-	"log"
 	"strings"
+	"time"
+
+	"github.com/void-linux/repo-exporter/cache"
+	"github.com/void-linux/repo-exporter/requests"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -17,7 +21,7 @@ const (
 	namespace = "repo"
 )
 
-func doProbe(w http.ResponseWriter, r *http.Request) {
+func (h handler) doProbe(w http.ResponseWriter, r *http.Request) {
 	target := r.URL.Query().Get("target")
 	if target == "" {
 		http.Error(w, "Target parameter is missing", http.StatusBadRequest)
@@ -28,22 +32,36 @@ func doProbe(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Arch parameter is missing", http.StatusBadRequest)
 		return
 	}
+	repodataURL := "https://" + target + "/" + arch + "-repodata"
+	headers, _, err := h.client.Head(repodataURL)
+	if err != nil {
+		log.Printf("Error fetching headers repodata: %s", err)
+		http.Error(w, "Error fetching headers repodata: "+err.Error(), http.StatusPreconditionFailed)
+	}
 
-	repodata, _, err := fetch("http://" + target + "/" + arch + "-repodata")
+	lastModified, err := time.Parse(time.RFC1123, headers["Last-Modified"][0])
+	if err != nil {
+		log.Printf("Error parsing last modified from repodata: %s", err)
+		http.Error(w, "Error parsing last modified from repodata: "+err.Error(), http.StatusPreconditionFailed)
+	}
+
+	var repodata []byte
+	key := fmt.Sprintf("%s{{%d}}", repodataURL, lastModified.Unix())
+	err = h.storage.Get(r.Context(), key, &repodata)
 	if err != nil {
 		log.Printf("Error fetching repodata: %s", err)
 		http.Error(w, "Error fetching repodata: "+err.Error(), http.StatusPreconditionFailed)
 	}
 
-	_, stagedataStatusCode, err := fetch("http://" + target + "/" + arch + "-stagedata")
+	_, stagedataStatusCode, err := h.client.Fetch("http://" + target + "/" + arch + "-stagedata")
 	if err != nil {
 		log.Printf("Error fetching stagedata: %s", err)
 		http.Error(w, "Error fetching stagedata: "+err.Error(), http.StatusPreconditionFailed)
 	}
 
-	otimes, c, err := fetch("http://" + target + "/otime")
+	otimes, c, err := h.client.Fetch("http://" + target + "/otime")
 	if err != nil {
-		log.Println("Error fetching origin timestamp file: %s", err)
+		log.Printf("Error fetching origin timestamp file: %s", err)
 		http.Error(w, "Error fetching origin time: "+err.Error(), http.StatusPreconditionFailed)
 	}
 	var otime float64
@@ -55,7 +73,7 @@ func doProbe(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	stimeStarts, c, err := fetch("http://" + target + "/stime-start")
+	stimeStarts, c, err := h.client.Fetch("http://" + target + "/stime-start")
 	if err != nil {
 		http.Error(w, "Error fetching origin time: "+err.Error(), http.StatusPreconditionFailed)
 	}
@@ -67,7 +85,7 @@ func doProbe(w http.ResponseWriter, r *http.Request) {
 			log.Println("Error parsing stimeStart", err)
 		}
 	}
-	stimeEnds, c, err := fetch("http://" + target + "/stime-end")
+	stimeEnds, c, err := h.client.Fetch("http://" + target + "/stime-end")
 	if err != nil {
 		http.Error(w, "Error fetching origin time: "+err.Error(), http.StatusPreconditionFailed)
 	}
@@ -131,31 +149,30 @@ func doProbe(w http.ResponseWriter, r *http.Request) {
 	promhttp.HandlerFor(registry, promhttp.HandlerOpts{}).ServeHTTP(w, r)
 }
 
-func fetch(url string) ([]byte, int, error) {
-	c := http.Client{Timeout: time.Second * 10}
-
-	resp, err := c.Get(url)
-	if err != nil {
-		return nil, 0, err
-	}
-	defer resp.Body.Close()
-
-	bytes, err := ioutil.ReadAll(resp.Body)
-	return bytes, resp.StatusCode, err
-}
-
-func main() {
-	http.Handle("/metrics", promhttp.Handler())
-	http.HandleFunc("/probe", doProbe)
-
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		_, _ = w.Write([]byte(`<html>
+func (handler) root(w http.ResponseWriter, r *http.Request) {
+	_, _ = w.Write([]byte(`<html>
                 <head><title>XBPS Repo Exporter</title></head>
                 <body>
                 <h1>XBPS Repo Exporter</h1>
                 </body>
                 </html>`))
-	})
+}
+
+func main() {
+	peers := flag.String("pool", "http://localhost:8080", "server pool list separated by commas")
+	flag.Parse()
+	client := requests.NewHTTPRequestHandler(20 * time.Second)
+	h := handler{
+		client: client,
+		storage: cache.NewRepoDataStorage("repodata", 64<<20, *peers, func(key string) ([]byte, error) {
+			b, _, err := client.Fetch(key)
+			return b, err
+		}),
+	}
+
+	http.Handle("/metrics", promhttp.Handler())
+	http.HandleFunc("/probe", h.doProbe)
+	http.HandleFunc("/", h.root)
 
 	http.ListenAndServe(":1234", nil)
 }

requests/http.go

@@ -0,0 +1,49 @@
+// Package requests contain functions for calling HTTP/HTTPS requests
+// for each HTTP method.
+package requests
+
+import (
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+// HTTPClient struct implements the RequestHandler
+// It needs to receive a expected timeout value for his clients.
+type HTTPClient struct {
+	timeout time.Duration
+}
+
+// NewHTTPRequestHandler build a new request handler
+func NewHTTPRequestHandler(timeout time.Duration) HTTPClient {
+	return HTTPClient{timeout: timeout}
+}
+
+// Fetch create a GET HTTP request asking for content
+// The request will timeout after 10s
+func (h HTTPClient) Fetch(url string) ([]byte, int, error) {
+	c := http.Client{Timeout: h.timeout}
+
+	resp, err := c.Get(url)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer resp.Body.Close()
+
+	bytes, err := ioutil.ReadAll(resp.Body)
+	return bytes, resp.StatusCode, err
+}
+
+// Head creates a HEAD HTTP request asking for headers
+// The request will timeout after 10s
+func (h HTTPClient) Head(url string) (map[string][]string, int, error) {
+	c := http.Client{Timeout: h.timeout}
+
+	resp, err := c.Head(url)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer resp.Body.Close()
+
+	return resp.Header, resp.StatusCode, err
+}