diff --git a/cmd/playground/main.go b/cmd/playground/main.go index bc9ed107..22483c02 100644 --- a/cmd/playground/main.go +++ b/cmd/playground/main.go @@ -15,6 +15,7 @@ import ( "github.com/x1unix/go-playground/internal/builder/storage" "github.com/x1unix/go-playground/internal/config" "github.com/x1unix/go-playground/internal/server" + "github.com/x1unix/go-playground/internal/server/backendinfo" "github.com/x1unix/go-playground/internal/server/webutil" "github.com/x1unix/go-playground/pkg/goplay" "github.com/x1unix/go-playground/pkg/util/cmdutil" @@ -77,11 +78,16 @@ func start(logger *zap.Logger, cfg *config.Config) error { go cleanupSvc.Start(ctx) } + backendsInfoSvc := backendinfo.NewBackendVersionService(zap.L(), playgroundClient, backendinfo.ServiceConfig{ + CacheFile: filepath.Join(cfg.Build.BuildDir, "go-versions.json"), + TTL: backendinfo.DefaultVersionCacheTTL, + }) + // Initialize API endpoints r := mux.NewRouter() apiRouter := r.PathPrefix("/api").Subrouter() svcCfg := server.ServiceConfig{Version: Version} - server.NewAPIv1Handler(svcCfg, playgroundClient, buildSvc). + server.NewAPIv1Handler(svcCfg, playgroundClient, buildSvc, backendsInfoSvc). Mount(apiRouter) apiv2Router := apiRouter.PathPrefix("/v2").Subrouter() @@ -90,7 +96,6 @@ func start(logger *zap.Logger, cfg *config.Config) error { Builder: buildSvc, BuildTimeout: cfg.Build.GoBuildTimeout, }).Mount(apiv2Router) - //server.NewAPIv2Handler(playgroundClient, buildSvc).Mount(apiv2Router) // Web UI routes tplVars := server.TemplateArguments{ diff --git a/internal/server/backendinfo/fallback.go b/internal/server/backendinfo/fallback.go new file mode 100644 index 00000000..1d89b260 --- /dev/null +++ b/internal/server/backendinfo/fallback.go @@ -0,0 +1,32 @@ +package backendinfo + +import ( + "strconv" + "strings" +) + +func prefillFallbacks(info *BackendVersions) { + if info.PreviousStable == "" { + info.PreviousStable = guessPreviousVersion(info.CurrentStable) + } + + if info.Nightly == "" { + info.Nightly = "devel" + } +} + +func guessPreviousVersion(baseVer string) string { + chunks := strings.Split(baseVer, ".") + if len(chunks) < 2 { + return baseVer + } + + minorVer, err := strconv.Atoi(chunks[1]) + if err != nil { + return baseVer + } + + minorVer = max(0, minorVer-1) + return chunks[0] + "." + strconv.Itoa(minorVer) + ".0" +} + diff --git a/internal/server/backendinfo/provider.go b/internal/server/backendinfo/provider.go new file mode 100644 index 00000000..09cc4fbf --- /dev/null +++ b/internal/server/backendinfo/provider.go @@ -0,0 +1,22 @@ +package backendinfo + +import "context" + +type BackendVersions struct { + // CurrentStable is latest stable Go version. + CurrentStable string + + // PreviousStable is previous stable Go version. + PreviousStable string + + // Nightly is latest unstable Go version (tip) version. + Nightly string +} + +type BackendVersionProvider interface { + // GetRemoteVersions returns Go version used on remote Go backends. + GetRemoteVersions(ctx context.Context) (*BackendVersions, error) + + // ServerVersion returns Go version used on server. + ServerVersion() string +} diff --git a/internal/server/resources/version.go.txt b/internal/server/backendinfo/resources/version.go.txt similarity index 100% rename from internal/server/resources/version.go.txt rename to internal/server/backendinfo/resources/version.go.txt diff --git a/internal/server/backendinfo/service.go b/internal/server/backendinfo/service.go new file mode 100644 index 00000000..4d3fca6a --- /dev/null +++ b/internal/server/backendinfo/service.go @@ -0,0 +1,274 @@ +package backendinfo + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/avast/retry-go" + "github.com/x1unix/go-playground/pkg/goplay" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +const ( + goVersionRetryAttempts = 3 + goVersionRetryDelay = time.Second + + DefaultVersionCacheTTL = 48 * time.Hour +) + +//go:embed resources/version.go.txt +var versionSnippet []byte + +const cacheFileVersion = 1 + +var _ BackendVersionProvider = (*BackendVersionService)(nil) + +type ServiceConfig struct { + // Version is cache file version + Version int + + // CacheFile is name of a file which will be used to cache Go playground versions. + CacheFile string + + // TTL is expiration interval. + TTL time.Duration +} + +type cacheEntry struct { + Version int + CreatedAt time.Time + Data BackendVersions +} + +// BackendVersionService provides information about used Go versions +// for all backends. +type BackendVersionService struct { + logger *zap.Logger + client *goplay.Client + cfg ServiceConfig + + memCache *cacheEntry +} + +func NewBackendVersionService(logger *zap.Logger, client *goplay.Client, cfg ServiceConfig) *BackendVersionService { + return &BackendVersionService{ + logger: logger, + client: client, + cfg: cfg, + } +} + +func (svc *BackendVersionService) ServerVersion() string { + return normalizeGoVersion(runtime.Version()) +} + +func (svc *BackendVersionService) visitCache() (*cacheEntry, error) { + if svc.memCache != nil { + return svc.memCache, nil + } + + if svc.cfg.CacheFile == "" { + return nil, fs.ErrNotExist + } + + f, err := os.Open(svc.cfg.CacheFile) + if err != nil { + return nil, err + } + + defer f.Close() + dst := &cacheEntry{} + err = json.NewDecoder(f).Decode(dst) + + return dst, err +} + +// GetVersions provides Go version information for all backends. +func (svc *BackendVersionService) GetRemoteVersions(ctx context.Context) (*BackendVersions, error) { + cached, err := svc.visitCache() + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + svc.logger.Error("failed to check Go versions cache", zap.Error(err)) + } + + return svc.populateVersionCache(ctx) + } + + if cached.Version != cacheFileVersion { + return nil, fs.ErrNotExist + } + + dt := time.Now().UTC().Sub(cached.CreatedAt.UTC()) + if dt >= svc.cfg.TTL { + return svc.populateVersionCache(ctx) + } + + return &cached.Data, nil +} + +func (svc *BackendVersionService) populateVersionCache(ctx context.Context) (*BackendVersions, error) { + versions, err := svc.pullBackendVersions(ctx) + if err != nil { + return nil, err + } + + if err := svc.cacheVersions(versions); err != nil { + svc.logger.Error("failed to cache Go versions", zap.Error(err)) + } + + return versions, nil +} + +func (svc *BackendVersionService) cacheVersions(versions *BackendVersions) error { + svc.memCache = &cacheEntry{ + Version: cacheFileVersion, + CreatedAt: time.Now().UTC(), + Data: *versions, + } + + if svc.cfg.CacheFile == "" { + return nil + } + + err := os.MkdirAll(filepath.Dir(svc.cfg.CacheFile), 0755) + if err != nil { + return fmt.Errorf("MkdirAll failed: %w", err) + } + + f, err := os.OpenFile(svc.cfg.CacheFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + + defer f.Close() + return json.NewEncoder(f).Encode(svc.memCache) +} + +func (svc *BackendVersionService) pullBackendVersions(ctx context.Context) (*BackendVersions, error) { + versionInfo := &BackendVersions{} + g, gCtx := errgroup.WithContext(ctx) + + mapping := [3]struct { + backend string + dst *string + }{ + { + backend: goplay.BackendGoCurrent, + dst: &versionInfo.CurrentStable, + }, + { + backend: goplay.BackendGoPrev, + dst: &versionInfo.PreviousStable, + }, + { + backend: goplay.BackendGoTip, + dst: &versionInfo.Nightly, + }, + } + + for _, e := range mapping { + b := e + g.Go(func() error { + svc.logger.Debug("Fetching go version for backend", zap.String("backend", e.backend)) + result, err := svc.fetchGoBackendVersionWithRetry(gCtx, e.backend) + if err != nil { + // Playground "gotip" and "goprev" backends are often broken + // and I'm getting tired of seeing 5xx responses if just one of them is dead. + // + // Throw only if stable version is down. For others - try to figure out fallback values. + if e.backend == goplay.BackendGoCurrent { + return fmt.Errorf("failed to get Go version from Go playground server for backend %q: %w", + b.backend, err) + } + + svc.logger.Warn( + "can't fetch Go version for backend, will use fallback", + zap.String("backend", e.backend), zap.Error(err), + ) + return nil + } + + // We don't afraid race condition because each backend is written to a separate address + *b.dst = result + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + + prefillFallbacks(versionInfo) + return versionInfo, nil +} + +func (svc *BackendVersionService) fetchGoBackendVersionWithRetry(ctx context.Context, backend goplay.Backend) (string, error) { + var result string + err := retry.Do( + func() error { + version, err := svc.getGoBackendVersion(ctx, backend) + if err != nil { + return err + } + + result = version + return nil + }, + retry.Attempts(goVersionRetryAttempts), + retry.Delay(goVersionRetryDelay), + retry.RetryIf(func(err error) bool { + httpErr, ok := goplay.IsHTTPError(err) + if !ok { + return false + } + + // Retry only on server issues + return httpErr.StatusCode >= 500 + }), + retry.OnRetry(func(n uint, err error) { + svc.logger.Error("failed to get Go version from Go playground, retrying...", + zap.Error(err), zap.String("backend", backend), zap.Uint("attempt", n)) + }), + ) + + return result, err +} + +func (svc *BackendVersionService) getGoBackendVersion(ctx context.Context, backend goplay.Backend) (string, error) { + // Dirty hack to fetch Go version for playground backend by running a simple program + // which returns Go version to stdout. + result, err := svc.client.Evaluate(ctx, goplay.CompileRequest{ + Version: goplay.DefaultVersion, + WithVet: false, + Body: versionSnippet, + }, backend) + + if err != nil { + return "", err + } + + if result.Errors != "" { + return "", fmt.Errorf("probe program returned an error: %s", result.Errors) + } + + if len(result.Events) == 0 { + return "", errors.New("missing output events from probe program") + } + + version := normalizeGoVersion(result.Events[0].Message) + return version, nil +} + +func normalizeGoVersion(str string) string { + return strings.TrimPrefix(str, "go") +} diff --git a/internal/server/backendinfo/service_test.go b/internal/server/backendinfo/service_test.go new file mode 100644 index 00000000..76726e05 --- /dev/null +++ b/internal/server/backendinfo/service_test.go @@ -0,0 +1,220 @@ +package backendinfo + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/x1unix/go-playground/pkg/goplay" + "github.com/x1unix/go-playground/pkg/testutil" +) + +func TestBackendVersionService_ServerVersion(t *testing.T) { + want := strings.TrimPrefix(runtime.Version(), "go") + + svc := NewBackendVersionService(nil, nil, ServiceConfig{}) + got := svc.ServerVersion() + require.Equal(t, got, want) +} + +func TestBackendVersionService_GetRemoteVersions(t *testing.T) { + cases := map[string]struct{ + constructor func(t *testing.T, expect BackendVersions) *BackendVersionService + onPostRun func(t *testing.T, svc *BackendVersionService, expect BackendVersions) + expect BackendVersions + expectErr string + }{ + "should use ram cache": { + expect: BackendVersions{ + CurrentStable: "1.23", + PreviousStable: "1.22", + Nightly: "test", + }, + constructor: func(t *testing.T, expect BackendVersions) *BackendVersionService { + svc := NewBackendVersionService(nil, nil, ServiceConfig{TTL: DefaultVersionCacheTTL}) + svc.memCache = &cacheEntry{ + Version: cacheFileVersion, + CreatedAt: time.Now(), + Data: expect, + } + return svc + }, + }, + "should fallback to filesystem cache": { + expect: BackendVersions{ + CurrentStable: "1", + PreviousStable: "2", + Nightly: "test", + }, + constructor: func(t *testing.T, expect BackendVersions) *BackendVersionService { + fname := writeTestJson(t, "cache.json", cacheEntry{ + CreatedAt: time.Now(), + Version: cacheFileVersion, + Data: expect, + }) + svc := NewBackendVersionService(nil, nil, ServiceConfig{ + TTL: DefaultVersionCacheTTL, + CacheFile: fname, + }) + + return svc + }, + }, + "should fetch data if not cached": { + expect: BackendVersions{ + CurrentStable: "1", + PreviousStable: "2", + Nightly: "test", + }, + constructor: func(t *testing.T, expect BackendVersions) *BackendVersionService { + fname := filepath.Join(t.TempDir(), "test", "store.json") + srv := setupTestServer(expect) + logger := testutil.GetLogger(t).Desugar() + c := goplay.NewClient(srv.URL, "", 5 * time.Second) + + svc := NewBackendVersionService(logger, c, ServiceConfig{ + TTL: DefaultVersionCacheTTL, + CacheFile: fname, + }) + + return svc + }, + onPostRun: func(t *testing.T, svc *BackendVersionService, expect BackendVersions) { + f, err := os.Open(svc.cfg.CacheFile) + require.NoError(t, err) + defer f.Close() + + dst := &cacheEntry{} + err = json.NewDecoder(f).Decode(dst) + require.Equal(t, expect, dst.Data) + }, + }, + "should fetch data on cache miss": { + expect: BackendVersions{ + CurrentStable: "1", + PreviousStable: "2", + Nightly: "test", + }, + constructor: func(t *testing.T, expect BackendVersions) *BackendVersionService { + fname := writeTestJson(t, "cache.json", cacheEntry{ + CreatedAt: time.Now().Add(-30 * time.Hour), + Version: cacheFileVersion, + }) + + srv := setupTestServer(expect) + logger := testutil.GetLogger(t).Desugar() + c := goplay.NewClient(srv.URL, "", 5 * time.Second) + + svc := NewBackendVersionService(logger, c, ServiceConfig{ + TTL: time.Hour, + CacheFile: fname, + }) + + return svc + }, + onPostRun: func(t *testing.T, svc *BackendVersionService, expect BackendVersions) { + f, err := os.Open(svc.cfg.CacheFile) + require.NoError(t, err) + defer f.Close() + + dst := &cacheEntry{} + err = json.NewDecoder(f).Decode(dst) + require.Equal(t, expect, dst.Data) + }, + }, + "should prefill fallbacks if one on backends is down": { + expect: BackendVersions{ + CurrentStable: "1.23.1", + PreviousStable: "1.22.0", + Nightly: "devel", + }, + constructor: func(t *testing.T, expect BackendVersions) *BackendVersionService { + srv := setupTestServer(BackendVersions{ + CurrentStable: expect.CurrentStable, + }) + + logger := testutil.GetLogger(t).Desugar() + c := goplay.NewClient(srv.URL, "", 5 * time.Second) + + svc := NewBackendVersionService(logger, c, ServiceConfig{ + TTL: time.Hour, + }) + + return svc + }, + }, + } + + for n, c := range cases { + t.Run(n, func(t *testing.T) { + if c.constructor == nil { + t.Fatal("missing constructor") + } + + svc := c.constructor(t, c.expect) + got, err := svc.GetRemoteVersions(context.TODO()) + if c.expectErr != "" { + require.Error(t, err) + require.EqualError(t, err, c.expectErr) + return + } + + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, *got, c.expect) + }) + } +} + +func writeTestJson(t *testing.T, name string, data any) string { + fname := filepath.Join(t.TempDir(), name) + f, err := os.OpenFile(fname, os.O_RDWR | os.O_CREATE | os.O_TRUNC, 0644) + require.NoError(t, err, "can't create test file") + + err = json.NewEncoder(f).Encode(data) + _ = f.Close() + + require.NoError(t, err, "can't wrote to a test file") + return fname +} + +func setupTestServer(expects BackendVersions) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var msg string + switch r.URL.Query().Get("backend") { + case goplay.BackendGoPrev: + msg = expects.PreviousStable + case goplay.BackendGoTip: + msg = expects.Nightly + case goplay.BackendGoCurrent: + msg = expects.CurrentStable + } + + // Simulate prod down if empty + if msg == "" { + w.WriteHeader(http.StatusBadGateway) + return + } + + rsp := goplay.CompileResponse{ + Events: []*goplay.CompileEvent{ + { + Kind: "stdout", + Message: msg, + }, + }, + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(rsp) + })) +} + diff --git a/internal/server/handler_v1.go b/internal/server/handler_v1.go index d0d5fad3..9173e2a2 100644 --- a/internal/server/handler_v1.go +++ b/internal/server/handler_v1.go @@ -11,6 +11,7 @@ import ( "github.com/gorilla/mux" "github.com/x1unix/go-playground/internal/builder" "github.com/x1unix/go-playground/internal/builder/storage" + "github.com/x1unix/go-playground/internal/server/backendinfo" "github.com/x1unix/go-playground/pkg/goplay" "go.uber.org/zap" "golang.org/x/time/rate" @@ -25,16 +26,12 @@ const ( artifactParamVal = "artifactId" ) -type BackendVersionProvider interface { - GetVersions(ctx context.Context) (*VersionsInformation, error) -} - // APIv1Handler is API v1 handler type APIv1Handler struct { config ServiceConfig log *zap.SugaredLogger compiler builder.BuildService - versionProvider BackendVersionProvider + versionProvider backendinfo.BackendVersionProvider client *goplay.Client limiter *rate.Limiter @@ -45,13 +42,13 @@ type ServiceConfig struct { } // NewAPIv1Handler is APIv1Handler constructor -func NewAPIv1Handler(cfg ServiceConfig, client *goplay.Client, builder builder.BuildService) *APIv1Handler { +func NewAPIv1Handler(cfg ServiceConfig, client *goplay.Client, builder builder.BuildService, versionProvider backendinfo.BackendVersionProvider) *APIv1Handler { return &APIv1Handler{ config: cfg, compiler: builder, client: client, log: zap.S().Named("api.v1"), - versionProvider: NewBackendVersionService(zap.L(), client, VersionCacheTTL), + versionProvider: versionProvider, limiter: rate.NewLimiter(rate.Every(frameTime), compileRequestsPerFrame), } } @@ -73,12 +70,25 @@ func (s *APIv1Handler) HandleGetVersion(w http.ResponseWriter, _ *http.Request) } func (s *APIv1Handler) HandleGetVersions(w http.ResponseWriter, r *http.Request) error { - versions, err := s.versionProvider.GetVersions(r.Context()) + versions, err := s.versionProvider.GetRemoteVersions(r.Context()) if err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + return err } - WriteJSON(w, versions) + rsp := VersionsInformation{ + WebAssembly: s.versionProvider.ServerVersion(), + Playground: &PlaygroundVersions{ + GoCurrent: versions.CurrentStable, + GoPrevious: versions.PreviousStable, + GoTip: versions.Nightly, + }, + } + + WriteJSON(w, rsp) return nil } diff --git a/internal/server/versions.go b/internal/server/versions.go deleted file mode 100644 index 012dfcd5..00000000 --- a/internal/server/versions.go +++ /dev/null @@ -1,167 +0,0 @@ -package server - -import ( - "context" - _ "embed" - "errors" - "fmt" - "runtime" - "strings" - "time" - - "github.com/avast/retry-go" - "github.com/x1unix/go-playground/internal/util/syncx" - "github.com/x1unix/go-playground/pkg/goplay" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - goVersionRetryAttempts = 3 - goVersionRetryDelay = time.Second - - VersionCacheTTL = 24 * time.Hour -) - -//go:embed resources/version.go.txt -var versionSnippet []byte - -var backends = []goplay.Backend{ - goplay.BackendGoCurrent, - goplay.BackendGoPrev, - goplay.BackendGoTip, -} - -var _ BackendVersionProvider = (*BackendVersionService)(nil) - -// BackendVersionService provides information about used Go versions -// for all backends. -type BackendVersionService struct { - logger *zap.Logger - client *goplay.Client - cachedValue *syncx.TTLValue[*VersionsInformation] -} - -func NewBackendVersionService(logger *zap.Logger, client *goplay.Client, cacheTTL time.Duration) *BackendVersionService { - return &BackendVersionService{ - logger: logger, - client: client, - cachedValue: syncx.NewTTLValue[*VersionsInformation](cacheTTL, nil), - } -} - -// GetVersions provides Go version information for all backends. -func (svc *BackendVersionService) GetVersions(ctx context.Context) (*VersionsInformation, error) { - value := svc.cachedValue.Get() - if value != nil { - return value, nil - } - - vers, err := svc.probeGoVersions(ctx) - if err != nil { - return nil, err - } - - svc.cachedValue.Set(vers) - return vers, nil -} - -func (svc *BackendVersionService) probeGoVersions(ctx context.Context) (*VersionsInformation, error) { - playgroundVers, err := svc.getPlaygroundVersions(ctx) - if err != nil { - return nil, err - } - - hostGoVersion := normalizeGoVersion(runtime.Version()) - return &VersionsInformation{ - Playground: playgroundVers, - WebAssembly: hostGoVersion, - }, nil -} - -func (svc *BackendVersionService) getPlaygroundVersions(ctx context.Context) (*PlaygroundVersions, error) { - versionInfo := &PlaygroundVersions{} - g, gCtx := errgroup.WithContext(ctx) - for _, backend := range backends { - b := backend - g.Go(func() error { - svc.logger.Debug("Fetching go version for backend", zap.String("backend", b)) - result, err := svc.fetchGoBackendVersionWithRetry(gCtx, b) - if err != nil { - return fmt.Errorf("failed to get Go version from Go playground server for backend %q: %w", - b, err) - } - - // We don't afraid race condition because each backend is written to a separate address - versionInfo.SetBackendVersion(b, result) - return nil - }) - } - - if err := g.Wait(); err != nil { - return nil, err - } - - return versionInfo, nil -} - -func (svc *BackendVersionService) fetchGoBackendVersionWithRetry(ctx context.Context, backend goplay.Backend) (string, error) { - var result string - err := retry.Do( - func() error { - version, err := svc.getGoBackendVersion(ctx, backend) - if err != nil { - return err - } - - result = version - return nil - }, - retry.Attempts(goVersionRetryAttempts), - retry.Delay(goVersionRetryDelay), - retry.RetryIf(func(err error) bool { - httpErr, ok := goplay.IsHTTPError(err) - if !ok { - return false - } - - // Retry only on server issues - return httpErr.StatusCode >= 500 - }), - retry.OnRetry(func(n uint, err error) { - svc.logger.Error("failed to get Go version from Go playground, retrying...", - zap.Error(err), zap.String("backend", backend), zap.Uint("attempt", n)) - }), - ) - - return result, err -} - -func (svc *BackendVersionService) getGoBackendVersion(ctx context.Context, backend goplay.Backend) (string, error) { - // Dirty hack to fetch Go version for playground backend by running a simple program - // which returns Go version to stdout. - result, err := svc.client.Evaluate(ctx, goplay.CompileRequest{ - Version: goplay.DefaultVersion, - WithVet: false, - Body: versionSnippet, - }, backend) - - if err != nil { - return "", err - } - - if result.Errors != "" { - return "", fmt.Errorf("probe program returned an error: %s", result.Errors) - } - - if len(result.Events) == 0 { - return "", errors.New("missing output events from probe program") - } - - version := normalizeGoVersion(result.Events[0].Message) - return version, nil -} - -func normalizeGoVersion(str string) string { - return strings.TrimPrefix(str, "go") -}