Merge pull request #260 from hearchco/go-chi-image-proxy

feat(router)!: switch to go-chi and add image proxy

go.mod

@@ -6,16 +6,12 @@ toolchain go1.22.0
 
 require (
 	github.com/alecthomas/kong v0.9.0
+	github.com/andybalholm/brotli v1.1.0
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/fxamacker/cbor/v2 v2.6.0
-	github.com/gin-contrib/cors v1.7.0
-	github.com/gin-contrib/graceful v1.0.0
-	github.com/gin-contrib/gzip v0.0.6
-	github.com/gin-contrib/pprof v1.4.0
-	github.com/gin-gonic/gin v1.9.1
-	github.com/goccy/go-json v0.10.2
+	github.com/go-chi/chi/v5 v5.0.12
+	github.com/go-chi/cors v1.2.1
 	github.com/gocolly/colly/v2 v2.1.1-0.20231020184023-3c987f1982ed
-	github.com/hearchco/logger v1.0.1
 	github.com/knadh/koanf/parsers/yaml v0.1.0
 	github.com/knadh/koanf/providers/env v0.1.0
 	github.com/knadh/koanf/providers/file v0.1.0
@@ -31,50 +27,35 @@ require (
 require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/bits-and-blooms/bitset v1.13.0 // indirect
-	github.com/bytedance/sonic v1.11.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
-	github.com/chenzhuoyu/iasm v0.9.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/fgprof v0.9.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-playground/locales v0.14.1 // indirect
-	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.19.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.2.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v23.5.26+incompatible // indirect
-	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/google/flatbuffers v24.3.25+incompatible // indirect
+	github.com/google/pprof v0.0.0-20240320155624-b11c3daa6f07 // indirect
 	github.com/klauspost/compress v1.17.7 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
-	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nlnwa/whatwg-url v0.4.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/rs/xid v1.5.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/mod v0.16.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

hearchco_example.yaml

@@ -2,6 +2,8 @@ server:
   frontendurls: http://localhost:5173,https://*hearch.co,https://*hearchco-frontend.pages.dev
   cache:
     type: none
+  proxy:
+    salt: changemepls
 # categories:
 #   science:
 #     engines:
@@ -16,4 +18,4 @@ server:
 #   general:
 #     engines:
 #       google:
-#         enabled: true
+#         enabled: true

src/anonymize/hash.go

@@ -16,3 +16,11 @@ func HashToSHA256B64(orig string) string {
 
 	return hashedString
 }
+
+func HashToSHA256B64Salted(orig string, salt string) string {
+	return HashToSHA256B64(orig + salt)
+}
+
+func CheckHash(hash string, orig string, salt string) bool {
+	return hash == HashToSHA256B64Salted(orig, salt)
+}

src/cli/climode.go

@@ -76,7 +76,7 @@ func Run(flags Flags, db cache.DB, conf config.Config) {
 
 	start := time.Now()
 
-	results, foundInDB := search.Search(flags.Query, options, db, conf.Settings, conf.Categories)
+	results, foundInDB := search.Search(flags.Query, options, db, conf.Settings, conf.Categories, conf.Server.Proxy.Salt)
 
 	duration := time.Since(start)
 	if !flags.Silent {
@@ -87,5 +87,5 @@ func Run(flags Flags, db cache.DB, conf config.Config) {
 		Int64("ms", duration.Milliseconds()).
 		Msg("Found results")
 
-	search.CacheAndUpdateResults(flags.Query, options, db, conf.Server.Cache.TTL, conf.Settings, conf.Categories, results, foundInDB)
+	search.CacheAndUpdateResults(flags.Query, options, db, conf.Server.Cache.TTL, conf.Settings, conf.Categories, results, foundInDB, conf.Server.Proxy.Salt)
 }

src/config/defaults.go

@@ -162,6 +162,13 @@ func New() Config {
 					Port: 6379,
 				},
 			},
+			Proxy: Proxy{
+				Timeouts: ProxyTimeouts{
+					Dial:         3 * time.Second,
+					KeepAlive:    3 * time.Second,
+					TLSHandshake: 2 * time.Second,
+				},
+			},
 		},
 		Settings: NewSettings(),
 		Categories: map[category.Name]Category{

src/config/load.go

@@ -22,19 +22,31 @@ var LogDumpLocation string = "dump/"
 
 // passed as pointer since config is modified
 func (c *Config) fromReader(rc ReaderConfig) {
+	if rc.Server.Proxy.Salt == "" {
+		log.Fatal().Msg("config.fromReader(): proxy salt is empty")
+	}
+
 	nc := Config{
 		Server: Server{
 			Port:         rc.Server.Port,
 			FrontendUrls: strings.Split(rc.Server.FrontendUrls, ","),
 			Cache: Cache{
 				Type: rc.Server.Cache.Type,
 				TTL: TTL{
-					Time:        moretime.ConvertFancyTime(rc.Server.Cache.TTL.Time),
-					RefreshTime: moretime.ConvertFancyTime(rc.Server.Cache.TTL.RefreshTime),
+					Time:        moretime.ConvertFromFancyTime(rc.Server.Cache.TTL.Time),
+					RefreshTime: moretime.ConvertFromFancyTime(rc.Server.Cache.TTL.RefreshTime),
 				},
 				Badger: rc.Server.Cache.Badger,
 				Redis:  rc.Server.Cache.Redis,
 			},
+			Proxy: Proxy{
+				Salt: rc.Server.Proxy.Salt,
+				Timeouts: ProxyTimeouts{
+					Dial:         moretime.ConvertFromFancyTime(rc.Server.Proxy.Timeouts.Dial),
+					KeepAlive:    moretime.ConvertFromFancyTime(rc.Server.Proxy.Timeouts.KeepAlive),
+					TLSHandshake: moretime.ConvertFromFancyTime(rc.Server.Proxy.Timeouts.TLSHandshake),
+				},
+			},
 		},
 		Settings:   map[engines.Name]Settings{},
 		Categories: map[category.Name]Category{},
@@ -83,6 +95,7 @@ func (c *Config) fromReader(rc ReaderConfig) {
 	*c = nc
 }
 
+// called when loading default config, before merging with yaml and env
 func (c Config) getReader() ReaderConfig {
 	rc := ReaderConfig{
 		Server: ReaderServer{
@@ -97,6 +110,14 @@ func (c Config) getReader() ReaderConfig {
 				Badger: c.Server.Cache.Badger,
 				Redis:  c.Server.Cache.Redis,
 			},
+			Proxy: ReaderProxy{
+				Salt: c.Server.Proxy.Salt,
+				Timeouts: ReaderProxyTimeouts{
+					Dial:         moretime.ConvertToFancyTime(c.Server.Proxy.Timeouts.Dial),
+					KeepAlive:    moretime.ConvertToFancyTime(c.Server.Proxy.Timeouts.KeepAlive),
+					TLSHandshake: moretime.ConvertToFancyTime(c.Server.Proxy.Timeouts.TLSHandshake),
+				},
+			},
 		},
 		RCategories: map[category.Name]ReaderCategory{},
 		Settings:    map[string]Settings{},

src/config/structs.go

@@ -81,6 +81,26 @@ type Cache struct {
 	Redis  Redis
 }
 
+type ReaderProxyTimeouts struct {
+	Dial         string `koanf:"dial"`
+	KeepAlive    string `koanf:"keepalive"`
+	TLSHandshake string `koanf:"tlshandshake"`
+}
+type ProxyTimeouts struct {
+	Dial         time.Duration
+	KeepAlive    time.Duration
+	TLSHandshake time.Duration
+}
+
+type ReaderProxy struct {
+	Salt     string              `koanf:"salt"`
+	Timeouts ReaderProxyTimeouts `koanf:"timeouts"`
+}
+type Proxy struct {
+	Salt     string
+	Timeouts ProxyTimeouts
+}
+
 // ReaderServer is format in which the config is read from the config file
 type ReaderServer struct {
 	// port on which the API server listens
@@ -89,11 +109,14 @@ type ReaderServer struct {
 	FrontendUrls string `koanf:"frontendurls"`
 	// cache settings
 	Cache ReaderCache `koanf:"cache"`
+	// salt used for image proxy
+	Proxy ReaderProxy `koanf:"proxy"`
 }
 type Server struct {
 	Port         int
 	FrontendUrls []string
 	Cache        Cache
+	Proxy        Proxy
 }
 
 // ReaderEngine is format in which the config is read from the config file

src/main.go

@@ -57,17 +57,8 @@ func main() {
 	if cliFlags.Cli {
 		cli.Run(cliFlags, db, conf)
 	} else {
-		rw, err := router.New(conf.Server, cliFlags.Verbosity, lgr)
-		if err != nil {
-			log.Fatal().Err(err).Msg("main.main(): failed creating a router")
-			// ^FATAL
-		}
-
-		err = rw.Start(ctx, db, conf, cliFlags.ServeProfiler)
-		if err != nil {
-			log.Fatal().Err(err).Msg("main.main(): failed starting the router")
-			// ^FATAL
-		}
+		rw := router.New(lgr, conf, db, cliFlags.ServeProfiler)
+		rw.Start(ctx)
 	}
 
 	// program cleanup

src/moretime/fancy.go

@@ -22,7 +22,15 @@ func convertToDurationWithoutLastChar(s string) time.Duration {
 	return time.Duration(handleAtoi(s[:len(s)-1]))
 }
 
-func ConvertFancyTime(fancy string) time.Duration {
+// converts 1y to 1 year
+// converts 2M to 2 month
+// converts 3w to 3 week
+// converts 4d to 4 day
+// converts 5h to 5 hour
+// converts 6m to 6 minute
+// converts 7s to 7 second
+// converts 8 to 8 millisecond
+func ConvertFromFancyTime(fancy string) time.Duration {
 	switch fancy[len(fancy)-1] {
 	case 'y':
 		return convertToDurationWithoutLastChar(fancy) * Year
@@ -43,6 +51,7 @@ func ConvertFancyTime(fancy string) time.Duration {
 	}
 }
 
+// converts to milliseconds
 func ConvertToFancyTime(d time.Duration) string {
 	return strconv.Itoa(int(d.Milliseconds()))
 }

src/router/compress.go

@@ -0,0 +1,24 @@
+package router
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/andybalholm/brotli"
+	"github.com/go-chi/chi/v5/middleware"
+)
+
+func compress(level int, types ...string) [](func(http.Handler) http.Handler) {
+	// deflate & gzip
+	dig := middleware.Compress(level, types...)
+
+	// brotli
+	br := middleware.NewCompressor(level, types...)
+	br.SetEncoder("br", func(w io.Writer, level int) io.Writer {
+		return brotli.NewWriterOptions(w, brotli.WriterOptions{
+			Quality: level,
+		})
+	})
+
+	return [](func(http.Handler) http.Handler){dig, br.Handler}
+}

src/router/logging.go

@@ -0,0 +1,53 @@
+package router
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/hlog"
+)
+
+func ignoredPath(path string, skipPaths []string) bool {
+	for _, p := range skipPaths {
+		if p == path {
+			return true
+		}
+	}
+	return false
+}
+
+func zerologMiddleware(lgr zerolog.Logger, skipPaths []string) [](func(http.Handler) http.Handler) {
+	newHandler := hlog.NewHandler(lgr)
+	fieldsHandler := hlog.AccessHandler(func(r *http.Request, status int, size int, duration time.Duration) {
+		// skip logging for ignored paths
+		if ignoredPath(r.URL.Path, skipPaths) {
+			return
+		}
+
+		// get logger from context
+		lgr := hlog.FromRequest(r)
+
+		// decide on log level
+		event := lgr.Info()
+		if status >= 500 {
+			event = lgr.Error()
+		} else if status >= 400 {
+			event = lgr.Warn()
+		}
+
+		// log
+		event.
+			Str("method", r.Method).
+			Str("path", r.URL.Path).
+			Int("status", status).
+			Dur("duration", duration).
+			Str("ip", r.RemoteAddr).
+			Msg("Request")
+	})
+
+	return [](func(http.Handler) http.Handler){
+		newHandler,
+		fieldsHandler,
+	}
+}