feat: Error handling

Makefile

@@ -12,3 +12,6 @@ test:
 update:
 	go get -u ./...
 	go mod tidy
+
+lint:
+	golangci-lint run

generate/searcher/searcher.go

@@ -72,6 +72,7 @@ func main() {
 	} else {
 		if len(tags) != 0 {
 			log.Fatal("-tags option applies only to directories, not when files are specified")
+			// ^FATAL
 		}
 		dir = path.Dir(args[0])
 	}
@@ -108,6 +109,7 @@ func main() {
 	err := os.WriteFile(outputName, src, 0644)
 	if err != nil {
 		log.Fatalf("writing output: %s", err)
+		// ^FATAL
 	}
 }
 
@@ -127,9 +129,11 @@ func (g *Generator) parsePackage(patterns []string, tags []string) {
 	pkgs, err := packages.Load(cfg, patterns...)
 	if err != nil {
 		log.Fatal(err)
+		// ^FATAL
 	}
 	if len(pkgs) != 1 {
 		log.Fatalf("error: %d packages matching %v", len(pkgs), strings.Join(patterns, " "))
+		// ^FATAL
 	}
 	g.addPackage(pkgs[0])
 }
@@ -167,6 +171,7 @@ func (g *Generator) generate(typeName string) {
 
 	if len(values) == 0 {
 		log.Fatalf("no values defined for type %s", typeName)
+		// ^FATAL
 	}
 
 	// Generate code for importing engines
@@ -271,19 +276,23 @@ func (f *File) genDecl(node ast.Node) bool {
 			obj, ok := f.pkg.defs[name]
 			if !ok {
 				log.Fatalf("no value for constant %s", name)
+				// ^FATAL
 			}
 			info := obj.Type().Underlying().(*types.Basic).Info()
 			if info&types.IsInteger == 0 {
 				log.Fatalf("can't handle non-integer constant type %s", typ)
+				// ^FATAL
 			}
 			value := obj.(*types.Const).Val() // Guaranteed to succeed as this is CONST.
 			if value.Kind() != constant.Int {
 				log.Fatalf("can't happen: constant is not an integer %s", name)
+				// ^FATAL
 			}
 			i64, isInt := constant.Int64Val(value)
 			u64, isUint := constant.Uint64Val(value)
 			if !isInt && !isUint {
 				log.Fatalf("internal error: value of %s is not an integer: %s", name, value.String())
+				// ^FATAL
 			}
 			if !isInt {
 				u64 = uint64(i64)

generate/searcher/util.go

@@ -24,6 +24,7 @@ func isDirectoryFatal(path string) bool {
 	info, err := os.Stat(path)
 	if err != nil {
 		log.Fatal(err)
+		// ^FATAL
 	}
 	return info.IsDir()
 }

go.mod

@@ -6,7 +6,7 @@ toolchain go1.21.3
 
 require (
 	github.com/alecthomas/kong v0.8.1
-	github.com/cockroachdb/pebble v0.0.0-20231106144427-a0b01b62e8f9
+	github.com/cockroachdb/pebble v0.0.0-20231206044450-348b3a068f94
 	github.com/fxamacker/cbor/v2 v2.5.0
 	github.com/gin-contrib/cors v1.5.0
 	github.com/gin-contrib/graceful v0.1.0
@@ -50,9 +50,9 @@ require (
 	github.com/go-playground/validator/v10 v10.16.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
+	github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -73,11 +73,11 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/arch v0.6.0 // indirect
 	golang.org/x/crypto v0.16.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect

go.sum

@@ -63,8 +63,10 @@ github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZ
 github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20231106144427-a0b01b62e8f9 h1:sCTdjoX7OXa5jjHNXw/jwR+gnmM/yLhwsEsAG1OFtBQ=
-github.com/cockroachdb/pebble v0.0.0-20231106144427-a0b01b62e8f9/go.mod h1:acMRUGd/BK8AUmQNK3spUCCGzFLZU2bSST3NMXSq2Kc=
+github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
+github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
+github.com/cockroachdb/pebble v0.0.0-20231206044450-348b3a068f94 h1:BnNrdQcwSlz6eJF62X/zP5gN6efSNoz0CBu1yQdpzLQ=
+github.com/cockroachdb/pebble v0.0.0-20231206044450-348b3a068f94/go.mod h1:BHuaMa/lK7fUe75BlsteiiTu8ptIG+qSAuDtGMArP18=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
@@ -155,8 +157,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
-github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk=
-github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 h1:PxlBVtIFHR/mtWk2i0gTEdCz+jBnqiuHNSki0epDbVs=
+github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -167,8 +169,8 @@ github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8Nz
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -278,8 +280,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -297,8 +299,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
 golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=

src/bucket/bucket.go

@@ -1,6 +1,7 @@
 package bucket
 
 import (
+	"fmt"
 	"sync"
 
 	"github.com/gocolly/colly/v2"
@@ -62,12 +63,12 @@ func AddSEResult(seResult *engines.RetrievedResult, seName engines.Name, relay *
 
 	if !exists && options.VisitPages {
 		if err := pagesCol.Visit(seResult.URL); err != nil {
-			log.Error().Err(err).Msgf("bucket: failed visiting %v", seResult.URL)
+			log.Error().Err(err).Msgf("bucket.AddSEResult(): failed visiting %v", seResult.URL)
 		}
 	}
 }
 
-func SetResultResponse(link string, response *colly.Response, relay *Relay, seName engines.Name) {
+func SetResultResponse(link string, response *colly.Response, relay *Relay, seName engines.Name) error {
 	log.Trace().Msgf("%v: Got Response -> %v", seName, link)
 
 	relay.Mutex.Lock()
@@ -76,12 +77,15 @@ func SetResultResponse(link string, response *colly.Response, relay *Relay, seNa
 	if !exists {
 		relay.Mutex.Unlock()
 		relay.Mutex.RLock()
-		log.Error().Msgf("URL not in map when adding response! Should not be possible. URL: %v.\nRelay: %v", link, relay)
+		err := fmt.Errorf("bucket.SetResultResponse(): URL not in map when adding response, should not be possible. URL: %v.\nRelay: %v", link, relay)
 		relay.Mutex.RUnlock()
+		return err
 	} else {
 		mapRes.Response = response
 		relay.Mutex.Unlock()
 	}
+
+	return nil
 }
 
 func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *engines.RetrievedResult {

src/cache/interfaces.go

@@ -2,8 +2,8 @@ package cache
 
 type DB interface {
 	Close()
-	Set(k string, v Value)
-	Get(k string, o Value)
+	Set(k string, v Value) error
+	Get(k string, o Value) error
 }
 
 type Value interface{}

src/cache/nocache/nocache.go

@@ -10,6 +10,6 @@ func New() *DB { return nil }
 
 func (db *DB) Close() {}
 
-func (db *DB) Set(k string, v cache.Value) {}
+func (db *DB) Set(k string, v cache.Value) error { return nil }
 
-func (db *DB) Get(k string, o cache.Value) {}
+func (db *DB) Get(k string, o cache.Value) error { return nil }

src/cache/pebble/pebble.go

@@ -1,13 +1,14 @@
 package pebble
 
 import (
+	"fmt"
 	"path"
 	"time"
 
 	"github.com/cockroachdb/pebble"
 	"github.com/fxamacker/cbor/v2"
-	"github.com/rs/zerolog/log"
 	"github.com/hearchco/hearchco/src/cache"
+	"github.com/rs/zerolog/log"
 )
 
 type DB struct {
@@ -19,7 +20,8 @@ func New(dataDirPath string) *DB {
 	pdb, err := pebble.Open(pebblePath, &pebble.Options{})
 
 	if err != nil {
-		log.Fatal().Err(err).Msgf("Error opening pebble at path: %v", pebblePath)
+		log.Fatal().Err(err).Msgf("pebble.New(): error opening pebble at path: %v", pebblePath)
+		// ^FATAL
 	} else {
 		log.Info().Msgf("Successfully opened pebble (path: %v)", pebblePath)
 	}
@@ -29,37 +31,43 @@ func New(dataDirPath string) *DB {
 
 func (db *DB) Close() {
 	if err := db.pdb.Close(); err != nil {
-		log.Fatal().Err(err).Msg("Error closing pebble")
+		log.Fatal().Err(err).Msg("pebble.Close(): error closing pebble")
+		// ^FATAL
 	} else {
 		log.Debug().Msg("Successfully closed pebble")
 	}
 }
 
-func (db *DB) Set(k string, v cache.Value) {
+func (db *DB) Set(k string, v cache.Value) error {
 	log.Debug().Msg("Caching...")
 	cacheTimer := time.Now()
 
 	if val, err := cbor.Marshal(v); err != nil {
-		log.Error().Err(err).Msg("Error marshaling value")
+		return fmt.Errorf("pebble.Set(): error marshaling value: %w", err)
 	} else if err := db.pdb.Set([]byte(k), val, pebble.NoSync); err != nil {
-		log.Fatal().Err(err).Msg("Error setting KV to pebble")
+		log.Fatal().Err(err).Msg("pebble.Set(): error setting KV to pebble")
+		// ^FATAL
 	} else {
 		cacheTimeSince := time.Since(cacheTimer)
 		log.Debug().Msgf("Cached results in %vms (%vns)", cacheTimeSince.Milliseconds(), cacheTimeSince.Nanoseconds())
 	}
+	return nil
 }
 
-func (db *DB) Get(k string, o cache.Value) {
+func (db *DB) Get(k string, o cache.Value) error {
 	v, c, err := db.pdb.Get([]byte(k))
 	val := []byte(v) // copy data before closing, casting needed for unmarshal
 
 	if err == pebble.ErrNotFound {
-		log.Trace().Msgf("Found no value in pebble for key %v", k)
+		log.Trace().Msgf("Found no value in pebble for key: \"%v\"", k)
 	} else if err != nil {
-		log.Fatal().Err(err).Msgf("Error getting value from pebble for key %v", k)
+		log.Fatal().Err(err).Msgf("pebble.Get(): error getting value from pebble for key %v", k)
+		// ^FATAL
 	} else if err := c.Close(); err != nil {
-		log.Fatal().Err(err).Msgf("Error closing io to pebble for key %v", k)
+		log.Fatal().Err(err).Msgf("pebble.Get(): error closing io to pebble for key %v", k)
+		// ^FATAL
 	} else if err := cbor.Unmarshal(val, o); err != nil {
-		log.Error().Err(err).Msgf("Failed unmarshaling value from pebble for key %v", k)
+		return fmt.Errorf("pebble.Get(): failed unmarshaling value from pebble for key %v: %w", k, err)
 	}
+	return nil
 }

src/cache/redis/redis.go

@@ -6,10 +6,10 @@ import (
 	"time"
 
 	"github.com/fxamacker/cbor/v2"
-	"github.com/redis/go-redis/v9"
-	"github.com/rs/zerolog/log"
 	"github.com/hearchco/hearchco/src/cache"
 	"github.com/hearchco/hearchco/src/config"
+	"github.com/redis/go-redis/v9"
+	"github.com/rs/zerolog/log"
 )
 
 type DB struct {
@@ -25,7 +25,8 @@ func New(ctx context.Context, config config.Redis) *DB {
 	})
 
 	if err := rdb.Ping(ctx).Err(); err != nil {
-		log.Fatal().Err(err).Msgf("Error connecting to redis with addr: %v:%v/%v", config.Host, config.Port, config.Database)
+		log.Fatal().Err(err).Msgf("redis.New(): error connecting to redis with addr: %v:%v/%v", config.Host, config.Port, config.Database)
+		// ^FATAL
 	} else {
 		log.Info().Msgf("Successful connection to redis (addr: %v:%v/%v)", config.Host, config.Port, config.Database)
 	}
@@ -35,35 +36,40 @@ func New(ctx context.Context, config config.Redis) *DB {
 
 func (db *DB) Close() {
 	if err := db.rdb.Close(); err != nil {
-		log.Fatal().Err(err).Msg("Error disconnecting from redis")
+		log.Fatal().Err(err).Msg("redis.Close(): error disconnecting from redis")
+		// ^FATAL
 	} else {
 		log.Debug().Msg("Successfully disconnected from redis")
 	}
 }
 
-func (db *DB) Set(k string, v cache.Value) {
+func (db *DB) Set(k string, v cache.Value) error {
 	log.Debug().Msg("Caching...")
 	cacheTimer := time.Now()
 
 	if val, err := cbor.Marshal(v); err != nil {
-		log.Error().Err(err).Msg("Error marshaling value")
+		return fmt.Errorf("redis.Set(): error marshaling value: %w", err)
 	} else if err := db.rdb.Set(db.ctx, k, val, 0).Err(); err != nil {
-		log.Fatal().Err(err).Msg("Error setting KV to redis")
+		log.Fatal().Err(err).Msg("redis.Set(): error setting KV to redis")
+		// ^FATAL
 	} else {
 		cacheTimeSince := time.Since(cacheTimer)
 		log.Debug().Msgf("Cached results in %vms (%vns)", cacheTimeSince.Milliseconds(), cacheTimeSince.Nanoseconds())
 	}
+	return nil
 }
 
-func (db *DB) Get(k string, o cache.Value) {
+func (db *DB) Get(k string, o cache.Value) error {
 	v, err := db.rdb.Get(db.ctx, k).Result()
 	val := []byte(v) // copy data before closing, casting needed for unmarshal
 
 	if err == redis.Nil {
-		log.Trace().Msgf("Found no value in redis for key %v", k)
+		log.Trace().Msgf("found no value in redis for key \"%v\"", k)
 	} else if err != nil {
-		log.Fatal().Err(err).Msgf("Error getting value from redis for key %v", k)
+		log.Fatal().Err(err).Msgf("redis.Get(): error getting value from redis for key %v", k)
+		// ^FATAL
 	} else if err := cbor.Unmarshal(val, o); err != nil {
-		log.Error().Err(err).Msgf("Failed unmarshaling value from redis for key %v", k)
+		return fmt.Errorf("redis.Set(): failed unmarshaling value from redis for key %v: %w", k, err)
 	}
+	return nil
 }