Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: image search #204

Closed
wants to merge 49 commits into from
Closed
Show file tree
Hide file tree
Changes from 46 commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
f447f2b
feat(images): added Google and Bing Images search, with category image
aleksasiriski Feb 8, 2024
38a90c5
chore(tests): use %q for printing
aleksasiriski Feb 8, 2024
95ce00e
chore(tests): added bingimages test
aleksasiriski Feb 8, 2024
edf7820
fix(googleimages): disable UA to good results
aleksasiriski Feb 8, 2024
9070c35
chore(tests): added googleimages test
aleksasiriski Feb 8, 2024
ff57af9
Merge branch 'refactor-search' into feat-image-search
aleksasiriski Feb 8, 2024
ab0315e
Merge branch 'refactor-search' into feat-image-search
aleksasiriski Feb 8, 2024
6166e36
chore(tests): fixed bingimages test
aleksasiriski Feb 8, 2024
e79fcdc
fix(bingimages,googleimages): no adding results w/o url, orig and thu…
aleksasiriski Feb 9, 2024
a5c893c
fix(bingimages,googleimages): universal field names for logs
aleksasiriski Feb 9, 2024
8aacd45
fix(result): add ImageResult from RetrievedResult to Result
aleksasiriski Feb 9, 2024
bb15d62
fix(googleimages): mistake in log field name
aleksasiriski Feb 9, 2024
a834abd
Merge branch 'refactor-search' into feat-image-search
aleksasiriski Feb 9, 2024
2d3d0a3
fix(imagetests): added larger timeout for IMAGE category in unit testing
aleksasiriski Feb 9, 2024
b378fd1
Merge branch 'main' into feat-image-search
aleksasiriski Feb 9, 2024
15edf2b
Merge branch 'main' into feat-image-search
aleksasiriski Feb 10, 2024
578a21d
fix(bingimages): skip videos
aleksasiriski Feb 13, 2024
468c693
Merge branch 'main' into feat-image-search
aleksasiriski Feb 13, 2024
b53a8fb
fix(bingimages): ineffectual assignment
aleksasiriski Feb 19, 2024
fd42b82
refactor(googleimages): return in error if statements
aleksasiriski Feb 19, 2024
0806168
refactor(bingimages): add more returns and remove else statements
aleksasiriski Feb 19, 2024
f2b7176
Merge branch 'main' into feat-image-search
aleksasiriski Feb 20, 2024
ddf1e3e
Merge branch 'main' into feat-image-search
aleksasiriski Feb 21, 2024
316d1c0
fix(bingimages): three types of html dom for thumbnail height and width
aleksasiriski Feb 21, 2024
c1e4f94
refactor(bingimages): moved dompaths in a separate file
aleksasiriski Feb 21, 2024
0afd06c
fix: change trace to error, since code is expected to always match pr…
k4lizen Feb 22, 2024
c1f39a7
fix: clean up json field names
k4lizen Feb 22, 2024
f9ea3a6
fix: make logger print the json data if unmarshal failed
k4lizen Feb 22, 2024
cf4e0d6
fix: more descriptive error message if image/json/thumbnail url is mi…
k4lizen Feb 22, 2024
11fb864
fix: add comment for titleText detection
k4lizen Feb 22, 2024
7f1397d
fix: height string was being used for width
k4lizen Feb 22, 2024
9756c24
fix: make result selector REALLY only select images
k4lizen Feb 22, 2024
3c1615f
fix: bing returns empty body if it gets too many requests, log this
k4lizen Feb 22, 2024
0aae515
fix: update comment on titleText fetching
k4lizen Feb 23, 2024
9cf5a85
fix(googleimages,bingimages): use original url as primary key
aleksasiriski Feb 24, 2024
788796d
Merge branch 'feat-image-search' into image-bing-rev
aleksasiriski Feb 24, 2024
c8901be
fix(bingimages): unit test
aleksasiriski Feb 24, 2024
511dd09
fix(googleimages): unit test
aleksasiriski Feb 24, 2024
c370ad4
Merge branch 'main' into feat-image-search
aleksasiriski Feb 24, 2024
855d9a3
Merge branch 'feat-image-search' into image-bing-rev
aleksasiriski Feb 24, 2024
ac8664e
fix: review and cleanup code relating to bing image search (#223)
aleksasiriski Feb 24, 2024
40a9a03
fix: msgf to msg
k4lizen Feb 24, 2024
464f2a9
fix: change format Error back to trace
k4lizen Feb 24, 2024
2c7781c
fix(bingimages,googlescholar): replace Msgf with fields
aleksasiriski Feb 24, 2024
4089304
refactor(images): MakeSEImageResult primitives, better formating and …
aleksasiriski Feb 25, 2024
079af81
fix(bucket): naming of field
aleksasiriski Feb 25, 2024
8711445
fix(googleimages): col.OnResponse log
aleksasiriski Feb 25, 2024
411decd
fix(images): renamed image category to images
aleksasiriski Mar 1, 2024
e61ce29
chore: better default example conf
aleksasiriski Mar 1, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cli/climode.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
func printResults(results []result.Result) {
fmt.Print("\n\tThe Search Results:\n\n")
for _, r := range results {
fmt.Printf("%v (%.2f) -----\n\t\"%v\"\n\t\"%v\"\n\t\"%v\"\n\t-", r.Rank, r.Score, r.Title, r.URL, r.Description)
fmt.Printf("%v (%.2f) -----\n\t%q\n\t%q\n\t%q\n\n%+v\n\t-", r.Rank, r.Score, r.Title, r.URL, r.Description, r)
for seInd := uint8(0); seInd < r.TimesReturned; seInd++ {
fmt.Printf("%v", r.EngineRanks[seInd].SearchEngine.ToLower())
if seInd != r.TimesReturned-1 {
Expand Down
21 changes: 21 additions & 0 deletions src/config/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ func NewSettings() map[engines.Name]Settings {
engines.BING: {
Shortcut: "bi",
},
engines.BINGIMAGES: {
Shortcut: "biimg",
},
engines.BRAVE: {
Shortcut: "br",
},
Expand All @@ -56,6 +59,9 @@ func NewSettings() map[engines.Name]Settings {
engines.GOOGLE: {
Shortcut: "g",
},
engines.GOOGLEIMAGES: {
Shortcut: "gimg",
},
engines.GOOGLESCHOLAR: {
Shortcut: "gs",
},
Expand Down Expand Up @@ -114,6 +120,13 @@ func NewGeneral() []engines.Name {
}
}

func NewImage() []engines.Name {
return []engines.Name{
engines.BINGIMAGES,
engines.GOOGLEIMAGES,
}
}

func NewInfo() []engines.Name {
return []engines.Name{
engines.BING,
Expand Down Expand Up @@ -158,6 +171,14 @@ func New() *Config {
PageTimeout: 1000 * time.Millisecond,
},
},
category.IMAGE: {
Engines: NewImage(),
Ranking: NewRanking(),
Timings: Timings{
Timeout: 1500 * time.Millisecond,
PageTimeout: 1500 * time.Millisecond,
},
},
category.INFO: {
Engines: NewInfo(),
Ranking: NewRanking(),
Expand Down
24 changes: 12 additions & 12 deletions src/router/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,37 +50,37 @@ func Search(c *gin.Context, conf *config.Config, db cache.DB) error {
} else {
maxPages, pageserr := strconv.Atoi(pages)
if pageserr != nil {
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert pages value (\"%v\") to int", pages))
return fmt.Errorf("router.Search(): cannot convert pages value \"%v\" to int: %w", pages, pageserr)
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert pages value (%q) to int", pages))
return fmt.Errorf("router.Search(): cannot convert pages value %q to int: %w", pages, pageserr)
}

visitPages, deeperr := strconv.ParseBool(deepSearch)
if deeperr != nil {
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert deep value (\"%v\") to bool", deepSearch))
return fmt.Errorf("router.Search(): cannot convert deep value \"%v\" to int: %w", deepSearch, deeperr)
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert deep value (%q) to bool", deepSearch))
return fmt.Errorf("router.Search(): cannot convert deep value %q to int: %w", deepSearch, deeperr)
}

if lerr := engines.ValidateLocale(locale); lerr != nil {
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Invalid locale value (\"%v\"), should be of the form \"en_US\"", locale))
return fmt.Errorf("router.Search(): invalid locale value \"%v\": %w", locale, lerr)
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Invalid locale value (%q), should be of the form \"en_US\"", locale))
return fmt.Errorf("router.Search(): invalid locale value %q: %w", locale, lerr)
}

ccateg = category.SafeFromString(categ)
if ccateg == category.UNDEFINED {
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Invalid category value (\"%v\")", categ))
return fmt.Errorf("router.Search(): invalid category value \"%v\"", categ)
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Invalid category value (%q)", categ))
return fmt.Errorf("router.Search(): invalid category value %q", categ)
}

safeSearchB, safeerr := strconv.ParseBool(safesearch)
if safeerr != nil {
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert safesearch value (\"%v\") to bool", safesearch))
return fmt.Errorf("router.Search(): cannot convert safesearch value \"%v\" to bool: %w", safesearch, safeerr)
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert safesearch value (%q) to bool", safesearch))
return fmt.Errorf("router.Search(): cannot convert safesearch value %q to bool: %w", safesearch, safeerr)
}

isMobile, mobileerr := strconv.ParseBool(mobile)
if mobileerr != nil {
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert mobile value (\"%v\") to bool", mobile))
return fmt.Errorf("router.Search(): cannot convert mobile value \"%v\" to bool: %w", mobile, mobileerr)
c.String(http.StatusUnprocessableEntity, fmt.Sprintf("Cannot convert mobile value (%q) to bool", mobile))
return fmt.Errorf("router.Search(): cannot convert mobile value %q to bool: %w", mobile, mobileerr)
}

options := engines.Options{
Expand Down
2 changes: 1 addition & 1 deletion src/search/bucket/addresult.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *R
Description: seResult.Description,
EngineRanks: engineRanks,
TimesReturned: 1,
Response: nil,
ImageResult: seResult.ImageResult,
}

relay.Mutex.Lock()
Expand Down
49 changes: 46 additions & 3 deletions src/search/bucket/makeresult.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,61 @@ import (
"github.com/hearchco/hearchco/src/search/result"
)

func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *result.RetrievedResult {
func MakeSEResult(
urll, title, desc string,
seName engines.Name, sePage, seOnPageRank int,
) *result.RetrievedResult {

ser := result.RetrievedRank{
SearchEngine: searchEngineName,
SearchEngine: seName,
Rank: 0,
Page: uint(sePage),
OnPageRank: uint(seOnPageRank),
}

res := result.RetrievedResult{
URL: urll,
Title: title,
Description: description,
Description: desc,
Rank: ser,
}

return &res
}

func MakeSEImageResult(
urll, title, desc string,
src, srcUrl, thmbUrl string,
origH, origW, thmbH, thmbW int,
seName engines.Name, sePage, seOnPageRank int,
) *result.RetrievedResult {

ser := result.RetrievedRank{
SearchEngine: seName,
Rank: 0,
Page: uint(sePage),
OnPageRank: uint(seOnPageRank),
}

res := result.RetrievedResult{
URL: urll,
Title: title,
Description: desc,
ImageResult: result.ImageResult{
Original: result.ImageFormat{
Height: uint(origH),
Width: uint(origW),
},
Thumbnail: result.ImageFormat{
Height: uint(thmbH),
Width: uint(thmbW),
},
ThumbnailURL: thmbUrl,
Source: src,
SourceURL: srcUrl,
},
Rank: ser,
}

return &res
}
1 change: 1 addition & 0 deletions src/search/category/category.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
var FromString = map[string]Name{
//main
"general": GENERAL,
"image": IMAGE,
"info": INFO,
"science": SCIENCE,
"news": NEWS,
Expand Down
1 change: 1 addition & 0 deletions src/search/category/name.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ type Name string
const (
UNDEFINED Name = "undefined"
GENERAL Name = "general"
IMAGE Name = "image"
INFO Name = "info"
SCIENCE Name = "science"
NEWS Name = "news"
Expand Down
7 changes: 7 additions & 0 deletions src/search/engines/_engines_test/structs.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,13 @@ func NewConfig(engineName engines.Name) *config.Config {
Timeout: 10000 * time.Millisecond, // colly default
},
},
category.IMAGE: {
Engines: []engines.Name{engineName},
Ranking: config.NewRanking(),
Timings: config.Timings{
Timeout: 10000 * time.Millisecond, // colly default
},
},
},
}
}
10 changes: 5 additions & 5 deletions src/search/engines/_engines_test/tester.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult
// TestCaseHasAnyResults
for _, tc := range tchar {
if results := search.PerformSearch(tc.Query, tc.Options, conf); len(results) == 0 {
defer t.Errorf("Got no results for %v", tc.Query)
defer t.Errorf("Got no results for %q", tc.Query)
}
}

// TestCaseContainsResults
for _, tc := range tccr {
results := search.PerformSearch(tc.Query, tc.Options, conf)
if len(results) == 0 {
defer t.Errorf("Got no results for %v", tc.Query)
defer t.Errorf("Got no results for %q", tc.Query)
} else {
for _, rURL := range tc.ResultURL {
found := false
Expand All @@ -35,7 +35,7 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult
}

if !found {
defer t.Errorf("Couldn't find %v (%v).\nThe results: %v", rURL, tc.Query, results)
defer t.Errorf("Couldn't find %q (%q).\nThe results: %q", rURL, tc.Query, results)
}
}
}
Expand All @@ -45,13 +45,13 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult
for _, tc := range tcrr {
results := search.PerformSearch(tc.Query, tc.Options, conf)
if len(results) == 0 {
defer t.Errorf("Got no results for %v", tc.Query)
defer t.Errorf("Got no results for %q", tc.Query)
} else if len(results) < len(tc.ResultURL) {
defer t.Errorf("Number of results is less than test case URLs.")
} else {
for i, rURL := range tc.ResultURL {
if !strings.Contains(results[i].URL, rURL) {
defer t.Errorf("Wrong result on rank %v: %v (%v).\nThe results: %v", i+1, rURL, tc.Query, results)
defer t.Errorf("Wrong result on rank %q: %q (%q).\nThe results: %q", i+1, rURL, tc.Query, results)
}
}
}
Expand Down
Loading
Loading