initial commit

.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: Test
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.22.1"]
+        os: [ubuntu-latest]
+    name: ${{ matrix.os }} Go ${{ matrix.go }} Tests
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+      - run: go test

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Radu Lucuț
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,68 @@
+# Search
+
+In-memory fuzzy search engine for Golang.
+
+[![Go Reference](https://pkg.go.dev/badge/github.com/radulucut/search.svg)](https://pkg.go.dev/github.com/radulucut/search)
+![Test](https://github.com/radulucut/search/actions/workflows/test.yml/badge.svg)
+
+## Installation
+
+```bash
+go get github.com/radulucut/search
+```
+
+## Usage
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/radulucut/search"
+)
+
+type Book struct {
+	Id   int64
+	Text string
+}
+
+var items = []Book{
+	{Id: 1, Text: "Ultima noapte de dragoste, întâia noapte de război de Camil Petrescu"},
+	{Id: 2, Text: "Pădurea spânzuraţilor de Liviu Rebreanu"},
+	{Id: 3, Text: "Moromeții I de Marin Preda"},
+	{Id: 4, Text: "Maitreyi de Mircea Eliade"},
+	{Id: 5, Text: "Enigma Otiliei de George Călinescu"},
+	{Id: 6, Text: "La țigănci de Mircea Eliade"},
+	{Id: 7, Text: "Moara cu noroc de Ioan Slavici"},
+	{Id: 8, Text: "Amintiri din copilărie de Ion Creangă"},
+	{Id: 9, Text: "Patul lui Procust de Camil Petrescu"},
+	{Id: 10, Text: "Elevul Dima dintr-a VII-A de Mihail Drumeș"},
+	{Id: 11, Text: "Întoarcerea din rai de Mircea Eliade"},
+	{Id: 12, Text: "La hanul lui Mânjoală de Ion Luca Caragiale"},
+	{Id: 13, Text: "O scrisoare pierdută de Ion Luca Caragiale"},
+	{Id: 14, Text: "Ion de Liviu Rebreanu"},
+	{Id: 15, Text: "Baltagul de Mihail Sadoveanu"},
+}
+
+func main() {
+	// Create a new search engine
+	engine := search.NewEngine(items, func(item Book) (int64, string) {
+		return item.Id, item.Text
+	}, nil)
+	engine.SetTolerance(2)
+
+	// Search for a book
+	results := engine.Search("Eliade", 5)
+
+	// Print the results
+	// Output: []int64{11, 6, 4}
+	fmt.Println(results)
+}
+
+```
+
+## TODO
+
+- [ ] Add stemming support
+- [ ] Add tf-idf support

go.mod

@@ -0,0 +1,5 @@
+module github.com/radulucut/search
+
+go 1.22.1
+
+require github.com/google/go-cmp v0.6.0

go.sum

@@ -0,0 +1,2 @@
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

search.go

@@ -0,0 +1,116 @@
+package search
+
+import (
+	"sort"
+	"sync"
+)
+
+type Engine struct {
+	items     map[int64][][]rune
+	tokenize  TokenizeFunc
+	tolerance int
+	mx        sync.RWMutex
+}
+
+// NewEngine creates a new search engine.
+//
+// items is a slice of items to be indexed.
+//
+// mapFunc is a function that maps an item to an id and a string.
+//
+// tokenizeFunc is an optional function that tokenizes a string into words.
+func NewEngine[T any](
+	items []T,
+	mapFunc func(T) (int64, string),
+	tokenizeFunc TokenizeFunc,
+) *Engine {
+	engine := &Engine{
+		items:     make(map[int64][][]rune),
+		tolerance: 1,
+		tokenize:  Tokenize,
+		mx:        sync.RWMutex{},
+	}
+	if tokenizeFunc != nil {
+		engine.tokenize = tokenizeFunc
+	}
+	for i := range items {
+		id, s := mapFunc(items[i])
+		engine.items[id] = engine.tokenize(s)
+	}
+	return engine
+}
+
+// SetTolerance sets the maximum number of typos per word allowed.
+// The default value is 1.
+func (e *Engine) SetTolerance(tolerance int) {
+	e.mx.Lock()
+	defer e.mx.Unlock()
+	e.tolerance = tolerance
+}
+
+// SetItem adds a new item to the search engine.
+func (e *Engine) SetItem(id int64, text string) {
+	e.mx.Lock()
+	defer e.mx.Unlock()
+	e.items[id] = e.tokenize(text)
+}
+
+// DeleteItem removes an item from the search engine.
+func (e *Engine) DeleteItem(id int64) {
+	e.mx.Lock()
+	defer e.mx.Unlock()
+	delete(e.items, id)
+}
+
+type itemScore struct {
+	id    int64
+	score int
+}
+
+// Search finds the most similar items to the given query.
+// limit is the maximum number of items to return.
+func (e *Engine) Search(query string, limit int) []int64 {
+	q := e.tokenize(query)
+	e.mx.RLock()
+	defer e.mx.RUnlock()
+	scores := make([]itemScore, 0)
+	for id := range e.items {
+		score := e.score(q, e.items[id])
+		if score == -1 {
+			continue
+		}
+		scores = append(scores, itemScore{id: id, score: score})
+	}
+	sort.Slice(scores, func(i, j int) bool {
+		if scores[i].score == scores[j].score {
+			return scores[i].id > scores[j].id
+		} else {
+			return scores[i].score < scores[j].score
+		}
+	})
+	limit = min(limit, len(scores))
+	res := make([]int64, 0, limit)
+	for i := 0; i < limit; i++ {
+		res = append(res, scores[i].id)
+	}
+	return res
+}
+
+func (e *Engine) score(q, b [][]rune) int {
+	var score int
+	skip := true
+	for i := range q {
+		best := (1<<63 - 1)
+		for j := range b {
+			best = min(best, LevenshteinDistance(q[i], b[j]))
+		}
+		if best <= e.tolerance {
+			skip = false
+		}
+		score += best
+	}
+	if skip {
+		return -1
+	}
+	return score
+}

search_test.go

@@ -0,0 +1,102 @@
+package search
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+type Book struct {
+	Id   int64
+	Text string
+}
+
+var items = []Book{
+	{Id: 1, Text: "Ultima noapte de dragoste, întâia noapte de război de Camil Petrescu"},
+	{Id: 2, Text: "Pădurea spânzuraţilor de Liviu Rebreanu"},
+	{Id: 3, Text: "Moromeții I de Marin Preda"},
+	{Id: 4, Text: "Maitreyi de Mircea Eliade"},
+	{Id: 5, Text: "Enigma Otiliei de George Călinescu"},
+	{Id: 6, Text: "La țigănci de Mircea Eliade"},
+	{Id: 7, Text: "Moara cu noroc de Ioan Slavici"},
+	{Id: 8, Text: "Amintiri din copilărie de Ion Creangă"},
+	{Id: 9, Text: "Patul lui Procust de Camil Petrescu"},
+	{Id: 10, Text: "Elevul Dima dintr-a VII-A de Mihail Drumeș"},
+	{Id: 11, Text: "Întoarcerea din rai de Mircea Eliade"},
+	{Id: 12, Text: "La hanul lui Mânjoală de Ion Luca Caragiale"},
+	{Id: 13, Text: "O scrisoare pierdută de Ion Luca Caragiale"},
+	{Id: 14, Text: "Ion de Liviu Rebreanu"},
+	{Id: 15, Text: "Baltagul de Mihail Sadoveanu"},
+}
+
+func Test_Engine(t *testing.T) {
+	engine := NewEngine(items, func(item Book) (int64, string) {
+		return item.Id, item.Text
+	}, nil)
+	engine.SetTolerance(2)
+	tests := []struct {
+		query    string
+		expected []int64
+	}{
+		{"maitreyi", []int64{4}},
+		{"eliade", []int64{11, 6, 4}},
+		{"Patul lui", []int64{9, 12, 11, 10, 7}},
+		{"spânzuraţilor", []int64{2}},
+		{"amintiri din copilărie", []int64{8, 11, 10, 5, 15}},
+		{"xyz zyx", []int64{}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			actual := engine.Search(test.query, 5)
+			if diff := cmp.Diff(test.expected, actual); diff != "" {
+				t.Errorf("mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func Test_Tokenize(t *testing.T) {
+	tokens := Tokenize("Țară, România, școală, mâine! 123.4 ăĂâÂîÎșşȘŞțţȚŢ")
+	expected := [][]rune{
+		{'t', 'a', 'r', 'a'},
+		{'r', 'o', 'm', 'a', 'n', 'i', 'a'},
+		{'s', 'c', 'o', 'a', 'l', 'a'},
+		{'m', 'a', 'i', 'n', 'e'},
+		{'1', '2', '3'},
+		{'4'},
+		{'a', 'a', 'a', 'a', 'i', 'i', 's', 's', 's', 's', 't', 't', 't', 't'},
+	}
+	if diff := cmp.Diff(expected, tokens); diff != "" {
+		t.Errorf("mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func Test_LevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		a        []rune
+		b        []rune
+		expected int
+	}{
+		{[]rune(""), []rune(""), 0},
+		{[]rune("a"), []rune(""), 1},
+		{[]rune(""), []rune("a"), 1},
+		{[]rune("a"), []rune("a"), 0},
+		{[]rune("a"), []rune("ab"), 1},
+		{[]rune("ab"), []rune("a"), 1},
+		{[]rune("sitting"), []rune("kitten"), 3},
+		{[]rune("kitten"), []rune("sitting"), 3},
+		{[]rune("sitting"), []rune("sitting"), 0},
+		{[]rune("lawn"), []rune("flaw"), 2},
+		{[]rune("flaw"), []rune("lawn"), 2},
+		{[]rune("kit"), []rune("kitten"), 3},
+		{[]rune("kitten"), []rune("kit"), 3},
+	}
+	for _, test := range tests {
+		t.Run("LevenshteinDistance", func(t *testing.T) {
+			if diff := cmp.Diff(test.expected, LevenshteinDistance(test.a, test.b)); diff != "" {
+				t.Errorf("mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}