Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

T2118 fuzzy search #4401

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

T2118 fuzzy search #4401

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
18aa5af
basic fuzziness support
AndyKilmory Jan 14, 2025
00d570d
extend range of fuzzy parameters
AndyKilmory Jan 21, 2025
952bff8
Merge branch 'main' into t2118-fuzzy-search
AndyKilmory Jan 29, 2025
303626c
remove unwanted comments
AndyKilmory Jan 29, 2025
c43513a
add unit test for basic fuzzy search config
AndyKilmory Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dev/script/generate-config/service-config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,9 @@ function getMediaApiConfig(config) {
|es6.replicas=${config.es6.replicas}
|quota.store.key="rcs-quota.json"
|security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}"
|search.fuzziness={
| enabled=true
|}
|metrics.request.enabled=false
|syndication.review.useRuntimeFieldsFix=true
|`;
Expand Down
17 changes: 17 additions & 0 deletions media-api/app/lib/MediaApiConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,21 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla
val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key")
val cloudFrontKeyPairId: Option[String] = stringOpt("cloudfront.keypair.id")

val fuzzySearchEnabled: Boolean = boolean("search.fuzziness.enabled")
val fuzzySearchPrefixLength: Int = stringOpt("search.fuzziness.prefixLength") match {
case Some(prefixLength) if convertToInt(prefixLength).isDefined => prefixLength.toInt
case _ => 1
}
val fuzzySearchEditDistance: String = stringOpt("search.fuzziness.editDistance") match {
case Some(editDistance) if convertToInt(editDistance).isDefined => editDistance
case Some(editDistance) if editDistance.contains("AUTO:") => editDistance //<- for non-default AUTO word boundaries
case _ => "AUTO"
}
val fuzzyMaxExpansions: Int = stringOpt("search.fuzziness.maxExpansions") match {
case Some(maxExpansions) if convertToInt(maxExpansions).isDefined => maxExpansions.toInt
case _ => 50
}

val rootUri: String = services.apiBaseUri
val kahunaUri: String = services.kahunaBaseUri
val cropperUri: String = services.cropperBaseUri
Expand Down Expand Up @@ -61,4 +76,6 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla

val restrictDownload: Boolean = boolean("restrictDownload")

private def convertToInt(s: String): Option[Int] = Try { s.toInt }.toOption

}
17 changes: 14 additions & 3 deletions media-api/app/lib/elasticsearch/QueryBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,21 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc
private def multiMatchPhraseQuery(value: String, fields: Seq[String]): MultiMatchQuery =
ElasticDsl.multiMatchQuery(value).fields(fields).matchType(MultiMatchQueryBuilderType.PHRASE)

private def multiMatchWordQuery(value: String, fields: Seq[String]): MultiMatchQuery = {
val multiMatchQuery = ElasticDsl.multiMatchQuery(value).fields(fields).operator(Operator.AND)

if (config.fuzzySearchEnabled) {
multiMatchQuery.matchType(MultiMatchQueryBuilderType.BEST_FIELDS)
.fuzziness(config.fuzzySearchEditDistance)
.maxExpansions(config.fuzzyMaxExpansions)
.prefixLength(config.fuzzySearchPrefixLength)
} else {
multiMatchQuery.matchType(MultiMatchQueryBuilderType.CROSS_FIELDS)
}
}

private def makeMultiQuery(value: Value, fields: Seq[String]): MultiMatchQuery = value match {
case Words(value) => ElasticDsl.multiMatchQuery(value).fields(fields).
operator(Operator.AND).
matchType(MultiMatchQueryBuilderType.CROSS_FIELDS)
case Words(value) => multiMatchWordQuery(value, fields)
case Phrase(string) => multiMatchPhraseQuery(string, fields)
// That's OK, we only do date queries on a single field at a time
case e => throw InvalidQuery(s"Cannot do multiQuery on $e")
Expand Down
Loading