Merge pull request #595 from EBISPOT/dev

Annotations_trimmed, zooma fix
EBISPOT · Nov 27, 2023 · 0bf3c7c · 0bf3c7c
2 parents 9a5614c + 4ccbc98
commit 0bf3c7c
Show file tree

Hide file tree

Showing 14 changed files with 156 additions and 14 deletions.
diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/SolrFieldMapper.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/SolrFieldMapper.java
@@ -76,8 +76,8 @@ public static List<String> mapFieldsList(Collection<String> ols3FieldNames) {
                 continue;
             }
 
-            if (legacyFieldName.equals("_json")) {
-                newFields.add("_json" + suffix);
+            if (legacyFieldName.equals("annotations_trimmed")) {
+                newFields.add(prefix + "searchableAnnotationValues" + suffix);
                 continue;
             }
         }

diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java b/backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v1/V1SearchController.java
@@ -20,13 +20,15 @@
 import uk.ac.ebi.spot.ols.repository.transforms.RemoveLiteralDatatypesTransform;
 import uk.ac.ebi.spot.ols.repository.v1.JsonHelper;
 import uk.ac.ebi.spot.ols.repository.v1.V1OntologyRepository;
+import uk.ac.ebi.spot.ols.repository.v1.mappers.AnnotationExtractor;
 
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * @author Simon Jupp
@@ -73,7 +75,7 @@ public void search(
         if (queryFields == null) {
             // if exact just search the supplied fields for exact matches
             if (exact) {
-                String[] fields = {"label_s", "synonym_s", "short_form_s", "obo_id_s", "iri_s", "_json"};
+                String[] fields = {"label_s", "synonym_s", "short_form_s", "obo_id_s", "iri_s", "annotations_trimmed"};
                 solrQuery.setQuery(
                         "((" +
                                 createUnionQuery(query.toLowerCase(), SolrFieldMapper.mapFieldsList(List.of(fields)).toArray(new String[0]), true)
@@ -85,7 +87,7 @@ public void search(
                 solrQuery.set("defType", "edismax");
                 solrQuery.setQuery(query);
 
-                String[] fields = {"label^5", "synonym^3", "definition", "short_form^2", "obo_id^2", "iri", "_json"};
+                String[] fields = {"label^5", "synonym^3", "definition", "short_form^2", "obo_id^2", "iri", "annotations_trimmed"};
 
                 solrQuery.set("qf", String.join(" ", SolrFieldMapper.mapFieldsList(List.of(fields))));
 
@@ -106,7 +108,10 @@ public void search(
             }
         }
 
-        solrQuery.setFields("_json");
+        if (fieldList.contains("score"))
+           solrQuery.setFields("_json","score");
+        else
+            solrQuery.setFields("_json");
 
         if (ontologies != null && !ontologies.isEmpty()) {
 
@@ -231,11 +236,26 @@ public void search(
             if (fieldList.contains("synonym")) outDoc.put("synonym", JsonHelper.getStrings(json, "synonym"));
             if (fieldList.contains("ontology_prefix")) outDoc.put("ontology_prefix", JsonHelper.getString(json, "ontologyPreferredPrefix"));
             if (fieldList.contains("subset")) outDoc.put("subset", JsonHelper.getStrings(json, "http://www.geneontology.org/formats/oboInOwl#inSubset"));
+            if (fieldList.contains("ontology_iri")) outDoc.put("ontology_iri", JsonHelper.getStrings(json, "ontologyIri").get(0));
+            if (fieldList.contains("score")) outDoc.put("score", res.get("score"));
+
+            // Include annotations that were specified with <field>_annotation
+            boolean anyAnnotations = fieldList.stream()
+                    .anyMatch(s -> s.endsWith("_annotation"));
+            if (anyAnnotations) {
+                Stream<String> annotationFields = fieldList.stream().filter(s -> s.endsWith("_annotation"));
+                Map<String, Object> termAnnotations = AnnotationExtractor.extractAnnotations(json);
+
+                annotationFields.forEach(annotationName -> {
+                    // Remove _annotation suffix to get plain annotation name
+                    String fieldName = annotationName.replaceFirst("_annotation$", "");
+                    outDoc.put(annotationName, termAnnotations.get(fieldName));
+                });
+            }
 
             docs.add(outDoc);
         }
 
-
         Map<String, Object> responseHeader = new HashMap<>();
         responseHeader.put("status", 0);
         responseHeader.put("QTime", qr.getQTime());

diff --git a/backend/src/main/java/uk/ac/ebi/spot/ols/repository/v2/helpers/V2SearchFieldsParser.java b/backend/src/main/java/uk/ac/ebi/spot/ols/repository/v2/helpers/V2SearchFieldsParser.java
@@ -20,6 +20,7 @@ public static void addSearchFieldsToQuery(OlsSolrQuery query, String searchField
             query.addSearchField("id", 1, SearchType.WHITESPACE_EDGES);
             query.addSearchField("oboId", 1, SearchType.WHITESPACE_EDGES);
             query.addSearchField("synonym", 1, SearchType.WHITESPACE_EDGES);
+            query.addSearchField("searchableAnnotationValues", 1, SearchType.WHITESPACE_EDGES);
         } else {
             for (ParsedField field : parseFieldsString(searchFields)) {
                 query.addSearchField(field.property, field.weight, SearchType.CASE_INSENSITIVE_TOKENS);

diff --git a/dataload/json2neo/src/main/java/OntologyWriter.java b/dataload/json2neo/src/main/java/OntologyWriter.java
@@ -23,7 +23,9 @@ public class OntologyWriter {
 
     public static final Set<String> PROPERTY_BLACKLIST = Set.of(
             // large and doesn't get queried
-            "appearsIn"
+            "appearsIn",
+            // all property values together, this is for solr and not useful in neo4j
+            "searchableAnnotationValues"
     );
 
     public static final Set<String> EDGE_BLACKLIST = Set.of(

diff --git a/dataload/rdf2json/src/main/java/uk/ac/ebi/rdf2json/OntologyGraph.java b/dataload/rdf2json/src/main/java/uk/ac/ebi/rdf2json/OntologyGraph.java
@@ -247,6 +247,7 @@ private String urlToFilename(String url) {
         long endTime = System.nanoTime();
         System.out.println("load ontology: " + ((endTime - startTime) / 1000 / 1000 / 1000));
 
+        SearchableAnnotationValuesAnnotator.annotateSearchableAnnotationValues(this);
         InverseOfAnnotator.annotateInverseOf(this);
         NegativePropertyAssertionAnnotator.annotateNegativePropertyAssertions(this);
         OboSynonymTypeNameAnnotator.annotateOboSynonymTypeNames(this); // n.b. this one labels axioms so must run before the ReifiedPropertyAnnotator

diff --git a/...json/src/main/java/uk/ac/ebi/rdf2json/annotators/SearchableAnnotationValuesAnnotator.java b/...json/src/main/java/uk/ac/ebi/rdf2json/annotators/SearchableAnnotationValuesAnnotator.java
@@ -0,0 +1,61 @@
+package uk.ac.ebi.rdf2json.annotators;
+
+import uk.ac.ebi.rdf2json.OntologyGraph;
+import uk.ac.ebi.rdf2json.OntologyNode;
+import uk.ac.ebi.rdf2json.properties.PropertyValue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static uk.ac.ebi.rdf2json.properties.PropertyValue.Type.LITERAL;
+
+public class SearchableAnnotationValuesAnnotator {
+
+    // Roughly equivalent to "annotations_trimmed" in OLS3.
+    //
+    // A field that contains a list of just the values (no predicates) of all of the "annotations" (which is not a well
+    // defined term, so we have to make it up) of an entity.
+    //
+    // This field is used for solr searching, so that you can search for the value of any property (regardless of how
+    // important OLS thinks it is), and still expect a result.
+    //
+    public static void annotateSearchableAnnotationValues(OntologyGraph graph) {
+
+        long startTime3 = System.nanoTime();
+        for(String id : graph.nodes.keySet()) {
+            OntologyNode c = graph.nodes.get(id);
+            if(c.types.contains(OntologyNode.NodeType.CLASS) ||
+                    c.types.contains(OntologyNode.NodeType.PROPERTY) ||
+                    c.types.contains(OntologyNode.NodeType.INDIVIDUAL) ||
+                    c.types.contains(OntologyNode.NodeType.ONTOLOGY)) {
+
+                List<PropertyValue> values = new ArrayList<>();
+
+                for(var predicate : c.properties.getPropertyPredicates()) {
+
+                    // namespaces that are NOT considered annotations for this exercise...
+                    //
+                    if(predicate.startsWith("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
+                            || predicate.startsWith("http://www.w3.org/2000/01/rdf-schema#")
+                            || predicate.startsWith("http://www.w3.org/2002/07/owl#")) {
+
+                        continue;
+                    }
+
+                    for(var value : c.properties.getPropertyValues(predicate)) {
+                        if(value.getType().equals(LITERAL)) {
+                            values.add(value);
+                        }
+                    }
+                }
+
+                for(var value : values) {
+                    c.properties.addProperty("searchableAnnotationValues", value);
+                }
+            }
+        }
+
+        long endTime3 = System.nanoTime();
+        System.out.println("annotate searchable annotation values: " + ((endTime3 - startTime3) / 1000 / 1000 / 1000));
+    }
+}
diff --git a/dataload/solr_config/ols4_entities/conf/managed-schema.xml b/dataload/solr_config/ols4_entities/conf/managed-schema.xml
@@ -135,7 +135,7 @@
     <!-- OLS4: we are using dynamic fields for everything so no need to specify much of a schema.
 	 however we do need to specify the fields we DON'T want to be indexed
     -->
-    <field name="_json" type="ols4json" indexed="true" stored="true" multiValued="false" />
+    <field name="_json" type="ols4json" indexed="false" stored="true" multiValued="false" />
 
     <!-- Type used for data-driven schema, to add a string copy for each text field -->
     <dynamicField name="str_*" type="strings" stored="false" docValues="true" indexed="false" useDocValuesAsStored="false"/>

diff --git a/frontend/index.html.in b/frontend/index.html.in
@@ -15,7 +15,7 @@
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"
-      content="Web site created using create-react-app"
+      content="OLS is a repository for biomedical ontologies that aims to provide a single point of access to the latest ontology versions"
     />
     <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
     <!--

diff --git a/frontend/package-lock.json b/frontend/package-lock.json
diff --git a/frontend/package.json b/frontend/package.json
@@ -30,13 +30,14 @@
     "multimap": "^1.1.0",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
+    "react-helmet": "^6.1.0",
     "react-redux": "^8.0.2",
     "react-router-dom": "^6.8.0",
     "react-twitter-widgets": "^1.11.0",
     "styled-components": "^5.2.1",
     "typescript": "^4.1.2",
-    "web-vitals": "^1.0.1",
-    "url-join": "^5.0.0"
+    "url-join": "^5.0.0",
+    "web-vitals": "^1.0.1"
   },
   "scripts": {
     "build": "dotenv -e .env.$REACT_APP_ENV -e .env -- node build.mjs",

diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
@@ -1,4 +1,4 @@
-import React, { useEffect } from "react";
+import React, { Fragment, useEffect } from "react";
 import {
   BrowserRouter,
   Navigate,
@@ -20,10 +20,16 @@ import OntologyPage from "./pages/ontologies/OntologyPage";
 import EntityPage from "./pages/ontologies/entities/EntityPage";
 import { getEntity } from "./pages/ontologies/ontologiesSlice";
 import Search from "./pages/search/Search";
+import {Helmet} from "react-helmet";
 
 class App extends React.Component {
   render() {
     return (
+      <Fragment>
+        <Helmet>
+          <meta charSet="utf-8" />
+          <title>Ontology Lookup Service (OLS)</title>
+        </Helmet>
       <BrowserRouter basename={process.env.PUBLIC_URL!}>
         <Routes>
           <Route path={`*`} element={<Error />} />
@@ -84,6 +90,7 @@ class App extends React.Component {
         </Routes>
         <Footer />
       </BrowserRouter>
+      </Fragment>
     );
   }
 }

diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx
@@ -1,7 +1,9 @@
 import { Link } from "react-router-dom";
 import urlJoin from "url-join";
+import { Helmet } from 'react-helmet';
 
 export default function Header({ section }: { section?: string }) {
+
   return (
     <header
       className="bg-black bg-right bg-cover"
@@ -12,6 +14,10 @@ export default function Header({ section }: { section?: string }) {
           "')",
       }}
     >
+        <Helmet>
+          <meta charSet="utf-8" />
+          <title>{caps(section)} - Ontology Lookup Service</title>
+        </Helmet>
       <div className="container mx-auto px-4 flex flex-col md:flex-row md:gap-10">
         <div className="py-6 self-center">
           <a href={urlJoin(process.env.PUBLIC_URL!, "/")}>
@@ -95,3 +101,8 @@ export default function Header({ section }: { section?: string }) {
     </header>
   );
 }
+
+function caps(str) {
+    return str[0].toUpperCase() + str.slice(1);
+}
+
diff --git a/frontend/src/pages/ontologies/OntologyPage.tsx b/frontend/src/pages/ontologies/OntologyPage.tsx
@@ -29,6 +29,7 @@ import EntityTree from "./entities/EntityTree";
 import MetadataTooltip from "./entities/entityPageSections/MetadataTooltip";
 import addLinksToText from "./entities/entityPageSections/addLinksToText";
 import { getOntology } from "./ontologiesSlice";
+import {Helmet} from 'react-helmet';
 
 export default function OntologyPage() {
   const params = useParams();
@@ -91,10 +92,18 @@ export default function OntologyPage() {
     if (errorMessage) navigate("/error", { state: { message: errorMessage } });
   }, [errorMessage, navigate]);
 
-  document.title = ontology?.getName() || ontologyId;
+
+  let pageTitle = ontology?.getName() || ontologyId.toUpperCase();
+  let pageDesc = ontology?.getDescription();
+
   return (
     <div>
       <Header section="ontologies" />
+        <Helmet>
+          <meta charSet="utf-8" />
+          {pageTitle && <title>{ontologyId.toUpperCase()} - Ontology Lookup Service</title>}
+          {pageDesc && <meta name="description" content={ontology?.getDescription()}/>}
+        </Helmet>
       <main className="container mx-auto px-4">
         {ontology ? (
           <div className="my-8">

diff --git a/frontend/src/pages/ontologies/entities/EntityPage.tsx b/frontend/src/pages/ontologies/entities/EntityPage.tsx
@@ -45,6 +45,7 @@ import PropertyCharacteristicsSection from "./entityPageSections/PropertyCharact
 import PropertyInverseOfSection from "./entityPageSections/PropertyInverseOfSection";
 import RangeSection from "./entityPageSections/RangeSection";
 import addLinksToText from "./entityPageSections/addLinksToText";
+import { Helmet } from 'react-helmet'
 
 export default function EntityPage({
   entityType,
@@ -143,10 +144,17 @@ export default function EntityPage({
     if (errorMessage) navigate("/error", { state: { message: errorMessage } });
   }, [errorMessage, navigate]);
 
-  document.title = entity?.getShortForm() || entity?.getName() || ontologyId;
+  let pageTitle = entity?.getShortForm() || entity?.getName() || ontologyId;
+  let pageDesc = entity?.getDescription() || entity?.getName();
+
   return (
     <div>
       <Header section="ontologies" />
+        <Helmet>
+          <meta charSet="utf-8" />
+          {pageTitle && <title>{pageTitle}</title>}
+          {pageDesc && <meta name="description" content={pageDesc}/>}
+        </Helmet>
       <main className="container mx-auto px-4">
         {ontology && entity ? (
           <div className="my-8">