Merge branch 'master' of https://github.com/RTXteam/RTX

.gitignore

@@ -51,3 +51,4 @@ code/ARAX/ARAXQuery/Infer/data/xCRG_data/*.pt
 code/ARAX/ARAXQuery/Infer/data/xCRG_data/*.npz
 
 code/UI/OpenAPI/python-flask-server/openapi_server/openapi/openapi.json
+code/UI/OpenAPI/specifications/export/KG2/*/openapi.json

README.md

@@ -82,7 +82,7 @@ it.
 
 ARAX is a tool for querying, manipulating, filtering, and exploring biomedical
 knowledge graphs. It is designed to be a type of middleware—an *autonomous
-relay agent*—within the Translator system. The top-level layer of
+relay agent*—within the [Translator system](https://github.com/NCATSTranslator/Translator-All/wiki). The top-level layer of
 Translator (which is called the *autonomous relay system*) will issue structured
 queries to ARAX via ARAX's web application programming interface. Then, based on the
 query type, ARAX will determine which *knowledge providers* it needs to consult in

code/ARAX/ARAXQuery/ARAX_expander.py

@@ -28,6 +28,7 @@
 from openapi_server.models.edge import Edge
 from openapi_server.models.attribute_constraint import AttributeConstraint
 from openapi_server.models.attribute import Attribute
+from openapi_server.models.retrieval_source import RetrievalSource
 from Expand.kg2_querier import KG2Querier
 from Expand.trapi_querier import TRAPIQuerier
 
@@ -318,7 +319,6 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
         # Expand any specified edges
         inferred_qedge_keys = [qedge_key for qedge_key, qedge in query_graph.edges.items() if
                                qedge.knowledge_type == "inferred"]
-        do_issue_2328_patch = True  # Turns on/off the patch for #2328 (treats refactor issue)
         if qedge_keys_to_expand:
             query_sub_graph = self._extract_query_subgraph(qedge_keys_to_expand, query_graph, log)
             log.debug(f"Query graph for this Expand() call is: {query_sub_graph.to_dict()}")
@@ -359,8 +359,8 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                 for kp in kp_selector.valid_kps:
                     response.update_query_plan(qedge_key, kp, 'Waiting', 'Prepping query to send to KP')
                 qedge = query_graph.edges[qedge_key]
-                alter_kg2_treats_edges = True if (do_issue_2328_patch and qedge_key in inferred_qedge_keys
-                                                  and "biolink:treats" in qedge.predicates) else False
+                be_creative_treats = True if (qedge_key in inferred_qedge_keys
+                                              and "biolink:treats" in qedge.predicates) else False
 
                 # Create a query graph for this edge (that uses curies found in prior steps)
                 one_hop_qg = self._get_query_graph_for_edge(qedge_key, query_graph, overarching_kg, log)
@@ -435,7 +435,7 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                                                      kp_selector,
                                                      log,
                                                      multiple_kps=True,
-                                                     alter_kg2_treats_edges=alter_kg2_treats_edges)
+                                                     be_creative_treats=be_creative_treats)
                              for kp_to_use in kps_to_query]
                     task_group = asyncio.gather(*tasks)
                     kp_answers = loop.run_until_complete(task_group)
@@ -507,17 +507,58 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                         for kedge_key in kedges_to_remove:
                             if kedge_key in overarching_kg.edges_by_qg_id[qedge_key]:
                                 del overarching_kg.edges_by_qg_id[qedge_key][kedge_key]
-                # Remove KG2 SemMedDB treats_or_applied-type edges if this is an inferred treats query
-                if alter_kg2_treats_edges:
+                # Handle Expand's creative treats predicate answers
+                if be_creative_treats and qedge_key in overarching_kg.edges_by_qg_id:  # Skip if no answers
+                    # First remove any SemMedDB treats_or_applied-type edges (not trustworthy)
                     edge_keys_to_remove = {edge_key for edge_key, edge in overarching_kg.edges_by_qg_id[qedge_key].items()
                                            if edge.predicate in self.treats_like_predicates and
-                                           any(source.resource_id == "infores:rtx-kg2" for source in edge.sources) and
                                            any(source.resource_id == "infores:semmeddb" for source in edge.sources)}
-                    log.debug(f"Removing {len(edge_keys_to_remove)} KG2 semmeddb treats_or_applied-type edges "
+                    log.debug(f"Removing {len(edge_keys_to_remove)} semmeddb treats_or_applied-type edges "
                               f"fulfilling {qedge_key}")
                     for edge_key in edge_keys_to_remove:
                         del overarching_kg.edges_by_qg_id[qedge_key][edge_key]
 
+                    # Use remaining treats-like edges as support for one merged 'treats' edge (per subj/obj pair)
+                    higher_level_treats_edges = {edge_key: edge
+                                                 for edge_key, edge in overarching_kg.edges_by_qg_id[qedge_key].items()
+                                                 if edge.predicate in self.treats_like_predicates}
+                    if higher_level_treats_edges:
+                        # Add a virtual edge to the QG to capture all higher-level treats edges ('support' edges)
+                        virtual_qedge_key = f"creative_expand_treats_{qedge_key}"
+                        virtual_qedge = QEdge(subject=qedge.subject,
+                                              object=qedge.object,
+                                              option_group_id=f"creative_expand_treats_group_{qedge_key}")
+                        virtual_qedge.filled = True  # Resultify needs this flag
+                        message.query_graph.edges[virtual_qedge_key] = virtual_qedge
+                        overarching_kg.edges_by_qg_id[virtual_qedge_key] = dict()
+
+                        # Lump the higher-level treats edges together by subject/object
+                        subj_obj_map = defaultdict(set)
+                        for higher_treats_edge_key, higher_treats_edge in higher_level_treats_edges.items():
+                            hash_key = (higher_treats_edge.subject, higher_treats_edge.object)
+                            subj_obj_map[hash_key].add(higher_treats_edge_key)
+
+                        for (subj_key, obj_key), higher_treats_edge_keys in subj_obj_map.items():
+                            # Create a lumped edge to represent all of these edges
+                            lumped_edge = Edge(subject=subj_key, object=obj_key, predicate="biolink:treats",
+                                               sources=[RetrievalSource(resource_id="infores:arax",
+                                                                        resource_role="primary_knowledge_source")],
+                                               attributes=[Attribute(attribute_type_id="biolink:agent_type",
+                                                                     value="computational_model",
+                                                                     attribute_source="infores:arax"),
+                                                           Attribute(attribute_type_id="biolink:knowledge_level",
+                                                                     value="not_provided",
+                                                                     attribute_source="infores:arax")])
+                            lumped_edge_key = f"creative_expand_treats_edge:{subj_key}--treats--{obj_key}--infores:arax"
+                            overarching_kg.edges_by_qg_id[qedge_key][lumped_edge_key] = lumped_edge
+
+                            # Move the higher-level treats edges so that they fulfill the virtual qedge instead
+                            for higher_treats_edge_key in higher_treats_edge_keys:
+                                higher_treats_edge = overarching_kg.edges_by_qg_id[qedge_key][higher_treats_edge_key]
+                                overarching_kg.edges_by_qg_id[virtual_qedge_key][higher_treats_edge_key] = higher_treats_edge
+                                del overarching_kg.edges_by_qg_id[qedge_key][higher_treats_edge_key]
+
+
                 if mode != "RTXKG2":
                     # Apply any kryptonite ("not") qedges
                     self._apply_any_kryptonite_edges(overarching_kg, message.query_graph,
@@ -586,30 +627,6 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
             decorator = ARAXDecorator()
             decorator.decorate_edges(response, kind="SEMMEDDB")
 
-        # Second half of patch for #2328; edit KG2 'treats_or_applied_or_studied_to_treat' edges to just 'treats'
-        response.info(f"Treats-like predicates are: {self.treats_like_predicates}")
-        response.info(f"BiolinkHelper Biolink version is: {self.bh.biolink_version}")
-        response.info(f"Mode is {mode}, do_issue_2328_patch is {do_issue_2328_patch}, "
-                      f"inferred_qedge_keys is {inferred_qedge_keys}")
-        if mode != "RTXKG2" and do_issue_2328_patch and inferred_qedge_keys:
-            response.info(f"Made it into block where KG2 treats-like edge alteration happens")
-            num_edges_altered = 0
-            for edge in message.knowledge_graph.edges.values():
-                is_kg2_edge = any(source.resource_id == "infores:rtx-kg2" for source in edge.sources)
-                if is_kg2_edge and edge.predicate in self.treats_like_predicates:
-                    # Record the original KG2 predicate in an attribute
-                    edge.attributes.append(Attribute(attribute_type_id="biolink:original_predicate",
-                                                     value=edge.predicate,
-                                                     value_type_id="biolink:predicate",
-                                                     description="Predicate as it appears in RTX-KG2, prior to "
-                                                                 "alteration by ARAX.",
-                                                     attribute_source="infores:arax"))
-                    # Then change the predicate to treats
-                    edge.predicate = "biolink:treats"
-                    num_edges_altered += 1
-            if num_edges_altered:
-                log.info(f"Modified the predicate of {num_edges_altered} KG2 edges to biolink:treats")
-
         # Map canonical curies back to the input curies in the QG (where applicable) #1622
         self._map_back_to_input_curies(message.knowledge_graph, query_graph, log)
         if mode == "RTXKG2":
@@ -772,7 +789,7 @@ async def _expand_edge_async(self, edge_qg: QueryGraph,
                                  kp_selector: KPSelector,
                                  log: ARAXResponse,
                                  multiple_kps: bool = False,
-                                 alter_kg2_treats_edges: bool = False) -> Tuple[QGOrganizedKnowledgeGraph, ARAXResponse]:
+                                 be_creative_treats: bool = False) -> Tuple[QGOrganizedKnowledgeGraph, ARAXResponse]:
         # This function answers a single-edge (one-hop) query using the specified knowledge provider
         qedge_key = next(qedge_key for qedge_key in edge_qg.edges)
         log.info(f"Expanding qedge {qedge_key} using {kp_to_use}")
@@ -799,7 +816,7 @@ async def _expand_edge_async(self, edge_qg: QueryGraph,
                                       kp_selector=kp_selector,
                                       force_local=force_local)
             answer_kg = await kp_querier.answer_one_hop_query_async(edge_qg,
-                                                                    alter_kg2_treats_edges=alter_kg2_treats_edges)
+                                                                    be_creative_treats=be_creative_treats)
         except Exception:
             tb = traceback.format_exc()
             error_type, error, _ = sys.exc_info()

code/ARAX/ARAXQuery/ARAX_filter.py

@@ -167,10 +167,10 @@ def main():
     actions = result.data['actions']
 
     #### Read message #2 from the database. This should be the acetaminophen proteins query result message
-    sys.path.append(os.path.dirname(os.path.abspath(__file__))+"/../../UI/Feedback")
-    from RTXFeedback import RTXFeedback
-    araxdb = RTXFeedback()
-    message_dict = araxdb.getMessage(2)
+    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../ResponseCache")
+    from response_cache import ResponseCache
+    response_cache = ResponseCache()
+    message_dict = response_cache.get_response(314204)
 
     #### The stored message comes back as a dict. Transform it to objects
     sys.path.append(os.path.dirname(os.path.abspath(__file__))+"/../OpenAPI/python-flask-server/")

code/ARAX/ARAXQuery/ARAX_filter_kg.py

@@ -1525,17 +1525,17 @@ def main():
     actions = result.data['actions']
 
     #### Read message #2 from the database. This should be the acetaminophen proteins query result message
-    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../UI/Feedback")
-    from RTXFeedback import RTXFeedback
-    araxdb = RTXFeedback()
+    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../ResponseCache")
+    from response_cache import ResponseCache
+    response_cache = ResponseCache()
 
     #message_dict = araxdb.getMessage(2)  # acetaminophen2proteins graph
     # message_dict = araxdb.getMessage(13)  # ibuprofen -> proteins -> disease # work computer
     # message_dict = araxdb.getMessage(14)  # pleuropneumonia -> phenotypic_feature # work computer
     # message_dict = araxdb.getMessage(16)  # atherosclerosis -> phenotypic_feature  # work computer
     # message_dict = araxdb.getMessage(5)  # atherosclerosis -> phenotypic_feature  # home computer
     # message_dict = araxdb.getMessage(10)
-    message_dict = araxdb.getMessage(40)
+    message_dict = response_cache.get_response(314204)
 
     #### The stored message comes back as a dict. Transform it to objects
     from ARAX_messenger import ARAXMessenger

code/ARAX/ARAXQuery/ARAX_filter_results.py

@@ -942,17 +942,17 @@ def main():
     actions = result.data['actions']
 
     #### Read message #2 from the database. This should be the acetaminophen proteins query result message
-    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../UI/Feedback")
-    from RTXFeedback import RTXFeedback
-    araxdb = RTXFeedback()
+    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../ResponseCache")
+    from response_cache import ResponseCache
+    response_cache = ResponseCache()
 
     #message_dict = araxdb.getMessage(2)  # acetaminophen2proteins graph
     # message_dict = araxdb.getMessage(13)  # ibuprofen -> proteins -> disease # work computer
     # message_dict = araxdb.getMessage(14)  # pleuropneumonia -> phenotypic_feature # work computer
     # message_dict = araxdb.getMessage(16)  # atherosclerosis -> phenotypic_feature  # work computer
     # message_dict = araxdb.getMessage(5)  # atherosclerosis -> phenotypic_feature  # home computer
     # message_dict = araxdb.getMessage(10)
-    message_dict = araxdb.getMessage(40)
+    message_dict = response_cache.get_response(314204)
 
     #### The stored message comes back as a dict. Transform it to objects
     from ARAX_messenger import ARAXMessenger

code/ARAX/ARAXQuery/ARAX_infer.py

@@ -857,8 +857,8 @@ def __chemical_gene_regulation_graph_expansion(self, describe=False):
                 self.response.warning(f"Could not get predicted genes for chemical {preferred_subject_curie}. Likely the model was not trained with this chemical.")
                 return self.response
             if top_paths is None or len(top_paths) == 0:
-                self.response.warning(f"Could not get any predicted paths for chemical {preferred_subject_curie}. Likely the model considers there is no reasonable path for this chemical.")
-
+                self.response.warning(f"Could not get any predicted paths for chemical {preferred_subject_curie}. Either Plover is not reachable or no paths found")
+                return self.response
             iu = InferUtilities()
             qedge_id = self.parameters.get('qedge_id')
             self.response, self.kedge_global_iter, self.qedge_global_iter, self.qnode_global_iter, self.option_global_iter = iu.genrete_regulate_subgraphs(self.response, normalized_subject_curie, None, top_predictions, top_paths, qedge_id, self.parameters['regulation_type'], self.kedge_global_iter, self.qedge_global_iter, self.qnode_global_iter, self.option_global_iter)
@@ -875,8 +875,8 @@ def __chemical_gene_regulation_graph_expansion(self, describe=False):
                 self.response.warning(f"Could not get predicted chemicals for gene {preferred_object_curie}. Likely the model was not trained with this gene.")
                 return self.response
             if top_paths is None or len(top_paths) == 0:
-                self.response.warning(f"Could not get any predicted paths for gene {preferred_object_curie}. Likely the model considers there is no reasonable path for this gene.")
-
+                self.response.warning(f"Could not get any predicted paths for gene {preferred_object_curie}. Either Plover is not reachable or no paths found")
+                return self.response
             iu = InferUtilities()
             qedge_id = self.parameters.get('qedge_id')
 

code/ARAX/ARAXQuery/ARAX_overlay.py

@@ -1038,9 +1038,9 @@ def main():
     actions = result.data['actions']
 
     #### Read message #2 from the database. This should be the acetaminophen proteins query result message
-    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../UI/Feedback")
-    from RTXFeedback import RTXFeedback
-    araxdb = RTXFeedback()
+    sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../ResponseCache")
+    from response_cache import ResponseCache
+    response_cache = ResponseCache()
 
     # message_dict = araxdb.getMessage(2)  # acetaminophen2proteins graph
     # message_dict = araxdb.getMessage(13)  # ibuprofen -> proteins -> disease # work computer
@@ -1050,7 +1050,7 @@ def main():
     # message_dict = araxdb.getMessage(10)
     # message_dict = araxdb.getMessage(36)  # test COHD obs/exp, via ARAX_query.py 16
     # message_dict = araxdb.getMessage(39)  # ngd virtual edge test
-    message_dict = araxdb.getMessage(1)
+    message_dict = response_cache.get_response(314204)
 
     #### The stored message comes back as a dict. Transform it to objects
     from ARAX_messenger import ARAXMessenger