Merge branch 'live' into rg/public-api-rate-limit

datastore/additional_data/sources/find_that_charity.py

@@ -124,17 +124,28 @@ class OrgTypeNotKnownError(Exception):
         pass
 
 
-def non_primary_org_ids_map():
-    """Returns a dict of all non-primary org-ids and their corresponding primary org-id"""
-    org_ids = {}
+def non_primary_org_ids_lookup_maps():
+    """
+    Returns a tuple of:
+    * a dict of all non-primary org-ids mapped to their corresponding primary org-id.
+    * a dict of all primary org-ids mapped to their corresponding non-primary org-ids.
+    """
+    non_primary_to_primary = {}
+    primary_to_non_primary = {}
+
+    # [[orgid, orgid], [orgid, orgid] ...]
     orgs = OrgInfoCache.objects.filter(org_ids__len__gt=1).values_list(
         "org_ids", flat=True
     )
-    # [[orgid, orgid], [orgid, orgid] ...]
+
     for org in orgs:
         # [ primary-org-id, secondary-org-id, ...org-id ]
         for non_primary_org_id in org[1:]:
-            org_ids[non_primary_org_id] = org[0]
             # { non_primary_org_id : primary_org_id }
+            non_primary_to_primary[non_primary_org_id] = org[0]
+
+        if len(org) > 1:
+            # { primary_org_id : [non_primary_org_ids] }
+            primary_to_non_primary[org[0]] = org[1:]
 
-    return org_ids
+    return non_primary_to_primary, primary_to_non_primary

datastore/api/org/api.py

@@ -117,12 +117,16 @@ def get_queryset(self):
         org_id = self.kwargs.get("org_id")
 
         # Raise 404 if the Org doesn't exist
-        if not models.Organisation.exists(org_id):
+        try:
+            org = models.Organisation.get(org_id)
+        except models.Organisation.DoesNotExist:
             raise rest_framework.exceptions.NotFound()
 
+        org_ids = [org.org_id] + [lo.org_id for lo in org.linked_orgs]
+
         return (
             db.Grant.objects.filter(source_file__latest__series=db.Latest.CURRENT)
-            .filter(funding_org_ids__contains=[org_id])
+            .filter(funding_org_ids__overlap=org_ids)
             .select_related("source_file")
         )
 
@@ -144,11 +148,15 @@ def get_queryset(self):
         org_id = self.kwargs.get("org_id")
 
         # Raise 404 if the Org doesn't exist
-        if not models.Organisation.exists(org_id):
+        try:
+            org = models.Organisation.get(org_id)
+        except models.Organisation.DoesNotExist:
             raise rest_framework.exceptions.NotFound()
 
+        org_ids = [org.org_id] + [lo.org_id for lo in org.linked_orgs]
+
         return (
             db.Grant.objects.filter(source_file__latest__series=db.Latest.CURRENT)
-            .filter(recipient_org_ids__contains=[org_id])
+            .filter(recipient_org_ids__overlap=org_ids)
             .select_related("source_file")
         )

datastore/api/org/models.py

@@ -1,15 +1,16 @@
-from typing import Optional
 from dataclasses import dataclass
+from typing import Optional, List
+
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
-from django.db.models.query import QuerySet
+from django.db.models.query import QuerySet, Q
 
 import db.models as db
 
 
 @dataclass
 class OrganisationRef:
     """
-    Represents a link/reference to an Organisation.
+    Represents a reference to an Organisation, i.e. an object with an org_id.
     """
 
     org_id: str
@@ -30,6 +31,7 @@ class Organisation:
     funder: Optional[db.Funder]
     recipient: Optional[db.Recipient]
     publisher: Optional[db.Publisher]
+    linked_orgs: List[OrganisationRef]
 
     def __post_init__(self):
         if self.org_id == "":
@@ -64,10 +66,12 @@ def exists(
                 getter_run__in=db.GetterRun.objects.in_use()
             )
 
-        if funder_queryset.filter(org_id=org_id).exists():
+        id_query = Q(org_id=org_id) | Q(non_primary_org_ids__contains=[org_id])
+
+        if funder_queryset.filter(id_query).exists():
             return True
 
-        if recipient_queryset.filter(org_id=org_id).exists():
+        if recipient_queryset.filter(id_query).exists():
             return True
 
         if publisher_queryset.order_by().filter(org_id=org_id).exists():
@@ -103,21 +107,45 @@ def get(
             )
 
         name = None
+        primary_org_id = org_id
+        linked_org_ids = set()
+
+        id_query = Q(org_id=org_id) | Q(non_primary_org_ids__contains=[org_id])
+
+        # Note that we are searching by both org_id (Primary Org ID) and Non-primary Org IDs
+        # If the user searches by a non-primary ID, we will instead show info about the Primary Org.
+        # (Recipient and Funder objects are only created for primary IDs)
 
         # is org a Recipient?
         try:
-            recipient = recipient_queryset.get(org_id=org_id)
+            recipients = recipient_queryset.filter(id_query)
+            # For now, replicate GrantNav behaviour by taking the first filter result as Primary
+            # https://github.com/ThreeSixtyGiving/grantnav/blob/ee696779d110ab491daa6694f5344c07dbbf98d2/grantnav/frontend/views.py#L1196
+            recipient = recipients[0]
             name = recipient.name
+            primary_org_id = recipient.org_id
+
+            for rt in recipients:
+                linked_org_ids.add(rt.org_id)
+                linked_org_ids.update(rt.non_primary_org_ids)
 
-        except db.Recipient.DoesNotExist:
+        except IndexError:
             recipient = None
 
         # is org a Funder?
         try:
-            funder = funder_queryset.get(org_id=org_id)
+            funders = funder_queryset.filter(id_query)
+            # For now, replicate GrantNav behaviour by taking the first filter result as Primary
+            # https://github.com/ThreeSixtyGiving/grantnav/blob/ee696779d110ab491daa6694f5344c07dbbf98d2/grantnav/frontend/views.py#L1205
+            funder = funders[0]
             name = funder.name
+            primary_org_id = funder.org_id
+
+            for fr in funders:
+                linked_org_ids.add(fr.org_id)
+                linked_org_ids.update(fr.non_primary_org_ids)
 
-        except db.Funder.DoesNotExist:
+        except IndexError:
             funder = None
 
         # is org a Publisher?
@@ -126,18 +154,24 @@ def get(
                 "-getter_run__datetime"
             )[0]
             name = publisher.name
+            # Publishers take precedence over Funders / Recipients when it comes to primary vs non-primary ID priority
+            primary_org_id = publisher.org_id
         except IndexError:
             publisher = None
 
         if funder is None and recipient is None and publisher is None:
             raise Organisation.DoesNotExist
 
+        # Don't include the primary org itsself in linked_orgs
+        linked_org_ids.discard(primary_org_id)
+
         return Organisation(
-            org_id=org_id,
+            org_id=primary_org_id,
             name=name,
             funder=funder,
             recipient=recipient,
             publisher=publisher,
+            linked_orgs=list(OrganisationRef(org_id=oid) for oid in linked_org_ids),
         )
 
 

datastore/data_quality/quality_data.py

@@ -106,16 +106,19 @@ def create(grants):
                 ],
             }
 
+    # Sort recipient / funder org-id lists to be deterministic for tests
     aggregates = {
         "count": cove_results["grants_aggregates"]["count"],
         "recipient_organisations": list(
-            cove_results["grants_aggregates"]["distinct_recipient_org_identifier"]
+            sorted(
+                cove_results["grants_aggregates"]["distinct_recipient_org_identifier"]
+            )
         ),
         "recipient_individuals": cove_results["grants_aggregates"][
             "recipient_individuals_count"
         ],
         "funders": list(
-            cove_results["grants_aggregates"]["distinct_funding_org_identifier"]
+            sorted(cove_results["grants_aggregates"]["distinct_funding_org_identifier"])
         ),
         "max_award_date": cove_results["grants_aggregates"]["max_award_date"],
         "min_award_date": cove_results["grants_aggregates"]["min_award_date"],
@@ -432,7 +435,6 @@ def get_pc_publishers_publishing_in_last(self, delta):
         )
 
     def get_pc_publishers_with_recipient_ext_org(self):
-
         ret = {}
         total_publishers = self.get_total_publishers()
 
@@ -468,7 +470,6 @@ def get_pc_publishers_with_recipient_ext_org(self):
         return ret
 
     def get_total_grants_awarded_in_last_ten_years(self):
-
         this_year_int = datetime.now().year
         award_years = {}
 
@@ -485,7 +486,6 @@ def get_total_grants_awarded_in_last_ten_years(self):
         return award_years
 
     def get_pc_publishers_with_grants_awarded_in_last_ten_years(self):
-
         this_year_int = datetime.now().year
         award_years = {}
 

datastore/db/management/commands/manage_entities_data.py

@@ -7,7 +7,7 @@
 
 import sys, json
 
-from additional_data.sources.find_that_charity import non_primary_org_ids_map
+from additional_data.sources.find_that_charity import non_primary_org_ids_lookup_maps
 
 
 def update_entities():
@@ -23,20 +23,28 @@ def update_entities():
 
     recipient_orgs_bulk = {}
     funder_orgs_bulk = {}
-    non_primary_org_ids_map_cache = non_primary_org_ids_map()
+    (
+        non_primary_to_primary_org_ids_lookup,
+        primary_to_non_primary_org_ids_lookup,
+    ) = non_primary_org_ids_lookup_maps()
 
     print("Analysing latest best grant data for entities")
 
     for grant in grants:
         for recipient in grant.get("recipientOrganization", []):
             # If the org-id provided is a non-primary org-id return the primary
             # otherwise return the specified org-id
-            org_id = non_primary_org_ids_map_cache.get(recipient["id"], recipient["id"])
+            org_id = non_primary_to_primary_org_ids_lookup.get(
+                recipient["id"], recipient["id"]
+            )
+            non_primary_org_ids = primary_to_non_primary_org_ids_lookup.get(org_id, [])
 
             try:
                 recipient_ob = recipient_orgs_bulk[org_id]
             except KeyError:
-                recipient_ob = db.Recipient(org_id=org_id)
+                recipient_ob = db.Recipient(
+                    org_id=org_id, non_primary_org_ids=non_primary_org_ids
+                )
                 recipient_orgs_bulk[org_id] = recipient_ob
 
             recipient_ob.add_name(recipient["name"])
@@ -47,13 +55,16 @@ def update_entities():
         for funder in grant["fundingOrganization"]:
             # If the org-id provided is a non-primary org-id return the primary
             # otherwise return the specified org-id
-            org_id = non_primary_org_ids_map_cache.get(funder["id"], funder["id"])
+            org_id = non_primary_to_primary_org_ids_lookup.get(
+                funder["id"], funder["id"]
+            )
+            non_primary_org_ids = primary_to_non_primary_org_ids_lookup.get(org_id, [])
 
             try:
                 funder_ob = funder_orgs_bulk[org_id]
             except KeyError:
                 funder_ob = db.Funder(
-                    org_id=org_id,
+                    org_id=org_id, non_primary_org_ids=non_primary_org_ids
                 )
                 funder_orgs_bulk[org_id] = funder_ob
 

datastore/db/migrations/0024_auto_20240610_1847.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.16 on 2024-06-10 18:47
+
+import django.contrib.postgres.fields
+import django.contrib.postgres.indexes
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("db", "0023_auto_20240318_1952"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="funder",
+            name="non_primary_org_ids",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.TextField(), default=[], size=None
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="recipient",
+            name="non_primary_org_ids",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.TextField(), default=[], size=None
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddIndex(
+            model_name="funder",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["non_primary_org_ids"], name="db_funder_non_pri_745675_gin"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="recipient",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["non_primary_org_ids"], name="db_recipien_non_pri_0e0ccb_gin"
+            ),
+        ),
+    ]

datastore/db/models.py

@@ -231,7 +231,8 @@ class Entity(models.Model):
     class Meta:
         abstract = True
 
-    org_id = models.CharField(max_length=200)  # Unique
+    org_id = models.CharField(max_length=200)  # Primary Org ID, Unique
+
     # Allowed to be null or blank for progressive building of the record
     name = models.TextField(null=True, blank=True)
 
@@ -359,15 +360,25 @@ class Meta:
         constraints = [
             models.UniqueConstraint(fields=["org_id"], name="recipient_unique_org_id")
         ]
-        indexes = [Index(fields=["org_id", "name"])]
+        indexes = [
+            GinIndex(fields=["non_primary_org_ids"]),
+            Index(fields=["org_id", "name"]),
+        ]
+
+    non_primary_org_ids = ArrayField(models.TextField())
 
 
 class Funder(Entity):
     class Meta:
         constraints = [
             models.UniqueConstraint(fields=["org_id"], name="funder_unique_org_id")
         ]
-        indexes = [Index(fields=["org_id", "name"])]
+        indexes = [
+            GinIndex(fields=["non_primary_org_ids"]),
+            Index(fields=["org_id", "name"]),
+        ]
+
+    non_primary_org_ids = ArrayField(models.TextField())
 
 
 class Grant(models.Model):