From 86d3c2d07204446b919c51d125a20d44f963cadf Mon Sep 17 00:00:00 2001 From: Marcos Lopez Gonzalez Date: Wed, 30 Oct 2024 09:19:04 +0100 Subject: [PATCH] #635 additional facets + fixes --- .../mapper/CollectionsSearchMapperIT.java | 132 ++++++++++++++++++ .../params/DescriptorsListParams.java | 11 +- .../params/InstitutionListParams.java | 4 + .../collections/CollectionsSearchMapper.xml | 77 +++++++--- .../collections/CollectionsSearchService.java | 5 +- 5 files changed, 204 insertions(+), 25 deletions(-) diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionsSearchMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionsSearchMapperIT.java index a8807e642..22054cf2c 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionsSearchMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionsSearchMapperIT.java @@ -26,19 +26,23 @@ import org.gbif.api.model.collections.Address; import org.gbif.api.model.collections.AlternativeCode; import org.gbif.api.model.collections.Collection; +import org.gbif.api.model.collections.Institution; import org.gbif.api.model.collections.descriptors.DescriptorGroup; import org.gbif.api.vocabulary.Country; import org.gbif.api.vocabulary.License; import org.gbif.api.vocabulary.collections.CollectionFacetParameter; +import org.gbif.api.vocabulary.collections.InstitutionFacetParameter; import org.gbif.registry.database.TestCaseDatabaseInitializer; import org.gbif.registry.persistence.mapper.collections.AddressMapper; import org.gbif.registry.persistence.mapper.collections.CollectionMapper; import org.gbif.registry.persistence.mapper.collections.CollectionsSearchMapper; import org.gbif.registry.persistence.mapper.collections.DescriptorsMapper; +import org.gbif.registry.persistence.mapper.collections.InstitutionMapper; import org.gbif.registry.persistence.mapper.collections.dto.CollectionSearchDto; import org.gbif.registry.persistence.mapper.collections.dto.DescriptorDto; import org.gbif.registry.persistence.mapper.collections.dto.FacetDto; import org.gbif.registry.persistence.mapper.collections.params.DescriptorsListParams; +import org.gbif.registry.persistence.mapper.collections.params.InstitutionListParams; import org.gbif.registry.search.test.EsManageServer; import org.gbif.registry.ws.it.BaseItTest; import org.gbif.ws.client.filter.SimplePrincipalProvider; @@ -53,6 +57,7 @@ public class CollectionsSearchMapperIT extends BaseItTest { new TestCaseDatabaseInitializer("collection"); private CollectionMapper collectionMapper; + private InstitutionMapper institutionMapper; private AddressMapper addressMapper; private DescriptorsMapper descriptorsMapper; private CollectionsSearchMapper collectionsSearchMapper; @@ -60,6 +65,7 @@ public class CollectionsSearchMapperIT extends BaseItTest { @Autowired public CollectionsSearchMapperIT( CollectionMapper collectionMapper, + InstitutionMapper institutionMapper, AddressMapper addressMapper, DescriptorsMapper descriptorsMapper, CollectionsSearchMapper collectionsSearchMapper, @@ -67,6 +73,7 @@ public CollectionsSearchMapperIT( EsManageServer esServer) { super(principalProvider, esServer); this.collectionMapper = collectionMapper; + this.institutionMapper = institutionMapper; this.addressMapper = addressMapper; this.descriptorsMapper = descriptorsMapper; this.collectionsSearchMapper = collectionsSearchMapper; @@ -168,6 +175,43 @@ public void searchTest() { @Test public void facetsTest() { + UUID i1Key = UUID.randomUUID(); + Institution i1 = new Institution(); + i1.setKey(i1Key); + i1.setName("i1"); + i1.setCode("i1"); + i1.setTypes(Arrays.asList("ty1", "ty2")); + i1.setDisciplines(Arrays.asList("di1", "di2")); + i1.setCreatedBy("test"); + i1.setModifiedBy("test"); + + Address ai1 = new Address(); + ai1.setCountry(Country.SPAIN); + ai1.setCity("Oviedo"); + addressMapper.create(ai1); + assertNotNull(ai1.getKey()); + i1.setAddress(ai1); + + institutionMapper.create(i1); + + UUID i2Key = UUID.randomUUID(); + Institution i2 = new Institution(); + i2.setKey(i2Key); + i2.setName("i2"); + i2.setCode("i2"); + i2.setTypes(Collections.singletonList("ty1")); + i2.setCreatedBy("test"); + i2.setModifiedBy("test"); + + Address ai2 = new Address(); + ai2.setCountry(Country.SPAIN); + ai2.setCity("Bilbao"); + addressMapper.create(ai2); + assertNotNull(ai2.getKey()); + i2.setMailingAddress(ai2); + + institutionMapper.create(i2); + UUID c1Key = UUID.randomUUID(); Collection c1 = new Collection(); c1.setKey(c1Key); @@ -184,6 +228,7 @@ public void facetsTest() { Address address = new Address(); address.setCountry(Country.SPAIN); + address.setCity("Oviedo"); addressMapper.create(address); assertNotNull(address.getKey()); c1.setAddress(address); @@ -207,6 +252,7 @@ public void facetsTest() { descriptorDto1.setDescriptorGroupKey(descriptorGroup.getKey()); descriptorDto1.setUsageName("aves"); descriptorDto1.setCountry(Country.DENMARK); + descriptorDto1.setRecordedBy(Arrays.asList("John", "Clint")); descriptorDto1.setKingdomKey(1); descriptorsMapper.createDescriptor(descriptorDto1); @@ -214,6 +260,7 @@ public void facetsTest() { descriptorDto2.setDescriptorGroupKey(descriptorGroup.getKey()); descriptorDto2.setKingdomKey(1); descriptorDto2.setCountry(Country.DENMARK); + descriptorDto2.setObjectClassificationName("obn1"); descriptorsMapper.createDescriptor(descriptorDto2); UUID c2Key = UUID.randomUUID(); @@ -233,6 +280,7 @@ public void facetsTest() { mailingAddress = new Address(); mailingAddress.setCountry(Country.SPAIN); + mailingAddress.setCity("Bilbao"); addressMapper.create(mailingAddress); assertNotNull(mailingAddress.getKey()); c2.setMailingAddress(mailingAddress); @@ -249,7 +297,9 @@ public void facetsTest() { DescriptorDto descriptorDtoC2 = new DescriptorDto(); descriptorDtoC2.setDescriptorGroupKey(descriptorGroupC2.getKey()); descriptorDtoC2.setKingdomKey(2); + descriptorDtoC2.setRecordedBy(Collections.singletonList("John")); descriptorDtoC2.setCountry(Country.DENMARK); + descriptorDtoC2.setObjectClassificationName("obn1"); descriptorsMapper.createDescriptor(descriptorDtoC2); List facetDtos = @@ -305,5 +355,87 @@ public void facetsTest() { collectionsSearchMapper.collectionFacet( DescriptorsListParams.builder().facet(CollectionFacetParameter.TYPE_STATUS).build()); assertEquals(0, facetDtos.size()); + + assertEquals( + 0, + collectionsSearchMapper.collectionFacetCardinality( + DescriptorsListParams.builder().facet(CollectionFacetParameter.TYPE_STATUS).build())); + + facetDtos = + collectionsSearchMapper.collectionFacet( + DescriptorsListParams.builder().facet(CollectionFacetParameter.CITY).build()); + assertEquals(2, facetDtos.size()); + assertEquals(2, facetDtos.stream().filter(f -> f.getCount() == 1).count()); + assertEquals( + 2, + collectionsSearchMapper.collectionFacetCardinality( + DescriptorsListParams.builder().facet(CollectionFacetParameter.CITY).build())); + + facetDtos = + collectionsSearchMapper.collectionFacet( + DescriptorsListParams.builder().facet(CollectionFacetParameter.RECORDED_BY).build()); + assertEquals(2, facetDtos.size()); + assertEquals(1, facetDtos.stream().filter(f -> f.getCount() == 1).count()); + assertEquals(1, facetDtos.stream().filter(f -> f.getCount() == 2).count()); + assertEquals( + 2, + collectionsSearchMapper.collectionFacetCardinality( + DescriptorsListParams.builder().facet(CollectionFacetParameter.RECORDED_BY).build())); + + facetDtos = + collectionsSearchMapper.collectionFacet( + DescriptorsListParams.builder() + .facet(CollectionFacetParameter.OBJECT_CLASSIFICATION) + .build()); + assertEquals(1, facetDtos.size()); + assertEquals(1, facetDtos.stream().filter(f -> f.getCount() == 2).count()); + assertEquals( + 1, + collectionsSearchMapper.collectionFacetCardinality( + DescriptorsListParams.builder() + .facet(CollectionFacetParameter.OBJECT_CLASSIFICATION) + .build())); + + // institution facets + facetDtos = + collectionsSearchMapper.institutionFacet( + InstitutionListParams.builder().facet(InstitutionFacetParameter.COUNTRY).build()); + assertEquals(1, facetDtos.size()); + assertEquals(1, facetDtos.stream().filter(f -> f.getCount() == 2).count()); + assertEquals( + 1, + collectionsSearchMapper.institutionFacetCardinality( + InstitutionListParams.builder().facet(InstitutionFacetParameter.COUNTRY).build())); + + facetDtos = + collectionsSearchMapper.institutionFacet( + InstitutionListParams.builder().facet(InstitutionFacetParameter.CITY).build()); + assertEquals(2, facetDtos.size()); + assertEquals(2, facetDtos.stream().filter(f -> f.getCount() == 1).count()); + assertEquals( + 2, + collectionsSearchMapper.institutionFacetCardinality( + InstitutionListParams.builder().facet(InstitutionFacetParameter.CITY).build())); + + facetDtos = + collectionsSearchMapper.institutionFacet( + InstitutionListParams.builder().facet(InstitutionFacetParameter.TYPE).build()); + assertEquals(2, facetDtos.size()); + assertEquals(1, facetDtos.stream().filter(f -> f.getCount() == 2).count()); + assertEquals(1, facetDtos.stream().filter(f -> f.getCount() == 1).count()); + assertEquals( + 2, + collectionsSearchMapper.institutionFacetCardinality( + InstitutionListParams.builder().facet(InstitutionFacetParameter.TYPE).build())); + + facetDtos = + collectionsSearchMapper.institutionFacet( + InstitutionListParams.builder().facet(InstitutionFacetParameter.DISCIPLINE).build()); + assertEquals(2, facetDtos.size()); + assertEquals(2, facetDtos.stream().filter(f -> f.getCount() == 1).count()); + assertEquals( + 2, + collectionsSearchMapper.institutionFacetCardinality( + InstitutionListParams.builder().facet(InstitutionFacetParameter.DISCIPLINE).build())); } } diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/DescriptorsListParams.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/DescriptorsListParams.java index 32d435d55..fc31ff775 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/DescriptorsListParams.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/DescriptorsListParams.java @@ -40,7 +40,9 @@ public boolean descriptorFacet() { || facet == CollectionFacetParameter.FAMILY_KEY || facet == CollectionFacetParameter.GENUS_KEY || facet == CollectionFacetParameter.SPECIES_KEY - || facet == CollectionFacetParameter.TYPE_STATUS; + || facet == CollectionFacetParameter.TYPE_STATUS + || facet == CollectionFacetParameter.RECORDED_BY + || facet == CollectionFacetParameter.OBJECT_CLASSIFICATION; } public boolean descriptorSearch() { @@ -75,4 +77,11 @@ public boolean descriptorSearchWithoutQuery() { || objectClassification != null || issues != null; } + + public boolean isArrayFieldFacet() { + return facet == CollectionFacetParameter.PRESERVATION_TYPE + || facet == CollectionFacetParameter.CONTENT_TYPE + || facet == CollectionFacetParameter.TYPE_STATUS + || facet == CollectionFacetParameter.RECORDED_BY; + } } diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/InstitutionListParams.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/InstitutionListParams.java index 621dd6ba1..f9572ba7f 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/InstitutionListParams.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/params/InstitutionListParams.java @@ -31,4 +31,8 @@ public class InstitutionListParams extends ListParams { // facets @Nullable InstitutionFacetParameter facet; + + public boolean isArrayFieldFacet() { + return facet == InstitutionFacetParameter.TYPE || facet == InstitutionFacetParameter.DISCIPLINE; + } } diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/CollectionsSearchMapper.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/CollectionsSearchMapper.xml index 0c28f7020..ca642c9d9 100644 --- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/CollectionsSearchMapper.xml +++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/CollectionsSearchMapper.xml @@ -452,16 +452,19 @@ SELECT - + LEFT JOIN address a ON a.key = i.address_key LEFT JOIN address ma ON ma.key = i.mailing_address_key @@ -481,6 +484,9 @@ CASE WHEN a.country is not null THEN a.country ELSE ma.country END + + CASE WHEN a.city is not null THEN a.city ELSE ma.city END + unnest(i.type) @@ -496,10 +502,14 @@ COUNT(DISTINCT CASE WHEN a.country is not null THEN a.country ELSE ma.country END) FROM institution i + + COUNT(DISTINCT CASE WHEN a.city is not null THEN a.city ELSE ma.city END) + FROM institution i + COUNT(DISTINCT facet) FROM institution i - CROSS JOIN unnest( + INNER JOIN unnest( i.type @@ -508,7 +518,7 @@ i.discipline - ) AS facet + ) facet ON true @@ -516,7 +526,7 @@ SELECT - + LEFT JOIN address a ON a.key = c.address_key LEFT JOIN address ma ON ma.key = c.mailing_address_key - - INNER JOIN collection_descriptor_group cdg ON cdg.collection_key = c.key - INNER JOIN collection_descriptor cd ON cd.collection_descriptor_group_key = cdg.key - @@ -555,6 +564,9 @@ CASE WHEN a.country is not null THEN a.country ELSE ma.country END + + CASE WHEN a.city is not null THEN a.city ELSE ma.city END + unnest(c.preservation_type) @@ -567,6 +579,9 @@ unnest(cd.type_status) + + unnest(cd.recorded_by) + c.accession_status @@ -594,21 +609,33 @@ cd.species_key + + cd.object_classification_name + - - COUNT(DISTINCT CASE WHEN a.country is not null THEN a.country ELSE ma.country END) + + COUNT(DISTINCT CASE WHEN a.${params.facet.name} is not null THEN a.${params.facet.name} ELSE ma.${params.facet.name} END) FROM collection c + + INNER JOIN collection_descriptor_group cdg ON cdg.collection_key = c.key + INNER JOIN collection_descriptor cd ON cd.collection_descriptor_group_key = cdg.key + + or params.facet.name.equals('TYPE_STATUS') + or params.facet.name.equals('RECORDED_BY')"> COUNT(DISTINCT facet) FROM collection c - CROSS JOIN unnest( + + INNER JOIN collection_descriptor_group cdg ON cdg.collection_key = c.key + INNER JOIN collection_descriptor cd ON cd.collection_descriptor_group_key = cdg.key + + INNER JOIN unnest( c.preservation_type @@ -617,10 +644,13 @@ c.content_type - c.type_status + cd.type_status + + + cd.recorded_by - ) AS facet + ) facet ON true COUNT(DISTINCT @@ -655,8 +685,15 @@ cd.species_key + + cd.object_classification_name + ) FROM collection c + + INNER JOIN collection_descriptor_group cdg ON cdg.collection_key = c.key + INNER JOIN collection_descriptor cd ON cd.collection_descriptor_group_key = cdg.key + diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java b/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java index 1dc1882df..afcead243 100644 --- a/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java +++ b/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java @@ -335,10 +335,7 @@ private CollectionFacet createFacet( F f, List facetDtos, long cardinality) { List facetCounts = facetDtos.stream() - .filter( - dto -> - !Strings.isNullOrEmpty(dto.getFacet()) - && !"null".equalsIgnoreCase(dto.getFacet())) + .filter(dto -> !Strings.isNullOrEmpty(dto.getFacet())) .map(dto -> new CollectionFacet.Count(dto.getFacet(), dto.getCount())) .collect(Collectors.toList());