Bug fix catalogue name and page search

python-libraries/data-platform-catalogue/CHANGELOG.md

@@ -7,6 +7,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.0] 2024-02-19
+
+### Changed
+
+- bugfix - search now returns correct page results, where start is
+individual search result index.
+- bugfix - `upsert_table` client method now adds dataset name to datahub.
+- bugfix - `upsert_table` client method no longer duplicates assets assocaited
+with data product.
+
 ## [0.13.0] 2024-02-14
 
 ### Changed

python-libraries/data-platform-catalogue/data_platform_catalogue/client/datahub/datahub_client.py

@@ -238,6 +238,7 @@ def upsert_table(
         )
 
         dataset_properties = DatasetPropertiesClass(
+            name=metadata.name,
             description=metadata.description,
             customProperties={
                 "sourceDatasetName": metadata.source_dataset_name,
@@ -316,7 +317,8 @@ def upsert_table(
                 and data_product_existing_properties.assets is not None
             ):
                 assets = data_product_existing_properties.assets[::]
-                assets.append(data_product_association)
+                if data_product_association not in assets:
+                    assets.append(data_product_association)
             else:
                 assets = [data_product_association]
 

python-libraries/data-platform-catalogue/data_platform_catalogue/client/datahub/search.py

@@ -52,7 +52,7 @@ def search(
         if page is None:
             start = 0
         else:
-            start = int(page)
+            start = int(page) * count
 
         types = self._map_result_types(result_types)
         formatted_filters = self._map_filters(filters)

python-libraries/data-platform-catalogue/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ministryofjustice-data-platform-catalogue"
-version = "0.12.0"
+version = "0.14.0"
 description = "Library to integrate the MoJ data platform with the catalogue component."
 authors = ["MoJ Data Platform Team <[email protected]>"]
 license = "MIT"

python-libraries/data-platform-catalogue/tests/snapshots/datahub_create_table.json

@@ -12,6 +12,7 @@
                 "sensitivityLevel": "TOP_SECRET",
                 "rowCount": "1177"
             },
+            "name": "my_table",
             "description": "bla bla",
             "tags": []
         }

python-libraries/data-platform-catalogue/tests/snapshots/datahub_create_table_with_metadata.json

@@ -12,6 +12,7 @@
                 "sensitivityLevel": "TOP_SECRET",
                 "rowCount": "1177"
             },
+            "name": "my_table",
             "description": "bla bla",
             "tags": []
         }

python-libraries/data-platform-catalogue/tests/snapshots/datahub_create_two_tables_with_metadata.json

@@ -12,6 +12,7 @@
                 "sensitivityLevel": "TOP_SECRET",
                 "rowCount": "1177"
             },
+            "name": "my_table",
             "description": "bla bla",
             "tags": []
         }
@@ -170,6 +171,7 @@
                 "sensitivityLevel": "OFFICIAL",
                 "rowCount": "1177"
             },
+            "name": "my_table2",
             "description": "this is a different table",
             "tags": []
         }

python-libraries/data-platform-catalogue/tests/test_integration_with_datahub_server.py

@@ -4,7 +4,7 @@
 Run with:
 export API_URL='https://catalogue.apps-tools.development.data-platform.service.justice.gov.uk/api'
 export JWT_TOKEN=******
-poetry run pytest tests/test_integration_with_server.py
+poetry run pytest tests/test_integration_with_datahub_server.py
 """
 
 import os
@@ -75,6 +75,21 @@ def test_upsert_test_hierarchy():
     assert client.graph.get_aspect(table_fqn, DatasetPropertiesClass)
     assert client.graph.get_aspect(table_fqn, SchemaMetadataClass)
 
+    dataset_properties = client.graph.get_aspect(
+        table_fqn, aspect_type=DatasetPropertiesClass
+    )
+    # check properties been loaded to datahub dataset
+    assert dataset_properties.description == table.description
+    assert dataset_properties.name == table.name
+    assert (
+        dataset_properties.customProperties["sourceDatasetName"]
+        == table.source_dataset_name
+    )
+    assert (
+        dataset_properties.customProperties["whereToAccessDataset"]
+        == table.where_to_access_dataset
+    )
+
 
 @runs_on_development_server
 def test_search():
@@ -245,3 +260,11 @@ def test_fetch_dataset_belonging_to_data_product():
     metadata = response.page_results[0].metadata
     assert metadata["total_data_products"] == 1
     assert metadata["data_products"][0]["name"] == "my_data_product"
+
+
+@runs_on_development_server
+def test_paginated_search_results_unique():
+    client = DataHubCatalogueClient(jwt_token=jwt_token, api_url=api_url)
+    results1 = client.search(page="1").page_results
+    results2 = client.search(page="2").page_results
+    assert not any(x in results1 for x in results2)