From 1b97ce2e072cd4844960c7aa9f5183724bdcec8e Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Wed, 3 Jul 2024 10:26:59 -0400 Subject: [PATCH] q_system_use: ignore codeable concepts that are all nulls inside Instead of checking "concept is not null" we should instead be using "utils.is_concept_valid()" because we want to ignore concepts that look like: {coding=null, id=null, text=null} which is what we can get with the fully-specified schema that Cumulus uses. --- cumulus_library_data_metrics/q_system_use/q_system_use.jinja | 2 +- tests/data/q_system_use/general/expected_procedure_code.csv | 5 +++++ tests/data/q_system_use/general/expected_summary.csv | 4 ++-- tests/data/q_system_use/general/procedure/0.ndjson | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/data/q_system_use/general/expected_procedure_code.csv diff --git a/cumulus_library_data_metrics/q_system_use/q_system_use.jinja b/cumulus_library_data_metrics/q_system_use/q_system_use.jinja index db217f3..866b165 100644 --- a/cumulus_library_data_metrics/q_system_use/q_system_use.jinja +++ b/cumulus_library_data_metrics/q_system_use/q_system_use.jinja @@ -36,7 +36,7 @@ SELECT {{ field }} FROM rejoined_src WHERE - {{ field }} IS NOT NULL + {{ utils.is_concept_valid(field) }} AND ( has_target_system IS NULL OR NOT has_target_system diff --git a/tests/data/q_system_use/general/expected_procedure_code.csv b/tests/data/q_system_use/general/expected_procedure_code.csv new file mode 100644 index 0000000..3ad8e1f --- /dev/null +++ b/tests/data/q_system_use/general/expected_procedure_code.csv @@ -0,0 +1,5 @@ +id,status,code +missing-system,,"{'id': NULL, 'coding': [{'id': NULL, 'code': NULL, 'display': No system, 'system': NULL, 'userSelected': NULL, 'version': NULL}], 'text': NULL}" +missing-coding,,"{'id': NULL, 'coding': NULL, 'text': No coding}" +empty-coding,,"{'id': NULL, 'coding': [], 'text': NULL}" +bad-system,,"{'id': NULL, 'coding': [{'id': NULL, 'code': NULL, 'display': NULL, 'system': nope, 'userSelected': NULL, 'version': NULL}], 'text': NULL}" diff --git a/tests/data/q_system_use/general/expected_summary.csv b/tests/data/q_system_use/general/expected_summary.csv index b230bf9..d7517d8 100644 --- a/tests/data/q_system_use/general/expected_summary.csv +++ b/tests/data/q_system_use/general/expected_summary.csv @@ -1,7 +1,7 @@ resource,field,numerator,denominator,percentage # Various edge cases here in procedure -Procedure,cumulus__all,4,12,33.33 -Procedure,code,4,12,33.33 +Procedure,cumulus__all,4,13,30.77 +Procedure,code,4,13,30.77 # Rest are often short happy-path checks to confirm that we look at the right json field Observation,valueCodeableConcept,0,2,0.00 Observation,cumulus__all,0,2,0.00 diff --git a/tests/data/q_system_use/general/procedure/0.ndjson b/tests/data/q_system_use/general/procedure/0.ndjson index 37820fe..c131d52 100644 --- a/tests/data/q_system_use/general/procedure/0.ndjson +++ b/tests/data/q_system_use/general/procedure/0.ndjson @@ -9,4 +9,5 @@ {"resourceType": "Procedure", "id": "missing-system", "code": {"coding": [{"display": "No system"}]}} {"resourceType": "Procedure", "id": "empty-coding", "code": {"coding": []}} {"resourceType": "Procedure", "id": "missing-coding", "code": {"text": "No coding"}} +{"resourceType": "Procedure", "id": "invalid-code", "code": {"id": "no content"}} {"resourceType": "Procedure", "id": "missing-code"}