Skip to content

Commit

Permalink
Merge pull request hapifhir#1527 from hapifhir/2023-12-gg-big-tx-rework
Browse files Browse the repository at this point in the history
2023 12 gg big tx rework
  • Loading branch information
grahamegrieve authored Dec 30, 2023
2 parents 2dc3a5b + 8abdcc6 commit 8ee621a
Show file tree
Hide file tree
Showing 199 changed files with 22,177 additions and 5,626 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package org.hl7.fhir.convertors.misc;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;

import javax.xml.parsers.ParserConfigurationException;

import org.fhir.ucum.Utilities;
import org.hl7.fhir.convertors.misc.CVXImporter.CVXSorter;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.r4.formats.IParser.OutputStyle;
import org.hl7.fhir.r4.formats.JsonParser;
import org.hl7.fhir.r4.model.CodeSystem;
import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
import org.hl7.fhir.r4.model.CodeType;
import org.hl7.fhir.r4.model.Coding;
import org.hl7.fhir.r4.model.DateTimeType;
import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.terminologies.CodeSystemUtilities;
import org.hl7.fhir.utilities.xml.XMLUtil;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;

/**
* To use this class, download the CVX definitions from
* https://www2a.cdc.gov/vaccines/iis/iisstandards/vaccines.asp?rpt=cvx
* using the XML-new format, and then execute this class with two parameters:
* - the name of the downloaded file
* - a local name for the file https://github.com/FHIR/packages/blob/master/packages/fhir.tx.support.r4/package/CodeSystem-cvx.json
*
* //.Users/grahamegrieve/work/packages/packages/fhir.tx.support.r4/package/CodeSystem-cvx.json
*/

public class CVXImporter {

public class CVXSorter implements Comparator<ConceptDefinitionComponent> {

@Override
public int compare(ConceptDefinitionComponent o1, ConceptDefinitionComponent o2) {
int i1 = Integer.parseInt(o1.getCode());
int i2 = Integer.parseInt(o2.getCode());
return i1-i2;
}

}

public static void main(String[] args) throws FHIRException, FileNotFoundException, IOException, ClassNotFoundException, SQLException, ParserConfigurationException, SAXException {
new CVXImporter().doUpdate(args[0], args[1]);

}

private void doUpdate(String source, String dest) throws FHIRFormatError, FileNotFoundException, IOException, ParserConfigurationException, SAXException {
CodeSystem cvx = (CodeSystem) new JsonParser().parse(new FileInputStream(dest));

String ldate = null;

Document xml = XMLUtil.parseFileToDom(source);
Element cvxCodes = xml.getDocumentElement();
for (Element cvsInfo : XMLUtil.getNamedChildren(cvxCodes, "CVXInfo")) {
String desc = XMLUtil.getNamedChildText(cvsInfo, "ShortDescription").trim();
String name = XMLUtil.getNamedChildText(cvsInfo, "FullVaccinename").trim();
String code = XMLUtil.getNamedChildText(cvsInfo, "CVXCode").trim();
String notes = XMLUtil.getNamedChildText(cvsInfo, "Notes");
String status = XMLUtil.getNamedChildText(cvsInfo, "Status").trim();
String date = XMLUtil.getNamedChildText(cvsInfo, "LastUpdated").trim();
ConceptDefinitionComponent def = findCVXCode(cvx, code);
if (def == null) {
def = cvx.addConcept();
def.setCode(code);
} else {
def.getDesignation().clear();
}
def.setDisplay(name);
def.addDesignation().setValue(desc).setLanguage("en").setUse(new Coding().setSystem("http://snomed.info/sct").setCode("900000000000013009").setDisplay("Synonym"));
if (!Utilities.noString(notes)) {
def.forceProperty("notes").setValue(new StringType(notes.trim()));
}
def.forceProperty("vaccine-status").setValue(new CodeType(status.trim()));
String[] d = date.split("\\/");
String vdate = d[2]+"-"+Utilities.padLeft(d[0], '0', 2)+"-"+Utilities.padLeft(d[1], '0', 2);
def.forceProperty("last-updated").setValue(new DateTimeType(vdate));
if (ldate == null || ldate.compareTo(vdate) < 0) {
ldate = vdate;
}
}
Collections.sort(cvx.getConcept(), new CVXSorter());
cvx.setDateElement(new DateTimeType(ldate));
cvx.setVersion(ldate.replace("-", ""));
new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(dest), cvx);
}

private ConceptDefinitionComponent findCVXCode(CodeSystem cvx, String code) {
for (ConceptDefinitionComponent t : cvx.getConcept()) {
if (code.equals(t.getCode())) {
return t;
}
}
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2567,6 +2567,15 @@ public String fhirType() {

}

public ConceptPropertyComponent forceProperty(String code) {
for (ConceptPropertyComponent prop : getProperty()) {
if (code.equals(prop.getCode())) {
return prop;
}
}
return addProperty().setCode(code);
}

}

@Block()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
import org.hl7.fhir.utilities.validation.ValidationOptions;
import org.hl7.fhir.utilities.validation.ValidationOptions.ValueSetMode;

import com.google.gson.JsonObject;

Expand Down Expand Up @@ -1053,8 +1052,11 @@ private void setTerminologyOptions(ValidationOptions options, Parameters pIn) {
if (options.hasLanguages()) {
pIn.addParameter("displayLanguage", options.getLanguages().toString());
}
if (options.getValueSetMode() != ValueSetMode.ALL_CHECKS) {
pIn.addParameter("valueSetMode", options.getValueSetMode().toString());
if (options.isMembershipOnly()) {
pIn.addParameter("valueset-membership-only", true);
}
if (options.isDisplayWarningMode()) {
pIn.addParameter("lenient-display-validation", true);
}
if (options.isVersionFlexible()) {
pIn.addParameter("default-to-latest-version", true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS
import org.hl7.fhir.utilities.i18n.I18nConstants;
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
import org.hl7.fhir.utilities.validation.ValidationOptions;
import org.hl7.fhir.utilities.validation.ValidationOptions.ValueSetMode;

public class ValueSetCheckerSimple extends ValueSetWorker implements ValueSetChecker {

Expand Down Expand Up @@ -122,7 +121,7 @@ public ValidationResult validateCode(CodeableConcept code) throws FHIRException
// first, we validate the codings themselves
List<String> errors = new ArrayList<String>();
List<String> warnings = new ArrayList<String>();
if (options.getValueSetMode() != ValueSetMode.CHECK_MEMERSHIP_ONLY) {
if (!options.isMembershipOnly()) {
for (Coding c : code.getCoding()) {
if (!c.hasSystem()) {
warnings.add(context.formatMessage(I18nConstants.CODING_HAS_NO_SYSTEM__CANNOT_VALIDATE));
Expand All @@ -141,7 +140,7 @@ public ValidationResult validateCode(CodeableConcept code) throws FHIRException
}
}
}
if (valueset != null && options.getValueSetMode() != ValueSetMode.NO_MEMBERSHIP_CHECK) {
if (valueset != null) {
Boolean result = false;
CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(",", " and ");
for (Coding c : code.getCoding()) {
Expand Down Expand Up @@ -191,7 +190,7 @@ public ValidationResult validateCode(Coding code) throws FHIRException {
boolean inExpansion = false;
boolean inInclude = false;
String system = code.hasSystem() ? code.getSystem() : getValueSetSystemOrNull();
if (options.getValueSetMode() != ValueSetMode.CHECK_MEMERSHIP_ONLY) {
if (!options.isMembershipOnly()) {
if (system == null && !code.hasDisplay()) { // dealing with just a plain code (enum)
system = systemForCodeInValueSet(code.getCode());
}
Expand Down Expand Up @@ -254,7 +253,7 @@ public ValidationResult validateCode(Coding code) throws FHIRException {
List<String> warnings = new ArrayList<>();

// then, if we have a value set, we check it's in the value set
if (valueset != null && options.getValueSetMode() != ValueSetMode.NO_MEMBERSHIP_CHECK) {
if (valueset != null) {
if ((res == null || res.isOk())) {
Boolean ok = codeInValueSet(system, code.getCode(), warnings);
if (ok == null || !ok) {
Expand Down Expand Up @@ -366,10 +365,10 @@ private ValidationResult validateCode(Coding code, CodeSystem cs) {
if (cc == null) {
if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
return new ValidationResult(IssueSeverity.WARNING,
context.formatMessage(I18nConstants.UNKNOWN_CODE__IN_FRAGMENT, code, cs.getUrl()));
context.formatMessage(I18nConstants.UNKNOWN_CODE_IN_FRAGMENT, code, cs.getUrl()));
} else {
return new ValidationResult(IssueSeverity.ERROR,
context.formatMessage(I18nConstants.UNKNOWN_CODE__IN_, code, cs.getUrl()));
context.formatMessage(I18nConstants.UNKNOWN_CODE_IN, code, cs.getUrl()));
}
}
if (code.getDisplay() == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS
import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesCodeSystemComponent;
import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesExpansionParameterComponent;
import org.hl7.fhir.r5.model.UriType;
import org.hl7.fhir.r5.model.UrlType;
import org.hl7.fhir.r5.model.ValueSet;
import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent;
import org.hl7.fhir.r5.model.Bundle.BundleType;
Expand Down Expand Up @@ -153,7 +154,6 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
import org.hl7.fhir.utilities.validation.ValidationOptions;
import org.hl7.fhir.utilities.validation.ValidationOptions.ValueSetMode;

import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
Expand Down Expand Up @@ -1106,7 +1106,7 @@ public void validateCodeBatch(ValidationOptions options, List<? extends CodingVa
BundleEntryComponent r = resp.getEntry().get(i);

if (r.getResource() instanceof Parameters) {
t.setResult(processValidationResult((Parameters) r.getResource(), vs != null ? vs.getUrl() : t.getVsObj() != null ? t.getVsObj().getUrl() : null));
t.setResult(processValidationResult((Parameters) r.getResource(), vs != null ? vs.getUrl() : t.getVsObj() != null ? t.getVsObj().getUrl() : null, tcc.getClient().getAddress()));
if (txCache != null) {
txCache.cacheValidation(t.getCacheToken(), t.getResult(), TerminologyCache.PERMANENT);
}
Expand Down Expand Up @@ -1211,7 +1211,7 @@ public void validateCodeBatchByRef(ValidationOptions options, List<? extends Cod
BundleEntryComponent r = resp.getEntry().get(i);

if (r.getResource() instanceof Parameters) {
t.setResult(processValidationResult((Parameters) r.getResource(), vsUrl));
t.setResult(processValidationResult((Parameters) r.getResource(), vsUrl, tcc.getClient().getAddress()));
if (txCache != null) {
txCache.cacheValidation(t.getCacheToken(), t.getResult(), TerminologyCache.PERMANENT);
}
Expand Down Expand Up @@ -1272,6 +1272,7 @@ public ValidationResult validateCode(final ValidationOptions optionsArg, String

String localError = null;
String localWarning = null;
TerminologyServiceErrorClass type = TerminologyServiceErrorClass.UNKNOWN;
if (options.isUseClient()) {
// ok, first we try to validate locally
try {
Expand All @@ -1294,6 +1295,7 @@ public ValidationResult validateCode(final ValidationOptions optionsArg, String
if (e.getIssues() != null) {
issues.addAll(e.getIssues());
}
type = e.getType();
} catch (TerminologyServiceProtectionException e) {
OperationOutcomeIssueComponent iss = new OperationOutcomeIssueComponent(org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.ERROR, e.getType());
iss.getDetails().setText(e.getMessage());
Expand Down Expand Up @@ -1343,6 +1345,9 @@ public ValidationResult validateCode(final ValidationOptions optionsArg, String
} catch (Exception e) {
res = new ValidationResult(IssueSeverity.ERROR, e.getMessage() == null ? e.getClass().getName() : e.getMessage(), null).setTxLink(txLog == null ? null : txLog.getLastId()).setErrorClass(TerminologyServiceErrorClass.SERVER_ERROR);
}
if (!res.isOk() && res.getErrorClass() == TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED && (localError != null && !localError.equals(ValueSetValidator.NO_TRY_THE_SERVER))) {
res = new ValidationResult(IssueSeverity.ERROR, localError, null).setTxLink(txLog == null ? null : txLog.getLastId()).setErrorClass(type);
}
if (!res.isOk() && localError != null) {
res.setDiagnostics("Local Error: "+localError.trim()+". Server Error: "+res.getMessage());
} else if (!res.isOk() && res.getUnknownSystems() != null && res.getUnknownSystems().contains(codeKey) && localWarning != null) {
Expand Down Expand Up @@ -1434,8 +1439,11 @@ private void setTerminologyOptions(ValidationOptions options, Parameters pIn) {
if (options.hasLanguages()) {
pIn.addParameter("displayLanguage", options.getLanguages().toString());
}
if (options.getValueSetMode() != ValueSetMode.ALL_CHECKS) {
pIn.addParameter("valueSetMode", options.getValueSetMode().toString());
if (options.isMembershipOnly()) {
pIn.addParameter("valueset-membership-only", true);
}
if (options.isDisplayWarningMode()) {
pIn.addParameter("lenient-display-validation", true);
}
if (options.isVersionFlexible()) {
pIn.addParameter("default-to-latest-version", true);
Expand Down Expand Up @@ -1552,7 +1560,7 @@ protected ValidationResult validateOnServer(ValueSet vs, Parameters pin, Validat
} else {
pOut = tcc.getClient().validateVS(pin);
}
return processValidationResult(pOut, vs == null ? null : vs.getUrl());
return processValidationResult(pOut, vs == null ? null : vs.getUrl(), tcc.getClient().getAddress());
}

protected void addServerValidationParameters(ValueSet vs, Parameters pin, ValidationOptions options) {
Expand Down Expand Up @@ -1650,7 +1658,7 @@ private boolean hasCanonicalResource(Parameters pin, String name, String vUrl) {
return false;
}

public ValidationResult processValidationResult(Parameters pOut, String vs) {
public ValidationResult processValidationResult(Parameters pOut, String vs, String server) {
boolean ok = false;
String message = "No Message returned";
String display = null;
Expand Down Expand Up @@ -1724,13 +1732,27 @@ public ValidationResult processValidationResult(Parameters pOut, String vs) {
} catch (FHIRException e) {
}
}
} else if (p.hasResource()) {
if (p.getName().equals("issues")) {
OperationOutcome oo = (OperationOutcome) p.getResource();
for (OperationOutcomeIssueComponent iss : oo.getIssue()) {
iss.addExtension(ToolingExtensions.EXT_ISSUE_SERVER, new UrlType(server));
issues.add(iss);
}
} else {
// nothing?
}
}
}
ValidationResult res = null;
if (!ok) {
res = new ValidationResult(IssueSeverity.ERROR, message+" (from "+tcc.getClient().getId()+")", err, null).setTxLink(txLog.getLastId());
res = new ValidationResult(IssueSeverity.ERROR, message, err, null).setTxLink(txLog.getLastId());
if (code != null) {
res.setDefinition(new ConceptDefinitionComponent().setDisplay(display).setCode(code));
res.setDisplay(display);
}
} else if (message != null && !message.equals("No Message returned")) {
res = new ValidationResult(IssueSeverity.WARNING, message+" (from "+tcc.getClient().getId()+")", system, version, new ConceptDefinitionComponent().setDisplay(display).setCode(code), display, null).setTxLink(txLog.getLastId());
res = new ValidationResult(IssueSeverity.WARNING, message, system, version, new ConceptDefinitionComponent().setDisplay(display).setCode(code), display, null).setTxLink(txLog.getLastId());
} else if (display != null) {
res = new ValidationResult(system, version, new ConceptDefinitionComponent().setDisplay(display).setCode(code), display).setTxLink(txLog.getLastId());
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,6 @@ public String toString() {

public ProfiledType(String n) {
uri = ns(n);
if (uri.equals("http://hl7.org/fhir/StructureDefinition/CDA")) {
System.out.println("!"); // #FIXME
}
}

public String getUri() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,7 @@ public boolean is(String system, String code) {
}

public String toString() {
String base = getSystem();
String base = hasSystem() ? getSystem() : "";
if (hasVersion())
base = base+"|"+getVersion();
base = base + "#"+getCode();
Expand Down
Loading

0 comments on commit 8ee621a

Please sign in to comment.