From b20c2a0dff6380df27c6c8fd2b7cbe4a0fee15bf Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 6 Apr 2022 14:39:24 +0200 Subject: [PATCH 01/12] version 1.0.1-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f2c0fc..f5e8745 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ de.samply.common adt2fhir - 1.0.0 + 1.0.1-SNAPSHOT jar From b85065ce79cbd2f6f0d48bd9362b48d1c604cb7f Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Fri, 8 Apr 2022 09:59:23 +0200 Subject: [PATCH 02/12] WIP --- pom.xml | 137 +++++++++--------- .../samply/adt2fhir/PatientPseudonymizer.java | 19 ++- 2 files changed, 89 insertions(+), 67 deletions(-) diff --git a/pom.xml b/pom.xml index f5e8745..3daca20 100644 --- a/pom.xml +++ b/pom.xml @@ -1,67 +1,72 @@ - - - 4.0.0 - - de.samply.common - adt2fhir - 1.0.1-SNAPSHOT - jar - - - 8 - 8 - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.2 - - - src/main/resources/META-INF/MANIFEST.MF - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.3.0 - - - package - - single - - - - - de.samply.adt2fhir.Adt2fhir - - - - jar-with-dependencies - - - - - - - - - - - net.sf.saxon - Saxon-HE - 9.8.0-15 - - - org.apache.httpcomponents - httpclient - 4.5.13 - compile - - + + + 4.0.0 + + de.samply.common + adt2fhir + 1.0.1-SNAPSHOT + jar + + + 8 + 8 + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.2 + + + src/main/resources/META-INF/MANIFEST.MF + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + package + + single + + + + + de.samply.adt2fhir.Adt2fhir + + + + jar-with-dependencies + + + + + + + + + + + net.sf.saxon + Saxon-HE + 9.8.0-15 + + + org.apache.httpcomponents + httpclient + 4.5.13 + compile + + + de.pseudonymisierung + mainzelliste-client + 2.1.0 + + \ No newline at end of file diff --git a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java index 4ed7a85..95f6bc9 100644 --- a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java +++ b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java @@ -1,5 +1,10 @@ package de.samply.adt2fhir; +import de.pseudonymisierung.mainzelliste.client.InvalidSessionException; +import de.pseudonymisierung.mainzelliste.client.MainzellisteConnection; +import de.pseudonymisierung.mainzelliste.client.MainzellisteNetworkException; +import de.pseudonymisierung.mainzelliste.client.Session; +import java.net.URISyntaxException; import net.sf.saxon.expr.XPathContext; import net.sf.saxon.lib.ExtensionFunctionCall; import net.sf.saxon.lib.ExtensionFunctionDefinition; @@ -45,7 +50,19 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { String surname = args[2].iterate().next().getStringValue(); String birthname = args[3].iterate().next().getStringValue(); String brithdate = args[4].iterate().next().getStringValue(); - output = DigestUtils.sha256Hex(gender+prename+surname+birthname+brithdate);//TODO create patientlist call +// output = DigestUtils.sha256Hex(gender+prename+surname+birthname+brithdate);//TODO create patientlist call + // More Information on: https://github.com/medicalinformatics/mainzelliste-client + try { + MainzellisteConnection mainzellisteConnection = new MainzellisteConnection("http://localhost:8080", "youReallyShouldChangeMe"); + Session session = mainzellisteConnection.createSession(); + String addPatientToken = session.getAddPatientToken(null, null); + } catch (MainzellisteNetworkException e) { + + } catch (URISyntaxException e) { + + } catch (InvalidSessionException e) { + // create new session here + } return StringValue.makeStringValue(output); } From a73e187d821911dabab6d7cc6c483523c03e53e2 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 26 Apr 2022 10:45:22 +0200 Subject: [PATCH 03/12] Add Mainzelliste support --- pom.xml | 6 ++ src/docker/adt2fhir.properties | 6 +- .../java/de/samply/adt2fhir/Adt2fhir.java | 16 ++-- .../java/de/samply/adt2fhir/ConfigReader.java | 8 ++ .../samply/adt2fhir/PatientPseudonymizer.java | 88 +++++++++++++++---- src/main/resources/MDS2FHIR.xsl | 3 +- src/main/resources/adt2fhir.properties | 6 +- src/main/resources/toSinglePatients.xsl | 3 +- 8 files changed, 108 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index 3daca20..a9e9fd2 100644 --- a/pom.xml +++ b/pom.xml @@ -68,5 +68,11 @@ mainzelliste-client 2.1.0 + + com.google.code.gson + gson + 2.9.0 + + \ No newline at end of file diff --git a/src/docker/adt2fhir.properties b/src/docker/adt2fhir.properties index 75d6151..a7e990b 100644 --- a/src/docker/adt2fhir.properties +++ b/src/docker/adt2fhir.properties @@ -1,3 +1,5 @@ file_path=/adt2fhir/clinical_data -store_path=http://host.docker.internal:8080/fhir -identifier_system=http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS \ No newline at end of file +store_path=http://host.docker.internal:8090/fhir +identifier_system=http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS +mainzelliste_url=http://host.docker.internal:8080 +mainzelliste_apikey=pleaseChangeMe \ No newline at end of file diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index ed17665..96402be 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -47,7 +47,9 @@ public static void main(String[] args) throws TransformerConfigurationException, System.out.print("initialize transformers... "); final TransformerFactoryImpl factory = (TransformerFactoryImpl) TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl", null); net.sf.saxon.Configuration saxonConfig = factory.getConfiguration(); - ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(new PatientPseudonymizer()); + PatientPseudonymizer patientPseudonymizer = new PatientPseudonymizer(); + patientPseudonymizer.setConfigReader(configReader); + ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(patientPseudonymizer); ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(new UniqueIdGenerator()); final Transformer ADT2singleADTtransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("toSinglePatients.xsl"))); @@ -95,9 +97,9 @@ private static void processXmlFiles (String inputData, Transformer transformer, else { int counter=1; for (File inputFile : listOfFiles) { - System.out.println("\u001B[AFile "+counter+" of "+listOfFiles.length); - counter+=1; if (inputFile.isFile() & inputFile.getName().toLowerCase().endsWith(".xml")) { + System.out.println("\u001B[AFile "+counter+" of "+(listOfFiles.length-1)); + counter+=1; if (transformer ==null){ try { postToFhirStore(inputFile, httppost); @@ -109,17 +111,19 @@ private static void processXmlFiles (String inputData, Transformer transformer, } else { //System.out.print("processing file " + inputFile.getName() + "..."); - String combinedADTfile = null; + String inputXml = null; try { - combinedADTfile = new String(Files.readAllBytes(Paths.get(String.valueOf(inputFile))), StandardCharsets.UTF_8); + inputXml = new String(Files.readAllBytes(Paths.get(String.valueOf(inputFile))), StandardCharsets.UTF_8); } catch (IOException e) { counter-=1; System.out.print("ERROR - reading: problem with file " + inputFile); e.printStackTrace(); } try { - String xmlResult = applyXslt(combinedADTfile, transformer); + transformer.setParameter("customPrefix", inputFile.getName()); + String xmlResult = applyXslt(inputXml, transformer); if(transformWrittenResults){ + transformer2.setParameter("customPrefix", inputFile.getName()); applyXslt(xmlResult, transformer2); inputFile.deleteOnExit(); } diff --git a/src/main/java/de/samply/adt2fhir/ConfigReader.java b/src/main/java/de/samply/adt2fhir/ConfigReader.java index df261d9..d62ce49 100644 --- a/src/main/java/de/samply/adt2fhir/ConfigReader.java +++ b/src/main/java/de/samply/adt2fhir/ConfigReader.java @@ -8,6 +8,8 @@ public class ConfigReader { public String file_path; public String store_path; public String identifier_system; + public String mainzelliste_url; + public String mainzelliste_apikey; public void init() throws IOException { try { @@ -31,6 +33,8 @@ public void init() throws IOException { this.file_path = prop.getProperty("file_path"); this.store_path = prop.getProperty("store_path"); this.identifier_system = prop.getProperty("identifier_system"); + this.mainzelliste_url = prop.getProperty("mainzelliste_url"); + this.mainzelliste_apikey = prop.getProperty("mainzelliste_apikey"); } catch (IOException e) { e.printStackTrace(); } @@ -45,4 +49,8 @@ public String getStore_path() { public String getIdentifier_system() { return identifier_system; } + public String getMainzelliste_url() { return mainzelliste_url; } + public String getMainzelliste_apikey() { + return mainzelliste_apikey; + } } \ No newline at end of file diff --git a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java index 95f6bc9..bb0f92a 100644 --- a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java +++ b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java @@ -1,10 +1,14 @@ package de.samply.adt2fhir; -import de.pseudonymisierung.mainzelliste.client.InvalidSessionException; -import de.pseudonymisierung.mainzelliste.client.MainzellisteConnection; -import de.pseudonymisierung.mainzelliste.client.MainzellisteNetworkException; -import de.pseudonymisierung.mainzelliste.client.Session; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import de.pseudonymisierung.mainzelliste.client.*; +import java.io.*; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import net.sf.saxon.expr.XPathContext; import net.sf.saxon.lib.ExtensionFunctionCall; import net.sf.saxon.lib.ExtensionFunctionDefinition; @@ -14,9 +18,20 @@ import net.sf.saxon.value.SequenceType; import net.sf.saxon.value.StringValue; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.protocol.HTTP; +import org.apache.http.util.EntityUtils; public class PatientPseudonymizer extends ExtensionFunctionDefinition { - + private String mainzelliste_url; + private String mainzelliste_apikey; + private boolean anonymize; @Override public net.sf.saxon.value.SequenceType[] getArgumentTypes() { @@ -50,22 +65,63 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { String surname = args[2].iterate().next().getStringValue(); String birthname = args[3].iterate().next().getStringValue(); String brithdate = args[4].iterate().next().getStringValue(); -// output = DigestUtils.sha256Hex(gender+prename+surname+birthname+brithdate);//TODO create patientlist call - // More Information on: https://github.com/medicalinformatics/mainzelliste-client - try { - MainzellisteConnection mainzellisteConnection = new MainzellisteConnection("http://localhost:8080", "youReallyShouldChangeMe"); - Session session = mainzellisteConnection.createSession(); - String addPatientToken = session.getAddPatientToken(null, null); - } catch (MainzellisteNetworkException e) { - - } catch (URISyntaxException e) { - } catch (InvalidSessionException e) { - // create new session here + if (mainzelliste_url.isEmpty()) { + output = DigestUtils.sha256Hex(gender + prename + surname + birthname + brithdate).substring(32); + } else { + try { + output=pseudonymizationCall(gender, prename, surname, birthname, brithdate); + } catch (URISyntaxException | MainzellisteNetworkException | InvalidSessionException | IOException e) { + e.printStackTrace(); + } } return StringValue.makeStringValue(output); } }; } + + private String pseudonymizationCall(String gender, String prename, String surname, String birthname, String brithdate) throws URISyntaxException, MainzellisteNetworkException, InvalidSessionException, IOException { + String pseudonym=""; + String[] brithdateParts = brithdate.split("[.]"); + MainzellisteConnection mainzellisteConnection = new MainzellisteConnection(mainzelliste_url, mainzelliste_apikey); + Session session = mainzellisteConnection.createSession(); + AddPatientToken token = new AddPatientToken(); + token.addIdType("pid"); + String addPatientToken = session.getToken(token); + HttpPost httppost = new HttpPost(mainzelliste_url+"/patients?tokenId="+addPatientToken); + httppost.addHeader("content-type", "application/x-www-form-urlencoded"); + httppost.addHeader("mainzellisteApiVersion", "3.2"); + List idat = new ArrayList(); + //idat.add(new BasicNameValuePair("gender", gender)); + idat.add(new BasicNameValuePair("vorname", prename)); + idat.add(new BasicNameValuePair("nachname", surname)); + idat.add(new BasicNameValuePair("geburtsname", birthname)); + idat.add(new BasicNameValuePair("geburtstag", brithdateParts[0])); + idat.add(new BasicNameValuePair("geburtsmonat", brithdateParts[1])); + idat.add(new BasicNameValuePair("geburtsjahr", brithdateParts[2])); + //idat.add(new BasicNameValuePair("plz", "")); + //idat.add(new BasicNameValuePair("ort", "")); + idat.add(new BasicNameValuePair("sureness", "true")); + httppost.setEntity(new UrlEncodedFormEntity(idat, HTTP.UTF_8)); + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpResponse response = httpclient.execute(httppost); + if (response.getStatusLine().getStatusCode()!=201) { + System.out.println("Error - Pseudonymization response: " + response.getStatusLine().getStatusCode()); + } + String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + httpclient.close(); + + JsonArray ids = new Gson ().fromJson(responseBody, JsonArray.class); + if(!ids.isEmpty()){ + JsonObject id = ids.get(0).getAsJsonObject(); + pseudonym=id.getAsJsonPrimitive("idString").getAsString(); + } + return pseudonym; + } + + public void setConfigReader (ConfigReader configReader){ + this.mainzelliste_url=configReader.getMainzelliste_url(); + this.mainzelliste_apikey=configReader.getMainzelliste_apikey(); + } } diff --git a/src/main/resources/MDS2FHIR.xsl b/src/main/resources/MDS2FHIR.xsl index fb27c52..0728112 100644 --- a/src/main/resources/MDS2FHIR.xsl +++ b/src/main/resources/MDS2FHIR.xsl @@ -24,6 +24,7 @@ + @@ -37,7 +38,7 @@ - + diff --git a/src/main/resources/adt2fhir.properties b/src/main/resources/adt2fhir.properties index 37ae743..43fe7e1 100644 --- a/src/main/resources/adt2fhir.properties +++ b/src/main/resources/adt2fhir.properties @@ -1,3 +1,5 @@ file_path=changeMeToAbsolutePath/adt2fhir/clinical_data -store_path=http://localhost:8080/fhir -identifier_system=http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS \ No newline at end of file +store_path=http://localhost:8090/fhir +identifier_system=http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS +mainzelliste_url=http://localhost:8080 +mainzelliste_apikey=pleaseChangeMe \ No newline at end of file diff --git a/src/main/resources/toSinglePatients.xsl b/src/main/resources/toSinglePatients.xsl index 244239e..ecefc7a 100644 --- a/src/main/resources/toSinglePatients.xsl +++ b/src/main/resources/toSinglePatients.xsl @@ -10,10 +10,11 @@ + - + From 4d3d04905b6eff2ff5e95ea46985f9bc20b6cbcf Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 27 Apr 2022 10:17:23 +0200 Subject: [PATCH 04/12] Fix docker configuration for adt2fhir.properties --- docker-compose.yml | 6 ++++++ src/docker/adt2fhir.properties | 10 +++++----- src/docker/start.sh | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d917bac..80359ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,12 @@ services: image: docker.verbis.dkfz.de/ccp/adt2fhir:test #image: adt2fhir #build this image locally using the Dockerfile "docker build -t adt2fhir ." + environment: + FILE_PATH: "/adt2fhir/clinical_data" + STORE_PATH: "http://host.docker.internal:8090/fhir" + IDENTIFIER_SYSTEM: "http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS" + MAINZELLIST_URL: "http://host.docker.internal:8080" + MAINZELLIST_APIKEY: "pleaseChangeMe" volumes: - ./clinical_data:/adt2fhir/clinical_data extra_hosts: diff --git a/src/docker/adt2fhir.properties b/src/docker/adt2fhir.properties index a7e990b..05bbb42 100644 --- a/src/docker/adt2fhir.properties +++ b/src/docker/adt2fhir.properties @@ -1,5 +1,5 @@ -file_path=/adt2fhir/clinical_data -store_path=http://host.docker.internal:8090/fhir -identifier_system=http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS -mainzelliste_url=http://host.docker.internal:8080 -mainzelliste_apikey=pleaseChangeMe \ No newline at end of file +file_path={file_path} +store_path={store_path} +identifier_system={identifier_system} +mainzelliste_url={mainzelliste_url} +mainzelliste_apikey={mainzelliste_apikey} \ No newline at end of file diff --git a/src/docker/start.sh b/src/docker/start.sh index cbb85dc..86ff05e 100644 --- a/src/docker/start.sh +++ b/src/docker/start.sh @@ -1,5 +1,19 @@ #!/usr/bin/env bash + +export FILES_TO_PARSE="/etc/samply/adt2fhir.properties" + +set -e + +for file in $FILES_TO_PARSE +do +sed -i "s|{file_path}|${FILE_PATH:-/adt2fhir/clinical_data}|" $file +sed -i "s|{store_path}|${STORE_PATH:-http://host.docker.internal:8090/fhir}|" $file +sed -i "s|{identifier_system}|${IDENTIFIER_SYSTEM:-http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS}|" $file +sed -i "s|{mainzelliste_url}|${MAINZELLIST_URL:-http://host.docker.internal:8080}|" $file +sed -i "s|{mainzelliste_apikey}|${MAINZELLIST_APIKEY}|" $file +done + if [ -d "/clinical_data/InputADT" ]; then echo "Found import dir" else echo "Error - can not find dir: create dir /clinical_data/InputADT/" From 1fb9d3692a90c9e9c0c76b57ffc4dfb79b53153a Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 3 May 2022 15:33:55 +0200 Subject: [PATCH 05/12] Refactoring and performance measures - Adt2fhir.java Optimize patientlist API calls - PatientPseudonymizer.java --- .../java/de/samply/adt2fhir/Adt2fhir.java | 45 ++++++++++--------- .../java/de/samply/adt2fhir/ConfigReader.java | 2 - .../samply/adt2fhir/PatientPseudonymizer.java | 34 +++++++++----- .../de/samply/adt2fhir/UniqueIdGenerator.java | 2 +- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index 96402be..46f25a2 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -48,7 +48,7 @@ public static void main(String[] args) throws TransformerConfigurationException, final TransformerFactoryImpl factory = (TransformerFactoryImpl) TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl", null); net.sf.saxon.Configuration saxonConfig = factory.getConfiguration(); PatientPseudonymizer patientPseudonymizer = new PatientPseudonymizer(); - patientPseudonymizer.setConfigReader(configReader); + patientPseudonymizer.initialize(configReader); ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(patientPseudonymizer); ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(new UniqueIdGenerator()); @@ -60,18 +60,25 @@ public static void main(String[] args) throws TransformerConfigurationException, MDS2FHIRtransformer.setParameter("identifier_system", configReader.getIdentifier_system()); System.out.println(ANSI_GREEN+"...done"+ANSI_RESET); + long startTime = System.nanoTime(); System.out.println("Transforming to single Patients... \n"); processXmlFiles(INPUT_ADT, ADT2singleADTtransformer, configReader); - System.out.println(ANSI_GREEN+"...done"+ANSI_RESET); + long stopTime = System.nanoTime(); + System.out.println(ANSI_GREEN+"...done "+ANSI_RESET+(stopTime - startTime)/1000000000+ " seconds"); + + startTime = System.nanoTime(); System.out.println("Transforming to FHIR... \n"); processXmlFiles(ADT_PATIENTS, ADT2MDStransformer, configReader, MDS2FHIRtransformer, true); - System.out.println(ANSI_GREEN+"...done"+ANSI_RESET); + stopTime = System.nanoTime(); + System.out.println(ANSI_GREEN+"...done "+ANSI_RESET+(stopTime - startTime)/1000000000+ " seconds"); + startTime = System.nanoTime(); System.out.println("posting fhir resources to blaze store...\n"); processXmlFiles(FHIR_PATIENTS, null, configReader, httppost); - System.out.println(ANSI_GREEN+"...done"+ANSI_RESET); + stopTime = System.nanoTime(); + System.out.println(ANSI_GREEN+"...done "+ANSI_RESET+(stopTime - startTime)/1000000000+ " seconds"); } @@ -95,11 +102,12 @@ private static void processXmlFiles (String inputData, Transformer transformer, System.out.println("ABORTING: empty "+ fileDir +" dir"); } else { - int counter=1; + int counter=0; for (File inputFile : listOfFiles) { + //System.out.println(inputFile); if (inputFile.isFile() & inputFile.getName().toLowerCase().endsWith(".xml")) { - System.out.println("\u001B[AFile "+counter+" of "+(listOfFiles.length-1)); counter+=1; + System.out.println("\u001B[AFile "+counter+" of "+(listOfFiles.length-1)); if (transformer ==null){ try { postToFhirStore(inputFile, httppost); @@ -120,17 +128,17 @@ private static void processXmlFiles (String inputData, Transformer transformer, e.printStackTrace(); } try { - transformer.setParameter("customPrefix", inputFile.getName()); + transformer.setParameter("customPrefix", counter); String xmlResult = applyXslt(inputXml, transformer); if(transformWrittenResults){ transformer2.setParameter("customPrefix", inputFile.getName()); applyXslt(xmlResult, transformer2); inputFile.deleteOnExit(); } - } catch (TransformerException | UnsupportedEncodingException e) { + } catch (UnsupportedEncodingException | TransformerException | RuntimeException e) { counter-=1; System.out.print("ERROR - transformation: problem with file " + inputFile); - e.printStackTrace(); + //e.printStackTrace(); } } } @@ -173,19 +181,12 @@ private static void postToFhirStore(File inputFile, HttpPost httppost) throws IO } - private static String applyXslt(String xmlString, Transformer adtPrime) throws TransformerException, UnsupportedEncodingException { - Source xmlSource = new StreamSource(new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8.name()))); - Writer outputWriter = new StringWriter(); - StreamResult transformed = new StreamResult(outputWriter); - - /*ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream("file.zip")); - factory.setAttribute("http://saxon.sf.net/feature/outputURIResolver", new ZipOutputURIReslover(zipOut)) - MemoryOutputURIResolver mem=new MemoryOutputURIResolver();*/ - /*factory.setAttribute("http://saxon.sf.net/feature/outputURIResolver", mem); - factory.setAttribute("http://saxon.sf.net/feature/allow-external-functions", new Boolean(true));*/ - adtPrime.transform(xmlSource, transformed); - String output = outputWriter.toString(); - + private static String applyXslt(String xmlString, Transformer transformer) throws UnsupportedEncodingException, TransformerException { + Source xmlSource = new StreamSource(new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8.name()))); + Writer outputWriter = new StringWriter(); + StreamResult transformed = new StreamResult(outputWriter); + transformer.transform(xmlSource, transformed); + String output = outputWriter.toString(); return output; } diff --git a/src/main/java/de/samply/adt2fhir/ConfigReader.java b/src/main/java/de/samply/adt2fhir/ConfigReader.java index d62ce49..5a3c22a 100644 --- a/src/main/java/de/samply/adt2fhir/ConfigReader.java +++ b/src/main/java/de/samply/adt2fhir/ConfigReader.java @@ -15,8 +15,6 @@ public void init() throws IOException { try { Properties prop = new Properties(); InputStream inputStream =null; - //try loading docker configuration - //("adt2fhir.properties"); File file = new File("/etc/samply/"+PROPERTY_FILE); if (file.isFile()){ inputStream =new FileInputStream(file); diff --git a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java index bb0f92a..a053553 100644 --- a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java +++ b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java @@ -4,7 +4,7 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import de.pseudonymisierung.mainzelliste.client.*; -import java.io.*; +import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -31,7 +31,11 @@ public class PatientPseudonymizer extends ExtensionFunctionDefinition { private String mainzelliste_url; private String mainzelliste_apikey; - private boolean anonymize; + private boolean anonymize=false; + private MainzellisteConnection mainzellisteConnection; + private Session session; + private AddPatientToken token; + private CloseableHttpClient httpclient; @Override public net.sf.saxon.value.SequenceType[] getArgumentTypes() { @@ -66,7 +70,7 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { String birthname = args[3].iterate().next().getStringValue(); String brithdate = args[4].iterate().next().getStringValue(); - if (mainzelliste_url.isEmpty()) { + if (anonymize) { output = DigestUtils.sha256Hex(gender + prename + surname + birthname + brithdate).substring(32); } else { try { @@ -84,10 +88,6 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { private String pseudonymizationCall(String gender, String prename, String surname, String birthname, String brithdate) throws URISyntaxException, MainzellisteNetworkException, InvalidSessionException, IOException { String pseudonym=""; String[] brithdateParts = brithdate.split("[.]"); - MainzellisteConnection mainzellisteConnection = new MainzellisteConnection(mainzelliste_url, mainzelliste_apikey); - Session session = mainzellisteConnection.createSession(); - AddPatientToken token = new AddPatientToken(); - token.addIdType("pid"); String addPatientToken = session.getToken(token); HttpPost httppost = new HttpPost(mainzelliste_url+"/patients?tokenId="+addPatientToken); httppost.addHeader("content-type", "application/x-www-form-urlencoded"); @@ -104,13 +104,11 @@ private String pseudonymizationCall(String gender, String prename, String surnam //idat.add(new BasicNameValuePair("ort", "")); idat.add(new BasicNameValuePair("sureness", "true")); httppost.setEntity(new UrlEncodedFormEntity(idat, HTTP.UTF_8)); - CloseableHttpClient httpclient = HttpClients.createDefault(); HttpResponse response = httpclient.execute(httppost); if (response.getStatusLine().getStatusCode()!=201) { - System.out.println("Error - Pseudonymization response: " + response.getStatusLine().getStatusCode()); + System.out.println("ERROR - Pseudonymization response: " + response.getStatusLine().getStatusCode()); } String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); - httpclient.close(); JsonArray ids = new Gson ().fromJson(responseBody, JsonArray.class); if(!ids.isEmpty()){ @@ -120,8 +118,22 @@ private String pseudonymizationCall(String gender, String prename, String surnam return pseudonym; } - public void setConfigReader (ConfigReader configReader){ + public void initialize (ConfigReader configReader){ this.mainzelliste_url=configReader.getMainzelliste_url(); this.mainzelliste_apikey=configReader.getMainzelliste_apikey(); + if (mainzelliste_apikey.isEmpty()){ + this.anonymize=true; + }else { + this.anonymize=false; + try { + this.mainzellisteConnection = new MainzellisteConnection(mainzelliste_url, mainzelliste_apikey); + this.session = mainzellisteConnection.createSession(); + this.token = new AddPatientToken(); + this.token.addIdType("pid"); + this.httpclient = HttpClients.createDefault(); + } catch (URISyntaxException | MainzellisteNetworkException e) { + throw new RuntimeException(e); + } + } } } diff --git a/src/main/java/de/samply/adt2fhir/UniqueIdGenerator.java b/src/main/java/de/samply/adt2fhir/UniqueIdGenerator.java index 0d512d8..56bf7cd 100644 --- a/src/main/java/de/samply/adt2fhir/UniqueIdGenerator.java +++ b/src/main/java/de/samply/adt2fhir/UniqueIdGenerator.java @@ -41,7 +41,7 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { String patient = args[0].iterate().next().getStringValue(); String diagnosis = args[1].iterate().next().getStringValue(); String id = args[2].iterate().next().getStringValue(); - output = DigestUtils.sha256Hex(patient+diagnosis+id).substring(32); + output = DigestUtils.sha256Hex(patient+diagnosis+id).substring(48); return StringValue.makeStringValue(output); } From 531564bc1deb8700bbb4233acfc92a4238ba4d70 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Tue, 3 May 2022 15:35:51 +0200 Subject: [PATCH 06/12] Transformation fixes - MDS2FHIR.xsl and toSinglePatients.xsl --- src/main/resources/MDS2FHIR.xsl | 30 ++++++++++++------------- src/main/resources/toSinglePatients.xsl | 3 ++- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/main/resources/MDS2FHIR.xsl b/src/main/resources/MDS2FHIR.xsl index 0728112..e7642d5 100644 --- a/src/main/resources/MDS2FHIR.xsl +++ b/src/main/resources/MDS2FHIR.xsl @@ -18,14 +18,14 @@ - - + + @@ -38,7 +38,7 @@ - + @@ -263,8 +263,8 @@ - - + " + @@ -600,7 +600,7 @@ - + @@ -759,11 +759,11 @@ - - - - - + + + + + @@ -1239,7 +1239,7 @@ - + @@ -1285,7 +1285,7 @@ - + @@ -1308,8 +1308,8 @@ - - + + diff --git a/src/main/resources/toSinglePatients.xsl b/src/main/resources/toSinglePatients.xsl index ecefc7a..1adedd7 100644 --- a/src/main/resources/toSinglePatients.xsl +++ b/src/main/resources/toSinglePatients.xsl @@ -2,6 +2,7 @@ @@ -14,7 +15,7 @@ - + From 920d8553c8d74087802f3bbdd87cda7cb592ccca Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Fri, 6 May 2022 16:09:27 +0200 Subject: [PATCH 07/12] Fix transformation remove adverse events - ADT2MDS_FHIR.xsl and MDS2FHIR.xsl --- .../java/de/samply/adt2fhir/Adt2fhir.java | 12 ++-- src/main/resources/ADT2MDS_FHIR.xsl | 19 +++--- src/main/resources/MDS2FHIR.xsl | 62 +++++++++++-------- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index 46f25a2..bfc1e76 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -4,11 +4,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Base64; import javax.xml.transform.*; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.Processor; +import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPost; @@ -36,12 +38,6 @@ public static void main(String[] args) throws TransformerConfigurationException, System.out.println(" failed"); e.printStackTrace(); } - HttpPost httppost = new HttpPost(configReader.getStore_path()); - RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT) - //.setProxy(new HttpHost("XXX.XXX.XXX.XXX", 8080)) - .build(); - httppost.setConfig(requestConfig); - httppost.addHeader("content-type", "application/xml+fhir"); System.out.println(ANSI_GREEN+"...done"+ANSI_RESET); System.out.print("initialize transformers... "); @@ -73,6 +69,10 @@ public static void main(String[] args) throws TransformerConfigurationException, stopTime = System.nanoTime(); System.out.println(ANSI_GREEN+"...done "+ANSI_RESET+(stopTime - startTime)/1000000000+ " seconds"); + HttpPost httppost = new HttpPost(configReader.getStore_path()); + RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT).build(); + httppost.setConfig(requestConfig); + httppost.addHeader("content-type", "application/xml+fhir"); startTime = System.nanoTime(); System.out.println("posting fhir resources to blaze store...\n"); diff --git a/src/main/resources/ADT2MDS_FHIR.xsl b/src/main/resources/ADT2MDS_FHIR.xsl index 54b4e99..ff9657e 100644 --- a/src/main/resources/ADT2MDS_FHIR.xsl +++ b/src/main/resources/ADT2MDS_FHIR.xsl @@ -578,7 +578,6 @@ - @@ -586,6 +585,15 @@ + + + + + + + + @@ -681,17 +689,12 @@ - - - - - - + - + diff --git a/src/main/resources/MDS2FHIR.xsl b/src/main/resources/MDS2FHIR.xsl index e7642d5..739bd2a 100644 --- a/src/main/resources/MDS2FHIR.xsl +++ b/src/main/resources/MDS2FHIR.xsl @@ -417,7 +417,7 @@ - + @@ -585,11 +585,11 @@ - + @@ -760,10 +760,10 @@ - - - - + + + + @@ -802,26 +802,34 @@ + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + From 981953aeeea8645fd7a09e353f7cc7b45aaa401e Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Mon, 9 May 2022 11:40:59 +0200 Subject: [PATCH 08/12] Add Blaze error reporting - Adt2fhir.java Fix id entity prefixes - ADT2MDS_FHIR.xsl Fix diagnosis references as "focus" in Observation Resources - MDS2FHIR.xsl --- .../java/de/samply/adt2fhir/Adt2fhir.java | 2 + src/main/resources/ADT2MDS_FHIR.xsl | 30 +-- src/main/resources/MDS2FHIR.xsl | 217 +++++++++--------- 3 files changed, 120 insertions(+), 129 deletions(-) diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index bfc1e76..a974dec 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -17,6 +17,7 @@ import org.apache.http.entity.FileEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; public class Adt2fhir { @@ -165,6 +166,7 @@ private static void postToFhirStore(File inputFile, HttpPost httppost) throws IO //System.out.println(response.getStatusLine()); if (!response.getStatusLine().getReasonPhrase().equals("OK")) {; System.out.println("Error - FHIR import: could not import file"+ inputFile.getName()); + System.out.println(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); } else { inputFile.deleteOnExit(); diff --git a/src/main/resources/ADT2MDS_FHIR.xsl b/src/main/resources/ADT2MDS_FHIR.xsl index ff9657e..bf37b85 100644 --- a/src/main/resources/ADT2MDS_FHIR.xsl +++ b/src/main/resources/ADT2MDS_FHIR.xsl @@ -118,7 +118,7 @@ gen:missing-ID-and-Date - + true @@ -145,7 +145,7 @@ - + @@ -161,7 +161,7 @@ - + @@ -300,7 +300,7 @@ - + @@ -322,7 +322,7 @@ - + @@ -344,7 +344,7 @@ - + p @@ -376,7 +376,7 @@ - + c @@ -408,7 +408,7 @@ - + @@ -442,7 +442,7 @@ - + @@ -519,7 +519,7 @@ - + @@ -574,7 +574,7 @@ gen:missing_ID_and_Date - + @@ -608,7 +608,7 @@ gen:missing_ID_and_Date - + @@ -649,7 +649,7 @@ - + @@ -661,7 +661,7 @@ - + @@ -694,7 +694,7 @@ - + diff --git a/src/main/resources/MDS2FHIR.xsl b/src/main/resources/MDS2FHIR.xsl index 739bd2a..09f7d9b 100644 --- a/src/main/resources/MDS2FHIR.xsl +++ b/src/main/resources/MDS2FHIR.xsl @@ -235,74 +235,74 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -464,6 +464,9 @@ + + + @@ -497,6 +500,9 @@ + + + @@ -856,6 +862,9 @@ + + + @@ -958,6 +967,9 @@ + + + @@ -983,6 +995,7 @@ + @@ -991,6 +1004,7 @@ + @@ -1035,6 +1049,9 @@ + + + @@ -1190,6 +1207,7 @@ + @@ -1211,7 +1229,13 @@ - + + + + + + + @@ -1244,53 +1268,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1313,6 +1294,9 @@ + + + @@ -1348,6 +1332,9 @@ + + + @@ -1371,16 +1358,18 @@ + - + + From dd206d0c57cdaf358b21d0d1818759faf6c53468 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 11 May 2022 08:36:37 +0200 Subject: [PATCH 09/12] Fix extension "valueReference" - MDS2FHIR.xsl --- src/main/java/de/samply/adt2fhir/Adt2fhir.java | 2 -- src/main/java/de/samply/adt2fhir/ConfigReader.java | 7 ++++--- src/main/resources/MDS2FHIR.xsl | 8 ++++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index a974dec..fb1a7a1 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -4,13 +4,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Base64; import javax.xml.transform.*; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.Processor; -import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPost; diff --git a/src/main/java/de/samply/adt2fhir/ConfigReader.java b/src/main/java/de/samply/adt2fhir/ConfigReader.java index 5a3c22a..43d7940 100644 --- a/src/main/java/de/samply/adt2fhir/ConfigReader.java +++ b/src/main/java/de/samply/adt2fhir/ConfigReader.java @@ -1,6 +1,9 @@ package de.samply.adt2fhir; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Properties; public class ConfigReader { @@ -25,8 +28,6 @@ public void init() throws IOException { } if (inputStream != null) { prop.load(inputStream); - } else { - throw new FileNotFoundException("property file '" + PROPERTY_FILE + "' not found in resouces"); } this.file_path = prop.getProperty("file_path"); this.store_path = prop.getProperty("store_path"); diff --git a/src/main/resources/MDS2FHIR.xsl b/src/main/resources/MDS2FHIR.xsl index 09f7d9b..c69fdad 100644 --- a/src/main/resources/MDS2FHIR.xsl +++ b/src/main/resources/MDS2FHIR.xsl @@ -345,12 +345,16 @@ - + + + - + + + From 4812d794a427778634cd1c69679d2dfbe6073247 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 11 May 2022 08:42:53 +0200 Subject: [PATCH 10/12] Update OpenJDK 8u282-slim to 8u332-slim - Dockerfile Update Saxon-HE to 11.3 - pom.xml --- Dockerfile | 2 +- docker-compose.yml | 1 + pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ad673b5..25f406f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG OPENJDK_VERSION=8u282-slim +ARG OPENJDK_VERSION=8u332-slim FROM openjdk:${OPENJDK_VERSION} ADD target/adt2fhir*jar-with-dependencies.jar /usr/local/bin/adt2fhir.jar ADD src/docker/adt2fhir.properties /etc/samply/ diff --git a/docker-compose.yml b/docker-compose.yml index 80359ee..dc8842b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,5 +15,6 @@ services: MAINZELLIST_APIKEY: "pleaseChangeMe" volumes: - ./clinical_data:/adt2fhir/clinical_data + #in case you run a separate docker-compose.yml extra_hosts: - "host.docker.internal:host-gateway" \ No newline at end of file diff --git a/pom.xml b/pom.xml index a9e9fd2..e8213cd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ de.samply.common adt2fhir - 1.0.1-SNAPSHOT + 1.0.1 jar @@ -55,7 +55,7 @@ net.sf.saxon Saxon-HE - 9.8.0-15 + 11.3 org.apache.httpcomponents From 78e7a4993f08213fbb7665fe9babff2dba5ac981 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 11 May 2022 10:26:43 +0200 Subject: [PATCH 11/12] Transfer processed files to "Processed" directory - Adt2fhir.java --- clinical_data/Processed/.gitignore | 2 ++ src/main/java/de/samply/adt2fhir/Adt2fhir.java | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 clinical_data/Processed/.gitignore diff --git a/clinical_data/Processed/.gitignore b/clinical_data/Processed/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/clinical_data/Processed/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index fb1a7a1..0a1b0a1 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -134,6 +134,9 @@ private static void processXmlFiles (String inputData, Transformer transformer, applyXslt(xmlResult, transformer2); inputFile.deleteOnExit(); } + else { + inputFile.renameTo(new File(configReader.getFile_path()+"/Processed/"+inputFile.getName())); + } } catch (UnsupportedEncodingException | TransformerException | RuntimeException e) { counter-=1; System.out.print("ERROR - transformation: problem with file " + inputFile); From 1e9e1060c76fbfeb04c3d8c6f7d263b775feb6f3 Mon Sep 17 00:00:00 2001 From: "p.delpy@dkfz-heidelberg.de" Date: Wed, 25 May 2022 08:39:32 +0200 Subject: [PATCH 12/12] Fix Error handling - Adt2fhir.java Add PatientIdentifier to pseudonymization - PatientPseudonymizer.java and ADT2MDS_FHIR.xsl Replace 8u332-slim with 8u332-jre-slim - Dockerfile --- .gitignore | 5 +- Dockerfile | 2 +- src/docker/start.sh | 4 +- .../java/de/samply/adt2fhir/Adt2fhir.java | 29 +++++--- .../samply/adt2fhir/PatientPseudonymizer.java | 71 +++++++++++++------ src/main/resources/ADT2MDS_FHIR.xsl | 5 +- 6 files changed, 76 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 2e19b14..771bd8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -./.idea/* -./clinical_data/* -./target +.idea/* +target *.properties \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 25f406f..f418a2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG OPENJDK_VERSION=8u332-slim +ARG OPENJDK_VERSION=8u332-jre-slim FROM openjdk:${OPENJDK_VERSION} ADD target/adt2fhir*jar-with-dependencies.jar /usr/local/bin/adt2fhir.jar ADD src/docker/adt2fhir.properties /etc/samply/ diff --git a/src/docker/start.sh b/src/docker/start.sh index 86ff05e..58c49b9 100644 --- a/src/docker/start.sh +++ b/src/docker/start.sh @@ -16,13 +16,13 @@ done if [ -d "/clinical_data/InputADT" ]; then echo "Found import dir" - else echo "Error - can not find dir: create dir /clinical_data/InputADT/" + #else echo "Error - can not find dir: create dir /clinical_data/InputADT/" #exit fi if [ -d "/clinical_data/FHIR_Patients" ]; then echo "Found output dir" - else echo "Error - can not find dir: create dir /clinical_data/FHIR_Patients/" + #else echo "Error - can not find dir: create dir /clinical_data/FHIR_Patients/" #exit fi diff --git a/src/main/java/de/samply/adt2fhir/Adt2fhir.java b/src/main/java/de/samply/adt2fhir/Adt2fhir.java index 0a1b0a1..3c10b15 100644 --- a/src/main/java/de/samply/adt2fhir/Adt2fhir.java +++ b/src/main/java/de/samply/adt2fhir/Adt2fhir.java @@ -28,7 +28,7 @@ public class Adt2fhir { public static final String ANSI_RED = "\u001B[31m"; public static final String ANSI_GREEN = "\u001B[32m"; - public static void main(String[] args) throws TransformerConfigurationException, IOException { + public static void main(String[] args) { System.out.print("load configuration... "); ConfigReader configReader = new ConfigReader(); try { @@ -47,12 +47,19 @@ public static void main(String[] args) throws TransformerConfigurationException, ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(patientPseudonymizer); ((Processor) saxonConfig.getProcessor()).registerExtensionFunction(new UniqueIdGenerator()); - final Transformer ADT2singleADTtransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("toSinglePatients.xsl"))); - ADT2singleADTtransformer.setParameter("filepath", configReader.getFile_path()); - final Transformer ADT2MDStransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("ADT2MDS_FHIR.xsl"))); - final Transformer MDS2FHIRtransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("MDS2FHIR.xsl"))); - MDS2FHIRtransformer.setParameter("filepath", configReader.getFile_path()); - MDS2FHIRtransformer.setParameter("identifier_system", configReader.getIdentifier_system()); + Transformer ADT2singleADTtransformer = null; + Transformer ADT2MDStransformer = null; + Transformer MDS2FHIRtransformer = null; + try { + ADT2singleADTtransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("toSinglePatients.xsl"))); + ADT2singleADTtransformer.setParameter("filepath", configReader.getFile_path()); + ADT2MDStransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("ADT2MDS_FHIR.xsl"))); + MDS2FHIRtransformer = factory.newTransformer(new StreamSource(Adt2fhir.class.getClassLoader().getResourceAsStream("MDS2FHIR.xsl"))); + MDS2FHIRtransformer.setParameter("filepath", configReader.getFile_path()); + MDS2FHIRtransformer.setParameter("identifier_system", configReader.getIdentifier_system()); + } catch (TransformerConfigurationException e) { + System.out.print("Transformer configurtaion error"); + } System.out.println(ANSI_GREEN+"...done"+ANSI_RESET); long startTime = System.nanoTime(); @@ -106,7 +113,7 @@ private static void processXmlFiles (String inputData, Transformer transformer, //System.out.println(inputFile); if (inputFile.isFile() & inputFile.getName().toLowerCase().endsWith(".xml")) { counter+=1; - System.out.println("\u001B[AFile "+counter+" of "+(listOfFiles.length-1)); + System.out.println("\u001B[AFile " + counter + " of " + (listOfFiles.length-1) + " / Filename: " + inputFile); if (transformer ==null){ try { postToFhirStore(inputFile, httppost); @@ -118,9 +125,9 @@ private static void processXmlFiles (String inputData, Transformer transformer, } else { //System.out.print("processing file " + inputFile.getName() + "..."); - String inputXml = null; + String inputFileString = null; try { - inputXml = new String(Files.readAllBytes(Paths.get(String.valueOf(inputFile))), StandardCharsets.UTF_8); + inputFileString = new String(Files.readAllBytes(Paths.get(String.valueOf(inputFile))), StandardCharsets.UTF_8); } catch (IOException e) { counter-=1; System.out.print("ERROR - reading: problem with file " + inputFile); @@ -128,7 +135,7 @@ private static void processXmlFiles (String inputData, Transformer transformer, } try { transformer.setParameter("customPrefix", counter); - String xmlResult = applyXslt(inputXml, transformer); + String xmlResult = applyXslt(inputFileString, transformer); if(transformWrittenResults){ transformer2.setParameter("customPrefix", inputFile.getName()); applyXslt(xmlResult, transformer2); diff --git a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java index a053553..b507b00 100644 --- a/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java +++ b/src/main/java/de/samply/adt2fhir/PatientPseudonymizer.java @@ -5,6 +5,7 @@ import com.google.gson.JsonObject; import de.pseudonymisierung.mainzelliste.client.*; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -35,6 +36,7 @@ public class PatientPseudonymizer extends ExtensionFunctionDefinition { private MainzellisteConnection mainzellisteConnection; private Session session; private AddPatientToken token; + private String addPatientToken; private CloseableHttpClient httpclient; @Override @@ -44,6 +46,7 @@ public net.sf.saxon.value.SequenceType[] getArgumentTypes() { net.sf.saxon.value.SequenceType.SINGLE_STRING, net.sf.saxon.value.SequenceType.SINGLE_STRING, net.sf.saxon.value.SequenceType.SINGLE_STRING, + net.sf.saxon.value.SequenceType.SINGLE_STRING, net.sf.saxon.value.SequenceType.SINGLE_STRING }; } @@ -69,12 +72,13 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { String surname = args[2].iterate().next().getStringValue(); String birthname = args[3].iterate().next().getStringValue(); String brithdate = args[4].iterate().next().getStringValue(); + String identifier = args[5].iterate().next().getStringValue(); if (anonymize) { - output = DigestUtils.sha256Hex(gender + prename + surname + birthname + brithdate).substring(32); + output = DigestUtils.sha256Hex(gender + prename + surname + birthname + brithdate + identifier).substring(32); } else { try { - output=pseudonymizationCall(gender, prename, surname, birthname, brithdate); + output=pseudonymizationCall(gender, prename, surname, birthname, brithdate, identifier); } catch (URISyntaxException | MainzellisteNetworkException | InvalidSessionException | IOException e) { e.printStackTrace(); } @@ -85,26 +89,24 @@ public Sequence call(XPathContext ctx, Sequence[] args) throws XPathException { }; } - private String pseudonymizationCall(String gender, String prename, String surname, String birthname, String brithdate) throws URISyntaxException, MainzellisteNetworkException, InvalidSessionException, IOException { + private String pseudonymizationCall(String gender, String prename, String surname, String birthname, String brithdate, String identifier) throws URISyntaxException, MainzellisteNetworkException, InvalidSessionException, IOException { String pseudonym=""; String[] brithdateParts = brithdate.split("[.]"); - String addPatientToken = session.getToken(token); - HttpPost httppost = new HttpPost(mainzelliste_url+"/patients?tokenId="+addPatientToken); - httppost.addHeader("content-type", "application/x-www-form-urlencoded"); - httppost.addHeader("mainzellisteApiVersion", "3.2"); - List idat = new ArrayList(); - //idat.add(new BasicNameValuePair("gender", gender)); - idat.add(new BasicNameValuePair("vorname", prename)); - idat.add(new BasicNameValuePair("nachname", surname)); - idat.add(new BasicNameValuePair("geburtsname", birthname)); - idat.add(new BasicNameValuePair("geburtstag", brithdateParts[0])); - idat.add(new BasicNameValuePair("geburtsmonat", brithdateParts[1])); - idat.add(new BasicNameValuePair("geburtsjahr", brithdateParts[2])); - //idat.add(new BasicNameValuePair("plz", "")); - //idat.add(new BasicNameValuePair("ort", "")); - idat.add(new BasicNameValuePair("sureness", "true")); - httppost.setEntity(new UrlEncodedFormEntity(idat, HTTP.UTF_8)); + this.addPatientToken = session.getToken(token); + HttpPost httppost = createHttpPost(prename, surname, birthname, brithdateParts[0], brithdateParts[1], brithdateParts[2]); HttpResponse response = httpclient.execute(httppost); + if (response.getStatusLine().getStatusCode()==401) { + createMainzellisteConnection(); + this.addPatientToken = session.getToken(token); + response = httpclient.execute(new HttpPost(mainzelliste_url+"/patients?tokenId="+addPatientToken)); + System.out.println("Creating new session"); + } + if (response.getStatusLine().getStatusCode()==400){ + this.httpclient = HttpClients.createDefault(); + httppost = createHttpPost(preprocessIDAT(prename), preprocessIDAT(surname), preprocessIDAT(birthname), brithdateParts[0], brithdateParts[1], brithdateParts[2]); + response = httpclient.execute(httppost); + System.out.println("\u001B[A"+"\u001B[100C" + "Unallowed character in patient "+ identifier + " ... autocorrected"); + } if (response.getStatusLine().getStatusCode()!=201) { System.out.println("ERROR - Pseudonymization response: " + response.getStatusLine().getStatusCode()); } @@ -127,13 +129,40 @@ public void initialize (ConfigReader configReader){ this.anonymize=false; try { this.mainzellisteConnection = new MainzellisteConnection(mainzelliste_url, mainzelliste_apikey); - this.session = mainzellisteConnection.createSession(); this.token = new AddPatientToken(); this.token.addIdType("pid"); this.httpclient = HttpClients.createDefault(); - } catch (URISyntaxException | MainzellisteNetworkException e) { + createMainzellisteConnection(); + } catch (URISyntaxException | MainzellisteNetworkException | InvalidSessionException e) { throw new RuntimeException(e); } } } + + private void createMainzellisteConnection() throws InvalidSessionException, MainzellisteNetworkException { + this.session = mainzellisteConnection.createSession(); + } + + private HttpPost createHttpPost (String prename, String surname, String birthname, String brithday, String brithmonth, String brithyear) throws UnsupportedEncodingException { + HttpPost httppost= new HttpPost(mainzelliste_url+"/patients?tokenId="+addPatientToken); + httppost.addHeader("content-type", "application/x-www-form-urlencoded"); + httppost.addHeader("mainzellisteApiVersion", "3.2"); + List idat = new ArrayList(); + //idat.add(new BasicNameValuePair("gender", gender)); + idat.add(new BasicNameValuePair("vorname", prename)); + idat.add(new BasicNameValuePair("nachname", surname)); + idat.add(new BasicNameValuePair("geburtsname", birthname)); + idat.add(new BasicNameValuePair("geburtstag", brithday)); + idat.add(new BasicNameValuePair("geburtsmonat", brithmonth)); + idat.add(new BasicNameValuePair("geburtsjahr", brithyear)); + //idat.add(new BasicNameValuePair("plz", "")); + //idat.add(new BasicNameValuePair("ort", "")); + idat.add(new BasicNameValuePair("sureness", "true")); + httppost.setEntity(new UrlEncodedFormEntity(idat, HTTP.UTF_8)); + return httppost; + } + + private String preprocessIDAT (String name){ + return name.replaceAll("[^a-zA-ZäÄöÖüÜßéÉ]", ""); + } } diff --git a/src/main/resources/ADT2MDS_FHIR.xsl b/src/main/resources/ADT2MDS_FHIR.xsl index bf37b85..5f47abe 100644 --- a/src/main/resources/ADT2MDS_FHIR.xsl +++ b/src/main/resources/ADT2MDS_FHIR.xsl @@ -41,7 +41,7 @@ - + @@ -994,7 +994,8 @@ - + +