From 26efb9e869e3288a5a27afdb080bc349dffc36b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne=20Martin?= Date: Thu, 22 Oct 2020 10:16:27 -0700 Subject: [PATCH 1/2] Use instanceID in bad submission filename for traceability It was previously impossible to track down empty submission files. --- src/org/opendatakit/briefcase/export/ExportToCsv.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/org/opendatakit/briefcase/export/ExportToCsv.java b/src/org/opendatakit/briefcase/export/ExportToCsv.java index b494aa1ea..bd352a2ef 100644 --- a/src/org/opendatakit/briefcase/export/ExportToCsv.java +++ b/src/org/opendatakit/briefcase/export/ExportToCsv.java @@ -37,7 +37,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; import org.bushe.swing.event.EventBus; import org.opendatakit.briefcase.model.FormStatus; import org.opendatakit.briefcase.model.form.FormMetadata; @@ -143,14 +142,13 @@ private static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetad } private static SubmissionExportErrorCallback buildParsingErrorCallback(Path errorsDir) { - AtomicInteger errorSeq = new AtomicInteger(1); // Remove errors from a previous export attempt if (exists(errorsDir)) deleteRecursive(errorsDir); return (path, message) -> { if (!exists(errorsDir)) createDirectories(errorsDir); - copy(path, errorsDir.resolve("failed_submission_" + errorSeq.getAndIncrement() + ".xml")); + copy(path, errorsDir.resolve("failed_submission_" + path.getParent().getFileName() + ".xml")); log.warn("A submission has been excluded from the export output due to some problem ({}). If you didn't expect this, please ask for support at https://forum.getodk.org/c/support", message); }; } From 04216f8d83ca5bb7bf2acefcb4251042998b1336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne=20Martin?= Date: Thu, 22 Oct 2020 10:16:57 -0700 Subject: [PATCH 2/2] Don't export submissions that can't be parsed --- .../briefcase/export/SubmissionParser.java | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/org/opendatakit/briefcase/export/SubmissionParser.java b/src/org/opendatakit/briefcase/export/SubmissionParser.java index 0ef0b2b0d..fc9d5af44 100644 --- a/src/org/opendatakit/briefcase/export/SubmissionParser.java +++ b/src/org/opendatakit/briefcase/export/SubmissionParser.java @@ -86,11 +86,12 @@ public static List getListOfSubmissionFiles(FormMetadata formMetadata, For .forEach(instanceDir -> { Path submissionFile = instanceDir.resolve("submission.xml"); try { - Optional submissionDate = readSubmissionDate(submissionFile, onParsingError); + Optional submissionDate = readSubmissionDate(submissionFile); paths.add(Pair.of(submissionFile, submissionDate.orElse(OffsetDateTime.of(1970, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)))); } catch (Throwable t) { - log.error("Can't read submission date", t); + log.error("Parse error attempting to read instance date", t); EventBus.publish(ExportEvent.failureSubmission(formDef, instanceDir.getFileName().toString(), t)); + onParsingError.accept(submissionFile, "Parse error attempting to read instance date"); } }); return paths.parallelStream() @@ -152,32 +153,26 @@ public static Optional parseSubmission(Path path, boolean isEncrypte }); } - private static Optional readSubmissionDate(Path path, SubmissionExportErrorCallback onParsingError) { + private static Optional readSubmissionDate(Path path) throws XMLStreamException { try (InputStream is = Files.newInputStream(path); InputStreamReader isr = new InputStreamReader(is, UTF_8)) { - return parseAttribute(path, isr, "submissionDate", onParsingError) + return parseAttribute(isr, "submissionDate") .map(Iso8601Helpers::parseDateTime); } catch (IOException e) { throw new CryptoException("Can't decrypt file", e); } } - private static Optional parseAttribute(Path submission, Reader ioReader, String attributeName, SubmissionExportErrorCallback onParsingError) { - try { - XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(ioReader); - while (reader.hasNext()) - if (reader.next() == START_ELEMENT) - for (int i = 0, c = reader.getAttributeCount(); i < c; ++i) - if (reader.getAttributeLocalName(i).equals(attributeName)) - return Optional.of(reader.getAttributeValue(i)); - } catch (XMLStreamException e) { - log.error("Can't parse submission", e); - onParsingError.accept(submission, "parsing error"); - } + private static Optional parseAttribute(Reader ioReader, String attributeName) throws XMLStreamException { + XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(ioReader); + while (reader.hasNext()) + if (reader.next() == START_ELEMENT) + for (int i = 0, c = reader.getAttributeCount(); i < c; ++i) + if (reader.getAttributeLocalName(i).equals(attributeName)) + return Optional.of(reader.getAttributeValue(i)); return Optional.empty(); } - private static Optional decrypt(Submission submission, SubmissionExportErrorCallback onError) { List mediaPaths = submission.getMediaPaths();