Skip to content
This repository has been archived by the owner on Apr 16, 2022. It is now read-only.

Commit

Permalink
Merge pull request #887 from lognaturel/issue-886
Browse files Browse the repository at this point in the history
  • Loading branch information
@lognaturel
lognaturel authored Oct 30, 2020
2 parents fbd9fd3 + 8a010b1 commit 97b57f4
Show file tree
Hide file tree
Showing 6 changed files with 242 additions and 71 deletions.
4 changes: 3 additions & 1 deletion src/org/opendatakit/briefcase/pull/aggregate/PullFromAggregate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,9 @@ public Job<Void> pull(FormStatus form, Optional<Cursor> lastCursor) {
submissionAttachments.parallelStream().forEach(attachment ->
downloadSubmissionAttachment(form, submission, attachment, rs, tracker, currentSubmissionNumber, totalSubmissions, submissionAttachmentNumber.getAndIncrement(), totalSubmissionAttachments)
);
db.putRecordedInstanceDirectory(submission.getInstanceId(), form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).toFile());
if (!rs.isCancelled()) {
db.putRecordedInstanceDirectory(submission.getInstanceId(), form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).toFile());
}
});
});

Expand Down
55 changes: 30 additions & 25 deletions src/org/opendatakit/briefcase/pull/central/PullFromCentral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
import org.opendatakit.briefcase.model.form.FormKey;
import org.opendatakit.briefcase.model.form.FormMetadataPort;
import org.opendatakit.briefcase.pull.PullEvent;
import org.opendatakit.briefcase.reused.Triple;
import org.opendatakit.briefcase.reused.Pair;
import org.opendatakit.briefcase.reused.http.Http;
import org.opendatakit.briefcase.reused.http.response.Response;
import org.opendatakit.briefcase.reused.job.Job;
Expand Down Expand Up @@ -98,30 +98,35 @@ public Job<Void> pull(FormStatus form) {
tracker.trackNoSubmissions();

submissions.stream()
.map(instanceId -> Triple.of(submissionNumber.getAndIncrement(), instanceId, db.hasRecordedInstance(instanceId) == null))
.peek(triple -> {
if (!triple.get3())
tracker.trackSubmissionAlreadyDownloaded(triple.get1(), totalSubmissions);
})
.filter(Triple::get3)
.forEach(triple -> {
int currentSubmissionNumber = triple.get1();
String instanceId = triple.get2();
downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
.map(instanceId -> Pair.of(submissionNumber.getAndIncrement(), instanceId))
.forEach(submissionNumberId -> {
int currentSubmissionNumber = submissionNumberId.getLeft();
String instanceId = submissionNumberId.getRight();

boolean inDb = db.hasRecordedInstance(instanceId) != null;
Path downloadedSubmissionPath = form.getSubmissionFile(briefcaseDir, instanceId);
if (downloadedSubmissionPath.toFile().exists()) {
XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
SubmissionMetaData metaData = new SubmissionMetaData(root);
metaData.getVersion().ifPresent(submissionVersions::add);
if (!inDb || !downloadedSubmissionPath.toFile().exists()) {
downloadSubmission(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
if (downloadedSubmissionPath.toFile().exists()) {
XmlElement root = XmlElement.from(new String(readAllBytes(downloadedSubmissionPath)));
SubmissionMetaData metaData = new SubmissionMetaData(root);
metaData.getVersion().ifPresent(submissionVersions::add);
}
} else {
tracker.trackSubmissionAlreadyDownloaded(currentSubmissionNumber, totalSubmissions);
}

List<CentralAttachment> attachments = getSubmissionAttachments(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
List<CentralAttachment> attachments = getSubmissionAttachmentList(form, instanceId, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions);
int totalAttachments = attachments.size();
AtomicInteger attachmentNumber = new AtomicInteger(1);
attachments.forEach(attachment ->
downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments)
);
db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
attachments.stream()
.filter(attachment -> !inDb || !form.getSubmissionMediaFile(briefcaseDir, instanceId, attachment.getName()).toFile().exists())
.forEach(attachment ->
downloadSubmissionAttachment(form, instanceId, attachment, token, runnerStatus, tracker, currentSubmissionNumber, totalSubmissions, attachmentNumber.getAndIncrement(), totalAttachments)
);
if (!runnerStatus.isCancelled() && !inDb) {
db.putRecordedInstanceDirectory(instanceId, form.getSubmissionDir(briefcaseDir, instanceId).toFile());
}
});
tracker.trackEnd();

Expand Down Expand Up @@ -218,21 +223,21 @@ void downloadSubmission(FormStatus form, String instanceId, String token, Runner
tracker.trackErrorDownloadingSubmission(submissionNumber, totalSubmissions, response);
}

List<CentralAttachment> getSubmissionAttachments(FormStatus form, String instanceId, String token, RunnerStatus runnerStatus, PullFromCentralTracker tracker, int submissionNumber, int totalSubmissions) {
List<CentralAttachment> getSubmissionAttachmentList(FormStatus form, String instanceId, String token, RunnerStatus runnerStatus, PullFromCentralTracker tracker, int submissionNumber, int totalSubmissions) {
if (runnerStatus.isCancelled()) {
tracker.trackCancellation("Get submission attachments of " + instanceId);
return emptyList();
}

tracker.trackStartGettingSubmissionAttachments(submissionNumber, totalSubmissions);
tracker.trackStartGettingSubmissionAttachmentList(submissionNumber, totalSubmissions);
Response<List<CentralAttachment>> response = http.execute(server.getSubmissionAttachmentListRequest(form.getFormId(), instanceId, token));
if (!response.isSuccess()) {
tracker.trackErrorGettingSubmissionAttachments(submissionNumber, totalSubmissions, response);
tracker.trackErrorGettingSubmissionAttachmentList(instanceId, response);
return emptyList();
}

List<CentralAttachment> attachments = response.get();
tracker.trackEndGettingSubmissionAttachments(submissionNumber, totalSubmissions);
tracker.trackEndGettingSubmissionAttachmentList(submissionNumber, totalSubmissions);
return attachments;
}

Expand All @@ -250,6 +255,6 @@ void downloadSubmissionAttachment(FormStatus form, String instanceId, CentralAtt
if (response.isSuccess())
tracker.trackEndDownloadingSubmissionAttachment(submissionNumber, totalSubmissions, attachmentNumber, totalAttachments);
else
tracker.trackErrorDownloadingSubmissionAttachment(submissionNumber, totalSubmissions, attachmentNumber, totalAttachments, response);
tracker.trackErrorDownloadingSubmissionAttachment(instanceId, attachment.getName(), response);
}
}
20 changes: 10 additions & 10 deletions src/org/opendatakit/briefcase/pull/central/PullFromCentralTracker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,46 +186,46 @@ void trackEndDownloadingSubmission(int submissionNumber, int totalSubmissions) {
notifyTrackingEvent();
}

void trackStartGettingSubmissionAttachments(int submissionNumber, int totalSubmissions) {
String message = "Start getting attachments of submission " + submissionNumber + " of " + totalSubmissions;
void trackStartGettingSubmissionAttachmentList(int submissionNumber, int totalSubmissions) {
String message = " Start getting attachment list of submission " + submissionNumber + " of " + totalSubmissions;
form.setStatusString(message);
log.info("Pull {} - {}", form.getFormName(), message);
notifyTrackingEvent();
}

void trackErrorGettingSubmissionAttachments(int submissionNumber, int totalSubmissions, Response response) {
void trackErrorGettingSubmissionAttachmentList(String instanceId, Response response) {
errored = true;
String message = "Error getting attachments of submission " + submissionNumber + " of " + totalSubmissions;
String message = " Error getting attachment list of submission " + instanceId;
form.setStatusString(message + ": " + response.getStatusPhrase());
log.error("Pull {} - {}: HTTP {} {}", form.getFormName(), message, response.getStatusCode(), response.getStatusPhrase());
notifyTrackingEvent();
}


void trackEndGettingSubmissionAttachments(int submissionNumber, int totalSubmissions) {
String message = "Got all the attachments of submission " + submissionNumber + " of " + totalSubmissions;
void trackEndGettingSubmissionAttachmentList(int submissionNumber, int totalSubmissions) {
String message = " Got attachment list of submission " + submissionNumber + " of " + totalSubmissions;
form.setStatusString(message);
log.info("Pull {} - {}", form.getFormName(), message);
notifyTrackingEvent();
}

void trackStartDownloadingSubmissionAttachment(int submissionNumber, int totalSubmissions, int attachmentNumber, int totalAttachments) {
String message = "Start downloading attachment " + attachmentNumber + " of " + totalAttachments + " of submission " + submissionNumber + " of " + totalSubmissions;
String message = " Start downloading attachment " + attachmentNumber + " of " + totalAttachments + " of submission " + submissionNumber + " of " + totalSubmissions;
form.setStatusString(message);
log.info("Pull {} - {}", form.getFormName(), message);
notifyTrackingEvent();
}

void trackEndDownloadingSubmissionAttachment(int submissionNumber, int totalSubmissions, int attachmentNumber, int totalAttachments) {
String message = "Attachment " + attachmentNumber + " of " + totalAttachments + " of submission " + submissionNumber + " of " + totalSubmissions + " downloaded";
String message = " Attachment " + attachmentNumber + " of " + totalAttachments + " of submission " + submissionNumber + " of " + totalSubmissions + " downloaded";
form.setStatusString(message);
log.info("Pull {} - {}", form.getFormName(), message);
notifyTrackingEvent();
}

void trackErrorDownloadingSubmissionAttachment(int submissionNumber, int totalSubmissions, int attachmentNumber, int totalAttachments, Response response) {
void trackErrorDownloadingSubmissionAttachment(String instanceId, String attachmentName, Response response) {
errored = true;
String message = "Error downloading attachment " + attachmentNumber + " of " + totalAttachments + " of submission " + submissionNumber + " of " + totalSubmissions;
String message = " Error downloading attachment " + attachmentName + " of submission " + instanceId;
form.setStatusString(message + ": " + response.getStatusPhrase());
log.error("Pull {} - {}: HTTP {} {}", form.getFormName(), message, response.getStatusCode(), response.getStatusPhrase());
notifyTrackingEvent();
Expand Down
Loading

0 comments on commit 97b57f4

Please sign in to comment.