Skip to content

Commit

Permalink
introduced a new processed tag for gradient scoring and set it when g…
Browse files Browse the repository at this point in the history 
…rad scoring is performed
  • Loading branch information
Cristian Goina committed Feb 28, 2024
1 parent a6bd4e6 commit f2fda03
Show file tree
Hide file tree
Showing 7 changed files with 60 additions and 12 deletions.
3 changes: 1 addition & 2 deletions colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/ProcessingType.java
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package org.janelia.colormipsearch.model;

import org.apache.commons.lang3.StringUtils;

public enum ProcessingType {
ColorDepthSearch,
GradientScore,
PPPMatch;
}
3 changes: 2 additions & 1 deletion colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/CDMIPsWriter.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.janelia.colormipsearch.dataio;

import java.util.Collection;
import java.util.List;
import java.util.Set;

Expand Down Expand Up @@ -36,7 +37,7 @@ public interface CDMIPsWriter {
* @param processingType
* @param tags
*/
void addProcessingTags(List<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags);
void addProcessingTags(Collection<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags);

/**
* Finish all writes and close the writer.
Expand Down
8 changes: 6 additions & 2 deletions ...rmipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsWriter.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.janelia.colormipsearch.dataio.db;

import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -35,9 +36,12 @@ public void writeOne(AbstractNeuronEntity neuronEntity) {
}

@Override
public void addProcessingTags(List<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags) {
public void addProcessingTags(Collection<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags) {
neuronMetadataDao.addProcessingTagsToMIPIDs(
neuronEntities.stream().map(AbstractNeuronEntity::getMipId).collect(Collectors.toSet()),
neuronEntities.stream()
.filter(AbstractNeuronEntity::hasMipID)
.map(AbstractNeuronEntity::getMipId)
.collect(Collectors.toSet()),
processingType,
tags);
}
Expand Down
9 changes: 6 additions & 3 deletions ...rch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCheckedCDMIPsWriter.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.janelia.colormipsearch.dataio.db;

import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.janelia.colormipsearch.dao.NeuronMetadataDao;
import org.janelia.colormipsearch.dataio.CDMIPsWriter;
import org.janelia.colormipsearch.model.AbstractNeuronEntity;
import org.janelia.colormipsearch.model.ComputeFileType;
import org.janelia.colormipsearch.model.ProcessingType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -37,9 +37,12 @@ public void writeOne(AbstractNeuronEntity neuronEntity) {
}

@Override
public void addProcessingTags(List<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags) {
public void addProcessingTags(Collection<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags) {
neuronMetadataDao.addProcessingTagsToMIPIDs(
neuronEntities.stream().map(AbstractNeuronEntity::getMipId).collect(Collectors.toSet()),
neuronEntities.stream()
.filter(AbstractNeuronEntity::hasMipID)
.map(AbstractNeuronEntity::getMipId)
.collect(Collectors.toSet()),
processingType,
tags);
}
Expand Down
3 changes: 2 additions & 1 deletion ...ipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONCDMIPsWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.io.UncheckedIOException;
import java.nio.channels.Channels;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Set;

Expand Down Expand Up @@ -68,7 +69,7 @@ public void writeOne(AbstractNeuronEntity neuronEntity) {
}

@Override
public void addProcessingTags(List<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags) {
public void addProcessingTags(Collection<? extends AbstractNeuronEntity> neuronEntities, ProcessingType processingType, Set<String> tags) {
// do nothing here
}

Expand Down
40 changes: 40 additions & 0 deletions ...search-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
Expand All @@ -28,9 +29,11 @@
import org.janelia.colormipsearch.cds.ShapeMatchScore;
import org.janelia.colormipsearch.cmd.cdsprocess.ColorMIPProcessUtils;
import org.janelia.colormipsearch.dao.DaosProvider;
import org.janelia.colormipsearch.dataio.CDMIPsWriter;
import org.janelia.colormipsearch.dataio.DataSourceParam;
import org.janelia.colormipsearch.dataio.NeuronMatchesReader;
import org.janelia.colormipsearch.dataio.NeuronMatchesWriter;
import org.janelia.colormipsearch.dataio.db.DBCheckedCDMIPsWriter;
import org.janelia.colormipsearch.dataio.db.DBNeuronMatchesReader;
import org.janelia.colormipsearch.dataio.db.DBNeuronMatchesWriter;
import org.janelia.colormipsearch.dataio.fs.JSONNeuronMatchesReader;
Expand All @@ -42,12 +45,14 @@
import org.janelia.colormipsearch.imageprocessing.ImageRegionDefinition;
import org.janelia.colormipsearch.mips.NeuronMIP;
import org.janelia.colormipsearch.mips.NeuronMIPUtils;
import org.janelia.colormipsearch.model.AbstractMatchEntity;
import org.janelia.colormipsearch.model.AbstractNeuronEntity;
import org.janelia.colormipsearch.model.CDMatchEntity;
import org.janelia.colormipsearch.model.ComputeFileType;
import org.janelia.colormipsearch.model.EMNeuronEntity;
import org.janelia.colormipsearch.model.FileData;
import org.janelia.colormipsearch.model.LMNeuronEntity;
import org.janelia.colormipsearch.model.ProcessingType;
import org.janelia.colormipsearch.results.GroupedMatchedEntities;
import org.janelia.colormipsearch.results.ItemsHandling;
import org.janelia.colormipsearch.results.MatchEntitiesGrouping;
Expand Down Expand Up @@ -134,9 +139,17 @@ static class GradientScoresArgs extends AbstractColorDepthMatchArgs {
description = "Number of best matches for each sample to be used for gradient scoring")
int numberOfBestMatchesPerSample;

@Parameter(names = {"--processing-tag"},
description = "Associate this tag with the run. Also all MIPs that are color depth searched will be stamped with this processing tag")
String processingTag;

GradientScoresArgs(CommonArgs commonArgs) {
super(commonArgs);
}

String getProcessingTag() {
return processingTag.trim();
}
}

private final GradientScoresArgs args;
Expand Down Expand Up @@ -217,6 +230,10 @@ private void calculateAllGradientScores() {
LOG.info("Completed grad scores for {} matches of {}", cdMatchesWithGradScores.size(), maskIdToProcess);
long writtenUpdates = updateCDMatches(cdMatchesWithGradScores);
LOG.info("Updated {} grad scores for {} matches of {}", writtenUpdates, cdMatchesWithGradScores.size(), maskIdToProcess);
if (StringUtils.isNotBlank(args.processingTag)) {
long updatesWithProcessedTag = updateProcessingTag(cdMatchesForMask);
LOG.info("Set processing tag {} for {} mips", args.getProcessingTag(), updatesWithProcessedTag);
}
});
LOG.info("Finished partition {} ({} items) in {}s - memory usage {}M out of {}M",
indexedPartition.getKey(),
Expand Down Expand Up @@ -272,6 +289,14 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> NeuronM
}
}

private Optional<CDMIPsWriter> getCDMipsWriter() {
if (args.commonArgs.resultsStorage == StorageType.DB) {
return Optional.of(new DBCheckedCDMIPsWriter(getDaosProvider().getNeuronMetadataDao()));
} else {
return Optional.empty();
}
}

/**
* The method calculates and updates the gradient scores for all color depth matches of the given mask MIP ID.
*
Expand Down Expand Up @@ -326,6 +351,21 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> long up
));
}

private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> long updateProcessingTag(List<CDMatchEntity<M, T>> cdMatches) {
Set<String> processingTags = Collections.singleton(args.getProcessingTag());
return getCDMipsWriter()
.map(cdmipsWriter -> {
Set<M> masksToUpdate = cdMatches.stream()
.map(AbstractMatchEntity::getMaskImage).collect(Collectors.toSet());
Set<T> targetsToUpdate = cdMatches.stream()
.map(AbstractMatchEntity::getMatchedImage).collect(Collectors.toSet());
cdmipsWriter.addProcessingTags(masksToUpdate, ProcessingType.GradientScore, Collections.singleton(args.getProcessingTag()));
cdmipsWriter.addProcessingTags(masksToUpdate, ProcessingType.GradientScore, Collections.singleton(args.getProcessingTag()));
return masksToUpdate.size() + targetsToUpdate.size();
})
.orElse(0);
}

private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity>
List<CDMatchEntity<M, T>> getCDMatchesForMask(NeuronMatchesReader<CDMatchEntity<M, T>> cdsMatchesReader, String maskCDMipId) {
LOG.info("Read all color depth matches for {}", maskCDMipId);
Expand Down
6 changes: 3 additions & 3 deletions colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ColorDepthSearchCmd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,12 +245,12 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void ru
} finally {
LOG.info("Set processing tags to {}:{}", ProcessingType.ColorDepthSearch, processingTags);
// update the mips processing tags
getCDMipsWriter().ifPresent(cdmiPsWriter -> {
cdmiPsWriter.addProcessingTags(
getCDMipsWriter().ifPresent(cdmipsWriter -> {
cdmipsWriter.addProcessingTags(
filterProcessedNeurons(maskMips, processingTags),
ProcessingType.ColorDepthSearch,
processingTags);
cdmiPsWriter.addProcessingTags(
cdmipsWriter.addProcessingTags(
filterProcessedNeurons(targetMips, processingTags),
ProcessingType.ColorDepthSearch,
processingTags);
Expand Down

0 comments on commit f2fda03

Please sign in to comment.