Changed neuronreader api

JaneliaSciComp · Jan 31, 2024 · 75b327a · 75b327a
1 parent 6ffff3a
commit 75b327a
Show file tree

Hide file tree

Showing 12 changed files with 99 additions and 16 deletions.
diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java
@@ -17,6 +17,7 @@ public class DataSourceParam {
     private Collection<String> mipIDs = new HashSet<>();
     private Collection<String> names = new HashSet<>();
     private Collection<String> tags = new HashSet<>();
+    private Collection<String> datasets = new HashSet<>();
     private Collection<String> excludedTags = new HashSet<>();
     private long offset;
     private int size;
@@ -81,6 +82,16 @@ public DataSourceParam addTags(Collection<String> tags) {
         return this;
     }
 
+    public Collection<String> getDatasets() {
+        return datasets;
+    }
+
+    public DataSourceParam addDatasets(Collection<String> datasets) {
+        if (datasets != null) datasets.stream().filter(StringUtils::isNotBlank).forEach(ds -> this.datasets.add(ds));
+        return this;
+
+    }
+
     public Collection<String> getExcludedTags() {
         return excludedTags;
     }
@@ -123,6 +134,7 @@ public Map<String, Object> asMap() {
         params.put("mipIDs", mipIDs);
         params.put("names", names);
         params.put("tags", tags);
+        params.put("datasets", datasets);
         params.put("offset", hasOffset() ? offset : null);
         params.put("size", hasSize() ? size : null);
         return params;
@@ -142,6 +154,7 @@ public boolean equals(Object o) {
                 .append(mipIDs, that.mipIDs)
                 .append(names, that.names)
                 .append(tags, that.tags)
+                .append(datasets, that.datasets)
                 .append(offset, that.offset)
                 .append(size, that.size)
                 .isEquals();

diff --git a/...ipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java b/...ipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java
@@ -27,11 +27,13 @@ public interface NeuronMatchesReader<R extends AbstractMatchEntity<? extends Abs
      * @param maskLibraries
      * @param maskPublishedNames
      * @param maskMipIds
+     * @param maskDatasets
      * @param maskTags
      * @param maskExcludedTags
      * @param targetLibraries
      * @param targetPublishedNames
      * @param targetMipIds
+     * @param targetDatasets
      * @param targetTags
      * @param targetExcludedTags
      * @param matchTags
@@ -44,11 +46,13 @@ List<R> readMatchesByMask(String alignmentSpace,
                               Collection<String> maskLibraries,
                               Collection<String> maskPublishedNames,
                               Collection<String> maskMipIds,
+                              Collection<String> maskDatasets,
                               Collection<String> maskTags,
                               Collection<String> maskExcludedTags,
                               Collection<String> targetLibraries,
                               Collection<String> targetPublishedNames,
                               Collection<String> targetMipIds,
+                              Collection<String> targetDatasets,
                               Collection<String> targetTags,
                               Collection<String> targetExcludedTags,
                               Collection<String> matchTags,
@@ -63,10 +67,14 @@ List<R> readMatchesByMask(String alignmentSpace,
      * @param maskLibraries
      * @param maskPublishedNames
      * @param maskMipIds
+     * @param maskDatasets
+     * @param maskTags
      * @param maskExcludedTags
      * @param targetLibraries
      * @param targetPublishedNames
      * @param targetMipIds
+     * @param targetDatasets
+     * @param targetTags
      * @param targetExcludedTags
      * @param matchTags
      * @param matchExcludedTags
@@ -78,10 +86,14 @@ List<R> readMatchesByTarget(String alignmentSpace,
                                 Collection<String> maskLibraries,
                                 Collection<String> maskPublishedNames,
                                 Collection<String> maskMipIds,
+                                Collection<String> maskDatasets,
+                                Collection<String> maskTags,
                                 Collection<String> maskExcludedTags,
                                 Collection<String> targetLibraries,
                                 Collection<String> targetPublishedNames,
                                 Collection<String> targetMipIds,
+                                Collection<String> targetDatasets,
+                                Collection<String> targetTags,
                                 Collection<String> targetExcludedTags,
                                 Collection<String> matchTags,
                                 Collection<String> matchExcludedTags,

diff --git a/...rch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBNeuronMatchesReader.java b/...rch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBNeuronMatchesReader.java
@@ -46,7 +46,8 @@ public List<String> listMatchesLocations(Collection<DataSourceParam> matchesSour
                                         .addLibraries(cdMatchInput.getLibraries())
                                         .addMipIDs(cdMatchInput.getMipIDs())
                                         .addNames(cdMatchInput.getNames())
-                                        .addTags(cdMatchInput.getTags()),
+                                        .addTags(cdMatchInput.getTags())
+                                        .addDatasetLabels(cdMatchInput.getDatasets()),
                                 new PagedRequest()
                                         .setFirstPageOffset(cdMatchInput.getOffset())
                                         .setPageSize(cdMatchInput.getSize())
@@ -68,28 +69,33 @@ public List<R> readMatchesByMask(String alignmentSpace,
                                      Collection<String> maskLibraries,
                                      Collection<String> maskPublishedNames,
                                      Collection<String> maskMipIds,
+                                     Collection<String> maskDatasets,
                                      Collection<String> maskTags,
                                      Collection<String> maskExcludedTags,
                                      Collection<String> targetLibraries,
                                      Collection<String> targetPublishedNames,
                                      Collection<String> targetMipIds,
+                                     Collection<String> targetDatasets,
                                      Collection<String> targetTags,
                                      Collection<String> targetExcludedTags,
                                      Collection<String> matchTags,
-                                     Collection<String> matchExcludedTags, ScoresFilter matchScoresFilter,
+                                     Collection<String> matchExcludedTags,
+                                     ScoresFilter matchScoresFilter,
                                      List<SortCriteria> sortCriteriaList) {
         NeuronSelector maskSelector = new NeuronSelector()
                 .setAlignmentSpace(alignmentSpace)
                 .addLibraries(maskLibraries)
                 .addNames(maskPublishedNames)
                 .addMipIDs(maskMipIds)
+                .addDatasetLabels(maskDatasets)
                 .addTags(maskTags)
                 .addExcludedTags(maskExcludedTags);
         NeuronSelector targetSelector = new NeuronSelector()
                 .setAlignmentSpace(alignmentSpace)
                 .addLibraries(targetLibraries)
                 .addNames(targetPublishedNames)
                 .addMipIDs(targetMipIds)
+                .addDatasetLabels(targetDatasets)
                 .addTags(targetTags)
                 .addExcludedTags(targetExcludedTags);
         List<Number> maskEntityIds = getNeuronEntityIds(maskSelector);
@@ -107,10 +113,14 @@ public List<R> readMatchesByTarget(String alignmentSpace,
                                        Collection<String> maskLibraries,
                                        Collection<String> maskPublishedNames,
                                        Collection<String> maskMipIds,
+                                       Collection<String> maskDatasets,
+                                       Collection<String> maskTags,
                                        Collection<String> maskExcludedTags,
                                        Collection<String> targetLibraries,
                                        Collection<String> targetPublishedNames,
                                        Collection<String> targetMipIds,
+                                       Collection<String> targetDatasets,
+                                       Collection<String> targetTags,
                                        Collection<String> targetExcludedTags,
                                        Collection<String> matchTags,
                                        Collection<String> matchExcludedTags,
@@ -121,12 +131,16 @@ public List<R> readMatchesByTarget(String alignmentSpace,
                 .addLibraries(maskLibraries)
                 .addNames(maskPublishedNames)
                 .addMipIDs(maskMipIds)
+                .addDatasetLabels(maskDatasets)
+                .addTags(maskTags)
                 .addExcludedTags(maskExcludedTags);
         NeuronSelector targetSelector = new NeuronSelector()
                 .setAlignmentSpace(alignmentSpace)
                 .addLibraries(targetLibraries)
                 .addNames(targetPublishedNames)
                 .addMipIDs(targetMipIds)
+                .addDatasetLabels(targetDatasets)
+                .addTags(targetTags)
                 .addExcludedTags(targetExcludedTags);
         List<Number> targetEntityIds = getNeuronEntityIds(targetSelector);
         NeuronsMatchFilter<R> neuronsMatchFilter = new NeuronsMatchFilter<R>()

diff --git a/...h-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONNeuronMatchesReader.java b/...h-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONNeuronMatchesReader.java
@@ -57,11 +57,13 @@ public List<R> readMatchesByMask(String alignmentSpace,
                                      Collection<String> maskLibraries,
                                      Collection<String> maskPublishedNames,
                                      Collection<String> maskMipIds,
+                                     Collection<String> maskDatasets,
                                      Collection<String> maskTags,
                                      Collection<String> maskExcludedTags,
                                      Collection<String> targetLibraries,
                                      Collection<String> targetPublishedNames,
                                      Collection<String> targetMipIds,
+                                     Collection<String> targetDatasets,
                                      Collection<String> targetTags,
                                      Collection<String> targetExcludedTags,
                                      Collection<String> matchTags,
@@ -83,11 +85,18 @@ public List<R> readMatchesByTarget(String alignmentSpace,
                                        Collection<String> maskLibraries,
                                        Collection<String> maskPublishedNames,
                                        Collection<String> maskMipIds,
-                                       Collection<String> maskExcludedTags, Collection<String> targetLibraries,
+                                       Collection<String> maskDatasets,
+                                       Collection<String> maskTags,
+                                       Collection<String> maskExcludedTags,
+                                       Collection<String> targetLibraries,
                                        Collection<String> targetPublishedNames,
                                        Collection<String> targetMipIds,
-                                       Collection<String> targetExcludedTags, Collection<String> matchTags,
-                                       Collection<String> matchExcludedTags, ScoresFilter matchScoresFilter,
+                                       Collection<String> targetDatasets,
+                                       Collection<String> targetTags,
+                                       Collection<String> targetExcludedTags,
+                                       Collection<String> matchTags,
+                                       Collection<String> matchExcludedTags,
+                                       ScoresFilter matchScoresFilter,
                                        List<SortCriteria> sortCriteriaList) {
         return (List<R>) targetMipIds.stream()
                 .flatMap(targetMipId -> CollectionUtils.isEmpty(targetLibraries)

diff --git a/...psearch-persist/src/test/java/org/janelia/colormipsearch/dataio/fs/JSONReadWriteTest.java b/...psearch-persist/src/test/java/org/janelia/colormipsearch/dataio/fs/JSONReadWriteTest.java
@@ -18,6 +18,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.janelia.colormipsearch.dataio.fileutils.FSUtils;
+import org.janelia.colormipsearch.model.AbstractMatchEntity;
 import org.janelia.colormipsearch.model.AbstractNeuronEntity;
 import org.janelia.colormipsearch.model.CDMatchEntity;
 import org.janelia.colormipsearch.model.EMNeuronEntity;
@@ -95,11 +96,13 @@ public void readWriteCDSResults() {
                                     /* maskLibraries */null,
                                     /* maskPublishedNames */null,
                                     Collections.singletonList(f),
+                                    /* maskDatasets */null,
                                     /* maskTags */null,
                                     /* maskExcludedTags */null,
                                     /* targetLibraries */null,
                                     /* targetPublishedNames */null,
                                     /* targetMIPIDs */null,
+                                    /* targetDatasets*/null,
                                     /* targetTags */null,
                                     /* targetExcludedTags */null,
                                     /* matchTags */null,
@@ -110,9 +113,7 @@ public void readWriteCDSResults() {
                     String mId = FilenameUtils.getBaseName(f);
                     List<CDMatchEntity<EMNeuronEntity, LMNeuronEntity>> testMatchesWithSameMask = cdMatches.stream()
                             .filter(cdsMatch -> cdsMatch.getMaskImage().getMipId().equals(mId))
-                            .peek(cdsMatch -> {
-                                cdsMatch.resetMatchComputeFiles();
-                            })
+                            .peek(AbstractMatchEntity::resetMatchComputeFiles)
                             .sorted(Comparator.comparingDouble(m -> -m.getMatchingPixels()))
                             .collect(Collectors.toList());
                     assertEquals("Results did not match for " + mId,

diff --git a/...search-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java b/...search-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java
@@ -79,10 +79,18 @@ static class GradientScoresArgs extends AbstractColorDepthMatchArgs {
         @Parameter(names = {"--masks-mips"}, description = "Selected mask MIPs", variableArity = true)
         List<String> masksMIPIDs;
 
+        @Parameter(names = {"--mask-datasets"}, description = "Datasets associated with the mask of the match to be scored",
+                variableArity = true)
+        List<String> maskDatasets = new ArrayList<>();
+
         @Parameter(names = {"--mask-tags"}, description = "Tags associated with the mask of the match to be scored",
                 variableArity = true)
         List<String> maskTags = new ArrayList<>();
 
+        @Parameter(names = {"--target-datasets"}, description = "Datasets associated with the target of the match to be scored",
+                variableArity = true)
+        List<String> targetDatasets = new ArrayList<>();
+
         @Parameter(names = {"--target-tags"}, description = "Tags associated with the target of the match to be scored",
                 variableArity = true)
         List<String> targetTags = new ArrayList<>();
@@ -314,11 +322,13 @@ List<CDMatchEntity<M, T>> getCDMatchesForMask(NeuronMatchesReader<CDMatchEntity<
                 /* maskLibraries */null,
                 /* maskPublishedNames */null,
                 Collections.singletonList(maskCDMipId),
+                args.maskDatasets,
                 args.maskTags,
                 /*maskExcludedTags*/null,
                 args.targetsLibraries,
                 args.targetsPublishedNames,
                 args.targetsMIPIDs,
+                args.targetDatasets,
                 args.targetTags,
                 /*targetExcludedTags*/null,
                 args.matchTags,

diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ColorDepthSearchCmd.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ColorDepthSearchCmd.java
@@ -88,9 +88,15 @@ static class ColorDepthSearchArgs extends AbstractColorDepthMatchArgs {
         @Parameter(names = {"--masks-tags"}, description = "Masks MIPs tags to be selected for CDS", variableArity = true)
         List<String> masksTags;
 
+        @Parameter(names = {"--masks-datasets"}, description = "Masks MIPs datasets to be selected for CDS", variableArity = true)
+        List<String> masksDatasets;
+
         @Parameter(names = {"--targets-tags"}, description = "Targets MIPs tags to be selected for CDS", variableArity = true)
         List<String> targetsTags;
 
+        @Parameter(names = {"--targets-datasets"}, description = "Targets MIPs datasets to be selected for CDS", variableArity = true)
+        List<String> targetsDatasets;
+
         @Parameter(names = {"--masks-published-names"}, description = "Masks MIPs published names to be selected for CDS", variableArity = true)
         List<String> masksPublishedNames;
 
@@ -160,13 +166,15 @@ private <M extends AbstractNeuronEntity, T extends AbstractNeuronEntity> void ru
                 args.masksLibraries,
                 args.masksPublishedNames,
                 args.masksTags,
+                args.masksDatasets,
                 args.masksStartIndex, args.masksLength,
                 args.maskMIPsFilter);
         @SuppressWarnings("unchecked")
         List<T> targetMips = (List<T>) readMIPs(cdmipsReader,
                 args.targetsLibraries,
                 args.targetsPublishedNames,
                 args.targetsTags,
+                args.targetsDatasets,
                 args.targetsStartIndex, args.targetsLength,
                 args.libraryMIPsFilter);
         if (maskMips.isEmpty() || targetMips.isEmpty()) {
@@ -290,6 +298,7 @@ private List<? extends AbstractNeuronEntity> readMIPs(CDMIPsReader mipsReader,
                                                           List<ListArg> mipsLibraries,
                                                           List<String> mipsPublishedNames,
                                                           List<String> mipsTags,
+                                                          List<String> mipsDatasets,
                                                           long startIndexArg, int length,
                                                           Set<String> filter) {
         long startIndex = startIndexArg > 0 ? startIndexArg : 0;
@@ -300,6 +309,7 @@ private List<? extends AbstractNeuronEntity> readMIPs(CDMIPsReader mipsReader,
                                 .addLibrary(libraryInput.input)
                                 .addNames(mipsPublishedNames)
                                 .addTags(mipsTags)
+                                .addDatasets(mipsDatasets)
                                 .setOffset(libraryInput.offset)
                                 .setSize(libraryInput.length)).stream())
                 .filter(neuronMetadata -> CollectionUtils.isEmpty(filter) ||

diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ImportV2CDMatchesCmd.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ImportV2CDMatchesCmd.java
@@ -204,16 +204,18 @@ List<CDMatchEntity<M, T>> getCDMatchesForMask(NeuronMatchesReader<CDMatchEntity<
                 /* maskLibraries */null,
                 /* maskPublishedNames */null,
                 Collections.singletonList(maskCDMipId),
+                /* maskDatasets */null,
                 /* maskTags */null,
                 /* maskExcludedTags */null,
                 /* targetLibraries */null,
                 /* targetPublishedNames */null,
                 /* targetMIPIDs */null,
+                /* targetDatasets */null,
                 /* targetTags */null,
                 /* targetExcludedTags */null,
                 /* matchTags */null,
                 /* matchExcludedTags */null,
-                /* matchScoresFilter */null ,
+                /* matchScoresFilter */null,
                 Collections.singletonList(
                         new SortCriteria("normalizedScore", SortDirection.DESC)
                 ));