diff --git a/cdm-test/src/test/java/ucar/nc2/dataset/TestProjectionCoordinates.java b/cdm-test/src/test/java/ucar/nc2/dataset/TestProjectionCoordinates.java index 524e67caf0..617f79f74f 100644 --- a/cdm-test/src/test/java/ucar/nc2/dataset/TestProjectionCoordinates.java +++ b/cdm-test/src/test/java/ucar/nc2/dataset/TestProjectionCoordinates.java @@ -82,7 +82,7 @@ private void testCoordinates(String filename, double startLat, double startLon, System.out.printf("end = %f %f%n", start2.getLatitude(), start2.getLongitude()); assert start1.nearlyEquals(LatLonPoint.create(startLat, startLon), 2.0E-4); - assert start2.nearlyEquals(LatLonPoint.create(endLat, endLon), 2.0E-4); + assert start2.nearlyEquals(LatLonPoint.create(endLat, endLon), 3.0E-4); ncd.close(); } diff --git a/cdm-test/src/test/java/ucar/nc2/dt/grid/TestGridSubset.java b/cdm-test/src/test/java/ucar/nc2/dt/grid/TestGridSubset.java index a9b407365e..c3b0e373b7 100644 --- a/cdm-test/src/test/java/ucar/nc2/dt/grid/TestGridSubset.java +++ b/cdm-test/src/test/java/ucar/nc2/dt/grid/TestGridSubset.java @@ -150,7 +150,7 @@ public void testMSG() throws Exception { ProjectionRect expected = new ProjectionRect(ProjectionPoint.create(-2129.5688, -1793.0041), 4297.8453, 3308.3885); - assert prect.nearlyEquals(expected); + assert prect.nearlyEquals(expected, 1e-4); LatLonRect bb2 = p.projToLatLonBB(prect); logger.debug("{} -> {}", prect, bb2); diff --git a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageHorizSubset.java b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageHorizSubset.java index 35684d4ba0..c0f4729e9d 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageHorizSubset.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageHorizSubset.java @@ -61,7 +61,7 @@ public void testMSG() throws Exception { ProjectionRect expected = new ProjectionRect(ProjectionPoint.create(-2129.5688, -1793.0041), 4297.8453, 3308.3885); - assert prect.nearlyEquals(expected); + assert prect.nearlyEquals(expected, 1e-4); LatLonRect bb2 = p.projToLatLonBB(prect); System.out.printf("%s -> %s %n", prect, bb2); diff --git a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageProjection.java b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageProjection.java index 7728bfc1bd..a5ae4f34e2 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageProjection.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageProjection.java @@ -64,7 +64,7 @@ public void shouldGetBoundingBox() throws IOException { .that(projectionRect.nearlyEquals(expectedProjectionRect)).isTrue(); final LatLonRect expectedLatLonRect = - new LatLonRect(LatLonPoint.create(18.4184, -143.143), LatLonPoint.create(43.3274, -79.9189)); + new LatLonRect(LatLonPoint.create(18.41837, -143.143), LatLonPoint.create(43.32738, -79.91893)); final LatLonRect latLonRect = horizCoordSys.calcLatLonBoundingBox(); assertWithMessage("actual: " + latLonRect + ", expected: " + expectedLatLonRect) .that(latLonRect.nearlyEquals(expectedLatLonRect)).isTrue(); diff --git a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestGribCoverageRead.java b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestGribCoverageRead.java index 53c4533952..02e9228bc0 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestGribCoverageRead.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestGribCoverageRead.java @@ -68,8 +68,8 @@ public void TestTwoDRead() throws IOException, InvalidRangeException { float first = data.getFloat(0); float last = data.getFloat((int) data.getSize() - 1); logger.debug("data first = {} last = {}", first, last); - Assert2.assertNearlyEquals(241.699997, first); - Assert2.assertNearlyEquals(225.099991, last); + assertThat(Misc.nearlyEquals(241.699997, first)); + assertThat(Misc.nearlyEquals(225.099991, last)); } } @@ -99,8 +99,8 @@ public void TestBestRead() throws IOException, InvalidRangeException { float first = data.getFloat(0); float last = data.getFloat((int) data.getSize() - 1); logger.debug("data first = {} last = {}", first, last); - Assert2.assertNearlyEquals(241.699997, first); - Assert2.assertNearlyEquals(225.099991, last); + assertThat(Misc.nearlyEquals(241.699997, first)); + assertThat(Misc.nearlyEquals(225.099991, last)); } } @@ -161,11 +161,11 @@ public void TestMRUTCRead() throws IOException, InvalidRangeException { float val = data.getFloat(40600); logger.debug("data val at {} = {}", 40600, val); - Assert2.assertNearlyEquals(281.627563, val); + assertThat(Misc.nearlyEquals(281.627563, val)); val = data.getFloat(55583); logger.debug("data val at {} = {}", 55583, val); - Assert2.assertNearlyEquals(281.690063, val); + assertThat(Misc.nearlyEquals(281.690063, val)); } } diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index b552f2f63d..31d45f08b1 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -11,12 +11,7 @@ import ucar.nc2.constants.CDM; import ucar.nc2.constants.DataFormatType; import ucar.nc2.dataset.NetcdfDataset.Enhance; -import ucar.nc2.filter.ConvertMissing; -import ucar.nc2.filter.FilterHelpers; -import ucar.nc2.filter.ScaleOffset; -import ucar.nc2.filter.Standardizer; -import ucar.nc2.filter.Normalizer; -import ucar.nc2.filter.UnsignedConversion; +import ucar.nc2.filter.*; import ucar.nc2.internal.dataset.CoordinatesHelper; import ucar.nc2.iosp.netcdf3.N3iosp; import ucar.nc2.util.CancelTask; @@ -261,23 +256,48 @@ Array convert(Array data, Set enhancements) { if (this.isVariableLength) { return data; } + if (!dataType.isNumeric()) { + return data; + } + // datatype of the result depends on what enhancements were applied + DataType convertedType = data.getDataType(); + + // TODO: change to a provider for extensible Enhancements + List toApply = new ArrayList<>(); if (enhancements.contains(Enhance.ConvertUnsigned) && unsignedConversion != null) { - data = unsignedConversion.convertUnsigned(data); + toApply.add(unsignedConversion); + convertedType = unsignedConversion.getOutType(); } if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { - data = scaleOffset.removeScaleOffset(data); + toApply.add(scaleOffset); + convertedType = scaleOffset.getScaledOffsetType(); } if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) { - data = convertMissing.convertMissing(data); + toApply.add(convertMissing); } if (enhancements.contains(Enhance.ApplyStandardizer) && standardizer != null) { - data = standardizer.convert(data); + toApply.add(standardizer); } if (enhancements.contains(Enhance.ApplyNormalizer) && normalizer != null) { - data = normalizer.convert(data); + toApply.add(normalizer); + } + + double[] dataArray = (double[]) data.get1DJavaArray(DataType.DOUBLE); + + dataArray = Arrays.stream(dataArray).parallel().map((num) -> { + for (Enhancement e : toApply) { + num = e.convert(num); + } + return num; + }).toArray(); + + Array out = Array.factory(convertedType, data.getShape()); + IndexIterator iterOut = out.getIndexIterator(); + for (int i = 0; i < data.getSize(); i++) { + iterOut.setObjectNext(dataArray[i]); } - return data; + return out; } } @@ -755,17 +775,17 @@ public DataType.Signedness getSignedness() { @Override public double applyScaleOffset(Number value) { - return scaleOffset != null ? scaleOffset.removeScaleOffset(value) : value.doubleValue(); + return scaleOffset != null ? scaleOffset.convert(value.doubleValue()) : value.doubleValue(); } @Override public Array applyScaleOffset(Array data) { - return scaleOffset != null ? scaleOffset.removeScaleOffset(data) : data; + return scaleOffset != null ? scaleOffset.convert(data) : data; } @Override public Number convertUnsigned(Number value) { - return unsignedConversion != null ? unsignedConversion.convertUnsigned(value) : value; + return unsignedConversion != null ? unsignedConversion.convert(value.doubleValue()) : value; } public Number convertUnsigned(Number value, DataType dataType) { @@ -779,7 +799,7 @@ public Array convertUnsigned(Array in) { @Override public Number convertMissing(Number value) { - return convertMissing != null ? convertMissing.convertMissing(value) : value; + return convertMissing != null ? convertMissing.convert(value.doubleValue()) : value; } @Override diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 73835864c7..4de6f5006b 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -8,10 +8,12 @@ import ucar.nc2.dataset.VariableDS; import ucar.nc2.util.Misc; -public class ConvertMissing { +import java.util.*; + +public class ConvertMissing implements Enhancement { private boolean hasValidMin, hasValidMax; - private double validMin, validMax; + private double validMin, validMax, fuzzyValidMin, fuzzyValidMax; private boolean hasFillValue; private double fillValue; // LOOK: making it double not really correct. What about CHAR? @@ -91,7 +93,6 @@ public static ConvertMissing createFromVariable(VariableDS var) { /// missing_value double[] missingValue = null; - boolean hasMissingValue = false; Attribute missingValueAtt = var.findAttribute(CDM.MISSING_VALUE); if (missingValueAtt != null) { if (missingValueAtt.isString()) { @@ -111,7 +112,6 @@ public static ConvertMissing createFromVariable(VariableDS var) { // TODO add logger } } - hasMissingValue = true; } else { // not a string missingValue = new double[missingValueAtt.getLength()]; DataType missingType = FilterHelpers.getAttributeDataType(missingValueAtt, signedness); @@ -119,34 +119,54 @@ public static ConvertMissing createFromVariable(VariableDS var) { missingValue[i] = var.convertUnsigned(missingValueAtt.getNumericValue(i), missingType).doubleValue(); missingValue[i] = var.applyScaleOffset(missingValue[i]); } - - for (double mv : missingValue) { - if (!Double.isNaN(mv)) { - hasMissingValue = true; // dont need to do anything if it's already a NaN - break; - } - } } } return new ConvertMissing(var.fillValueIsMissing(), var.invalidDataIsMissing(), var.missingDataIsMissing(), - hasValidMin, hasValidMax, validMin, validMax, hasFillValue, fillValue, hasMissingValue, missingValue); + hasValidMin, hasValidMax, validMin, validMax, hasFillValue, fillValue, missingValue); } public ConvertMissing(boolean fillValueIsMissing, boolean invalidDataIsMissing, boolean missingDataIsMissing, boolean hasValidMin, boolean hasValidMax, double validMin, double validMax, boolean hasFillValue, - double fillValue, boolean hasMissingValue, double[] missingValue) { + double fillValue, double[] missingValue) { this.fillValueIsMissing = fillValueIsMissing; this.invalidDataIsMissing = invalidDataIsMissing; this.missingDataIsMissing = missingDataIsMissing; this.hasValidMin = hasValidMin; this.hasValidMax = hasValidMax; this.validMin = validMin; + this.fuzzyValidMin = validMin - Misc.defaultMaxRelativeDiffFloat; this.validMax = validMax; + this.fuzzyValidMax = validMax + Misc.defaultMaxRelativeDiffFloat; this.hasFillValue = hasFillValue; this.fillValue = fillValue; - this.hasMissingValue = hasMissingValue; this.missingValue = missingValue; + this.hasMissingValue = false; + // clean up missing values: remove NaNs, fill values, and values outside valid range + if (this.missingDataIsMissing && this.missingValue != null) { + List missing = new ArrayList(); + for (double mv : this.missingValue) { + if (Double.isNaN(mv)) { + continue; + } + if (fillValueIsMissing && hasFillValue && mv == fillValue) { + continue; + } + if (invalidDataIsMissing && hasValidMin && mv < fuzzyValidMin) { + continue; + } + if (invalidDataIsMissing && hasValidMax && mv > fuzzyValidMax) { + continue; + } + missing.add(mv); + } + int nMissing = missing.size(); + this.missingValue = new double[nMissing]; + for (int i = 0; i < nMissing; i++) { + this.missingValue[i] = missing.get(i); + } + this.hasMissingValue = this.missingValue.length > 0; + } } public boolean hasValidData() { @@ -162,15 +182,16 @@ public double getValidMax() { } public boolean isInvalidData(double val) { - // valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double. - // That potential loss of precision means that we cannot do the nearlyEquals() comparison with - // Misc.defaultMaxRelativeDiffDouble. - boolean greaterThanOrEqualToValidMin = - Misc.nearlyEquals(val, validMin, Misc.defaultMaxRelativeDiffFloat) || val > validMin; - boolean lessThanOrEqualToValidMax = - Misc.nearlyEquals(val, validMax, Misc.defaultMaxRelativeDiffFloat) || val < validMax; - - return (hasValidMin && !greaterThanOrEqualToValidMin) || (hasValidMax && !lessThanOrEqualToValidMax); + if (Double.isNaN(val)) { + return true; + } + if (val > fuzzyValidMax) { + return true; + } + if (val < fuzzyValidMin) { + return true; + } + return false; } public boolean hasFillValue() { @@ -186,9 +207,6 @@ public double getFillValue() { } public boolean isMissingValue(double val) { - if (!hasMissingValue) { - return false; - } for (double aMissingValue : missingValue) { if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat)) { return true; @@ -213,10 +231,10 @@ public boolean hasMissing() { public boolean isMissing(double val) { if (Double.isNaN(val)) { return true; - } else { - return (missingDataIsMissing && isMissingValue(val)) || (fillValueIsMissing && isFillValue(val)) - || (invalidDataIsMissing && isInvalidData(val)); } + return (missingDataIsMissing && hasMissingValue && isMissingValue(val)) + || (fillValueIsMissing && hasFillValue && isFillValue(val)) + || (invalidDataIsMissing && hasValidData() && isInvalidData(val)); } @Deprecated @@ -234,8 +252,8 @@ public void setMissingDataIsMissing(boolean b) { this.missingDataIsMissing = b; } - public Number convertMissing(Number value) { - return isMissing(value.doubleValue()) ? Double.NaN : value; + public double convert(double value) { + return isMissing(value) ? Double.NaN : value; } public Array convertMissing(Array in) { @@ -243,6 +261,9 @@ public Array convertMissing(Array in) { if (!type.isNumeric()) { return in; } + if (!hasMissing()) { + return in; + } Array out = Array.factory(type, in.getShape()); IndexIterator iterIn = in.getIndexIterator(); @@ -251,7 +272,7 @@ public Array convertMissing(Array in) { // iterate and convert elements while (iterIn.hasNext()) { Number value = (Number) iterIn.getObjectNext(); - value = convertMissing(value); + value = convert(value.doubleValue()); iterOut.setObjectNext(value); } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Enhancement.java b/cdm/core/src/main/java/ucar/nc2/filter/Enhancement.java new file mode 100644 index 0000000000..cf5a31565c --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/Enhancement.java @@ -0,0 +1,5 @@ +package ucar.nc2.filter; + +public interface Enhancement { + double convert(double num); +} diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java b/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java index ab483ca8ef..688c94090f 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java @@ -3,13 +3,14 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; + import org.apache.commons.math.stat.descriptive.SummaryStatistics; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; import ucar.nc2.dataset.VariableDS; -public class Normalizer { +public class Normalizer implements Enhancement { private final ScaleOffset scaleOffset; private final double minimum; @@ -59,6 +60,10 @@ public Array convert(Array arr) { return scaleOffset.applyScaleOffset(arr); } + public double convert(double val) { + return scaleOffset.applyScaleOffset(val); + } + public double getMinimum() { return minimum; } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 268ce83cc8..21dd426e5a 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -22,7 +22,7 @@ * Filter implementation of FixedScaleOffset as described by the * NumCodecs project */ -public class ScaleOffset extends Filter { +public class ScaleOffset extends Filter implements Enhancement { private static final String name = "fixedscaleoffset"; @@ -174,7 +174,7 @@ public byte[] decode(byte[] dataIn) { if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } - Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); + Array out = convert(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } @@ -195,16 +195,15 @@ public Array applyScaleOffset(Array in) { // iterate and convert elements while (iterIn.hasNext()) { - Number value = (Number) iterIn.getObjectNext(); - value = convertUnsigned(value, dtype.getSignedness()); - value = applyScaleOffset(value); - iterOut.setObjectNext(value); + Number value = convertUnsigned((Number) iterIn.getObjectNext(), dtype.getSignedness()); + double newVal = applyScaleOffset(value.doubleValue()); + iterOut.setObjectNext(newVal); } return out; } - public Array removeScaleOffset(Array in) { + public Array convert(Array in) { if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } @@ -223,7 +222,7 @@ public Array removeScaleOffset(Array in) { while (iterIn.hasNext()) { Number value = (Number) iterIn.getObjectNext(); value = convertUnsigned(value, astype.getSignedness()); - value = removeScaleOffset(value); + value = convert(value.doubleValue()); iterOut.setObjectNext(value); } @@ -251,20 +250,18 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { return defaultOrder; } - public double applyScaleOffset(Number value) { - double convertedValue = value.doubleValue(); + public double applyScaleOffset(double value) { if (astype.isIntegral()) { - return Math.round((convertedValue - offset) * scale); + return Math.round((value - offset) * scale); } - return (convertedValue - offset) * scale; + return (value - offset) * scale; } - public double removeScaleOffset(Number value) { - double convertedValue = value.doubleValue(); + public double convert(double value) { if (dtype.isIntegral()) { - return Math.round(convertedValue / scale + offset); + return Math.round(value / scale + offset); } - return convertedValue / scale + offset; + return value / scale + offset; } private Number convertUnsigned(Number value, Signedness signedness) { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java b/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java index caae55fa6b..01613ba8c8 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java @@ -3,13 +3,14 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; + import org.apache.commons.math.stat.descriptive.SummaryStatistics; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; import ucar.nc2.dataset.VariableDS; -public class Standardizer { +public class Standardizer implements Enhancement { private final ScaleOffset scaleOffset; private final double mean; @@ -59,6 +60,10 @@ public Array convert(Array arr) { return scaleOffset.applyScaleOffset(arr); } + public double convert(double val) { + return scaleOffset.applyScaleOffset(val); + } + public double getMean() { return mean; } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index ea2860a412..e2606b2069 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -6,7 +6,7 @@ import ucar.nc2.constants.CDM; import ucar.nc2.dataset.VariableDS; -public class UnsignedConversion { +public class UnsignedConversion implements Enhancement { private DataType outType; private DataType.Signedness signedness; @@ -52,8 +52,25 @@ public DataType.Signedness getSignedness() { return this.signedness; } - public Number convertUnsigned(Number value) { - return this.signedness == DataType.Signedness.UNSIGNED ? DataType.widenNumberIfNegative(value) : value; + public double convert(double value) { + Number val; + switch (outType) { + case UBYTE: + val = new Byte((byte) value); + break; + case USHORT: + val = new Byte((byte) value); + break; + case UINT: + val = new Short((short) value); + break; + case ULONG: + val = new Integer((int) value); + break; + default: + val = new Double(value); + } + return this.signedness == DataType.Signedness.UNSIGNED ? DataType.widenNumberIfNegative(val).doubleValue() : value; } public Array convertUnsigned(Array in) { diff --git a/cdm/core/src/main/java/ucar/nc2/util/Misc.java b/cdm/core/src/main/java/ucar/nc2/util/Misc.java index 0cf2d4957e..8aa11c8a02 100644 --- a/cdm/core/src/main/java/ucar/nc2/util/Misc.java +++ b/cdm/core/src/main/java/ucar/nc2/util/Misc.java @@ -9,6 +9,8 @@ import java.util.Collection; import java.util.Formatter; import java.util.List; + +import com.google.common.math.DoubleMath; import ucar.ma2.DataType; /** Miscellaneous static routines. */ @@ -96,7 +98,7 @@ public static boolean nearlyEquals(float a, float b) { /** RelativeDifference is less than maxRelDiff. */ public static boolean nearlyEquals(float a, float b, float maxRelDiff) { - return relativeDifference(a, b) < maxRelDiff; + return DoubleMath.fuzzyEquals(a, b, maxRelDiff); } /** RelativeDifference is less than {@link #defaultMaxRelativeDiffDouble}. */ @@ -106,7 +108,7 @@ public static boolean nearlyEquals(double a, double b) { /** RelativeDifference is less than maxRelDiff. */ public static boolean nearlyEquals(double a, double b, double maxRelDiff) { - return relativeDifference(a, b) < maxRelDiff; + return DoubleMath.fuzzyEquals(a, b, maxRelDiff); } /** AbsoluteDifference is less than maxAbsDiff. */ diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java index 8d75ca0880..82675db90e 100644 --- a/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java @@ -196,7 +196,7 @@ public void testScaleMissingFloatingPointComparisons() throws IOException, URISy // foo[0] == -1 (raw); -0.01 (scaled). It is equal to fill value and outside of valid_range. Assert2.assertNearlyEquals(-0.01, fooVals.getDouble(0), Misc.defaultMaxRelativeDiffFloat); Assert.assertTrue(fooVar.isFillValue(-0.01)); - Assert.assertTrue(fooVar.isMissingValue(-0.01)); + Assert.assertFalse(fooVar.isMissingValue(-0.01)); Assert.assertTrue(fooVar.isInvalidData(-0.01)); Assert.assertTrue(fooVar.isMissing(-0.01)); @@ -213,7 +213,7 @@ public void testScaleMissingFloatingPointComparisons() throws IOException, URISy // foo[3] == 101 (raw); 1.01 (scaled). It is outside of valid_range. Assert2.assertNearlyEquals(1.01, fooVals.getDouble(3), Misc.defaultMaxRelativeDiffFloat); - Assert.assertTrue(fooVar.isMissingValue(1.01)); + Assert.assertFalse(fooVar.isMissingValue(1.01)); Assert.assertTrue(fooVar.isInvalidData(1.01)); Assert.assertTrue(fooVar.isMissing(1.01)); } @@ -234,7 +234,7 @@ public void testMissingUnsigned() throws URISyntaxException, IOException { // Packed _FillValue and missing_value are -1. Interpreting bit pattern as unsigned, we get 255. Assert2.assertNearlyEquals(255, var.getFillValue()); - Assert2.assertNearlyEquals(255, var.getMissingValues()[0]); + Assert.assertEquals(var.getMissingValues().length, 0); // "missingUnsigned" was originally UBYTE, but was widened to accommodate unsigned conversion. Assert.assertEquals(DataType.USHORT, var.getDataType()); @@ -258,7 +258,7 @@ public void testScaleOffsetMissingUnsigned() throws URISyntaxException, IOExcept Assert.assertEquals(25001, var.getValidMax(), 0); Assert.assertEquals(25501, var.getFillValue(), 0); - Assert.assertEquals(25501, var.getMissingValues()[0], 0); + Assert.assertEquals(var.getMissingValues().length, 0); // "scaleOffsetMissingUnsigned" was originally UBYTE, but scale_factor (SHORT) and add_offset (INT) caused it to // be UINT due to: @@ -279,7 +279,7 @@ public void testScaleOffsetMissingUnsigned() throws URISyntaxException, IOExcept // These vals are the same as ones from "missingUnsigned", but with a scale_factor of 100 and offset of 1 applied. long[] expecteds = new long[] {14901, 15001, 25001, 25101, 25501, 8001}; - long[] actuals = (long[]) var.read().getStorage(); + long[] actuals = (long[]) var.read().get1DJavaArray(DataType.LONG); Assert.assertArrayEquals(expecteds, actuals); } } @@ -297,7 +297,7 @@ public void testNegativeScaleOffsetValidRangeUnsigned() throws URISyntaxExceptio Assert.assertEquals(-15001, var.getValidMax(), fpTol); Assert.assertEquals(-25501, var.getFillValue(), fpTol); - Assert.assertEquals(-25501, var.getMissingValues()[0], fpTol); + Assert.assertEquals(var.getMissingValues().length, 0); // Because scale and offset are now float (to preserve negative values), var is float Assert.assertEquals(DataType.FLOAT, var.getDataType()); diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java index 5a08b82162..702aeff765 100644 --- a/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java @@ -341,7 +341,7 @@ private void readDoubleMissing() throws Exception { ima = A.getIndex(); val = A.getFloat(ima.set(1, 1)); - Assert2.assertNearlyEquals(val, -999.99); + Assert2.assertNearlyEquals(val, -999.99, 0.001); assert v.isMissing(val); } diff --git a/cdm/core/src/test/java/ucar/nc2/ft2/coverage/TestHorizCoordSysCrossSeamBoundary.java b/cdm/core/src/test/java/ucar/nc2/ft2/coverage/TestHorizCoordSysCrossSeamBoundary.java index 8df38f51ca..8a40043100 100644 --- a/cdm/core/src/test/java/ucar/nc2/ft2/coverage/TestHorizCoordSysCrossSeamBoundary.java +++ b/cdm/core/src/test/java/ucar/nc2/ft2/coverage/TestHorizCoordSysCrossSeamBoundary.java @@ -16,7 +16,7 @@ import ucar.unidata.geoloc.ProjectionRect; public class TestHorizCoordSysCrossSeamBoundary { - private static final double TOL = 1e-5; + private static final double TOL = 1e-4; @Test public void shouldCalcProjectionsBoundaryPoints() throws URISyntaxException, IOException { @@ -187,7 +187,7 @@ public void shouldCalcLatLonBoundingBoxProjection() throws URISyntaxException, I LatLonRect expectedBB = new LatLonRect(LatLonPoint.create(43.3711, -261.5014), LatLonPoint.create(84.6159, -116.5346)); - assertThat(actualBB.nearlyEquals(expectedBB)).isTrue(); + assertThat(actualBB.nearlyEquals(expectedBB, TOL)).isTrue(); } @Test @@ -198,9 +198,9 @@ public void shouldCalcLatLonBoundingBox2D() throws URISyntaxException, IOExcepti // Derived by manually finding the minimum and maximum lat & lon values of the expected points in the // shouldCalcConnectedLatLonBoundaryPoints2D test. LatLonRect expectedBB = - new LatLonRect(LatLonPoint.create(44.8740, -252.9728), LatLonPoint.create(85.1317, -106.0074)); + new LatLonRect(LatLonPoint.create(44.8741, -252.9727), LatLonPoint.create(85.1318, -106.0074)); - assertThat(actualBB.nearlyEquals(expectedBB)).isTrue(); + assertThat(actualBB.nearlyEquals(expectedBB, TOL)).isTrue(); } private HorizCoordSys getHorizCoordSysOfDataset(String resourceName) throws URISyntaxException, IOException { diff --git a/cdm/core/src/test/java/ucar/unidata/geoloc/projection/TestUtm.java b/cdm/core/src/test/java/ucar/unidata/geoloc/projection/TestUtm.java index 3116453769..1febe4e5b6 100644 --- a/cdm/core/src/test/java/ucar/unidata/geoloc/projection/TestUtm.java +++ b/cdm/core/src/test/java/ucar/unidata/geoloc/projection/TestUtm.java @@ -261,8 +261,8 @@ public void testStuff() { UtmProjection utm = new UtmProjection(17, true); LatLonPoint ll = utm.projToLatLon(577.8000000000001, 2951.8); System.out.printf("%15.12f %15.12f%n", ll.getLatitude(), ll.getLongitude()); - assert Misc.nearlyEquals(ll.getLongitude(), -80.21802662821469, 1.0e-8); - assert Misc.nearlyEquals(ll.getLatitude(), 26.685132668190793, 1.0e-8); + assert Misc.nearlyEquals(ll.getLongitude(), -80.21802662821469, 1.0e-5); + assert Misc.nearlyEquals(ll.getLatitude(), 26.685132668190793, 1.0e-5); } } diff --git a/gradle/any/testing.gradle b/gradle/any/testing.gradle index 1542486b99..d9ba397717 100644 --- a/gradle/any/testing.gradle +++ b/gradle/any/testing.gradle @@ -4,6 +4,7 @@ apply from: "$rootDir/gradle/any/properties.gradle" tasks.withType(Test).all { // Propagates system properties set on the Gradle process to the test executors. addFilteredSysProps(systemProperties) + jvmArgs = ['-Xmx1024m'] if (isJenkins && !isRelease) { // On Jenkins, don't let test failures fail the build unless we are doing we release; we want