diff --git a/src/main/java/htsjdk/samtools/cram/compression/rans/Constants.java b/src/main/java/htsjdk/samtools/cram/compression/rans/Constants.java index 3ae46a76a1..f970582f48 100644 --- a/src/main/java/htsjdk/samtools/cram/compression/rans/Constants.java +++ b/src/main/java/htsjdk/samtools/cram/compression/rans/Constants.java @@ -5,7 +5,6 @@ final public class Constants { public static final int TOTAL_FREQ = (1 << TOTAL_FREQ_SHIFT); // 4096 public static final int NUMBER_OF_SYMBOLS = 256; public static final int RANS_4x8_LOWER_BOUND = 1 << 23; - public static final int RANS_4x8_NUM_INTERLEAVED_STREAMS = 4; public static final int RANS_4x8_ORDER_BYTE_LENGTH = 1; public static final int RANS_4x8_COMPRESSED_BYTE_LENGTH = 4; public static final int RANS_4x8_RAW_BYTE_LENGTH = 4; diff --git a/src/main/java/htsjdk/samtools/cram/compression/rans/rans4x8/RANS4x8Encode.java b/src/main/java/htsjdk/samtools/cram/compression/rans/rans4x8/RANS4x8Encode.java index 0d962baf82..827fd4b0c7 100644 --- a/src/main/java/htsjdk/samtools/cram/compression/rans/rans4x8/RANS4x8Encode.java +++ b/src/main/java/htsjdk/samtools/cram/compression/rans/rans4x8/RANS4x8Encode.java @@ -14,7 +14,7 @@ public class RANS4x8Encode extends RANSEncode { // streams smaller than this value don't have sufficient symbol context for ORDER-1 encoding, // so always use ORDER-0 - private static final int MINIMUM__ORDER_1_SIZE = 4; + private static final int MINIMUM_ORDER_1_SIZE = 4; private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0); public ByteBuffer compress(final ByteBuffer inBuffer, final RANS4x8Params params) { @@ -22,7 +22,7 @@ public ByteBuffer compress(final ByteBuffer inBuffer, final RANS4x8Params params return EMPTY_BUFFER; } initializeRANSEncoder(); - if (inBuffer.remaining() < MINIMUM__ORDER_1_SIZE) { + if (inBuffer.remaining() < MINIMUM_ORDER_1_SIZE) { // ORDER-1 encoding of less than 4 bytes is not permitted, so just use ORDER-0 return compressOrder0Way4(inBuffer); } diff --git a/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Decode.java b/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Decode.java index afc0eee3ba..62fc3a911d 100644 --- a/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Decode.java +++ b/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Decode.java @@ -166,7 +166,6 @@ private void uncompressOrder1WayN( final ByteBuffer inBuffer, final ByteBuffer outBuffer, final RANSNx16Params ransNx16Params) { - initializeRANSDecoder(); // read the first byte final int frequencyTableFirstByte = (inBuffer.get() & 0xFF); @@ -189,11 +188,19 @@ private void uncompressOrder1WayN( freqTableSource = ByteBuffer.allocate(uncompressedLength); final ByteBuffer compressedFrequencyTableBuffer = ByteBuffer.wrap(compressedFreqTable); compressedFrequencyTableBuffer.order(ByteOrder.LITTLE_ENDIAN); - uncompressOrder0WayN(compressedFrequencyTableBuffer, freqTableSource, uncompressedLength,new RANSNx16Params(0x00)); // format flags = 0 + + // Uncompress using RANSNx16 Order 0, Nway = 4. + // formatFlags = (~RANSNx16Params.ORDER_FLAG_MASK & ~RANSNx16Params.N32_FLAG_MASK) = ~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK) + uncompressOrder0WayN(compressedFrequencyTableBuffer, freqTableSource, uncompressedLength,new RANSNx16Params(~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK))); // format flags = 0 } else { freqTableSource = inBuffer; } + + // Moving initializeRANSDecoder() from the beginning of this method to this point in the code + // due to the nested call to uncompressOrder0WayN, which also invokes the initializeRANSDecoder() method. + // TODO: we should work on a more permanent solution for this issue! + initializeRANSDecoder(); final int shift = frequencyTableFirstByte >> 4; readFrequencyTableOrder1(freqTableSource, shift); final int outputSize = outBuffer.remaining(); diff --git a/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Encode.java b/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Encode.java index ecdfa8801f..cd6ba4c3ee 100644 --- a/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Encode.java +++ b/src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Encode.java @@ -16,7 +16,6 @@ public class RANSNx16Encode extends RANSEncode { ///////////////////////////////////////////////////////////////////////////////////////////////// private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0); - private static final int MINIMUM__ORDER_1_SIZE = 4; public ByteBuffer compress(final ByteBuffer inBuffer, final RANSNx16Params ransNx16Params) { if (inBuffer.remaining() == 0) { @@ -78,8 +77,8 @@ public ByteBuffer compress(final ByteBuffer inBuffer, final RANSNx16Params ransN return outBuffer; } - // if after encoding pack and rle, the inputBuffer size < 4, then use order 0 - if (inputBuffer.remaining() < MINIMUM__ORDER_1_SIZE && ransNx16Params.getOrder() == RANSParams.ORDER.ONE) { + // if after encoding pack and rle, the inputBuffer size < Nway, then use order 0 + if (inputBuffer.remaining() < ransNx16Params.getNumInterleavedRANSStates() && ransNx16Params.getOrder() == RANSParams.ORDER.ONE) { // set order flag to "0" in the first byte of the outBuffer outBuffer.put(0,(byte)(outBuffer.get(0) & ~RANSNx16Params.ORDER_FLAG_MASK)); @@ -191,7 +190,6 @@ private void compressOrder1WayN ( final ByteBuffer inBuffer, final RANSNx16Params ransNx16Params, final ByteBuffer outBuffer) { - initializeRANSEncoder(); final int[][] frequencies = buildFrequenciesOrder1(inBuffer, ransNx16Params.getNumInterleavedRANSStates()); // normalise frequencies with a variable shift calculated @@ -208,9 +206,15 @@ private void compressOrder1WayN ( frequencyTable.limit(uncompressedFrequencyTableSize); frequencyTable.rewind(); - // compressed frequency table using RANS Nx16 Order 0 - compressOrder0WayN(frequencyTable, new RANSNx16Params(0x00), compressedFrequencyTable); + // Compress using RANSNx16 Order 0, Nway = 4. + // formatFlags = (~RANSNx16Params.ORDER_FLAG_MASK & ~RANSNx16Params.N32_FLAG_MASK) = ~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK) + compressOrder0WayN(frequencyTable, new RANSNx16Params(~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK)), compressedFrequencyTable); frequencyTable.rewind(); + + // Moving initializeRANSEncoder() from the beginning of this method to this point in the code + // due to the nested call to compressOrder0WayN, which also invokes the initializeRANSEncoder() method. + // TODO: we should work on a more permanent solution for this issue! + initializeRANSEncoder(); final int compressedFrequencyTableSize = compressedFrequencyTable.limit(); final ByteBuffer cp = outBuffer.slice(); diff --git a/src/test/java/htsjdk/samtools/cram/compression/rans/RansTest.java b/src/test/java/htsjdk/samtools/cram/compression/rans/RansTest.java index 78ce092ff5..dd63e45c2a 100644 --- a/src/test/java/htsjdk/samtools/cram/compression/rans/RansTest.java +++ b/src/test/java/htsjdk/samtools/cram/compression/rans/RansTest.java @@ -51,6 +51,8 @@ public Object[][] getRansTestData() { { new TestDataEnvelope(new byte[] {0, 1}) }, { new TestDataEnvelope(new byte[] {0, 1, 2}) }, { new TestDataEnvelope(new byte[] {0, 1, 2, 3}) }, + { new TestDataEnvelope(new byte[] {1, 2, 3, 4}) }, + { new TestDataEnvelope(new byte[] {1, 2, 3, 4, 5}) }, { new TestDataEnvelope(new byte[1000]) }, { new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> (byte) 1)) }, { new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> Byte.MIN_VALUE)) }, @@ -58,6 +60,10 @@ public Object[][] getRansTestData() { { new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> (byte) index.intValue())) }, { new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> index < n / 2 ? (byte) 0 : (byte) 1)) }, { new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> index < n % 2 ? (byte) 0 : (byte) 1)) }, + { new TestDataEnvelope(randomBytesFromGeometricDistribution(10, 0.1)) }, + { new TestDataEnvelope(randomBytesFromGeometricDistribution(31, 0.1)) }, + { new TestDataEnvelope(randomBytesFromGeometricDistribution(32, 0.1)) }, + { new TestDataEnvelope(randomBytesFromGeometricDistribution(33, 0.1)) }, { new TestDataEnvelope(randomBytesFromGeometricDistribution(1000, 0.1)) }, { new TestDataEnvelope(randomBytesFromGeometricDistribution(1000, 0.01)) }, { new TestDataEnvelope(randomBytesFromGeometricDistribution(10 * 1000 * 1000 + 1, 0.01)) },