Skip to content

Commit

Permalink
Addressing the feedback from Nov 7 and Nov 20 - part 3
Browse files Browse the repository at this point in the history
  • Loading branch information
yash-puligundla committed Dec 6, 2023
1 parent d2802b1 commit e6b06a5
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 11 deletions.
1 change: 0 additions & 1 deletion src/main/java/htsjdk/samtools/cram/compression/rans/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ final public class Constants {
public static final int TOTAL_FREQ = (1 << TOTAL_FREQ_SHIFT); // 4096
public static final int NUMBER_OF_SYMBOLS = 256;
public static final int RANS_4x8_LOWER_BOUND = 1 << 23;
public static final int RANS_4x8_NUM_INTERLEAVED_STREAMS = 4;
public static final int RANS_4x8_ORDER_BYTE_LENGTH = 1;
public static final int RANS_4x8_COMPRESSED_BYTE_LENGTH = 4;
public static final int RANS_4x8_RAW_BYTE_LENGTH = 4;
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/htsjdk/samtools/cram/compression/rans/rans4x8/RANS4x8Encode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ public class RANS4x8Encode extends RANSEncode<RANS4x8Params> {

// streams smaller than this value don't have sufficient symbol context for ORDER-1 encoding,
// so always use ORDER-0
private static final int MINIMUM__ORDER_1_SIZE = 4;
private static final int MINIMUM_ORDER_1_SIZE = 4;
private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0);

public ByteBuffer compress(final ByteBuffer inBuffer, final RANS4x8Params params) {
if (inBuffer.remaining() == 0) {
return EMPTY_BUFFER;
}
initializeRANSEncoder();
if (inBuffer.remaining() < MINIMUM__ORDER_1_SIZE) {
if (inBuffer.remaining() < MINIMUM_ORDER_1_SIZE) {
// ORDER-1 encoding of less than 4 bytes is not permitted, so just use ORDER-0
return compressOrder0Way4(inBuffer);
}
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Decode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ private void uncompressOrder1WayN(
final ByteBuffer inBuffer,
final ByteBuffer outBuffer,
final RANSNx16Params ransNx16Params) {
initializeRANSDecoder();

// read the first byte
final int frequencyTableFirstByte = (inBuffer.get() & 0xFF);
Expand All @@ -189,11 +188,19 @@ private void uncompressOrder1WayN(
freqTableSource = ByteBuffer.allocate(uncompressedLength);
final ByteBuffer compressedFrequencyTableBuffer = ByteBuffer.wrap(compressedFreqTable);
compressedFrequencyTableBuffer.order(ByteOrder.LITTLE_ENDIAN);
uncompressOrder0WayN(compressedFrequencyTableBuffer, freqTableSource, uncompressedLength,new RANSNx16Params(0x00)); // format flags = 0

// Uncompress using RANSNx16 Order 0, Nway = 4.
// formatFlags = (~RANSNx16Params.ORDER_FLAG_MASK & ~RANSNx16Params.N32_FLAG_MASK) = ~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK)
uncompressOrder0WayN(compressedFrequencyTableBuffer, freqTableSource, uncompressedLength,new RANSNx16Params(~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK))); // format flags = 0
}
else {
freqTableSource = inBuffer;
}

// Moving initializeRANSDecoder() from the beginning of this method to this point in the code
// due to the nested call to uncompressOrder0WayN, which also invokes the initializeRANSDecoder() method.
// TODO: we should work on a more permanent solution for this issue!
initializeRANSDecoder();
final int shift = frequencyTableFirstByte >> 4;
readFrequencyTableOrder1(freqTableSource, shift);
final int outputSize = outBuffer.remaining();
Expand Down
16 changes: 10 additions & 6 deletions src/main/java/htsjdk/samtools/cram/compression/rans/ransnx16/RANSNx16Encode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ public class RANSNx16Encode extends RANSEncode<RANSNx16Params> {
/////////////////////////////////////////////////////////////////////////////////////////////////

private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0);
private static final int MINIMUM__ORDER_1_SIZE = 4;

public ByteBuffer compress(final ByteBuffer inBuffer, final RANSNx16Params ransNx16Params) {
if (inBuffer.remaining() == 0) {
Expand Down Expand Up @@ -78,8 +77,8 @@ public ByteBuffer compress(final ByteBuffer inBuffer, final RANSNx16Params ransN
return outBuffer;
}

// if after encoding pack and rle, the inputBuffer size < 4, then use order 0
if (inputBuffer.remaining() < MINIMUM__ORDER_1_SIZE && ransNx16Params.getOrder() == RANSParams.ORDER.ONE) {
// if after encoding pack and rle, the inputBuffer size < Nway, then use order 0
if (inputBuffer.remaining() < ransNx16Params.getNumInterleavedRANSStates() && ransNx16Params.getOrder() == RANSParams.ORDER.ONE) {

// set order flag to "0" in the first byte of the outBuffer
outBuffer.put(0,(byte)(outBuffer.get(0) & ~RANSNx16Params.ORDER_FLAG_MASK));
Expand Down Expand Up @@ -191,7 +190,6 @@ private void compressOrder1WayN (
final ByteBuffer inBuffer,
final RANSNx16Params ransNx16Params,
final ByteBuffer outBuffer) {
initializeRANSEncoder();
final int[][] frequencies = buildFrequenciesOrder1(inBuffer, ransNx16Params.getNumInterleavedRANSStates());

// normalise frequencies with a variable shift calculated
Expand All @@ -208,9 +206,15 @@ private void compressOrder1WayN (
frequencyTable.limit(uncompressedFrequencyTableSize);
frequencyTable.rewind();

// compressed frequency table using RANS Nx16 Order 0
compressOrder0WayN(frequencyTable, new RANSNx16Params(0x00), compressedFrequencyTable);
// Compress using RANSNx16 Order 0, Nway = 4.
// formatFlags = (~RANSNx16Params.ORDER_FLAG_MASK & ~RANSNx16Params.N32_FLAG_MASK) = ~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK)
compressOrder0WayN(frequencyTable, new RANSNx16Params(~(RANSNx16Params.ORDER_FLAG_MASK | RANSNx16Params.N32_FLAG_MASK)), compressedFrequencyTable);
frequencyTable.rewind();

// Moving initializeRANSEncoder() from the beginning of this method to this point in the code
// due to the nested call to compressOrder0WayN, which also invokes the initializeRANSEncoder() method.
// TODO: we should work on a more permanent solution for this issue!
initializeRANSEncoder();
final int compressedFrequencyTableSize = compressedFrequencyTable.limit();
final ByteBuffer cp = outBuffer.slice();

Expand Down
6 changes: 6 additions & 0 deletions src/test/java/htsjdk/samtools/cram/compression/rans/RansTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,19 @@ public Object[][] getRansTestData() {
{ new TestDataEnvelope(new byte[] {0, 1}) },
{ new TestDataEnvelope(new byte[] {0, 1, 2}) },
{ new TestDataEnvelope(new byte[] {0, 1, 2, 3}) },
{ new TestDataEnvelope(new byte[] {1, 2, 3, 4}) },
{ new TestDataEnvelope(new byte[] {1, 2, 3, 4, 5}) },
{ new TestDataEnvelope(new byte[1000]) },
{ new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> (byte) 1)) },
{ new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> Byte.MIN_VALUE)) },
{ new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> Byte.MAX_VALUE)) },
{ new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> (byte) index.intValue())) },
{ new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> index < n / 2 ? (byte) 0 : (byte) 1)) },
{ new TestDataEnvelope(getNBytesWithValues(1000, (n, index) -> index < n % 2 ? (byte) 0 : (byte) 1)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(10, 0.1)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(31, 0.1)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(32, 0.1)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(33, 0.1)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(1000, 0.1)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(1000, 0.01)) },
{ new TestDataEnvelope(randomBytesFromGeometricDistribution(10 * 1000 * 1000 + 1, 0.01)) },
Expand Down

0 comments on commit e6b06a5

Please sign in to comment.