Skip to content

Commit

Permalink
Make Snappy technically an optional dependency
Browse files Browse the repository at this point in the history
* Separate SnappyLoader into SnappyLoader and SnappyLoaderInternal
  This allows SnappyLoader to function (and correclty respond that snappy is not available)
  in the case where the Snappy libraryis completely missing.  Previously it identified and
  worked around cases where the native code was missing or incompatable, but the java code was required.

* Snappy is still marked as a dependency in gradle so downstream projects have to actively opt out of it.
  • Loading branch information
lbergelson committed Sep 17, 2024
1 parent 46f6f0f commit 4e736c6
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 27 deletions.
49 changes: 22 additions & 27 deletions src/main/java/htsjdk/samtools/util/SnappyLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,22 @@

import htsjdk.samtools.Defaults;
import htsjdk.samtools.SAMException;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* Checks if Snappy is available, and provides methods for wrapping InputStreams and OutputStreams with Snappy if so.
* Checks if Snappy is available, and provides methods for wrapping InputStreams and OutputStreams with Snappy if it is.
*
* @implNote this class must not import Snappy code in order to prevent exceptions if the Snappy Library is not available.
* Snappy code is handled by {@link SnappyLoaderInternal}.
*/
public class SnappyLoader {
private static final int SNAPPY_BLOCK_SIZE = 32768; // keep this as small as can be without hurting compression ratio.
private static final Log logger = Log.getInstance(SnappyLoader.class);

private final boolean snappyAvailable;


public SnappyLoader() {
this(Defaults.DISABLE_SNAPPY_COMPRESSOR);
}
Expand All @@ -52,24 +49,18 @@ public SnappyLoader() {
if (disableSnappy) {
logger.debug("Snappy is disabled via system property.");
snappyAvailable = false;
}
else {
boolean tmpSnappyAvailable = false;
try (final OutputStream test = new SnappyOutputStream(new ByteArrayOutputStream(1000))){
test.write("Hello World!".getBytes());
tmpSnappyAvailable = true;
logger.debug("Snappy successfully loaded.");
}
/*
* ExceptionInInitializerError: thrown by Snappy if native libs fail to load.
* IllegalStateException: thrown within the `test.write` call above if no UTF-8 encoder is found.
* IOException: potentially thrown by the `test.write` and `test.close` calls.
* SnappyError: potentially thrown for a variety of reasons by Snappy.
*/
catch (final ExceptionInInitializerError | IllegalStateException | IOException | SnappyError e) {
logger.warn(e, "Snappy native library failed to load.");
} else {
boolean tmpAvailable;
try {
//This triggers trying to import Snappy code, which causes an exception if the library is missing.
tmpAvailable = SnappyLoaderInternal.tryToLoadSnappy();
} catch (NoClassDefFoundError e){
tmpAvailable = false;
logger.error(e, "Snappy java library was requested but not found. If Snappy is " +
"intentionally missing, this message may be suppressed by setting " +
"-D"+ Defaults.SAMJDK_PREFIX + Defaults.DISABLE_SNAPPY_PROPERTY_NAME + "=true " );
}
snappyAvailable = tmpSnappyAvailable;
snappyAvailable = tmpAvailable;
}
}

Expand All @@ -81,18 +72,21 @@ public SnappyLoader() {
* @throws SAMException if Snappy is not available will throw an exception.
*/
public InputStream wrapInputStream(final InputStream inputStream) {
return wrapWithSnappyOrThrow(inputStream, SnappyInputStream::new);
return wrapWithSnappyOrThrow(inputStream, SnappyLoaderInternal.getInputStreamWrapper());
}

/**
* Wrap an OutputStream in a SnappyOutputStream.
* @throws SAMException if Snappy is not available
*/
public OutputStream wrapOutputStream(final OutputStream outputStream) {
return wrapWithSnappyOrThrow(outputStream, (stream) -> new SnappyOutputStream(stream, SNAPPY_BLOCK_SIZE));
return wrapWithSnappyOrThrow(outputStream, SnappyLoaderInternal.getOutputStreamWrapper());
}

private interface IOFunction<T,R> {
/**
* Function which can throw IOExceptions
*/
interface IOFunction<T,R> {
R apply(T input) throws IOException;
}

Expand All @@ -111,4 +105,5 @@ private <T,R> R wrapWithSnappyOrThrow(T stream, IOFunction<T, R> wrapper){
throw new SAMException(errorMessage);
}
}

}
69 changes: 69 additions & 0 deletions src/main/java/htsjdk/samtools/util/SnappyLoaderInternal.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package htsjdk.samtools.util;

import htsjdk.annotations.InternalAPI;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* This class is the only one which should actually import Snappy Classes. It is separated from SnappyLoader to allow
* snappy to be an optional dependency. Referencing snappy classes directly if the library is unavailable causes a
* NoClassDefFoundError, so use this instead.
*
* This should only be referenced by {@link SnappyLoader} in order to prevent accidental imports of Snappy classes.
*
*/
@InternalAPI
class SnappyLoaderInternal {
private static final Log logger = Log.getInstance(SnappyLoaderInternal.class);
private static final int SNAPPY_BLOCK_SIZE = 32768; // keep this as small as can be without hurting compression ratio.

/**
* Try to load Snappy's native library.
*
* Note that calling this when snappy is not available will throw NoClassDefFoundError!
*
* @return true iff Snappy's native libraries are loaded and functioning.
*/
static boolean tryToLoadSnappy() {
final boolean snappyAvailable;
boolean tmpSnappyAvailable = false;
try (final OutputStream test = new SnappyOutputStream(new ByteArrayOutputStream(1000))){
test.write("Hello World!".getBytes());
tmpSnappyAvailable = true;
logger.debug("Snappy successfully loaded.");
}
/*
* ExceptionInInitializerError: thrown by Snappy if native libs fail to load.
* IllegalStateException: thrown within the `test.write` call above if no UTF-8 encoder is found.
* IOException: potentially thrown by the `test.write` and `test.close` calls.
* SnappyError: potentially thrown for a variety of reasons by Snappy.
*/
catch (final ExceptionInInitializerError | IllegalStateException | IOException | SnappyError e) {
logger.warn(e, "Snappy native library failed to load.");
}
snappyAvailable = tmpSnappyAvailable;
return snappyAvailable;
}


/**
* @return a function which wraps an InputStream in a new SnappyInputStream
*/
static SnappyLoader.IOFunction<InputStream, InputStream> getInputStreamWrapper(){
return SnappyInputStream::new;
}

/**
* @return a function which wraps an OutputStream in a new SnappyOutputStream with an appropriate block size
*/
static SnappyLoader.IOFunction<OutputStream, OutputStream> getOutputStreamWrapper(){
return (stream) -> new SnappyOutputStream(stream, SNAPPY_BLOCK_SIZE);
}

}

0 comments on commit 4e736c6

Please sign in to comment.