diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java index 2a6823d36..04188c212 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java @@ -18,33 +18,34 @@ package eu.fasten.analyzer.javacgopal; -import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.INCLUDING_ALL_SUBTYPES; - import eu.fasten.analyzer.javacgopal.data.CGAlgorithm; import eu.fasten.analyzer.javacgopal.data.OPALCallGraphConstructor; -import eu.fasten.core.data.PartialJavaCallGraph; -import eu.fasten.core.data.opal.MavenCoordinate; import eu.fasten.analyzer.javacgopal.data.OPALPartialCallGraphConstructor; import eu.fasten.core.data.DirectedGraph; import eu.fasten.core.data.JSONUtils; +import eu.fasten.core.data.PartialJavaCallGraph; +import eu.fasten.core.data.opal.MavenCoordinate; import eu.fasten.core.maven.utils.MavenUtilities; import eu.fasten.core.merge.CGMerger; import eu.fasten.core.merge.CallGraphUtils; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.util.Collection; import org.json.JSONObject; import org.json.JSONTokener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; + import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.IOException; import java.nio.file.Paths; import java.text.DecimalFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.INCLUDING_ALL_SUBTYPES; + /** * Makes javacg-opal module runnable from command line. */ @@ -53,7 +54,7 @@ public class Main implements Runnable { private static final Logger logger = LoggerFactory.getLogger(Main.class); - @CommandLine.Option(names = { "-o", + @CommandLine.Option(names = {"-o", "--output" }, paramLabel = "OUT", description = "Output directory path", defaultValue = "") String output; @@ -274,8 +275,8 @@ private PartialJavaCallGraph generatePCG(final T artifact, final String arti revisionCallGraph = generatePCGFromFile(getArtifact(artifact), artifactName, algorithm); } else { revisionCallGraph = OPALPartialCallGraphConstructor - .createPartialJavaCG((MavenCoordinate) artifact, algorithm, - Long.parseLong(timestamp), getArtifactRepo(), INCLUDING_ALL_SUBTYPES); + .createPartialJavaCG((MavenCoordinate) artifact, algorithm, + Long.parseLong(timestamp), getArtifactRepo(), null, INCLUDING_ALL_SUBTYPES); } return revisionCallGraph; } diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java index 406f8eeed..12509fad1 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java @@ -18,21 +18,6 @@ package eu.fasten.analyzer.javacgopal; -import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA; -import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES; -import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO; -import static java.lang.System.currentTimeMillis; - -import java.io.File; -import java.util.Optional; - -import org.json.JSONObject; -import org.pf4j.Extension; -import org.pf4j.Plugin; -import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import eu.fasten.analyzer.javacgopal.data.OPALPartialCallGraphConstructor; import eu.fasten.core.data.Constants; import eu.fasten.core.data.JSONUtils; @@ -42,6 +27,22 @@ import eu.fasten.core.data.opal.exceptions.MissingArtifactException; import eu.fasten.core.data.opal.exceptions.OPALException; import eu.fasten.core.plugins.AbstractKafkaPlugin; +import eu.fasten.core.plugins.DataRW; +import org.json.JSONObject; +import org.pf4j.Extension; +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Optional; + +import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA; +import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES; +import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO; +import static java.lang.System.currentTimeMillis; public class OPALPlugin extends Plugin { @@ -50,12 +51,13 @@ public OPALPlugin(PluginWrapper wrapper) { } @Extension - public static class OPAL extends AbstractKafkaPlugin { + public static class OPAL extends AbstractKafkaPlugin implements DataRW { private final Logger logger = LoggerFactory.getLogger(getClass()); private PartialJavaCallGraph graph; private String outputPath; + private static String baseDir; @Override public void consume(String kafkaRecord, ProcessingLane l) { @@ -71,14 +73,17 @@ public void consume(String kafkaRecord, ProcessingLane l) { } var artifactRepository = fixResetAndGetArtifactRepo(json); final var mavenCoordinate = new MavenCoordinate(json); + final var mavenArtifactFile = new File(Paths.get(baseDir, ".m2", "repository", mavenCoordinate.toPath().toString()).toString()); long startTime = System.currentTimeMillis(); try { // Generate CG and measure construction duration. logger.info("[CG-GENERATION] [UNPROCESSED] [-1] [" + mavenCoordinate.getCoordinate() + "] [NONE] "); long date = json.optLong("releaseDate", -1); - this.graph = OPALPartialCallGraphConstructor.createPartialJavaCG(mavenCoordinate, - CHA, date, artifactRepository, ONLY_STATIC_CALLSITES); - long duration = currentTimeMillis() - startTime; + + this.graph = OPALPartialCallGraphConstructor.createPartialJavaCG(mavenCoordinate, + CHA, date, artifactRepository, mavenArtifactFile, ONLY_STATIC_CALLSITES); + + long duration = currentTimeMillis() - startTime; if (this.graph.isCallGraphEmpty()) { throw new EmptyCallGraphException(); @@ -149,5 +154,10 @@ public long getMaxConsumeTimeout() { public long getSessionTimeout() { return 1800000; // Due to static membership we also want to tune the session timeout to 30 minutes. } + + @Override + public void setBaseDir(String baseDir) { + OPAL.baseDir = baseDir; + } } } diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java index 843c6368d..817399689 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java @@ -18,18 +18,16 @@ package eu.fasten.analyzer.javacgopal.data; +import com.google.common.collect.Lists; +import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy; +import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod; +import eu.fasten.analyzer.javacgopal.data.analysis.OPALType; +import eu.fasten.core.data.Constants; +import eu.fasten.core.data.JavaGraph; import eu.fasten.core.data.PartialJavaCallGraph; -import java.io.File; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - +import eu.fasten.core.data.opal.MavenArtifactDownloader; +import eu.fasten.core.data.opal.MavenCoordinate; +import eu.fasten.core.data.opal.exceptions.OPALException; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.StringEscapeUtils; import org.opalj.br.Annotation; @@ -45,20 +43,20 @@ import org.opalj.value.ValueInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy; -import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod; -import eu.fasten.analyzer.javacgopal.data.analysis.OPALType; -import eu.fasten.core.data.Constants; -import eu.fasten.core.data.JavaGraph; -import eu.fasten.core.data.opal.MavenArtifactDownloader; -import eu.fasten.core.data.opal.MavenCoordinate; -import eu.fasten.core.data.opal.exceptions.OPALException; import scala.Function1; import scala.collection.JavaConverters; +import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + /** * Call graphs that are not still fully resolved. i.e. isolated call graphs which within-artifact * calls (edges) are known as internal calls and Cross-artifact calls are known as external calls. @@ -68,6 +66,7 @@ public class OPALPartialCallGraphConstructor { private static final Logger logger = LoggerFactory.getLogger(OPALPartialCallGraph.class); private OPALPartialCallGraph pcg; + private static boolean isTempJarFileUsed = false; /** * Given a file, algorithm and main class (in case of application package) @@ -95,29 +94,52 @@ public OPALPartialCallGraph construct(OPALCallGraph ocg, CallPreservationStrateg } throw e; } - + return pcg; } + /** + * Downloads the Maven artifact if it does not exist. + * + * @param artifactFile If `null` provided, the Jar file would be downloaded to a temp file. + * @return + */ + private static File downloadMavenArtifactIfNeeded(final MavenCoordinate coordinate, final String artifactRepo, File artifactFile) { + // Download the Jar file if the given file does not exist. + if (artifactFile != null) { + if (!artifactFile.exists()) { + logger.warn("Couldn't find the local JAR on filesystem for " + coordinate.toString()); + logger.info("About to download the artifact file for {} from {}", coordinate, artifactRepo); + artifactFile = new MavenArtifactDownloader(coordinate, artifactFile).downloadArtifact(artifactRepo); + } + } else { + // Downloads to a temporary file + logger.info("Downloading the artifact to a temporary file"); + artifactFile = new MavenArtifactDownloader(coordinate).downloadArtifact(artifactRepo); + isTempJarFileUsed = true; + } + return artifactFile; + } + /** * Creates RevisionCallGraph using OPAL call graph generator for a given maven * coordinate. It also sets the forge to "mvn". * - * @param coordinate maven coordinate of the revision to be processed - * @param timestamp timestamp of the revision release + * @param coordinate maven coordinate of the revision to be processed + * @param timestamp timestamp of the revision release + * @param artifactRepo the repository from which the artifact/Jar should be downloaded from + * @param artifactFile The path to an existing Jar file. * @return RevisionCallGraph of the given coordinate. */ public static PartialJavaCallGraph createPartialJavaCG( - final MavenCoordinate coordinate, - CGAlgorithm algorithm, final long timestamp, final String artifactRepo, CallPreservationStrategy callSiteOnly) { + final MavenCoordinate coordinate, + CGAlgorithm algorithm, final long timestamp, final String artifactRepo, File artifactFile, + CallPreservationStrategy callSiteOnly) { - File file = null; + artifactFile = downloadMavenArtifactIfNeeded(coordinate, artifactRepo, artifactFile); try { - logger.info("About to download {} from {}", coordinate, artifactRepo); - - file = new MavenArtifactDownloader(coordinate).downloadArtifact(artifactRepo); - final var opalCG = new OPALCallGraphConstructor().construct(file, algorithm); + final var opalCG = new OPALCallGraphConstructor().construct(artifactFile, algorithm); final var partialCallGraph = new OPALPartialCallGraphConstructor().construct(opalCG, callSiteOnly); return new PartialJavaCallGraph(Constants.mvnForge, coordinate.getProduct(), @@ -126,9 +148,9 @@ public static PartialJavaCallGraph createPartialJavaCG( partialCallGraph.classHierarchy, partialCallGraph.graph); } finally { - if (file != null) { - // TODO use apache commons FileUtils instead - file.delete(); + if (isTempJarFileUsed) { + // TODO use apache commons FileUtils instead + artifactFile.delete(); } } } diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/OPALPluginTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/OPALPluginTest.java index eabdda5d1..71758b8b6 100644 --- a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/OPALPluginTest.java +++ b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/OPALPluginTest.java @@ -18,38 +18,55 @@ package eu.fasten.analyzer.javacgopal; -import static eu.fasten.core.plugins.KafkaPlugin.ProcessingLane.NORMAL; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; - +import eu.fasten.core.data.PartialJavaCallGraph; +import eu.fasten.core.data.opal.exceptions.EmptyCallGraphException; +import eu.fasten.core.data.opal.exceptions.MissingArtifactException; +import org.apache.commons.io.FileUtils; import org.json.JSONException; import org.json.JSONObject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import eu.fasten.core.data.PartialJavaCallGraph; -import eu.fasten.core.data.opal.exceptions.EmptyCallGraphException; -import eu.fasten.core.data.opal.exceptions.MissingArtifactException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static eu.fasten.core.plugins.KafkaPlugin.ProcessingLane.NORMAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class OPALPluginTest { private OPALPlugin.OPAL plugin; + private static Path baseDirPath; + + @BeforeAll + public static void setUpBaseDir() throws IOException { + baseDirPath = Files.createTempDirectory("tmp-opal").getFileName().toAbsolutePath(); + } + + @AfterAll + public static void CleanUpBaseDir() throws IOException { + FileUtils.deleteDirectory(baseDirPath.toFile()); + } @BeforeEach public void setUp() { plugin = new OPALPlugin.OPAL(); + plugin.setBaseDir(String.valueOf(baseDirPath)); } @Test public void testConsumerTopicNotSetByDefault() { - assertThrows(RuntimeException.class, ()->{ + assertThrows(RuntimeException.class, () -> { plugin.consumeTopic(); }); } diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/JSONUtilsTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/JSONUtilsTest.java index cdb857c90..e59453676 100644 --- a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/JSONUtilsTest.java +++ b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/JSONUtilsTest.java @@ -1,17 +1,12 @@ package eu.fasten.analyzer.javacgopal.data; -import static eu.fasten.core.utils.TestUtils.getTestResource; - +import eu.fasten.core.data.JSONUtils; import eu.fasten.core.data.PartialJavaCallGraph; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - +import eu.fasten.core.data.opal.MavenCoordinate; +import eu.fasten.core.data.opal.exceptions.MissingArtifactException; +import eu.fasten.core.data.opal.exceptions.OPALException; +import eu.fasten.core.maven.utils.MavenUtilities; +import eu.fasten.core.merge.CGMerger; import org.jooq.tools.csv.CSVReader; import org.json.JSONObject; import org.junit.jupiter.api.Assertions; @@ -23,12 +18,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.fasten.core.data.JSONUtils; -import eu.fasten.core.data.opal.MavenCoordinate; -import eu.fasten.core.data.opal.exceptions.MissingArtifactException; -import eu.fasten.core.data.opal.exceptions.OPALException; -import eu.fasten.core.maven.utils.MavenUtilities; -import eu.fasten.core.merge.CGMerger; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static eu.fasten.core.utils.TestUtils.getTestResource; class JSONUtilsTest { private static final Logger logger = LoggerFactory.getLogger(JSONUtilsTest.class); @@ -43,17 +42,17 @@ static void setUp() throws IOException, OPALException, MissingArtifactException var coordinate = new MavenCoordinate("com.github.shoothzj", "java-tool", "3.0.30.RELEASE", "jar"); graph = OPALPartialCallGraphConstructor.createPartialJavaCG(coordinate, - CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, CallPreservationStrategy.ONLY_STATIC_CALLSITES); + CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, null, CallPreservationStrategy.ONLY_STATIC_CALLSITES); coordinate = - new MavenCoordinate("abbot", "costello", "1.4.0", "jar"); + new MavenCoordinate("abbot", "costello", "1.4.0", "jar"); artifact = OPALPartialCallGraphConstructor.createPartialJavaCG(coordinate, - CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, CallPreservationStrategy.ONLY_STATIC_CALLSITES); + CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, null, CallPreservationStrategy.ONLY_STATIC_CALLSITES); coordinate = - new MavenCoordinate("abbot", "abbot", "1.4.0", "jar"); + new MavenCoordinate("abbot", "abbot", "1.4.0", "jar"); dependency = OPALPartialCallGraphConstructor.createPartialJavaCG(coordinate, - CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, CallPreservationStrategy.ONLY_STATIC_CALLSITES); + CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, null, CallPreservationStrategy.ONLY_STATIC_CALLSITES); final var deps = new ArrayList<>(Collections.singletonList(dependency)); deps.add(artifact); final var merger = new CGMerger(deps); @@ -102,7 +101,7 @@ void batchOfCGsTest() throws IOException { for (int i = 0; i < coordsSize; i++) { MavenCoordinate coord = coords.get(i); final var cg = OPALPartialCallGraphConstructor.createPartialJavaCG(coord, - CGAlgorithm.CHA, 1574072773, MavenUtilities.getRepos().get(0), CallPreservationStrategy.ONLY_STATIC_CALLSITES); + CGAlgorithm.CHA, 1574072773, MavenUtilities.getRepos().get(0), null, CallPreservationStrategy.ONLY_STATIC_CALLSITES); logger.debug("Serialization for: {}", coord.getCoordinate()); final var ser1 = avgConsumption(cg, "direct", "direct", 20, 20); diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java index ca541927e..9e3b80c9c 100644 --- a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java +++ b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java @@ -18,16 +18,15 @@ package eu.fasten.analyzer.javacgopal.data; -import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA; -import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES; -import static eu.fasten.core.utils.TestUtils.getTestResource; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; -import java.util.HashMap; - +import eu.fasten.core.data.Constants; +import eu.fasten.core.data.FastenJavaURI; +import eu.fasten.core.data.FastenURI; +import eu.fasten.core.data.JavaScope; +import eu.fasten.core.data.opal.MavenCoordinate; +import eu.fasten.core.data.opal.exceptions.MissingArtifactException; +import eu.fasten.core.data.opal.exceptions.OPALException; +import eu.fasten.core.maven.utils.MavenUtilities; +import it.unimi.dsi.fastutil.ints.IntIntPair; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -49,20 +48,20 @@ import org.opalj.collection.immutable.UIDSet; import org.opalj.collection.immutable.UIDSet1; import org.opalj.tac.cg.CallGraph; - -import eu.fasten.core.data.Constants; -import eu.fasten.core.data.FastenJavaURI; -import eu.fasten.core.data.FastenURI; -import eu.fasten.core.data.JavaScope; -import eu.fasten.core.data.opal.MavenCoordinate; -import eu.fasten.core.data.opal.exceptions.MissingArtifactException; -import eu.fasten.core.data.opal.exceptions.OPALException; -import eu.fasten.core.maven.utils.MavenUtilities; -import it.unimi.dsi.fastutil.ints.IntIntPair; import scala.Option; import scala.collection.Iterator; import scala.collection.mutable.HashSet; +import java.io.File; +import java.util.HashMap; + +import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA; +import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES; +import static eu.fasten.core.utils.TestUtils.getTestResource; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + class OPALOPALPartialCallGraphConstructorTest { private static OPALPartialCallGraph singleCallCG; @@ -162,8 +161,8 @@ void getNodeCount() { void createExtendedRevisionJavaCallGraph() throws MissingArtifactException, OPALException { var coordinate = new MavenCoordinate("org.slf4j", "slf4j-api", "1.7.29", "jar"); var cg = OPALPartialCallGraphConstructor - .createPartialJavaCG(coordinate, CGAlgorithm.CHA, 1574072773, - MavenUtilities.MAVEN_CENTRAL_REPO, CallPreservationStrategy.ONLY_STATIC_CALLSITES); + .createPartialJavaCG(coordinate, CGAlgorithm.CHA, 1574072773, + MavenUtilities.MAVEN_CENTRAL_REPO, null, CallPreservationStrategy.ONLY_STATIC_CALLSITES); assertNotNull(cg); Assertions.assertEquals(Constants.mvnForge, cg.forge); Assertions.assertEquals("1.7.29", cg.version); diff --git a/analyzer/parallel-vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/parallelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java b/analyzer/parallel-vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/parallelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java index 14c563164..5607ad377 100644 --- a/analyzer/parallel-vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/parallelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java +++ b/analyzer/parallel-vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/parallelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java @@ -18,13 +18,15 @@ package eu.fasten.analyzer.parallelvulnerabilitycacheinvalidationplugin; -import java.io.File; -import java.sql.Timestamp; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; - +import eu.fasten.core.maven.data.Revision; +import eu.fasten.core.maven.resolution.IMavenResolver; +import eu.fasten.core.maven.resolution.MavenResolver; +import eu.fasten.core.maven.resolution.MavenResolverIO; +import eu.fasten.core.maven.utils.MavenUtilities; +import eu.fasten.core.plugins.DataRW; +import eu.fasten.core.plugins.DependencyGraphUser; +import eu.fasten.core.plugins.KafkaPlugin; +import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet; import org.jooq.DSLContext; import org.json.JSONArray; import org.json.JSONObject; @@ -34,15 +36,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.fasten.core.maven.data.Revision; -import eu.fasten.core.maven.resolution.IMavenResolver; -import eu.fasten.core.maven.resolution.MavenResolver; -import eu.fasten.core.maven.resolution.MavenResolverIO; -import eu.fasten.core.maven.utils.MavenUtilities; -import eu.fasten.core.plugins.DataWriter; -import eu.fasten.core.plugins.DependencyGraphUser; -import eu.fasten.core.plugins.KafkaPlugin; -import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet; +import java.io.File; +import java.sql.Timestamp; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; public class ParallelVulnerabilityCacheInvalidationPlugin extends Plugin { public ParallelVulnerabilityCacheInvalidationPlugin(PluginWrapper wrapper) { @@ -50,7 +49,7 @@ public ParallelVulnerabilityCacheInvalidationPlugin(PluginWrapper wrapper) { } @Extension - public static class ParallelVulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataWriter { + public static class ParallelVulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataRW { private final Logger logger = LoggerFactory.getLogger(ParallelVulnerabilityCacheInvalidationExtension.class.getName()); diff --git a/analyzer/repo-cloner-plugin/src/main/java/eu/fasten/analyzer/repoclonerplugin/RepoClonerPlugin.java b/analyzer/repo-cloner-plugin/src/main/java/eu/fasten/analyzer/repoclonerplugin/RepoClonerPlugin.java index f9a047318..349e656b5 100644 --- a/analyzer/repo-cloner-plugin/src/main/java/eu/fasten/analyzer/repoclonerplugin/RepoClonerPlugin.java +++ b/analyzer/repo-cloner-plugin/src/main/java/eu/fasten/analyzer/repoclonerplugin/RepoClonerPlugin.java @@ -25,7 +25,7 @@ import eu.fasten.analyzer.repoclonerplugin.utils.SvnCloner; import eu.fasten.core.data.Constants; import eu.fasten.core.plugins.AbstractKafkaPlugin; -import eu.fasten.core.plugins.DataWriter; +import eu.fasten.core.plugins.DataRW; import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; import org.apache.commons.math3.util.Pair; @@ -46,7 +46,7 @@ public RepoClonerPlugin(PluginWrapper wrapper) { } @Extension - public static class RepoCloner extends AbstractKafkaPlugin implements DataWriter { + public static class RepoCloner extends AbstractKafkaPlugin implements DataRW { private final Logger logger = LoggerFactory.getLogger(RepoCloner.class.getName()); private String repoPath = null; diff --git a/analyzer/vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/vulnerabilitycacheinvalidationplugin/VulnerabilityCacheInvalidationPlugin.java b/analyzer/vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/vulnerabilitycacheinvalidationplugin/VulnerabilityCacheInvalidationPlugin.java index d52413e73..3b572152e 100644 --- a/analyzer/vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/vulnerabilitycacheinvalidationplugin/VulnerabilityCacheInvalidationPlugin.java +++ b/analyzer/vulnerability-cache-invalidation-plugin/src/main/java/eu/fasten/analyzer/vulnerabilitycacheinvalidationplugin/VulnerabilityCacheInvalidationPlugin.java @@ -18,24 +18,6 @@ package eu.fasten.analyzer.vulnerabilitycacheinvalidationplugin; -import java.io.File; -import java.sql.Timestamp; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import org.jooq.DSLContext; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.pf4j.Extension; -import org.pf4j.Plugin; -import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import eu.fasten.core.data.Constants; import eu.fasten.core.maven.data.ResolvedRevision; import eu.fasten.core.maven.data.Revision; @@ -44,9 +26,26 @@ import eu.fasten.core.maven.resolution.MavenResolver; import eu.fasten.core.maven.resolution.MavenResolverIO; import eu.fasten.core.maven.utils.MavenUtilities; -import eu.fasten.core.plugins.DataWriter; +import eu.fasten.core.plugins.DataRW; import eu.fasten.core.plugins.DependencyGraphUser; import eu.fasten.core.plugins.KafkaPlugin; +import org.jooq.DSLContext; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.pf4j.Extension; +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.sql.Timestamp; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; public class VulnerabilityCacheInvalidationPlugin extends Plugin { public VulnerabilityCacheInvalidationPlugin(PluginWrapper wrapper) { @@ -54,7 +53,7 @@ public VulnerabilityCacheInvalidationPlugin(PluginWrapper wrapper) { } @Extension - public static class VulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataWriter { + public static class VulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataRW { private final Logger logger = LoggerFactory.getLogger(VulnerabilityCacheInvalidationExtension.class.getName()); private List consumeTopics = new LinkedList<>(Collections.singletonList("fasten.CallableIndexExtension.out")); diff --git a/core/src/main/java/eu/fasten/core/data/opal/MavenArtifactDownloader.java b/core/src/main/java/eu/fasten/core/data/opal/MavenArtifactDownloader.java index e9582949a..c12081d42 100644 --- a/core/src/main/java/eu/fasten/core/data/opal/MavenArtifactDownloader.java +++ b/core/src/main/java/eu/fasten/core/data/opal/MavenArtifactDownloader.java @@ -15,6 +15,11 @@ */ package eu.fasten.core.data.opal; +import eu.fasten.core.data.opal.exceptions.MissingArtifactException; +import eu.fasten.core.maven.utils.MavenUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -22,16 +27,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.Arrays; import java.util.LinkedList; import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import eu.fasten.core.data.opal.exceptions.MissingArtifactException; -import eu.fasten.core.maven.utils.MavenUtilities; - /** * A set of methods for downloading POM and JAR files given Maven coordinates. */ @@ -42,11 +40,13 @@ public class MavenArtifactDownloader { private static final String[] defaultPackaging = {"zip", "aar", "tar.gz", "jar"}; private boolean foundPackage = false; private Optional artifactFile = Optional.empty(); - private MavenCoordinate mavenCoordinate; - private LinkedList mavenRepos; + private final MavenCoordinate mavenCoordinate; + private final LinkedList mavenRepos; private long startTime; /** + * This method downloads the Maven artifact to a temporary file. + * * @param mavenCoordinate A Maven coordinate in the for "groupId:artifactId:version" */ public MavenArtifactDownloader(final MavenCoordinate mavenCoordinate) { @@ -54,14 +54,28 @@ public MavenArtifactDownloader(final MavenCoordinate mavenCoordinate) { this.mavenRepos = mavenCoordinate.getMavenRepos(); } + /** + * This method downloads the Maven artifact to a given file. + * + * @param mavenCoordinate A Maven coordinate in the for "groupId:artifactId:version" + * @param artifactFilePath A path to store the downloaded Maven artifact to. + */ + public MavenArtifactDownloader(final MavenCoordinate mavenCoordinate, final File artifactFilePath) { + this.mavenCoordinate = mavenCoordinate; + this.mavenRepos = mavenCoordinate.getMavenRepos(); + this.artifactFile = Optional.of(artifactFilePath); + this.artifactFile.ifPresent(file -> file.getParentFile().mkdirs()); + } + /** * It tries to download the Maven artifact with the specified extension. E.g. jar */ private File trySpecifiedPackaging(int repoNumber) throws MissingArtifactException { try { - if (Arrays.asList(packaging).contains(mavenCoordinate.getPackaging())) { - foundPackage = true; + if (artifactFile.isEmpty()) { artifactFile = httpGetFile(mavenCoordinate.toProductUrl()); + } else { + httpGetToFile(mavenCoordinate.toProductUrl(), artifactFile.get()); } } catch (MissingArtifactException e) { foundPackage = false; @@ -70,7 +84,7 @@ private File trySpecifiedPackaging(int repoNumber) throws MissingArtifactExcepti logger.warn("[ARTIFACT-DOWNLOAD] [UNPROCESSED] [" + duration + "] [" + mavenCoordinate.getCoordinate() + "] [" + e.getClass().getSimpleName() + "] Artifact couldn't be retrieved for repo: " + mavenRepos.get(repoNumber), e); } - if (artifactFile.isPresent()) { + if (artifactFile.isPresent() && artifactFile.get().exists()) { long duration = computeDurationInMs(startTime); logger.info("[ARTIFACT-DOWNLOAD] [SUCCESS] [" + duration + "] [" + mavenCoordinate.getCoordinate() + "] [NONE] Artifact retrieved from repo: " + mavenRepos.get(repoNumber)); return artifactFile.get(); @@ -91,7 +105,11 @@ private File tryDefaultPackaging(int repoNumber) throws MissingArtifactException startTime = System.nanoTime(); try { foundPackage = true; - artifactFile = httpGetFile(mavenCoordinate.toProductUrl()); + if (artifactFile.isEmpty()) { + artifactFile = httpGetFile(mavenCoordinate.toProductUrl()); + } else { + httpGetToFile(mavenCoordinate.toProductUrl(), artifactFile.get()); + } } catch (MissingArtifactException e) { foundPackage = false; @@ -99,7 +117,7 @@ private File tryDefaultPackaging(int repoNumber) throws MissingArtifactException logger.warn("[ARTIFACT-DOWNLOAD] [UNPROCESSED] [" + duration + "] [" + mavenCoordinate.getCoordinate() + "] [" + e.getClass().getSimpleName() + "] Artifact couldn't be retrieved for repo: " + mavenRepos.get(repoNumber), e); } - if (artifactFile.isPresent()) { + if (artifactFile.isPresent() && artifactFile.get().exists()) { long duration = computeDurationInMs(startTime); logger.info("[ARTIFACT-DOWNLOAD] [SUCCESS] [" + duration + "] [" + mavenCoordinate.getCoordinate() + "] [NONE] Artifact retrieved from repo: " + mavenRepos.get(repoNumber)); return artifactFile.get(); @@ -159,7 +177,23 @@ private static Optional httpGetFile(final String url) throws MissingArtifa return Optional.of(new File(tempFile.toAbsolutePath().toString())); } catch (IOException e) { - if (tempFile != null) {tempFile.toFile().delete();} + if (tempFile != null) { + tempFile.toFile().delete(); + } + throw new MissingArtifactException(e.getMessage(), e.getCause()); + } + } + + /** + * Utility function that stores the contents of GET request to a given file + */ + private static void httpGetToFile(final String url, final File file) throws MissingArtifactException { + logger.info("Downloading artifact from URL: {} to File: {}", url, file.getAbsolutePath()); + try { + final InputStream in = new URL(url).openStream(); + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + in.close(); + } catch (IOException e) { throw new MissingArtifactException(e.getMessage(), e.getCause()); } } diff --git a/core/src/main/java/eu/fasten/core/data/opal/MavenCoordinate.java b/core/src/main/java/eu/fasten/core/data/opal/MavenCoordinate.java index 6933563b4..7fe761e9d 100644 --- a/core/src/main/java/eu/fasten/core/data/opal/MavenCoordinate.java +++ b/core/src/main/java/eu/fasten/core/data/opal/MavenCoordinate.java @@ -15,16 +15,18 @@ */ package eu.fasten.core.data.opal; -import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO; +import eu.fasten.core.data.Constants; +import eu.fasten.core.maven.utils.MavenUtilities; +import org.json.JSONException; +import org.json.JSONObject; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.LinkedList; import java.util.List; -import org.json.JSONException; -import org.json.JSONObject; - -import eu.fasten.core.data.Constants; -import eu.fasten.core.maven.utils.MavenUtilities; +import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO; /** * Maven coordinate as g:a:v e.g. "com.google.guava:guava:jar:28.1-jre". @@ -156,9 +158,14 @@ public String toProductUrl() { var repo = mavenRepos.get(0); return this.toURL(repo) + "/" + artifactID + "-" + versionConstraint + "." + packaging; } - + @Override public String toString() { - return getCoordinate(); + return getCoordinate(); + } + + public Path toPath() { + return Paths.get(groupID.replace('.', File.pathSeparatorChar), artifactID, versionConstraint, + artifactID + "-" + versionConstraint + "." + packaging); } } \ No newline at end of file diff --git a/core/src/main/java/eu/fasten/core/plugins/DataWriter.java b/core/src/main/java/eu/fasten/core/plugins/DataRW.java similarity index 81% rename from core/src/main/java/eu/fasten/core/plugins/DataWriter.java rename to core/src/main/java/eu/fasten/core/plugins/DataRW.java index e46f06340..52454e56f 100644 --- a/core/src/main/java/eu/fasten/core/plugins/DataWriter.java +++ b/core/src/main/java/eu/fasten/core/plugins/DataRW.java @@ -19,12 +19,12 @@ package eu.fasten.core.plugins; /** - * A plug-in that needs to write to disk some data should implement this interface. + * A plug-in that needs to read/write from/to disk some data should implement this interface. */ -public interface DataWriter extends FastenPlugin { +public interface DataRW extends FastenPlugin { /** - * Sets base directory into which the data will be written. + * Sets base directory into which the data will be read/written. * * @param baseDir Path to base directory */ diff --git a/server/src/main/java/eu/fasten/server/FastenServer.java b/server/src/main/java/eu/fasten/server/FastenServer.java index 8674ebee2..a27c6ba84 100644 --- a/server/src/main/java/eu/fasten/server/FastenServer.java +++ b/server/src/main/java/eu/fasten/server/FastenServer.java @@ -24,7 +24,7 @@ import eu.fasten.core.plugins.CallableIndexConnector; import eu.fasten.core.plugins.CallableIndexReader; import eu.fasten.core.plugins.DBConnector; -import eu.fasten.core.plugins.DataWriter; +import eu.fasten.core.plugins.DataRW; import eu.fasten.core.plugins.DependencyGraphUser; import eu.fasten.core.plugins.FastenPlugin; import eu.fasten.core.plugins.KafkaPlugin; @@ -184,7 +184,7 @@ public void run() { var dbPlugins = pluginManager.getExtensions(DBConnector.class); var kafkaPlugins = pluginManager.getExtensions(KafkaPlugin.class); var graphDbPlugins = pluginManager.getExtensions(CallableIndexConnector.class); - var dataWriterPlugins = pluginManager.getExtensions(DataWriter.class); + var dataRWPlugins = pluginManager.getExtensions(DataRW.class); var graphResolverUserPlugins = pluginManager.getExtensions(DependencyGraphUser.class); var graphDbReaderPlugins = pluginManager.getExtensions(CallableIndexReader.class); @@ -193,7 +193,7 @@ public void run() { registerDBConnections(dbPlugins); makeGraphDBConnection(graphDbPlugins); - setBaseDirectory(dataWriterPlugins); + setBaseDirectory(dataRWPlugins); loadDependencyGraphResolvers(graphResolverUserPlugins); makeReadOnlyGraphDBConnection(graphDbReaderPlugins); @@ -390,15 +390,15 @@ private void makeReadOnlyGraphDBConnection(List graphDbPlug /** * Sets base directory to Data Writer plugins * - * @param dataWriterPlugins list of Data Writer plugins + * @param dataRWPlugins list of Data Writer plugins */ - private void setBaseDirectory(List dataWriterPlugins) { - dataWriterPlugins.forEach((p) -> { + private void setBaseDirectory(List dataRWPlugins) { + dataRWPlugins.forEach((p) -> { if (ObjectUtils.allNotNull(baseDir)) { p.setBaseDir(baseDir); } else { logger.error("Couldn't set a base directory. Make sure that you have " - + "provided a valid path to base directory."); + + "provided a valid path to base directory."); } }); }