Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generate Java call graphs from local JARs in OPALPlugin #459

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
+282 −195
Open

Generate Java call graphs from local JARs in OPALPlugin #459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
076ba74
Change the `DataWriter` interface to `DataRW` to read/write from/to t…
mir-am May 21, 2022
c2b6d0a
Add the `toPath` method to `MavenCoordinate`
mir-am May 21, 2022
05016df
In `OPALPlugin`, generate a call graph from a local JAR file if avail…
mir-am May 21, 2022
3b12da6
Make `toPath()` cross-platform and return `Path` in `MavenCoordinate`
mir-am Sep 9, 2022
b79f313
Extend `MavenArtifactDownloader` to download Maven artifacts into a g…
mir-am Sep 9, 2022
2c4d825
Adapt `OPALPartialCallGraphConstructor`'s `createPartialJavaCG` to us…
mir-am Sep 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,34 @@

package eu.fasten.analyzer.javacgopal;

import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.INCLUDING_ALL_SUBTYPES;

import eu.fasten.analyzer.javacgopal.data.CGAlgorithm;
import eu.fasten.analyzer.javacgopal.data.OPALCallGraphConstructor;
import eu.fasten.core.data.PartialJavaCallGraph;
import eu.fasten.core.data.opal.MavenCoordinate;
import eu.fasten.analyzer.javacgopal.data.OPALPartialCallGraphConstructor;
import eu.fasten.core.data.DirectedGraph;
import eu.fasten.core.data.JSONUtils;
import eu.fasten.core.data.PartialJavaCallGraph;
import eu.fasten.core.data.opal.MavenCoordinate;
import eu.fasten.core.maven.utils.MavenUtilities;
import eu.fasten.core.merge.CGMerger;
import eu.fasten.core.merge.CallGraphUtils;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.Collection;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.INCLUDING_ALL_SUBTYPES;

/**
* Makes javacg-opal module runnable from command line.
*/
Expand All @@ -53,7 +54,7 @@ public class Main implements Runnable {

private static final Logger logger = LoggerFactory.getLogger(Main.class);

@CommandLine.Option(names = { "-o",
@CommandLine.Option(names = {"-o",
"--output" }, paramLabel = "OUT", description = "Output directory path", defaultValue = "")
String output;

Expand Down Expand Up @@ -274,8 +275,8 @@ private <T> PartialJavaCallGraph generatePCG(final T artifact, final String arti
revisionCallGraph = generatePCGFromFile(getArtifact(artifact), artifactName, algorithm);
} else {
revisionCallGraph = OPALPartialCallGraphConstructor
.createPartialJavaCG((MavenCoordinate) artifact, algorithm,
Long.parseLong(timestamp), getArtifactRepo(), INCLUDING_ALL_SUBTYPES);
.createPartialJavaCG((MavenCoordinate) artifact, algorithm,
Long.parseLong(timestamp), getArtifactRepo(), null, INCLUDING_ALL_SUBTYPES);
}
return revisionCallGraph;
}
Expand Down
48 changes: 29 additions & 19 deletions analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,6 @@

package eu.fasten.analyzer.javacgopal;

import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA;
import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES;
import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;
import static java.lang.System.currentTimeMillis;

import java.io.File;
import java.util.Optional;

import org.json.JSONObject;
import org.pf4j.Extension;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.fasten.analyzer.javacgopal.data.OPALPartialCallGraphConstructor;
import eu.fasten.core.data.Constants;
import eu.fasten.core.data.JSONUtils;
Expand All @@ -42,6 +27,22 @@
import eu.fasten.core.data.opal.exceptions.MissingArtifactException;
import eu.fasten.core.data.opal.exceptions.OPALException;
import eu.fasten.core.plugins.AbstractKafkaPlugin;
import eu.fasten.core.plugins.DataRW;
import org.json.JSONObject;
import org.pf4j.Extension;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.nio.file.Paths;
import java.util.Optional;

import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA;
import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES;
import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;
import static java.lang.System.currentTimeMillis;

public class OPALPlugin extends Plugin {

Expand All @@ -50,12 +51,13 @@ public OPALPlugin(PluginWrapper wrapper) {
}

@Extension
public static class OPAL extends AbstractKafkaPlugin {
public static class OPAL extends AbstractKafkaPlugin implements DataRW {

private final Logger logger = LoggerFactory.getLogger(getClass());

private PartialJavaCallGraph graph;
private String outputPath;
private static String baseDir;

@Override
public void consume(String kafkaRecord, ProcessingLane l) {
Expand All @@ -71,14 +73,17 @@ public void consume(String kafkaRecord, ProcessingLane l) {
}
var artifactRepository = fixResetAndGetArtifactRepo(json);
final var mavenCoordinate = new MavenCoordinate(json);
final var mavenArtifactFile = new File(Paths.get(baseDir, ".m2", "repository", mavenCoordinate.toPath().toString()).toString());
long startTime = System.currentTimeMillis();
try {
// Generate CG and measure construction duration.
logger.info("[CG-GENERATION] [UNPROCESSED] [-1] [" + mavenCoordinate.getCoordinate() + "] [NONE] ");
long date = json.optLong("releaseDate", -1);
this.graph = OPALPartialCallGraphConstructor.createPartialJavaCG(mavenCoordinate,
CHA, date, artifactRepository, ONLY_STATIC_CALLSITES);
long duration = currentTimeMillis() - startTime;

this.graph = OPALPartialCallGraphConstructor.createPartialJavaCG(mavenCoordinate,
CHA, date, artifactRepository, mavenArtifactFile, ONLY_STATIC_CALLSITES);

long duration = currentTimeMillis() - startTime;

if (this.graph.isCallGraphEmpty()) {
throw new EmptyCallGraphException();
Expand Down Expand Up @@ -149,5 +154,10 @@ public long getMaxConsumeTimeout() {
public long getSessionTimeout() {
return 1800000; // Due to static membership we also want to tune the session timeout to 30 minutes.
}

@Override
public void setBaseDir(String baseDir) {
OPAL.baseDir = baseDir;
}
}
}
92 changes: 57 additions & 35 deletions ...pal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,16 @@

package eu.fasten.analyzer.javacgopal.data;

import com.google.common.collect.Lists;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALType;
import eu.fasten.core.data.Constants;
import eu.fasten.core.data.JavaGraph;
import eu.fasten.core.data.PartialJavaCallGraph;
import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

import eu.fasten.core.data.opal.MavenArtifactDownloader;
import eu.fasten.core.data.opal.MavenCoordinate;
import eu.fasten.core.data.opal.exceptions.OPALException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.text.StringEscapeUtils;
import org.opalj.br.Annotation;
Expand All @@ -45,20 +43,20 @@
import org.opalj.value.ValueInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Lists;

import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALType;
import eu.fasten.core.data.Constants;
import eu.fasten.core.data.JavaGraph;
import eu.fasten.core.data.opal.MavenArtifactDownloader;
import eu.fasten.core.data.opal.MavenCoordinate;
import eu.fasten.core.data.opal.exceptions.OPALException;
import scala.Function1;
import scala.collection.JavaConverters;

import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

/**
* Call graphs that are not still fully resolved. i.e. isolated call graphs which within-artifact
* calls (edges) are known as internal calls and Cross-artifact calls are known as external calls.
Expand All @@ -68,6 +66,7 @@ public class OPALPartialCallGraphConstructor {
private static final Logger logger = LoggerFactory.getLogger(OPALPartialCallGraph.class);

private OPALPartialCallGraph pcg;
private static boolean isTempJarFileUsed = false;

/**
* Given a file, algorithm and main class (in case of application package)
Expand Down Expand Up @@ -95,29 +94,52 @@ public OPALPartialCallGraph construct(OPALCallGraph ocg, CallPreservationStrateg
}
throw e;
}

return pcg;
}

/**
* Downloads the Maven artifact if it does not exist.
*
* @param artifactFile If `null` provided, the Jar file would be downloaded to a temp file.
* @return
*/
private static File downloadMavenArtifactIfNeeded(final MavenCoordinate coordinate, final String artifactRepo, File artifactFile) {
// Download the Jar file if the given file does not exist.
if (artifactFile != null) {
if (!artifactFile.exists()) {
logger.warn("Couldn't find the local JAR on filesystem for " + coordinate.toString());
logger.info("About to download the artifact file for {} from {}", coordinate, artifactRepo);
artifactFile = new MavenArtifactDownloader(coordinate, artifactFile).downloadArtifact(artifactRepo);
}
} else {
// Downloads to a temporary file
logger.info("Downloading the artifact to a temporary file");
artifactFile = new MavenArtifactDownloader(coordinate).downloadArtifact(artifactRepo);
isTempJarFileUsed = true;
}
return artifactFile;
}

/**
* Creates RevisionCallGraph using OPAL call graph generator for a given maven
* coordinate. It also sets the forge to "mvn".
*
* @param coordinate maven coordinate of the revision to be processed
* @param timestamp timestamp of the revision release
* @param coordinate maven coordinate of the revision to be processed
* @param timestamp timestamp of the revision release
* @param artifactRepo the repository from which the artifact/Jar should be downloaded from
* @param artifactFile The path to an existing Jar file.
* @return RevisionCallGraph of the given coordinate.
*/
public static PartialJavaCallGraph createPartialJavaCG(
final MavenCoordinate coordinate,
CGAlgorithm algorithm, final long timestamp, final String artifactRepo, CallPreservationStrategy callSiteOnly) {
final MavenCoordinate coordinate,
CGAlgorithm algorithm, final long timestamp, final String artifactRepo, File artifactFile,
CallPreservationStrategy callSiteOnly) {

File file = null;
artifactFile = downloadMavenArtifactIfNeeded(coordinate, artifactRepo, artifactFile);
try {
logger.info("About to download {} from {}", coordinate, artifactRepo);

file = new MavenArtifactDownloader(coordinate).downloadArtifact(artifactRepo);
final var opalCG = new OPALCallGraphConstructor().construct(file, algorithm);

final var opalCG = new OPALCallGraphConstructor().construct(artifactFile, algorithm);
final var partialCallGraph = new OPALPartialCallGraphConstructor().construct(opalCG, callSiteOnly);

return new PartialJavaCallGraph(Constants.mvnForge, coordinate.getProduct(),
Expand All @@ -126,9 +148,9 @@ public static PartialJavaCallGraph createPartialJavaCG(
partialCallGraph.classHierarchy,
partialCallGraph.graph);
} finally {
if (file != null) {
// TODO use apache commons FileUtils instead
file.delete();
if (isTempJarFileUsed) {
// TODO use apache commons FileUtils instead
artifactFile.delete();
}
}
}
Expand Down
45 changes: 31 additions & 14 deletions analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/OPALPluginTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,38 +18,55 @@

package eu.fasten.analyzer.javacgopal;

import static eu.fasten.core.plugins.KafkaPlugin.ProcessingLane.NORMAL;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.Collections;
import java.util.List;
import java.util.Optional;

import eu.fasten.core.data.PartialJavaCallGraph;
import eu.fasten.core.data.opal.exceptions.EmptyCallGraphException;
import eu.fasten.core.data.opal.exceptions.MissingArtifactException;
import org.apache.commons.io.FileUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

import eu.fasten.core.data.PartialJavaCallGraph;
import eu.fasten.core.data.opal.exceptions.EmptyCallGraphException;
import eu.fasten.core.data.opal.exceptions.MissingArtifactException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Optional;

import static eu.fasten.core.plugins.KafkaPlugin.ProcessingLane.NORMAL;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

class OPALPluginTest {

private OPALPlugin.OPAL plugin;
private static Path baseDirPath;

@BeforeAll
public static void setUpBaseDir() throws IOException {
baseDirPath = Files.createTempDirectory("tmp-opal").getFileName().toAbsolutePath();
}

@AfterAll
public static void CleanUpBaseDir() throws IOException {
FileUtils.deleteDirectory(baseDirPath.toFile());
}

@BeforeEach
public void setUp() {
plugin = new OPALPlugin.OPAL();
plugin.setBaseDir(String.valueOf(baseDirPath));
}

@Test
public void testConsumerTopicNotSetByDefault() {
assertThrows(RuntimeException.class, ()->{
assertThrows(RuntimeException.class, () -> {
plugin.consumeTopic();
});
}
Expand Down
Loading