Skip to content

Commit

Permalink
Refactored Experiments starter
Browse files Browse the repository at this point in the history
  • Loading branch information
@jodavimehran
jodavimehran committed Sep 6, 2021
1 parent 87adf63 commit 1ef356f
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 55 deletions.
59 changes: 43 additions & 16 deletions src/main/java/org/codetracker/experiment/AbstractExperimentStarter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,18 @@
import static org.codetracker.util.FileUtil.writeToFile;

public abstract class AbstractExperimentStarter {
private static final String RESULT_FINAL_CSV = "result/%s/final.csv";
protected final static String FOLDER_TO_CLONE = "H:\\Projects\\";
protected static final String SUMMARY_RESULT_FILE_NAME_FORMAT = "result/%s/summary-%s-%s.csv";
protected static final String SUMMARY_RESULT_FILE_NAME_FORMAT = "experiments/tracking-accuracy/%s/%s/summary-%s-%s.csv";
protected final static String SUMMARY_RESULT_HEADER;

protected static final String DETAILED_RESULT_HEADER = "repository,element_key,parent_commit_id,commit_id,commit_time, change_type,element_file_before,element_file_after,element_name_before,element_name_after,result,comment" + System.lineSeparator();
protected static final String DETAILED_RESULT_FILE_NAME_FORMAT = "result/%s/detailed-%s-%s.csv";
protected static final String DETAILED_RESULT_HEADER = "file_name, repository,element_key,parent_commit_id,commit_id,commit_time, change_type,element_file_before,element_file_after,element_name_before,element_name_after,result,comment" + System.lineSeparator();
protected static final String DETAILED_RESULT_FILE_NAME_FORMAT = "experiments/tracking-accuracy/%s/%s/detailed-%s-%s.csv";
protected static final String FINAL_RESULT_HEADER = "tool,oracle,level,processing_time_avg,processing_time_median,tp,fp,fn,precision,recall" + System.lineSeparator();
protected static final String DETAILED_CONTENT_FORMAT = "\"%s\",\"%s\",%s,%s,%d,%s,%s,%s,\"%s\",\"%s\",%s,\"%s\"" + System.lineSeparator();
protected static final String DETAILED_CONTENT_FORMAT = "%s,\"%s\",\"%s\",%s,%s,%d,%s,%s,%s,\"%s\",\"%s\",%s,\"%s\"" + System.lineSeparator();
protected static final String FINAL_RESULT_FORMAT = "%s,%s,%s,%f,%d,%d,%d,%d,%f,%f" + System.lineSeparator();
protected static final String ERROR_FILE_NAME_FORMAT = "result/%s/error-%s-%s-%s.txt";
protected static final String ERROR_FILE_NAME_FORMAT = "experiments/tracking-accuracy/%s/%s/error-%s-%s-%s.txt";
protected static final ObjectMapper MAPPER = new ObjectMapper();

private static final String PROCESSED_FILE_NAME_FORMAT = "result/%s/processed-%s-%s.csv";
private static final String RESULT_FINAL_CSV = "experiments/tracking-accuracy/%s/%s/final.csv";
private static final String PROCESSED_FILE_NAME_FORMAT = "experiments/tracking-accuracy/%s/%s/processed-%s-%s.csv";

static {
StringBuilder header = new StringBuilder();
Expand Down Expand Up @@ -81,7 +79,7 @@ protected static HashMap<String, ChangeHistory> oracle(List<ChangeHistory> expec
}

protected String getProcessedFilePath(String oracleName, String toolName) {
return String.format(PROCESSED_FILE_NAME_FORMAT, getCodeElementName(), toolName, oracleName);
return String.format(PROCESSED_FILE_NAME_FORMAT, getCodeElementName(), toolName, toolName, oracleName);
}

protected Set<String> getAllProcessedSamples(String oracleName, String toolName) throws IOException {
Expand All @@ -91,12 +89,13 @@ protected Set<String> getAllProcessedSamples(String oracleName, String toolName)
}

protected abstract String getCodeElementName();

protected void calculateFinalResults(String oracleName, String toolName) {
Map<String, Set<String>> commitLevelExpected = new HashMap<>();
Map<String, Set<String>> commitLevelActual = new HashMap<>();
try {
List<Integer> processingTime = new ArrayList<>();
List<String[]> summaryResults = readResults(String.format(SUMMARY_RESULT_FILE_NAME_FORMAT, getCodeElementName(), toolName, oracleName));
List<String[]> summaryResults = readResults(getSummaryResultFileName(oracleName, toolName));
int tp = 0, fp = 0, fn = 0;
for (String[] result : summaryResults) {
tp += Integer.parseInt(result[result.length - 3]);
Expand All @@ -111,13 +110,13 @@ protected void calculateFinalResults(String oracleName, String toolName) {
middle = middle > 0 && middle % 2 == 0 ? middle - 1 : middle;
int processingTimeMedian = processingTime.get(middle);
double processingTimeAverage = processingTime.stream().mapToInt(Integer::intValue).average().getAsDouble();
writeToFile(String.format(RESULT_FINAL_CSV, getCodeElementName()), FINAL_RESULT_HEADER, String.format(FINAL_RESULT_FORMAT, toolName, oracleName, "change", processingTimeAverage, processingTimeMedian, tp, fp, fn, ((double) tp / (tp + fp)) * 100, ((double) tp / (tp + fn)) * 100), StandardOpenOption.APPEND);
writeToFile(String.format(RESULT_FINAL_CSV, getCodeElementName(), toolName), FINAL_RESULT_HEADER, String.format(FINAL_RESULT_FORMAT, toolName, oracleName, "change", processingTimeAverage, processingTimeMedian, tp, fp, fn, ((double) tp / (tp + fp)) * 100, ((double) tp / (tp + fn)) * 100), StandardOpenOption.APPEND);

List<String[]> detailedResults = readResults(String.format(DETAILED_RESULT_FILE_NAME_FORMAT, getCodeElementName(), toolName, oracleName));
List<String[]> detailedResults = readResults(getDetailedResultFileName(oracleName, toolName));

for (String[] result : detailedResults) {
String elementKey = result[1];
String commitId = result[3];
String elementKey = result[2];
String commitId = result[4];
if (result[result.length - 2].equals("TP")) {
commitLevelExpected.get(elementKey).add(commitId);
commitLevelActual.get(elementKey).add(commitId);
Expand Down Expand Up @@ -149,9 +148,37 @@ protected void calculateFinalResults(String oracleName, String toolName) {
int commitLevelTp = actualSize - commitLevelFp;
sumTp += commitLevelTp;
}
writeToFile(String.format(RESULT_FINAL_CSV, getCodeElementName()), FINAL_RESULT_HEADER, String.format(FINAL_RESULT_FORMAT, toolName, oracleName, "commit", processingTimeAverage, processingTimeMedian, sumTp, sumFP, sumFn, ((double) sumTp / (sumTp + sumFP)) * 100, ((double) sumTp / (sumTp + sumFn)) * 100), StandardOpenOption.APPEND);
writeToFile(String.format(RESULT_FINAL_CSV, getCodeElementName(), toolName), FINAL_RESULT_HEADER, String.format(FINAL_RESULT_FORMAT, toolName, oracleName, "commit", processingTimeAverage, processingTimeMedian, sumTp, sumFP, sumFn, ((double) sumTp / (sumTp + sumFP)) * 100, ((double) sumTp / (sumTp + sumFn)) * 100), StandardOpenOption.APPEND);
} catch (Exception exception) {
exception.printStackTrace();
}
}

private String getSummaryResultFileName(String oracleName, String toolName) {
return String.format(SUMMARY_RESULT_FILE_NAME_FORMAT, getCodeElementName(), toolName, toolName, oracleName);
}

protected void writeToSummaryFile(String oracleName, String toolName, String content) throws IOException {
writeToFile(getSummaryResultFileName(oracleName, toolName), SUMMARY_RESULT_HEADER, content, StandardOpenOption.APPEND);
}

private String getDetailedResultFileName(String oracleName, String toolName) {
return String.format(DETAILED_RESULT_FILE_NAME_FORMAT, getCodeElementName(), toolName, toolName, oracleName);
}

protected void writeToDetailedFile(String oracleName, String toolName, String oracleFileName,
String repositoryWebURL, String elementKey, String parentCommitId, String commitId,
long commitTime, String changeType, String elementFileBefore, String elementFileAfter,
String elementNameBefore, String elementNameAfter, String resultType, String comment) throws IOException {

writeToFile(getDetailedResultFileName(oracleName, toolName), DETAILED_RESULT_HEADER,
getDetailedResultContent(oracleFileName, repositoryWebURL, elementKey, parentCommitId, commitId, commitTime, changeType, elementFileBefore, elementFileAfter, elementNameBefore, elementNameAfter, resultType, comment), StandardOpenOption.APPEND);
}

private String getDetailedResultContent(String oracleFileName, String repositoryWebURL, String elementKey, String parentCommitId, String commitId, long commitTime, String changeType, String elementFileBefore, String elementFileAfter, String elementNameBefore, String elementNameAfter, String resultType, String comment) {
return String.format(DETAILED_CONTENT_FORMAT, oracleFileName, repositoryWebURL, elementKey, parentCommitId,
commitId, commitTime, changeType, elementFileBefore, elementFileAfter, elementNameBefore, elementNameAfter,
resultType, comment);
}

}
29 changes: 9 additions & 20 deletions src/main/java/org/codetracker/experiment/MethodExperimentStarter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public static void main(String[] args) throws IOException {
}

public void start() throws IOException {
createDirectory(new String[]{"result", "result/method", "result/method/shovel", "result/method/shovel/test", "result/method/shovel/training"});
// createDirectory(new String[]{"result", "result/method", "result/method/shovel", "result/method/shovel/test", "result/method/shovel/training"});
String[] toolNames = new String[]{"tracker", "shovel", "shovel-oracle"};
List<MethodOracle> oracles = MethodOracle.all();
for (String toolName : toolNames) {
Expand Down Expand Up @@ -149,7 +149,7 @@ private void codeTracker(MethodOracle methodOracle, String toolName) throws IOEx

content.append(tp).append(",").append(fp).append(",").append(fn).append(System.lineSeparator());

writeToFile(String.format(SUMMARY_RESULT_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, toolName, oracleName), SUMMARY_RESULT_HEADER, content.toString(), StandardOpenOption.APPEND);
writeToSummaryFile(oracleName, toolName, content.toString());

List<ChangeHistory> historyResults = allChanges.values().stream().sorted(Comparator.comparing(ChangeHistory::getCommitTime).reversed().thenComparing(ChangeHistory::getCommitId)).collect(Collectors.toList());
for (ChangeHistory changeHistory : historyResults) {
Expand All @@ -164,28 +164,17 @@ else if (falseDetectedChanges.containsKey(changeKey))
else
resultType = "UN!";

writeToFile(String.format(DETAILED_RESULT_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, toolName, oracleName),
DETAILED_RESULT_HEADER,
String.format(DETAILED_CONTENT_FORMAT,
methodHistoryInfo.getRepositoryWebURL(),
methodHistoryInfo.getFunctionKey(),
changeHistory.getParentCommitId(),
changeHistory.getCommitId(),
changeHistory.getCommitTime(),
changeHistory.getChangeType(),
changeHistory.getElementFileBefore(),
changeHistory.getElementFileAfter(),
changeHistory.getElementNameBefore(),
changeHistory.getElementNameAfter(),
resultType,
changeHistory.getComment()
),
StandardOpenOption.APPEND);
writeToDetailedFile(oracleName, toolName, fileName, methodHistoryInfo.getRepositoryWebURL(),
methodHistoryInfo.getFunctionKey(), changeHistory.getParentCommitId(), changeHistory.getCommitId(),
changeHistory.getCommitTime(), changeHistory.getChangeType(), changeHistory.getElementFileBefore(),
changeHistory.getElementFileAfter(), changeHistory.getElementNameBefore(), changeHistory.getElementNameAfter()
, resultType, changeHistory.getComment()
);
}
writeToFile(getProcessedFilePath(oracleName, toolName), "file_name" + System.lineSeparator(), fileName + System.lineSeparator(), StandardOpenOption.APPEND);

} catch (Exception exception) {
try (FileWriter fw = new FileWriter(String.format(ERROR_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, toolName, oracleName, fileName), false)) {
try (FileWriter fw = new FileWriter(String.format(ERROR_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, toolName, toolName, oracleName, fileName), false)) {
try (PrintWriter pw = new PrintWriter(fw)) {
pw.println(exception.getMessage());
pw.println("====================================================================================");
Expand Down
27 changes: 8 additions & 19 deletions src/main/java/org/codetracker/experiment/VariableExperimentStarter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private void codeTracker(VariableOracle variableOracle) throws IOException {

content.append(tp).append(",").append(fp).append(",").append(fn).append(System.lineSeparator());

writeToFile(String.format(SUMMARY_RESULT_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, TOOL_NAME, oracleName), SUMMARY_RESULT_HEADER, content.toString(), StandardOpenOption.APPEND);
writeToSummaryFile(oracleName, TOOL_NAME, content.toString());

List<ChangeHistory> historyResults = allChanges.values().stream().sorted(Comparator.comparing(ChangeHistory::getCommitTime).reversed().thenComparing(ChangeHistory::getCommitId)).collect(Collectors.toList());
for (ChangeHistory changeHistory : historyResults) {
Expand All @@ -178,27 +178,16 @@ else if (falseDetectedChanges.containsKey(changeKey))
else
resultType = "UN!";

writeToFile(String.format(DETAILED_RESULT_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, TOOL_NAME, oracleName),
DETAILED_RESULT_HEADER,
String.format(DETAILED_CONTENT_FORMAT,
variableHistoryInfo.getRepositoryWebURL(),
variableHistoryInfo.getVariableKey(),
changeHistory.getParentCommitId(),
changeHistory.getCommitId(),
changeHistory.getCommitTime(),
changeHistory.getChangeType(),
changeHistory.getElementFileBefore(),
changeHistory.getElementFileAfter(),
changeHistory.getElementNameBefore(),
changeHistory.getElementNameAfter(),
resultType,
changeHistory.getComment()
),
StandardOpenOption.APPEND);
writeToDetailedFile(oracleName, TOOL_NAME, fileName, variableHistoryInfo.getRepositoryWebURL(),
variableHistoryInfo.getVariableKey(), changeHistory.getParentCommitId(), changeHistory.getCommitId(),
changeHistory.getCommitTime(), changeHistory.getChangeType(), changeHistory.getElementFileBefore(),
changeHistory.getElementFileAfter(), changeHistory.getElementNameBefore(), changeHistory.getElementNameAfter()
, resultType, changeHistory.getComment()
);
}
writeToFile(getProcessedFilePath(oracleName, TOOL_NAME), "file_name" + System.lineSeparator(), fileName + System.lineSeparator(), StandardOpenOption.APPEND);
} catch (Exception exception) {
try (FileWriter fw = new FileWriter(String.format(ERROR_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, TOOL_NAME, oracleName, fileName), false)) {
try (FileWriter fw = new FileWriter(String.format(ERROR_FILE_NAME_FORMAT, CODE_ELEMENT_NAME, TOOL_NAME, TOOL_NAME, oracleName, fileName), false)) {
try (PrintWriter pw = new PrintWriter(fw)) {
pw.println(exception.getMessage());
pw.println("====================================================================================");
Expand Down

0 comments on commit 1ef356f

Please sign in to comment.