Skip to content

Commit

Permalink
Merge pull request #122 from ge-high-assurance/RITE-HYPERLINKING
Browse files Browse the repository at this point in the history 
hyperlinking, removing model vs data graph distinction in clear RACK
  • Loading branch information
@saratge
saratge authored Sep 1, 2023
2 parents 8a88d30 + 4c41aa3 commit 0cad9d2
Show file tree
Hide file tree
Showing 6 changed files with 571 additions and 430 deletions.
3 changes: 2 additions & 1 deletion tools/rack/rack.plugin/META-INF/MANIFEST.MF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ Require-Bundle: com.ge.research.jena,
org.eclipse.ui.console,
org.eclipse.ui.forms,
org.slf4j.api,
org.yaml.snakeyaml
org.yaml.snakeyaml,
org.eclipse.debug.ui
Bundle-ClassPath: .,
lib/connectionUtils.jar,
lib/graphviz-java.jar,
Expand Down
27 changes: 8 additions & 19 deletions tools/rack/rack.plugin/src/com/ge/research/rack/ClearAllHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@
import com.ge.research.rack.utils.ConnectionUtil;
import com.ge.research.rack.utils.RackConsole;
import com.ge.research.rack.views.ClearRackDialog;
import com.ge.research.rack.views.SelectDataGraphsDialog;
import com.ge.research.rack.views.ViewUtils;
import com.ge.research.semtk.nodeGroupStore.client.NodeGroupStoreRestClient;
import com.ge.research.semtk.resultSet.Table;
import com.ge.research.semtk.resultSet.TableResultSet;
Expand All @@ -44,15 +42,9 @@
import org.eclipse.core.commands.ExecutionEvent;
import org.eclipse.core.commands.ExecutionException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.*;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.swt.widgets.Display;
import org.eclipse.ui.console.IConsoleConstants;
import org.eclipse.ui.console.IConsoleView;
import org.eclipse.ui.handlers.HandlerUtil;

import java.util.ArrayList;

Expand Down Expand Up @@ -112,18 +104,15 @@ public static void deleteAllNodegroups(IProgressMonitor monitor) throws Exceptio

@Override
public Object execute(ExecutionEvent event) throws ExecutionException {

String queryNodegroup = "";

ClearRackDialog dialog =
new ClearRackDialog(
Display.getDefault().getActiveShell(), queryNodegroup);


ClearRackDialog dialog =
new ClearRackDialog(Display.getDefault().getActiveShell());
dialog.run();


return null;

/* try {

return null;

/* try {
String consoleViewId = IConsoleConstants.ID_CONSOLE_VIEW;
IConsoleView consoleView =
Expand Down
111 changes: 82 additions & 29 deletions tools/rack/rack.plugin/src/com/ge/research/rack/IngestInstanceDataHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
import org.eclipse.jface.viewers.ISelection;
import org.eclipse.jface.viewers.TreePath;
import org.eclipse.jface.viewers.TreeSelection;
import org.eclipse.swt.widgets.Display;
import org.eclipse.ui.handlers.HandlerUtil;

import java.io.File;
Expand Down Expand Up @@ -92,7 +91,7 @@ private IngestionStatus uploadModelFromYAML(String yamlPath, IProgressMonitor mo
}

if (dedupSteps.contains(yamlPath)) {
RackConsole.getConsole().print("Skipping previously executed step at: " + yamlPath);
RackConsole.getConsole().warning("Skipping previously executed step at: " + yamlPath);
return IngestionStatus.DONE;
}

Expand Down Expand Up @@ -129,6 +128,27 @@ private IngestionStatus uploadModelFromYAML(String yamlPath, IProgressMonitor mo
}

ArrayList<String> steps = (ArrayList<String>) oList;
String modelGraph = mGraphs.get(0);
if (yamlMap.containsKey("model-graphs")) {
Object oDataGraph = yamlMap.get("model-graphs");
if (oDataGraph instanceof List) {
if (((List) oDataGraph).size() > 1) {
RackConsole.getConsole().warning("We currently support ingesting only using a single model-graph");
// return IngestionStatus.FAILED;
}
modelGraph = ((List<String>) oDataGraph).get(0);
if (modelGraph.isEmpty()) {
modelGraph = mGraphs.get(0);
}
// validate target graph against footprint
if (!mGraphs.contains(modelGraph)) {
RackConsole.getConsole()
.error("Specified target graph " + modelGraph + " not declared in footprint");
RackConsole.getConsole().error("YAML file: " + yamlPath);
return IngestionStatus.FAILED;
}
}
}

for (String owl : steps) {
if (monitor.isCanceled()) {
Expand All @@ -144,16 +164,17 @@ private IngestionStatus uploadModelFromYAML(String yamlPath, IProgressMonitor mo
try {

RackConsole.getConsole()
.print("Uploading owl file " + owlFile.getName() + " to " + mGraphs.get(0) + " ... ");
SparqlQueryClient qAuthClient = ConnectionUtil.getOntologyUploadClient(mGraphs.get(0));
.print("Uploading owl file " + owlFile.getAbsolutePath() + " to " + modelGraph + " ... ");
SparqlQueryClient qAuthClient = ConnectionUtil.getOntologyUploadClient(modelGraph);
qAuthClient.uploadOwl(owlFile);
RackConsole.getConsole().printOK();
} catch (Exception e) {
RackConsole.getConsole().printFAIL();
RackConsole.getConsole().error(e.getLocalizedMessage());
RackConsole.getConsole().error(
"Ontology processing/upload failed, make sure you are connected to RACK or RACK-BOX instance");
RackConsole.getConsole().error("Upload of owl failed, OWL: " + owlFile.getAbsolutePath());
return IngestionStatus.FAILED;

}
}
return IngestionStatus.DONE;
Expand All @@ -166,7 +187,7 @@ private IngestionStatus uploadDataFromYAML(String yamlPath, IProgressMonitor mon
}

if (dedupSteps.contains(yamlPath)) {
RackConsole.getConsole().print("Skipping previously executed step at: " + yamlPath);
RackConsole.getConsole().warning("Skipping previously executed step at: " + yamlPath);
return IngestionStatus.DONE;
}

Expand Down Expand Up @@ -223,8 +244,8 @@ private IngestionStatus uploadDataFromYAML(String yamlPath, IProgressMonitor mon
Object oDataGraph = yamlMap.get("model-graphs");
if (oDataGraph instanceof List) {
if (((List) oDataGraph).size() > 1) {
RackConsole.getConsole().error("We currently support ingesting only using a single model-graph");
return IngestionStatus.FAILED;
RackConsole.getConsole().warning("We currently support ingesting only using a single model-graph");
// return IngestionStatus.FAILED;
}
modelGraph = ((List<String>) oDataGraph).get(0);
if (modelGraph.isEmpty()) {
Expand Down Expand Up @@ -280,6 +301,13 @@ private IngestionStatus uploadDataFromYAML(String yamlPath, IProgressMonitor mon
ArrayList<String> colsList = new ArrayList<>();
ArrayList<ArrayList<String>> tabData = new ArrayList<>();

if (dedupSteps.contains(csvFile.getAbsolutePath())) {
RackConsole.getConsole().warning("Skipping already processed file: " + csvFile.getAbsolutePath());
continue;
} else {
dedupSteps.add(csvFile.getAbsolutePath());
}

try {

FileReader reader = new FileReader(csvFile.getAbsolutePath());
Expand Down Expand Up @@ -332,18 +360,24 @@ private IngestionStatus uploadDataFromYAML(String yamlPath, IProgressMonitor mon
}
final String dGraph = dataGraph;

if (dedupSteps.contains(owl.getAbsolutePath())) {
RackConsole.getConsole().warning("Skipping already processed file: " + owl.getAbsolutePath());
continue;
} else {
dedupSteps.add(owl.getAbsolutePath());
}

try {
RackConsole.getConsole().print("Uploading owl file " + owlFile + " to " + dGraph + " ... ");
RackConsole.getConsole()
.print("Uploading owl file " + owl.getAbsolutePath() + " to " + dGraph + " ... ");
SparqlQueryClient qAuthClient = ConnectionUtil.getOntologyUploadClient(dGraph);
qAuthClient.uploadOwl(owl);
RackConsole.getConsole().printOK();
} catch (Exception e) {
RackConsole.getConsole().printFAIL();
RackConsole.getConsole().error(
"Ontology processing/upload failed, make sure you are connected to RACK or RACK-BOX instance");
RackConsole.getConsole().error("Upload of owl filed, OWL: " + owl.getAbsolutePath());
return IngestionStatus.FAILED;
}

}
}

Expand All @@ -359,7 +393,7 @@ private IngestionStatus uploadNodegroupsFromYAML(String ngPath, IProgressMonitor
}

if (dedupSteps.contains(ngPath)) {
RackConsole.getConsole().print("Skipping previously executed step at: " + ngPath);
RackConsole.getConsole().warning("Skipping previously executed step at: " + ngPath);
return IngestionStatus.DONE;
}

Expand All @@ -373,6 +407,7 @@ private IngestionStatus uploadNodegroupsFromYAML(String ngPath, IProgressMonitor
if (!csvNgStore.exists()) {
RackConsole.getConsole()
.error("Nodegroup csv store is missing, cannot ingest nodegroups specified in folder:" + ngPath);
RackConsole.getConsole().error("Store data CSV: " + csvNgStore.getAbsolutePath());
}
ArrayList<String> colsList = new ArrayList<>();
ArrayList<ArrayList<String>> tabData = new ArrayList<>();
Expand Down Expand Up @@ -440,18 +475,19 @@ private IngestionStatus uploadNodegroupsFromCSV(String ngPath, ArrayList<ArrayLi

} catch (Exception e) {
RackConsole.getConsole().printFAIL();
RackConsole.getConsole().error("Upload of nodegroup: " + nodegroupId + ".json " + "failed");
RackConsole.getConsole().error(e.getLocalizedMessage());
RackConsole.getConsole()
.error("Upload of nodegroup" + nodegroupId + " failed, JSON:" + ngJson.getAbsolutePath());
return IngestionStatus.FAILED;
}

}
return IngestionStatus.DONE;
}

private IngestionStatus uploadDataFromManifestYAML(String yamlPath, IProgressMonitor monitor) throws Exception {

if (dedupSteps.contains(yamlPath)) {
RackConsole.getConsole().print("Skipping previously executed step at: " + yamlPath);
RackConsole.getConsole().warning("Skipping previously executed step at: " + yamlPath);
return IngestionStatus.DONE;
}

Expand All @@ -473,19 +509,22 @@ private IngestionStatus uploadDataFromManifestYAML(String yamlPath, IProgressMon
}
if (oYaml == null || !(oYaml instanceof Map)) {
RackConsole.getConsole().error("Ill formed manifest at " + dir + "/" + file.getName() + ", please check");
RackConsole.getConsole().error("Check YAML: " + file.getAbsolutePath());
return IngestionStatus.FAILED;
}

HashMap<String, Object> yamlMap = (HashMap) oYaml;

if (!yamlMap.containsKey("steps")) {
RackConsole.getConsole().warning(dir + "/" + file.getName() + " contains no ingestion step, done");
RackConsole.getConsole().error(dir + "/" + file.getName() + " contains no ingestion step, done");
RackConsole.getConsole().error("Check YAML: " + file.getAbsolutePath());
return IngestionStatus.FAILED;
}

Object oList = yamlMap.get("steps");
if (!(oList instanceof List)) {
RackConsole.getConsole().error("steps in" + dir + "/" + file.getName() + " is ill formed, please check");
RackConsole.getConsole().error("Check YAML: " + file.getAbsolutePath());
return IngestionStatus.FAILED;
}
ArrayList<Map<String, Object>> steps = (ArrayList<Map<String, Object>>) oList;
Expand Down Expand Up @@ -580,6 +619,8 @@ private IStatus ingestInstanceData(IProgressMonitor monitor) {
if (oYaml == null || !(oYaml instanceof Map)) {
RackConsole.getConsole()
.error("Ill formed manifest at " + dir + "/" + ingestionYaml.getName() + ", please check");
RackConsole.getConsole().error("Check YAML: " + ingestionYaml.getAbsolutePath());

return Status.CANCEL_STATUS;
}
// read footprint
Expand Down Expand Up @@ -608,10 +649,9 @@ private IStatus ingestInstanceData(IProgressMonitor monitor) {
}

try {

setRunning(true);
IngestionStatus value = uploadDataFromManifestYAML(manifestPath, monitor);
monitor.worked(100);
setRunning(false);
switch (value) {
case DONE:
RackConsole.getConsole().print(MANIFEST_SUCCESS);
Expand All @@ -624,7 +664,9 @@ private IStatus ingestInstanceData(IProgressMonitor monitor) {
}

} catch (Exception e) {
RackConsole.getConsole().error("Ingestion failed using manifest yaml");
RackConsole.getConsole().error("Ingestion failed, check YAML: " + ingestionYaml.getAbsolutePath());
} finally {
setRunning(false);
}

return Status.OK_STATUS;
Expand Down Expand Up @@ -655,16 +697,31 @@ private IngestionStatus uploadInstanceDataCSV(String ingestionId, ArrayList<Stri
RackConsole.getConsole().print("Uploading CSV at " + path + " as class " + ingestionId + "... ");
if (bUriIngestion == false) {
client.dispatchIngestFromCsvStringsByIdSync(ingestionId, sCSV,
ConnectionUtil.getSparqlConnection(modelGraph, dataGraph, dataGraphs));
ConnectionUtil.getSparqlConnection(mGraphs, dataGraph, dataGraphs));

} else {
client.dispatchIngestFromCsvStringsByClassTemplateSync(ingestionId, "identifier", sCSV,
ConnectionUtil.getSparqlConnection(modelGraph, dataGraph, dataGraphs));

/*
* String error = client.execFromCsvUsingClassTemplate( ingestionId,
* "identifier", sCSV, ConnectionUtil.getSparqlConnection(mGraphs, dataGraph,
* dataGraphs).toJson().toJSONString(), false, "override");
*
*/
String status = client.dispatchIngestFromCsvStringsByClassTemplateSync(ingestionId, "identifier", sCSV,
ConnectionUtil.getSparqlConnection(mGraphs, dataGraph, dataGraphs));
}
RackConsole.getConsole().printOK();
} catch (Exception e) {

List<String> warnings = client.getWarnings();
if (warnings != null) {
for (String warning : warnings) {
RackConsole.getConsole().warning(warning);
}
}

} catch (Exception e) {
RackConsole.getConsole().printFAIL();
RackConsole.getConsole().error(e.getLocalizedMessage());
RackConsole.getConsole().error("Upload of " + ingestionId + " failed, " + "CSV: " + path);
return IngestionStatus.FAILED;
}
Expand All @@ -689,12 +746,8 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
protected IStatus run(IProgressMonitor monitor) {
ViewUtils.showProgressView();

if (!isRunning()) {
setRunning(true);
RackConsole.getConsole().clearConsole();
} else {
if (isRunning()) {
RackConsole.getConsole().error(MANIFEST_IN_PROGRESS);

return Status.CANCEL_STATUS;
}

Expand Down
Loading

0 comments on commit 0cad9d2

Please sign in to comment.