From fa4d55f24bac2fa90f927ac8a5541cee64b859e7 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Thu, 18 May 2023 11:25:10 -0400 Subject: [PATCH 01/11] fix: safe --- docker-compose.yml | 5 +- portal/.classpath | 6 -- portal/pom.xml | 2 +- .../portal/controllers/RunController.java | 92 +++++++++---------- 4 files changed, 46 insertions(+), 59 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3edf03cb..cf3eb261 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,10 @@ -version: '3' +version: "3" services: wings: image: "ikcap/wings:latest" + build: + context: . + dockerfile: wings-docker/docker/default/Dockerfile environment: WINGS_MODE: dind volumes: diff --git a/portal/.classpath b/portal/.classpath index 6330fa28..c2153bf1 100644 --- a/portal/.classpath +++ b/portal/.classpath @@ -39,17 +39,11 @@ - - - - - - diff --git a/portal/pom.xml b/portal/pom.xml index b9c6ad16..2228f7c4 100644 --- a/portal/pom.xml +++ b/portal/pom.xml @@ -17,7 +17,7 @@ - 1.2.4 + 1.2.6 2.5 4.5.2 2.27 diff --git a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java index 142c5eb7..73d8ea26 100644 --- a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java +++ b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java @@ -30,6 +30,8 @@ import javax.ws.rs.core.Response; import edu.isi.kcap.wings.opmm.Catalog; +import edu.isi.kcap.wings.opmm.FilePublisher; +import edu.isi.kcap.wings.opmm.Mapper; import edu.isi.wings.portal.classes.config.Publisher; import edu.isi.wings.portal.classes.config.ServerDetails; import edu.isi.wings.portal.classes.util.ComponentExecutingThread; @@ -48,6 +50,7 @@ import edu.isi.kcap.wings.opmm.WorkflowExecutionExport; import edu.isi.kcap.wings.opmm.WorkflowTemplateExport; +import edu.isi.kcap.wings.opmm.DataTypes.Links; import edu.isi.kcap.ontapi.KBTriple; import edu.isi.wings.catalog.component.ComponentFactory; import edu.isi.wings.catalog.data.DataFactory; @@ -528,6 +531,10 @@ private String urlToString(URL url) { return null; } + /** + * @param runid + * @return + */ public String publishRun(String runid) { HashMap retmap = new HashMap(); ExecutionMonitorAPI monitor = config.getDomainExecutionMonitor(); @@ -537,21 +544,24 @@ public String publishRun(String runid) { } else try { // Mapper opmm = new Mapper(); - + String webServerDirectory = "tmp/"; + String webServerDomain = "http://localhost"; + FilePublisher filePublisher = new FilePublisher(webServerDirectory, webServerDomain); + String domain = config.getDomainId(); + String catalogRepositoryDirectory = "domains"; + String endpointQueryURI = "https://endpoint.mint.isi.edu/provenance/query"; + String endpointPostURI = "https://endpoint.mint.isi.edu/provenance/query"; Publisher publisher = config.getPublisher(); - ServerDetails publishUrl = publisher.getUploadServer(); String tstoreurl = publisher.getTstorePublishUrl(); - String tstorequery = publisher.getTstoreQueryUrl(); - String exportName = publisher.getExportName(); + String endpointStoreUri = publisher.getTstoreQueryUrl(); + String exportPrefix = publisher.getExportName(); String exportUrl = publisher.getUrl(); - String uploadURL = publishUrl.getUrl(); String uploadUsername = publishUrl.getUsername(); String uploadDirectory = publishUrl.getDirectory(); String uploadPassword = publishUrl.getPassword(); long uploadMaxSize = publishUrl.getMaxUploadSize(); - // opmm.setPublishExportPrefix(puburl); String rname = runid.substring(runid.indexOf('#') + 1); // String runurl = opmm.getRunUrl(rname); @@ -623,11 +633,11 @@ public String publishRun(String runid) { // write aclibfie and rplanfile aclibdata += abslibdata + "\n"; - File aclibfile = new File(acontdir.getAbsolutePath() + "/library.owl"); - File rplanfile = new File(execsdir.getAbsolutePath() + "/" + + File componentLibraryFilePath = new File(acontdir.getAbsolutePath() + "/library.owl"); + File planFilePath = new File(execsdir.getAbsolutePath() + "/" + plan.getName() + ".owl"); - FileUtils.write(aclibfile, aclibdata); - FileUtils.write(rplanfile, rplandata); + FileUtils.write(componentLibraryFilePath, aclibdata); + FileUtils.write(planFilePath, rplandata); // workflow file? URL otplurl = new URL(plan.getOriginalTemplateID()); @@ -636,51 +646,31 @@ public String publishRun(String runid) { String otpldata = urlToString(otplurl); FileUtils.write(otplfile, otpldata); - Catalog catalog = new Catalog(config.getDomainId(), exportName, - publisher.getDomainsDir(), aclibfile.getAbsolutePath()); - - WorkflowExecutionExport exp = new WorkflowExecutionExport( - rplanfile.getAbsolutePath(), catalog, exportUrl, exportName, tstorequery, config.getDomainId()); - exp.setUploadURL(uploadURL); - exp.setUploadUsername(uploadUsername); - exp.setUploadPassword(uploadPassword); - exp.setUploadMaxSize(uploadMaxSize); - exp.setUploadDirectory(uploadDirectory); - String serialization = "TURTLE"; - - // publish the catalog - String domainPath = catalog.exportCatalog(null, serialization); - File domainFile = new File(domainPath); - this.publishFile(tstoreurl, catalog.getDomainGraphURI(), domainFile.getAbsolutePath()); - - // execution String executionFilePath = run_exportdir + File.separator + "execution"; - String graphUri = exp.exportAsOPMW(executionFilePath, serialization); - if (!exp.isExecPublished()) { - this.publishFile(tstoreurl, graphUri, executionFilePath); - - // expandedTemplate - String expandedTemplateFilePath = run_exportdir + File.separator + "expandedTemplate"; - String expandedTemplateGraphUri = exp.getConcreteTemplateExport().exportAsOPMW(expandedTemplateFilePath, - serialization); - if (!exp.getConcreteTemplateExport().isTemplatePublished()) - this.publishFile(tstoreurl, expandedTemplateGraphUri, expandedTemplateFilePath); - - // abstract - WorkflowTemplateExport abstractTemplateExport = exp.getConcreteTemplateExport().getAbstractTemplateExport(); - if (abstractTemplateExport != null) { - String abstractFilePath = run_exportdir + File.separator + "abstract"; - String abstractGraphUri = abstractTemplateExport.exportAsOPMW(abstractFilePath, serialization); - if (!abstractTemplateExport.isTemplatePublished()) - this.publishFile(tstoreurl, abstractGraphUri, abstractFilePath); - } + String expandedTemplateFilePath = run_exportdir + File.separator + "expandedTemplate"; + String abstractFilePath = run_exportdir + File.separator + "abstract"; + // OPMW MAPPER CODE + // domain + // exportName + // catalogRepositoryDirectory + // componentLibraryFile + // + String serialization = "turtle"; + File file = new File("tmp/" + serialization); + if (!file.exists()) { + file.mkdir(); + } + try { + HashMap response = Mapper.main(domain, exportPrefix, exportUrl, catalogRepositoryDirectory, + componentLibraryFilePath.getAbsolutePath(), planFilePath.getAbsolutePath(), + endpointQueryURI, endpointPostURI, executionFilePath, expandedTemplateFilePath, abstractFilePath, + filePublisher, serialization); + return json.toJson(response); + } catch (Exception e) { + e.printStackTrace(); } - - retmap.put("url", exp.getTransformedExecutionURI()); - } catch (Exception e) { e.printStackTrace(); - retmap.put("error", e.getMessage()); } return json.toJson(retmap); } From 2a713d2d76eac3607349a525ae5584a6ebd7fc2f Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Thu, 18 May 2023 16:54:15 -0400 Subject: [PATCH 02/11] fix: clean code to publish --- .../portal/controllers/RunController.java | 28 ++--- .../wings/portal/resources/RunResource.java | 102 ++++++++++-------- 2 files changed, 70 insertions(+), 60 deletions(-) diff --git a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java index 73d8ea26..5e0ce89b 100644 --- a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java +++ b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java @@ -47,6 +47,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.jena.base.Sys; import edu.isi.kcap.wings.opmm.WorkflowExecutionExport; import edu.isi.kcap.wings.opmm.WorkflowTemplateExport; @@ -534,13 +535,15 @@ private String urlToString(URL url) { /** * @param runid * @return + * @throws Exception */ - public String publishRun(String runid) { + public HashMap publishRun(String runid) throws Exception { HashMap retmap = new HashMap(); ExecutionMonitorAPI monitor = config.getDomainExecutionMonitor(); RuntimePlan plan = monitor.getRunDetails(runid); + HashMap response = null; if (plan.getRuntimeInfo().getStatus() != Status.SUCCESS) { - retmap.put("error", "Can only publish successfully completed runs"); + throw new Exception("Can only publish successfully completed runs"); } else try { // Mapper opmm = new Mapper(); @@ -649,30 +652,29 @@ public String publishRun(String runid) { String executionFilePath = run_exportdir + File.separator + "execution"; String expandedTemplateFilePath = run_exportdir + File.separator + "expandedTemplate"; String abstractFilePath = run_exportdir + File.separator + "abstract"; - // OPMW MAPPER CODE - // domain - // exportName - // catalogRepositoryDirectory - // componentLibraryFile - // String serialization = "turtle"; File file = new File("tmp/" + serialization); if (!file.exists()) { file.mkdir(); } try { - HashMap response = Mapper.main(domain, exportPrefix, exportUrl, catalogRepositoryDirectory, + response = Mapper.main(domain, exportPrefix, exportUrl, catalogRepositoryDirectory, componentLibraryFilePath.getAbsolutePath(), planFilePath.getAbsolutePath(), endpointQueryURI, endpointPostURI, executionFilePath, expandedTemplateFilePath, abstractFilePath, filePublisher, serialization); - return json.toJson(response); + System.out.println(executionFilePath); + System.out.println(expandedTemplateFilePath); + System.out.println(abstractFilePath); + + System.out.println("Response: " + response); + System.out.println("Published ended"); } catch (Exception e) { - e.printStackTrace(); + throw new Exception("Error publishing run: " + e.getMessage()); } } catch (Exception e) { - e.printStackTrace(); + throw new Exception("Error publishing run: " + e.getMessage()); } - return json.toJson(retmap); + return response; } /* diff --git a/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java b/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java index afb3c2c8..95874f62 100644 --- a/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java +++ b/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -15,7 +16,9 @@ import javax.ws.rs.core.Response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.Gson; +import edu.isi.kcap.wings.opmm.DataTypes.Links; import edu.isi.wings.portal.classes.util.TemplateBindings; import edu.isi.wings.portal.controllers.RunController; @@ -26,102 +29,97 @@ public class RunResource extends WingsResource { @PostConstruct public void init() { super.init(); - if(this.hasPermissions() && !this.isPage("intro")) + if (this.hasPermissions() && !this.isPage("intro")) this.rc = new RunController(config); } - + @PreDestroy public void destroy() { - if(this.rc != null) { + if (this.rc != null) { this.rc.end(); } } - + @GET @Produces(MediaType.TEXT_HTML) public String getHTML( @QueryParam("run_id") final String run_id) { - if(this.rc != null) { + if (this.rc != null) { request.setAttribute("controller", this.rc); request.setAttribute("run_id", run_id); return this.callViewer("RunViewer"); } return null; } - + @GET @Path("intro") public void getIntroduction() { this.loadIntroduction("ManageRuns"); } - + @GET @Path("getRunList") @Produces(MediaType.APPLICATION_JSON) public String getRunList( - @QueryParam("pattern") final String pattern, - @QueryParam("status") final String status, - @QueryParam("start") int start, - @QueryParam("limit") int limit - ) { - if(this.rc != null) + @QueryParam("pattern") final String pattern, + @QueryParam("status") final String status, + @QueryParam("start") int start, + @QueryParam("limit") int limit) { + if (this.rc != null) return this.rc.getRunListJSON(pattern, status, start, limit); return null; } - @GET @Path("getRunListSimple") @Produces(MediaType.APPLICATION_JSON) public String getRunListSimple( - @QueryParam("pattern") final String pattern, - @QueryParam("status") final String status, - @QueryParam("start") int start, - @QueryParam("limit") int limit, - @QueryParam("started_after") final long started_after - ) { - if(this.rc != null) { + @QueryParam("pattern") final String pattern, + @QueryParam("status") final String status, + @QueryParam("start") int start, + @QueryParam("limit") int limit, + @QueryParam("started_after") final long started_after) { + if (this.rc != null) { Date startdate = null; - if(started_after != 0) - startdate = new Date(started_after*1000); + if (started_after != 0) + startdate = new Date(started_after * 1000); return this.rc.getRunListSimpleJSON(pattern, status, start, limit, startdate); } return null; } - @POST @Path("getRunDetails") @Produces(MediaType.APPLICATION_JSON) public String getRunDetails( @FormParam("run_id") String run_id) { - if(this.rc != null) + if (this.rc != null) return this.rc.getRunJSON(run_id); return null; } - + @POST @Path("getRunPlan") @Produces(MediaType.APPLICATION_JSON) public String getRunPlan( @FormParam("run_id") String run_id) { - if(this.rc != null) + if (this.rc != null) return this.rc.getRunPlanJSON(run_id); return null; } - + @POST @Path("expandAndRunWorkflow") @Produces(MediaType.APPLICATION_JSON) public ArrayList expandAndRunWorkflow( @JsonProperty("template_bindings") final TemplateBindings tbindings) { - if(this.rc != null) { + if (this.rc != null) { return rc.expandAndRunTemplate(tbindings, this.context); } return null; } - @POST @Path("runWorkflow") @Produces(MediaType.TEXT_PLAIN) @@ -131,61 +129,71 @@ public String runWorkflow( @FormParam("constraints_json") String constraints_json, @FormParam("seed_json") String seed_json, @FormParam("seed_constraints_json") String seed_constraints_json) { - if(this.rc != null) + if (this.rc != null) return rc.runExpandedTemplate(template_id, json, constraints_json, seed_json, seed_constraints_json, null, this.context); return null; } - + @POST @Path("reRunWorkflow") @Produces("application/json") public Response reRunWorkflow( - @FormParam("run_id") String run_id) { - if(this.rc != null){ - return rc.reRunPlan(run_id, this.context); + @FormParam("run_id") String run_id) { + if (this.rc != null) { + return rc.reRunPlan(run_id, this.context); } return null; } - + @POST @Path("deleteRun") @Produces(MediaType.TEXT_PLAIN) public String deleteRun(@FormParam("json") String json) { - if(this.rc != null) + if (this.rc != null) return rc.deleteRun(json, context); return null; } - + @POST @Path("deleteRuns") @Produces(MediaType.TEXT_PLAIN) public String deleteRuns(@FormParam("json") String json) { - if(this.rc != null) + if (this.rc != null) return rc.deleteRuns(json, context); return null; } - + @POST @Path("stopRun") @Produces(MediaType.TEXT_PLAIN) public String stopRun( @FormParam("run_id") String run_id) { - if(this.rc != null) + if (this.rc != null) return this.rc.stopRun(run_id, context) + ""; return "false"; } - + @POST @Path("publishRun") @Produces(MediaType.TEXT_PLAIN) public String publishRun( - @FormParam("run_id") String run_id) { - if(this.rc != null) - return this.rc.publishRun(run_id); + @FormParam("run_id") String run_id) { + if (this.rc != null) { + try { + HashMap responseQuery = this.rc.publishRun(run_id); + Gson gson = new Gson(); + String json = gson.toJson(responseQuery); + Links catalog = responseQuery.get("catalog"); + System.out.println(json); + return json; + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } return null; } - - } \ No newline at end of file From 99f06c6a355e5f7b85c9a170798caef15bdd60d3 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 12 Jun 2023 19:42:58 -0400 Subject: [PATCH 03/11] fix: publish the files as internal files --- planner/pom.xml | 2 +- portal/pom.xml | 2 +- .../wings/portal/classes/config/Config.java | 287 +++++++++--------- .../portal/controllers/RunController.java | 11 +- .../wings/portal/resources/RunResource.java | 5 +- .../servlets/ProvenanceFileServlet.java | 65 ++++ portal/src/main/webapp/WEB-INF/web.xml | 37 ++- 7 files changed, 247 insertions(+), 162 deletions(-) create mode 100644 portal/src/main/java/edu/isi/wings/portal/servlets/ProvenanceFileServlet.java diff --git a/planner/pom.xml b/planner/pom.xml index d2337291..eade5f75 100644 --- a/planner/pom.xml +++ b/planner/pom.xml @@ -17,7 +17,7 @@ 0.1.53 - 1.3.0 + 1.3.1 0.8.11 2.2.4 4.5 diff --git a/portal/pom.xml b/portal/pom.xml index 2228f7c4..978c993e 100644 --- a/portal/pom.xml +++ b/portal/pom.xml @@ -17,7 +17,7 @@ - 1.2.6 + 1.2.7 2.5 4.5.2 2.27 diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java index 2ed15c51..ed65d5a7 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java @@ -62,6 +62,8 @@ public class Config { private String storageDirectory; private String tdbDirectory; private String logsDirectory; + private String provenanceDirectory; + private String dotFile; private String serverUrl; private String workflowOntologyUrl; @@ -83,10 +85,10 @@ public class Config { private boolean isSandboxed = false; private PlannerConfig plannerConfig = new PlannerConfig(); - + // Comma separated list of spellbook client hosts private String clients; - + // The following are set from the "request" variable private String viewerId; private String userId; @@ -123,22 +125,24 @@ public void setUserDomainUrl(String userDomainUrl) { this.userDomainUrl = userDomainUrl; } - public Config() {} - + public Config() { + } + public Config(HttpServletRequest request, String userid, String domain) { - // Initialize UserDatabase - this.initializeUserDatabase(); + // Initialize UserDatabase + this.initializeUserDatabase(); - // Initialize portal config - this.initializePortalConfig(request); + // Initialize portal config + this.initializePortalConfig(request); - // Initialize user config - this.initializeUserConfig(request, userid, domain); - } + // Initialize user config + this.initializeUserConfig(request, userid, domain); + } public void getPermissions() { // Check domain, user & viewerid - // Return Permissions (canRead=true/false, canWrite=true/false, canExecute=true/false) + // Return Permissions (canRead=true/false, canWrite=true/false, + // canExecute=true/false) } public boolean checkUser(HttpServletResponse response) { @@ -165,7 +169,7 @@ public boolean checkUser(HttpServletResponse response) { } public void showError(HttpServletRequest request, - HttpServletResponse response, String message) { + HttpServletResponse response, String message) { try { response.setContentType("text/html"); request.setAttribute("message", message); @@ -177,12 +181,12 @@ public void showError(HttpServletRequest request, } public boolean checkDomain(HttpServletRequest request, - HttpServletResponse response) { + HttpServletResponse response) { return this.checkDomain(request, response, true); } public boolean checkDomain(HttpServletRequest request, - HttpServletResponse response, boolean show_error) { + HttpServletResponse response, boolean show_error) { if (!this.checkUser(response)) return false; @@ -209,7 +213,7 @@ public boolean checkDomain(HttpServletRequest request, && !this.viewerId.equals(this.userId)) { // Check domain permissions // TODO: Check read, write & execute permission based on input - // For now: all or none permissions + // For now: all or none permissions Permission perm = this.domain.getPermissionForUser(this.viewerId); if (!perm.canRead()) { if (show_error) @@ -232,8 +236,8 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str this.scriptPath = request.getRequestURI(); if (this.domainId != null) - this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() - + "/" + this.getUserId() + "/" + this.getDomainId(); + this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() + + "/" + this.getUserId() + "/" + this.getDomainId(); this.sessionId = request.getSession().getId(); @@ -247,8 +251,8 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str if (!this.checkUser(null)) return; - this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir - + "/" + userId; + this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir + + "/" + userId; this.userDir = storageDirectory + File.separator + usersRelativeDir + File.separator + userId; this.userPath = contextRootPath + "/" + usersRelativeDir + "/" + userId; @@ -277,9 +281,9 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str } if (this.domain != null) { - this.domainId = this.domain.getDomainName(); - this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() - + "/" + this.getUserId() + "/" + this.domain.getDomainName(); + this.domainId = this.domain.getDomainName(); + this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() + + "/" + this.getUserId() + "/" + this.domain.getDomainName(); } } @@ -333,50 +337,57 @@ private void initializePortalConfig(HttpServletRequest request) { this.workflowOntologyUrl = serverConfig.getString("ontology.workflow"); this.executionOntologyUrl = serverConfig.getString("ontology.execution"); this.resourceOntologyUrl = serverConfig.getString("ontology.resource"); - - if(serverConfig.containsKey("metaworkflows")) - this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); - - if(serverConfig.containsKey("light-reasoner")) - plannerConfig.dataValidation = !serverConfig.getBoolean("light-reasoner"); - - if(serverConfig.containsKey("planner.data-validation")) - plannerConfig.dataValidation = serverConfig.getBoolean("planner.data-validation"); - if(serverConfig.containsKey("planner.specialization")) - plannerConfig.specialization = serverConfig.getBoolean("planner.specialization"); - if(serverConfig.containsKey("planner.use-rules")) - plannerConfig.useRules = serverConfig.getBoolean("planner.use-rules"); - - if(serverConfig.containsKey("planner.max-queue-size")) - plannerConfig.maxQueueSize = serverConfig.getInt("planner.max-queue-size", 1000); + + if (serverConfig.containsKey("metaworkflows")) + this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); + + if (serverConfig.containsKey("light-reasoner")) + plannerConfig.dataValidation = !serverConfig.getBoolean("light-reasoner"); + + if (serverConfig.containsKey("planner.data-validation")) + plannerConfig.dataValidation = serverConfig.getBoolean("planner.data-validation"); + if (serverConfig.containsKey("planner.specialization")) + plannerConfig.specialization = serverConfig.getBoolean("planner.specialization"); + if (serverConfig.containsKey("planner.use-rules")) + plannerConfig.useRules = serverConfig.getBoolean("planner.use-rules"); + + if (serverConfig.containsKey("planner.max-queue-size")) + plannerConfig.maxQueueSize = serverConfig.getInt("planner.max-queue-size", 1000); else - plannerConfig.maxQueueSize = 1000; - if(serverConfig.containsKey("planner.parallelism")) - plannerConfig.parallelism = serverConfig.getInt("planner.parallelism", 10); + plannerConfig.maxQueueSize = 1000; + if (serverConfig.containsKey("planner.parallelism")) + plannerConfig.parallelism = serverConfig.getInt("planner.parallelism", 10); else - plannerConfig.parallelism = 10; - - if(serverConfig.containsKey("storage.logs")) { - this.logsDirectory = serverConfig.getString("storage.logs"); - } - else { - this.logsDirectory = this.storageDirectory + File.separator + "logs"; + plannerConfig.parallelism = 10; + + if (serverConfig.containsKey("storage.logs")) { + this.logsDirectory = serverConfig.getString("storage.logs"); + } else { + this.logsDirectory = this.storageDirectory + File.separator + "logs"; } - if(serverConfig.containsKey("storage.delete-run-outputs")) { - this.deleteRunOutputs = serverConfig.getBoolean("storage.delete-run-outputs"); + if (serverConfig.containsKey("storage.provenance")) { + this.provenanceDirectory = serverConfig.getString("storage.provenance"); + } else { + this.provenanceDirectory = this.storageDirectory + File.separator + "provenance"; } - else { - this.deleteRunOutputs = false; + if (serverConfig.containsKey("storage.delete-run-outputs")) { + this.deleteRunOutputs = serverConfig.getBoolean("storage.delete-run-outputs"); + } else { + this.deleteRunOutputs = false; } // Create logsDir (if it doesn't exist) File logdir = new File(this.logsDirectory); if (!logdir.exists() && !logdir.mkdirs()) System.err.println("Cannot create logs directory : " + logdir.getAbsolutePath()); - + // Create provenanceDirectory (if it doesn't exist) + File provdir = new File(this.provenanceDirectory); + if (!provdir.exists() && !provdir.mkdirs()) + System.err.println("Cannot create provenance directory : " + provdir.getAbsolutePath()); + this.exportCommunityUrl = serverUrl + contextRootPath + exportServletPath + "/" - + communityRelativeDir; + + communityRelativeDir; this.communityPath = contextRootPath + "/" + usersRelativeDir + "/" + communityRelativeDir; - + this.communityDir = storageDirectory + File.separator + communityRelativeDir; // Create communityDir (if it doesn't exist) @@ -405,67 +416,66 @@ private void initializePortalConfig(HttpServletRequest request) { } // Load publishing configuration - String publishUrl = serverConfig.getString("publisher.url"); - String publishExportName = serverConfig.getString("publisher.name"); - String tstorePublishUrl = serverConfig.getString("publisher.triple-store.publish"); - String tstoreQueryUrl = serverConfig.getString("publisher.triple-store.query"); - String domainsDir = serverConfig.getString("publisher.triple-store.domains-directory"); - String uploadUrl = serverConfig.getString("publisher.upload-server.url"); - String uploadUsername = serverConfig.getString("publisher.upload-server.username"); - String uploadPassword = serverConfig.getString("publisher.upload-server.password"); - String uploadDir = serverConfig.getString("publisher.upload-server.directory"); - String uploadHost = serverConfig.getString("publisher.upload-server.host"); - String uploadUserId = serverConfig.getString("publisher.upload-server.userid"); - String uploadKey = serverConfig.getString("publisher.upload-server.private-key"); - String sizeString = serverConfig.getString("publisher.upload-server.max-upload-size"); - - this.publisher = new Publisher(); - this.publisher.setUrl(publishUrl); - this.publisher.setExportName(publishExportName); - this.publisher.setDomainsDir(domainsDir); - this.publisher.setTstorePublishUrl(tstorePublishUrl); - this.publisher.setTstoreQueryUrl(tstoreQueryUrl); - - - ServerDetails upserver = new ServerDetails(); - upserver.setUrl(uploadUrl); - upserver.setUsername(uploadUsername); - upserver.setPassword(uploadPassword); - upserver.setHostUserId(uploadUserId); - upserver.setDirectory(uploadDir); - upserver.setHost(uploadHost); - upserver.setPrivateKey(uploadKey); - if(sizeString != null) { + String publishUrl = serverConfig.getString("publisher.url"); + String publishExportName = serverConfig.getString("publisher.name"); + String tstorePublishUrl = serverConfig.getString("publisher.triple-store.publish"); + String tstoreQueryUrl = serverConfig.getString("publisher.triple-store.query"); + String domainsDir = serverConfig.getString("publisher.triple-store.domains-directory"); + String uploadUrl = serverConfig.getString("publisher.upload-server.url"); + String uploadUsername = serverConfig.getString("publisher.upload-server.username"); + String uploadPassword = serverConfig.getString("publisher.upload-server.password"); + String uploadDir = serverConfig.getString("publisher.upload-server.directory"); + String uploadHost = serverConfig.getString("publisher.upload-server.host"); + String uploadUserId = serverConfig.getString("publisher.upload-server.userid"); + String uploadKey = serverConfig.getString("publisher.upload-server.private-key"); + String sizeString = serverConfig.getString("publisher.upload-server.max-upload-size"); + + this.publisher = new Publisher(); + this.publisher.setUrl(publishUrl); + this.publisher.setExportName(publishExportName); + this.publisher.setDomainsDir(domainsDir); + this.publisher.setTstorePublishUrl(tstorePublishUrl); + this.publisher.setTstoreQueryUrl(tstoreQueryUrl); + + ServerDetails upserver = new ServerDetails(); + upserver.setUrl(uploadUrl); + upserver.setUsername(uploadUsername); + upserver.setPassword(uploadPassword); + upserver.setHostUserId(uploadUserId); + upserver.setDirectory(uploadDir); + upserver.setHost(uploadHost); + upserver.setPrivateKey(uploadKey); + if (sizeString != null) { long size = this.getSizeFromString(sizeString); upserver.setMaxUploadSize(size); - } - this.publisher.setUploadServer(upserver); + } + this.publisher.setUploadServer(upserver); } - + private long getSizeFromString(String sizeString) { - long kb = 1024; - long mb = kb*kb; - long gb = kb*mb; - long tb = kb*gb; - - Pattern pat = Pattern.compile("(\\d+)\\s*([KkMmGgTt])[Bb]?"); - Matcher mat = pat.matcher(sizeString); - if(mat.find()) { - long size=Long.parseLong(mat.group(1)); - if(mat.groupCount() > 1) { - String units = mat.group(2).toLowerCase(); - if(units.equals("k")) - return size*kb; - if(units.equals("m")) - return size*mb; - if(units.equals("g")) - return size*gb; - if(units.equals("t")) - return size*tb; + long kb = 1024; + long mb = kb * kb; + long gb = kb * mb; + long tb = kb * gb; + + Pattern pat = Pattern.compile("(\\d+)\\s*([KkMmGgTt])[Bb]?"); + Matcher mat = pat.matcher(sizeString); + if (mat.find()) { + long size = Long.parseLong(mat.group(1)); + if (mat.groupCount() > 1) { + String units = mat.group(2).toLowerCase(); + if (units.equals("k")) + return size * kb; + if (units.equals("m")) + return size * mb; + if (units.equals("g")) + return size * gb; + if (units.equals("t")) + return size * tb; + } + return size; } - return size; - } - return 0; + return 0; } private void initializeUserDatabase() { @@ -478,7 +488,7 @@ private ExeEngine getExeEngine(HierarchicalConfiguration node) { String impl = node.getString("implementation"); ExeEngine.Type type = ExeEngine.Type.valueOf(node.getString("type")); ExeEngine engine = new ExeEngine(name, impl, type); - for (Iterator it = node.getKeys("properties"); it.hasNext(); ) { + for (Iterator it = node.getKeys("properties"); it.hasNext();) { String key = (String) it.next(); String value = node.getString(key); engine.addProperty(key.replace("properties.", ""), value); @@ -530,7 +540,7 @@ private void createDefaultPortalConfig(HttpServletRequest request) { else storageDir = System.getProperty("java.io.tmpdir") + File.separator + "wings" + File.separator + "storage"; - + File storageDirFile = new File(storageDir); if (!storageDirFile.exists() && !storageDirFile.mkdirs()) System.err.println("Cannot create storage directory: " + storageDir); @@ -554,11 +564,13 @@ private void createDefaultPortalConfig(HttpServletRequest request) { this.addEngineConfig(config, new ExeEngine("Distributed", DistributedExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); - /*this.addEngineConfig(config, new ExeEngine("OODT", - OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); - - this.addEngineConfig(config, new ExeEngine("Pegasus", - PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN));*/ + /* + * this.addEngineConfig(config, new ExeEngine("OODT", + * OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); + * + * this.addEngineConfig(config, new ExeEngine("Pegasus", + * PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); + */ try { config.save(this.configFile); @@ -612,15 +624,15 @@ public Properties getProperties(Domain domain) { if (this.getResourceOntologyUrl() == null) this.setResourceOntologyUrl(ontdirurl + "/resource.owl"); props.setProperty("ont.resource.url", this.getResourceOntologyUrl()); - + props.setProperty("lib.resource.url", this.getExportCommunityUrl() + "/resource/library.owl"); if (domain != null && !domain.getUseSharedTripleStore()) - props.setProperty("lib.resource.map", - "file:" + domain.getDomainDirectory() + File.separator + "ontology" - + File.separator + "resource" + File.separator + "library.owl"); - + props.setProperty("lib.resource.map", + "file:" + domain.getDomainDirectory() + File.separator + "ontology" + + File.separator + "resource" + File.separator + "library.owl"); + props.setProperty("lib.provenance.url", this.getExportCommunityUrl() + "/provenance/library.owl"); @@ -766,15 +778,15 @@ public boolean isAdminViewer() { public void setAdminViewer(boolean isAdminViewer) { this.isAdminViewer = isAdminViewer; } - + public boolean hasMetaWorkflows() { - return hasMetaWorkflows; + return hasMetaWorkflows; } - + public void setMetaWorkflows(boolean hasMetaWorkflows) { this.hasMetaWorkflows = hasMetaWorkflows; } - + public void setExportUserUrl(String exportUserUrl) { this.exportUserUrl = exportUserUrl; } @@ -855,12 +867,16 @@ public void setTripleStoreDir(String tripleStoreDir) { this.tdbDirectory = tripleStoreDir; } + public String getProvenanceDirectory() { + return provenanceDirectory; + } + public String getLogsDirectory() { - return logsDirectory; + return logsDirectory; } public void setLogsDirectory(String logsDirectory) { - this.logsDirectory = logsDirectory; + this.logsDirectory = logsDirectory; } public String getSessionId() { @@ -888,11 +904,11 @@ public void setDotFile(String dotFile) { } public boolean isDeleteRunOutputs() { - return deleteRunOutputs; + return deleteRunOutputs; } public void setDeleteRunOutputs(boolean deleteRunOutputs) { - this.deleteRunOutputs = deleteRunOutputs; + this.deleteRunOutputs = deleteRunOutputs; } public String getClients() { @@ -902,7 +918,7 @@ public String getClients() { public void setClients(String clients) { this.clients = clients; } - + public String getStorageDirectory() { return storageDirectory; } @@ -912,15 +928,14 @@ public void setStorageDirectory(String storageDirectory) { } public PlannerConfig getPlannerConfig() { - return plannerConfig; + return plannerConfig; } public void setPlannerConfig(PlannerConfig plannerConfig) { - this.plannerConfig = plannerConfig; + this.plannerConfig = plannerConfig; } public Set getEnginesList() { return this.engines.keySet(); } } - diff --git a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java index 5e0ce89b..46900e15 100644 --- a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java +++ b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java @@ -52,6 +52,7 @@ import edu.isi.kcap.wings.opmm.WorkflowExecutionExport; import edu.isi.kcap.wings.opmm.WorkflowTemplateExport; import edu.isi.kcap.wings.opmm.DataTypes.Links; +import edu.isi.kcap.wings.opmm.DataTypes.ProvenanceResponseSchema; import edu.isi.kcap.ontapi.KBTriple; import edu.isi.wings.catalog.component.ComponentFactory; import edu.isi.wings.catalog.data.DataFactory; @@ -537,11 +538,11 @@ private String urlToString(URL url) { * @return * @throws Exception */ - public HashMap publishRun(String runid) throws Exception { + public ProvenanceResponseSchema publishRun(String runid) throws Exception { HashMap retmap = new HashMap(); ExecutionMonitorAPI monitor = config.getDomainExecutionMonitor(); RuntimePlan plan = monitor.getRunDetails(runid); - HashMap response = null; + ProvenanceResponseSchema response = null; if (plan.getRuntimeInfo().getStatus() != Status.SUCCESS) { throw new Exception("Can only publish successfully completed runs"); } else @@ -662,12 +663,6 @@ public HashMap publishRun(String runid) throws Exception { componentLibraryFilePath.getAbsolutePath(), planFilePath.getAbsolutePath(), endpointQueryURI, endpointPostURI, executionFilePath, expandedTemplateFilePath, abstractFilePath, filePublisher, serialization); - System.out.println(executionFilePath); - System.out.println(expandedTemplateFilePath); - System.out.println(abstractFilePath); - - System.out.println("Response: " + response); - System.out.println("Published ended"); } catch (Exception e) { throw new Exception("Error publishing run: " + e.getMessage()); } diff --git a/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java b/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java index 95874f62..922513c5 100644 --- a/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java +++ b/portal/src/main/java/edu/isi/wings/portal/resources/RunResource.java @@ -19,6 +19,7 @@ import com.google.gson.Gson; import edu.isi.kcap.wings.opmm.DataTypes.Links; +import edu.isi.kcap.wings.opmm.DataTypes.ProvenanceResponseSchema; import edu.isi.wings.portal.classes.util.TemplateBindings; import edu.isi.wings.portal.controllers.RunController; @@ -181,11 +182,9 @@ public String publishRun( @FormParam("run_id") String run_id) { if (this.rc != null) { try { - HashMap responseQuery = this.rc.publishRun(run_id); + ProvenanceResponseSchema responseQuery = this.rc.publishRun(run_id); Gson gson = new Gson(); String json = gson.toJson(responseQuery); - Links catalog = responseQuery.get("catalog"); - System.out.println(json); return json; } catch (Exception e) { // TODO Auto-generated catch block diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ProvenanceFileServlet.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ProvenanceFileServlet.java new file mode 100644 index 00000000..e3c4f381 --- /dev/null +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ProvenanceFileServlet.java @@ -0,0 +1,65 @@ +package edu.isi.wings.portal.servlets; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import edu.isi.wings.portal.classes.config.Config; + +public class ProvenanceFileServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + Config config = new Config(request, null, null); + String provenanceDirectoryPath = config.getProvenanceDirectory(); + File provenanceDirectory = new File(provenanceDirectoryPath); + + String requestFile = request.getPathInfo(); + String filePath = provenanceDirectoryPath + File.separator + requestFile; + System.out.println("FileServlet: " + filePath); + + File file = new File(filePath); + if (!file.exists()) { + response.sendError(HttpServletResponse.SC_NOT_FOUND); // 404. + return; + } + // check if filepath is in allowed directory + if (!isInsideDirectory(file, provenanceDirectory)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + // Set content type and headers for the response + response.setContentType("application/octet-stream"); + response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getName() + "\""); + + // Read the file and write its contents to the response output stream + try (FileInputStream fis = new FileInputStream(filePath); + OutputStream out = response.getOutputStream()) { + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = fis.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + // Handle exception + e.printStackTrace(); + } + } + + private static boolean isInsideDirectory(File file, File directory) { + File parent = file.getParentFile(); + if (parent == null) { + return false; + } else if (parent.equals(directory)) { + return true; + } else { + return isInsideDirectory(parent, directory); + } + } +} \ No newline at end of file diff --git a/portal/src/main/webapp/WEB-INF/web.xml b/portal/src/main/webapp/WEB-INF/web.xml index f211dfc2..11c49601 100644 --- a/portal/src/main/webapp/WEB-INF/web.xml +++ b/portal/src/main/webapp/WEB-INF/web.xml @@ -1,8 +1,8 @@ + xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee" + xmlns:web="http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd" + version="3.1" xmlns="http://xmlns.jcp.org/xml/ns/javaee"> Wings Portal Login page @@ -14,7 +14,7 @@ Login /login/* - + Servlet to Export Graph Graph Exporter @@ -26,6 +26,17 @@ /export/* + + Servlet to obtain the Provenance Files + Provenance Files + ProvenanceFile + edu.isi.wings.portal.servlets.ProvenanceFileServlet + + + ProvenanceFile + /provenance/* + + SparqlEndpoint @@ -36,7 +47,7 @@ SparqlEndpoint /sparql - + Wings Config Wings config @@ -46,9 +57,9 @@ ViewConfig /config - - - @@ -78,11 +89,11 @@ REQUEST FORWARD - + 43200 - + WingsUser @@ -111,7 +122,7 @@ WingsAdmin - + FORM @@ -119,5 +130,5 @@ /jsp/login/login_failed.jsp - - + + \ No newline at end of file From e9cce591f200c954e2941e1ecc66889ca9f724f1 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Tue, 13 Jun 2023 16:58:38 -0400 Subject: [PATCH 04/11] fix: save --- planner/.classpath | 11 ++++ portal/pom.xml | 2 +- wings-docker/config/default/portal.properties | 52 ------------------ .../config/local-mosorio/portal.properties | 54 ------------------- .../config/local-mosorio/wings-portal.xml | 5 -- wings-docker/config/pegasus/portal.properties | 53 ------------------ wings-docker/docker/default/Dockerfile | 1 - 7 files changed, 12 insertions(+), 166 deletions(-) delete mode 100644 wings-docker/config/default/portal.properties delete mode 100644 wings-docker/config/local-mosorio/portal.properties delete mode 100644 wings-docker/config/local-mosorio/wings-portal.xml delete mode 100644 wings-docker/config/pegasus/portal.properties diff --git a/planner/.classpath b/planner/.classpath index 94fadc55..9fdde8b4 100644 --- a/planner/.classpath +++ b/planner/.classpath @@ -54,5 +54,16 @@ + + + + + + + + + + + diff --git a/portal/pom.xml b/portal/pom.xml index 978c993e..9cdcf2f4 100644 --- a/portal/pom.xml +++ b/portal/pom.xml @@ -21,7 +21,7 @@ 2.5 4.5.2 2.27 - 2.8.6 + 2.14.0-rc2 1.1.0.Final 1.3.3 diff --git a/wings-docker/config/default/portal.properties b/wings-docker/config/default/portal.properties deleted file mode 100644 index dace1762..00000000 --- a/wings-docker/config/default/portal.properties +++ /dev/null @@ -1,52 +0,0 @@ -{ - storage = - { - local = /opt/wings/storage/default; - tdb = /opt/wings/storage/default/TDB; - } - - server = http://localhost:8080; - graphviz = /usr/bin/dot; - light-reasoner = false; - ontology = - { - data = http://www.wings-workflows.org/ontology/data.owl; - component = http://www.wings-workflows.org/ontology/component.owl; - workflow = http://www.wings-workflows.org/ontology/workflow.owl; - execution = http://www.wings-workflows.org/ontology/execution.owl; - } - - execution = - { - engine = - { - name = Local; - implementation = edu.isi.wings.execution.engine.api.impl.local.LocalExecutionEngine; - type = BOTH; - } - - engine = - { - name = Distributed; - implementation = edu.isi.wings.execution.engine.api.impl.distributed.DistributedExecutionEngine; - type = BOTH; - } - } - - publisher = - { - url = "https://publisher.mint.isi.edu"; - name = "exportTest" - upload-server = - { - url = "https://publisher.mint.isi.edu"; - username = "user"; - password = "password"; - } - triple-store = { - publish = http://ontosoft.isi.edu:3030/provenance/data; - query = http://ontosoft.isi.edu:3030/provenance/sparql; - domains-directory = /opt/wings/storage/default; - } - } -} diff --git a/wings-docker/config/local-mosorio/portal.properties b/wings-docker/config/local-mosorio/portal.properties deleted file mode 100644 index 1ec6f442..00000000 --- a/wings-docker/config/local-mosorio/portal.properties +++ /dev/null @@ -1,54 +0,0 @@ -{ - - - storage = - { - local = /Users/mosorio/Downloads/tomcat/wings/storage/default; - tdb = /Users/mosorio/Downloads/tomcat/wings/storage/default/TDB; - } - - server = http://localhost:8080; - graphviz = /usr/bin/dot; - light-reasoner = false; - ontology = - { - data = http://www.wings-workflows.org/ontology/data.owl; - component = http://www.wings-workflows.org/ontology/component.owl; - workflow = http://www.wings-workflows.org/ontology/workflow.owl; - execution = http://www.wings-workflows.org/ontology/execution.owl; - } - - execution = - { - engine = - { - name = Local; - implementation = edu.isi.wings.execution.engine.api.impl.local.LocalExecutionEngine; - type = BOTH; - } - - engine = - { - name = Distributed; - implementation = edu.isi.wings.execution.engine.api.impl.distributed.DistributedExecutionEngine; - type = BOTH; - } - } - - publisher = - { - url = "http://ontosoft.isi.edu:7071"; - name = "exportTest" - upload-server = - { - url = "http://ontosoft.isi.edu:7071"; - username = "user"; - password = "password"; - } - triple-store = { - publish = http://ontosoft.isi.edu:3030/provenance/data; - query = http://ontosoft.isi.edu:3030/provenance/sparql; - domains-directory = /Users/mosorio/Downloads/tomcat/wings/storage/default; - } - } -} diff --git a/wings-docker/config/local-mosorio/wings-portal.xml b/wings-docker/config/local-mosorio/wings-portal.xml deleted file mode 100644 index 272e5c07..00000000 --- a/wings-docker/config/local-mosorio/wings-portal.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/wings-docker/config/pegasus/portal.properties b/wings-docker/config/pegasus/portal.properties deleted file mode 100644 index c05f48bc..00000000 --- a/wings-docker/config/pegasus/portal.properties +++ /dev/null @@ -1,53 +0,0 @@ -{ - storage = - { - local = /lfs/wings/storage/default; - tdb = /lfs/wings/storage/default/TDB; - } - - server = http://localhost:8080; - graphviz = /usr/bin/dot; - light-reasoner = false; - ontology = - { - data = http://www.wings-workflows.org/ontology/data.owl; - component = http://www.wings-workflows.org/ontology/component.owl; - workflow = http://www.wings-workflows.org/ontology/workflow.owl; - execution = http://www.wings-workflows.org/ontology/execution.owl; - } - - execution = - { - engine = - { - name = Local; - implementation = edu.isi.wings.execution.engine.api.impl.local.LocalExecutionEngine; - type = BOTH; - } - - engine = - { - name = Distributed; - implementation = edu.isi.wings.execution.engine.api.impl.distributed.DistributedExecutionEngine; - type = BOTH; - } - - - engine = - { - name = Pegasus; - implementation = edu.isi.wings.execution.engine.api.impl.pegasus.PegasusExecutionEngine; - type = BOTH; - properties = - { - pegasus = - { - home = /usr; - storage-dir = /lfs/pegasus/storage; - site-catalog = /etc/pegasus/local.sites.xml - site = local; - } - } - } - } -} diff --git a/wings-docker/docker/default/Dockerfile b/wings-docker/docker/default/Dockerfile index fd9f3fee..a1738c90 100644 --- a/wings-docker/docker/default/Dockerfile +++ b/wings-docker/docker/default/Dockerfile @@ -49,6 +49,5 @@ RUN jar -xvf wings-portal-*.war # Add wings context file to the wings portal webapp COPY ./wings-docker/config/default/wings-portal.xml /usr/local/tomcat/webapps/wings-portal/META-INF/context.xml -COPY ./wings-docker/config/default/portal.properties /etc/wings/portal.properties WORKDIR /usr/local/tomcat From 84735f5c2be6d7b0a3ba6e87cd2e07e37753dab4 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Tue, 20 Jun 2023 12:09:14 -0400 Subject: [PATCH 05/11] fix: maintain jackson 2.8.6 --- portal/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portal/pom.xml b/portal/pom.xml index 85c372e0..51cc1824 100644 --- a/portal/pom.xml +++ b/portal/pom.xml @@ -21,7 +21,7 @@ 2.5 4.5.2 2.27 - 2.14.0-rc2 + 2.8.6 1.1.0.Final 1.3.3 From 8b6b2dd1c46008b6558b32c04ad85befddb4991f Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Tue, 25 Jul 2023 11:40:09 -0400 Subject: [PATCH 06/11] fix: safe point --- .../wings/portal/classes/config/Config.java | 941 ------------------ .../portal/classes/config/MainConfig.java | 2 - .../portal/controllers/RunController.java | 15 - 3 files changed, 958 deletions(-) delete mode 100644 portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java deleted file mode 100644 index ed65d5a7..00000000 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ /dev/null @@ -1,941 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edu.isi.wings.portal.classes.config; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.configuration.HierarchicalConfiguration; -import org.apache.commons.configuration.plist.PropertyListConfiguration; - -import edu.isi.wings.execution.engine.ExecutionFactory; -import edu.isi.wings.execution.engine.api.PlanExecutionEngine; -import edu.isi.wings.execution.engine.api.StepExecutionEngine; -import edu.isi.wings.execution.engine.api.impl.distributed.DistributedExecutionEngine; -import edu.isi.wings.execution.engine.api.impl.local.LocalExecutionEngine; -import edu.isi.wings.execution.tools.ExecutionToolsFactory; -import edu.isi.wings.execution.tools.api.ExecutionLoggerAPI; -import edu.isi.wings.execution.tools.api.ExecutionMonitorAPI; -import edu.isi.wings.execution.tools.api.ExecutionResourceAPI; -import edu.isi.wings.portal.classes.domains.Domain; -import edu.isi.wings.portal.classes.domains.Permission; -import edu.isi.wings.portal.classes.users.User; -import edu.isi.wings.portal.classes.users.UsersDB; -import edu.isi.wings.portal.controllers.DomainController; - -public class Config { - // The Portal configuration properties file. Order of checking: - // 1. Check "config.file" servlet context parameter - // 2. Check ${user.home}/.wings/portal.properties if ${user.home} is present - // 3. Check /etc/wings/portal.properties - private String configFile; - - // The following are loaded from the config file - private String storageDirectory; - private String tdbDirectory; - private String logsDirectory; - private String provenanceDirectory; - - private String dotFile; - private String serverUrl; - private String workflowOntologyUrl; - private String dataOntologyUrl; - private String componentOntologyUrl; - private String executionOntologyUrl; - private String resourceOntologyUrl; - private boolean deleteRunOutputs; - - private String ontdirurl = "http://www.wings-workflows.org/ontology"; - - private HashMap engines; - - private Publisher publisher; - - // Some hardcoded values (TODO: override from config file) - private String usersRelativeDir = "users"; - private String exportServletPath = "/export"; - private boolean isSandboxed = false; - - private PlannerConfig plannerConfig = new PlannerConfig(); - - // Comma separated list of spellbook client hosts - private String clients; - - // The following are set from the "request" variable - private String viewerId; - private String userId; - private String sessionId; - private String contextRootPath; - private String scriptPath; - private String[] scriptArguments; - - private String communityRelativeDir = "common"; - private String communityPath; - private String communityDir; - private String exportCommunityUrl; - - // This following are user/domain specific properties - private String userPath; - private String userDir; - private ArrayList usersList; - private boolean isAdminViewer; - private boolean hasMetaWorkflows; - - private Domain domain; - private String domainId; - private ArrayList domainsList; - private String userDomainUrl; - private String exportUserUrl; - - private UsersDB userapi; - - public String getUserDomainUrl() { - return userDomainUrl; - } - - public void setUserDomainUrl(String userDomainUrl) { - this.userDomainUrl = userDomainUrl; - } - - public Config() { - } - - public Config(HttpServletRequest request, String userid, String domain) { - // Initialize UserDatabase - this.initializeUserDatabase(); - - // Initialize portal config - this.initializePortalConfig(request); - - // Initialize user config - this.initializeUserConfig(request, userid, domain); - } - - public void getPermissions() { - // Check domain, user & viewerid - // Return Permissions (canRead=true/false, canWrite=true/false, - // canExecute=true/false) - } - - public boolean checkUser(HttpServletResponse response) { - /* Check that such a user exists */ - try { - if (!this.userapi.hasUser(this.userId)) { - if (response != null) { - // If userId is not present in server - response.setContentType("text/html"); - response.getWriter().println("No such user: " + this.userId + " !"); - } - return false; - } - // Get viewer roles - if (this.viewerId == null) - return false; - User viewer = this.userapi.getUser(this.viewerId); - this.isAdminViewer = viewer.isAdmin(); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - return true; - } - - public void showError(HttpServletRequest request, - HttpServletResponse response, String message) { - try { - response.setContentType("text/html"); - request.setAttribute("message", message); - request.setAttribute("nohome", true); - request.getRequestDispatcher("/").forward(request, response); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public boolean checkDomain(HttpServletRequest request, - HttpServletResponse response) { - return this.checkDomain(request, response, true); - } - - public boolean checkDomain(HttpServletRequest request, - HttpServletResponse response, boolean show_error) { - if (!this.checkUser(response)) - return false; - - try { - // For a non-owner viewer, if there is no domain available, - // then return a message - if (this.domain == null && !this.viewerId.equals(this.userId)) { - if (show_error) - this.showError(request, response, "No Domains shared by " + userId + " !"); - return false; - } - - // Check that a domain is provided in the URL, or a default domain exists - String redirectUrl = this.getUserPath() + "/domains"; - if (this.domain == null && !this.scriptPath.equals(redirectUrl)) { - response.setContentType("text/html"); - response.setHeader("Refresh", "5; URL=" + redirectUrl); - response.getWriter().println("No such domain !
" - + "See list of domains at " + redirectUrl + ". " - + "Redirecting in 5 seconds"); - return false; - } else if (this.domain != null - && !this.scriptPath.equals(redirectUrl) - && !this.viewerId.equals(this.userId)) { - // Check domain permissions - // TODO: Check read, write & execute permission based on input - // For now: all or none permissions - Permission perm = this.domain.getPermissionForUser(this.viewerId); - if (!perm.canRead()) { - if (show_error) - this.showError(request, response, "No Permission !"); - return false; - } - } - } catch (IOException e) { - e.printStackTrace(); - } - return true; - } - - private void initializeUserConfig(HttpServletRequest request, String userid, String domainid) { - this.userId = userid; - this.domainId = domainid; - this.viewerId = request.getRemoteUser(); - - // Set default script values - this.scriptPath = request.getRequestURI(); - - if (this.domainId != null) - this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() - + "/" + this.getUserId() + "/" + this.getDomainId(); - - this.sessionId = request.getSession().getId(); - - if (this.viewerId == null) - return; - - // If no userId specified, then set the viewer as the user - if (this.userId == null) - this.userId = this.viewerId; - - if (!this.checkUser(null)) - return; - - this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir - + "/" + userId; - this.userDir = storageDirectory + File.separator + usersRelativeDir + File.separator + userId; - - this.userPath = contextRootPath + "/" + usersRelativeDir + "/" + userId; - - // Create userDir (if it doesn't exist) - File uf = new File(this.userDir); - if (!uf.exists() && !uf.mkdirs()) - System.err.println("Cannot create user directory : " + uf.getAbsolutePath()); - - // Get domain and user list - DomainController dc = new DomainController(this); - this.domainsList = dc.getReadableDomainsList(); - this.usersList = this.userapi.getUsersList(); - - // Get user's selected domain - this.domain = dc.getUserDomain(); - - // If the domain isn't a part of the readable domain list, - // then choose the first one - if (this.domain == null || - !domainsList.contains(this.domain.getDomainName())) { - if (domainsList.size() > 0) - this.domain = dc.getDomain(domainsList.get(0)); - else - this.domain = null; - } - - if (this.domain != null) { - this.domainId = this.domain.getDomainName(); - this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() - + "/" + this.getUserId() + "/" + this.domain.getDomainName(); - } - } - - public String getViewerId() { - return viewerId; - } - - public void setViewerId(String viewerId) { - this.viewerId = viewerId; - } - - public String[] getScriptArguments() { - return scriptArguments; - } - - public void setScriptArguments(String[] scriptArguments) { - this.scriptArguments = scriptArguments; - } - - public String getDomainId() { - return domainId; - } - - public void setDomainId(String domainId) { - this.domainId = domainId; - } - - public ArrayList getDomainsList() { - return domainsList; - } - - public ArrayList getUsersList() { - return usersList; - } - - public Publisher getPublisher() { - return this.publisher; - } - - private void initializePortalConfig(HttpServletRequest request) { - this.contextRootPath = request.getContextPath(); - - PropertyListConfiguration serverConfig = getPortalConfiguration(request); - this.storageDirectory = serverConfig.getString("storage.local"); - this.tdbDirectory = serverConfig.getString("storage.tdb"); - this.serverUrl = serverConfig.getString("server"); - this.dotFile = serverConfig.getString("graphviz"); - this.clients = serverConfig.getString("clients"); - this.dataOntologyUrl = serverConfig.getString("ontology.data"); - this.componentOntologyUrl = serverConfig.getString("ontology.component"); - this.workflowOntologyUrl = serverConfig.getString("ontology.workflow"); - this.executionOntologyUrl = serverConfig.getString("ontology.execution"); - this.resourceOntologyUrl = serverConfig.getString("ontology.resource"); - - if (serverConfig.containsKey("metaworkflows")) - this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); - - if (serverConfig.containsKey("light-reasoner")) - plannerConfig.dataValidation = !serverConfig.getBoolean("light-reasoner"); - - if (serverConfig.containsKey("planner.data-validation")) - plannerConfig.dataValidation = serverConfig.getBoolean("planner.data-validation"); - if (serverConfig.containsKey("planner.specialization")) - plannerConfig.specialization = serverConfig.getBoolean("planner.specialization"); - if (serverConfig.containsKey("planner.use-rules")) - plannerConfig.useRules = serverConfig.getBoolean("planner.use-rules"); - - if (serverConfig.containsKey("planner.max-queue-size")) - plannerConfig.maxQueueSize = serverConfig.getInt("planner.max-queue-size", 1000); - else - plannerConfig.maxQueueSize = 1000; - if (serverConfig.containsKey("planner.parallelism")) - plannerConfig.parallelism = serverConfig.getInt("planner.parallelism", 10); - else - plannerConfig.parallelism = 10; - - if (serverConfig.containsKey("storage.logs")) { - this.logsDirectory = serverConfig.getString("storage.logs"); - } else { - this.logsDirectory = this.storageDirectory + File.separator + "logs"; - } - if (serverConfig.containsKey("storage.provenance")) { - this.provenanceDirectory = serverConfig.getString("storage.provenance"); - } else { - this.provenanceDirectory = this.storageDirectory + File.separator + "provenance"; - } - if (serverConfig.containsKey("storage.delete-run-outputs")) { - this.deleteRunOutputs = serverConfig.getBoolean("storage.delete-run-outputs"); - } else { - this.deleteRunOutputs = false; - } - // Create logsDir (if it doesn't exist) - File logdir = new File(this.logsDirectory); - if (!logdir.exists() && !logdir.mkdirs()) - System.err.println("Cannot create logs directory : " + logdir.getAbsolutePath()); - // Create provenanceDirectory (if it doesn't exist) - File provdir = new File(this.provenanceDirectory); - if (!provdir.exists() && !provdir.mkdirs()) - System.err.println("Cannot create provenance directory : " + provdir.getAbsolutePath()); - - this.exportCommunityUrl = serverUrl + contextRootPath + exportServletPath + "/" - + communityRelativeDir; - this.communityPath = contextRootPath + "/" + usersRelativeDir + "/" + communityRelativeDir; - - this.communityDir = storageDirectory + File.separator - + communityRelativeDir; - // Create communityDir (if it doesn't exist) - File uf = new File(this.communityDir); - if (!uf.exists() && !uf.mkdirs()) - System.err.println("Cannot create community directory : " + uf.getAbsolutePath()); - - // Load engine configurations - this.engines = new HashMap(); - List enginenodes = serverConfig.configurationsAt("execution.engine"); - for (HierarchicalConfiguration enode : enginenodes) { - ExeEngine engine = this.getExeEngine(enode); - this.engines.put(engine.getName(), engine); - } - // Add in the distributed engine if it doesn't already exist - if (!this.engines.containsKey("Distributed")) { - ExeEngine distengine = new ExeEngine("Distributed", - DistributedExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH); - this.engines.put(distengine.getName(), distengine); - this.addEngineConfig(serverConfig, distengine); - try { - serverConfig.save(this.configFile); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // Load publishing configuration - String publishUrl = serverConfig.getString("publisher.url"); - String publishExportName = serverConfig.getString("publisher.name"); - String tstorePublishUrl = serverConfig.getString("publisher.triple-store.publish"); - String tstoreQueryUrl = serverConfig.getString("publisher.triple-store.query"); - String domainsDir = serverConfig.getString("publisher.triple-store.domains-directory"); - String uploadUrl = serverConfig.getString("publisher.upload-server.url"); - String uploadUsername = serverConfig.getString("publisher.upload-server.username"); - String uploadPassword = serverConfig.getString("publisher.upload-server.password"); - String uploadDir = serverConfig.getString("publisher.upload-server.directory"); - String uploadHost = serverConfig.getString("publisher.upload-server.host"); - String uploadUserId = serverConfig.getString("publisher.upload-server.userid"); - String uploadKey = serverConfig.getString("publisher.upload-server.private-key"); - String sizeString = serverConfig.getString("publisher.upload-server.max-upload-size"); - - this.publisher = new Publisher(); - this.publisher.setUrl(publishUrl); - this.publisher.setExportName(publishExportName); - this.publisher.setDomainsDir(domainsDir); - this.publisher.setTstorePublishUrl(tstorePublishUrl); - this.publisher.setTstoreQueryUrl(tstoreQueryUrl); - - ServerDetails upserver = new ServerDetails(); - upserver.setUrl(uploadUrl); - upserver.setUsername(uploadUsername); - upserver.setPassword(uploadPassword); - upserver.setHostUserId(uploadUserId); - upserver.setDirectory(uploadDir); - upserver.setHost(uploadHost); - upserver.setPrivateKey(uploadKey); - if (sizeString != null) { - long size = this.getSizeFromString(sizeString); - upserver.setMaxUploadSize(size); - } - this.publisher.setUploadServer(upserver); - } - - private long getSizeFromString(String sizeString) { - long kb = 1024; - long mb = kb * kb; - long gb = kb * mb; - long tb = kb * gb; - - Pattern pat = Pattern.compile("(\\d+)\\s*([KkMmGgTt])[Bb]?"); - Matcher mat = pat.matcher(sizeString); - if (mat.find()) { - long size = Long.parseLong(mat.group(1)); - if (mat.groupCount() > 1) { - String units = mat.group(2).toLowerCase(); - if (units.equals("k")) - return size * kb; - if (units.equals("m")) - return size * mb; - if (units.equals("g")) - return size * gb; - if (units.equals("t")) - return size * tb; - } - return size; - } - return 0; - } - - private void initializeUserDatabase() { - this.userapi = new UsersDB(); - } - - @SuppressWarnings("rawtypes") - private ExeEngine getExeEngine(HierarchicalConfiguration node) { - String name = node.getString("name"); - String impl = node.getString("implementation"); - ExeEngine.Type type = ExeEngine.Type.valueOf(node.getString("type")); - ExeEngine engine = new ExeEngine(name, impl, type); - for (Iterator it = node.getKeys("properties"); it.hasNext();) { - String key = (String) it.next(); - String value = node.getString(key); - engine.addProperty(key.replace("properties.", ""), value); - } - return engine; - } - - public PropertyListConfiguration getPortalConfiguration(HttpServletRequest request) { - ServletContext app = request.getSession().getServletContext(); - this.configFile = app.getInitParameter("config.file"); - if (this.configFile == null) { - String home = System.getProperty("user.home"); - if (home != null && !home.equals("")) - this.configFile = home + File.separator + ".wings" - + File.separator + "portal.properties"; - else - this.configFile = "/etc/wings/portal.properties"; - } - // Create configFile if it doesn't exist (portal.properties) - File cfile = new File(this.configFile); - if (!cfile.exists()) { - File configDir = cfile.getParentFile(); - if (!configDir.exists() && !configDir.mkdirs()) { - System.err.println("Cannot create config file directory : " + configDir); - return null; - } - if (request != null) - createDefaultPortalConfig(request); - } - - // Load properties from configFile - PropertyListConfiguration props = new PropertyListConfiguration(); - try { - props.load(this.configFile); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - return props; - } - - private void createDefaultPortalConfig(HttpServletRequest request) { - String server = request.getScheme() + "://" + request.getServerName() + ":" - + request.getServerPort(); - String storageDir = null; - String home = System.getProperty("user.home"); - if (home != null && !home.equals("")) - storageDir = home + File.separator + ".wings" + File.separator + "storage"; - else - storageDir = System.getProperty("java.io.tmpdir") + - File.separator + "wings" + File.separator + "storage"; - - File storageDirFile = new File(storageDir); - if (!storageDirFile.exists() && !storageDirFile.mkdirs()) - System.err.println("Cannot create storage directory: " + storageDir); - - PropertyListConfiguration config = new PropertyListConfiguration(); - config.addProperty("storage.local", storageDir); - config.addProperty("storage.tdb", storageDir + File.separator + "TDB"); - config.addProperty("server", server); - - File loc1 = new File("/usr/bin/dot"); - File loc2 = new File("/usr/local/bin/dot"); - config.addProperty("graphviz", loc2.exists() ? loc2.getAbsolutePath() : loc1.getAbsolutePath()); - config.addProperty("ontology.data", ontdirurl + "/data.owl"); - config.addProperty("ontology.component", ontdirurl + "/component.owl"); - config.addProperty("ontology.workflow", ontdirurl + "/workflow.owl"); - config.addProperty("ontology.execution", ontdirurl + "/execution.owl"); - config.addProperty("ontology.resource", ontdirurl + "/resource.owl"); - - this.addEngineConfig(config, new ExeEngine("Local", - LocalExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); - this.addEngineConfig(config, new ExeEngine("Distributed", - DistributedExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); - - /* - * this.addEngineConfig(config, new ExeEngine("OODT", - * OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); - * - * this.addEngineConfig(config, new ExeEngine("Pegasus", - * PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); - */ - - try { - config.save(this.configFile); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void addEngineConfig(PropertyListConfiguration config, ExeEngine engine) { - config.addProperty("execution.engine(-1).name", engine.getName()); - config.addProperty("execution.engine.implementation", engine.getImplementation()); - config.addProperty("execution.engine.type", engine.getType()); - for (Entry entry : engine.getProperties().entrySet()) - config.addProperty("execution.engine.properties." + entry.getKey(), entry.getValue()); - } - - public Properties getProperties() { - return this.getProperties(this.domain); - } - - // Return Properties that are currently used by catalogs & planners - public Properties getProperties(Domain domain) { - Properties props = new Properties(); - if (domain != null) { - props = domain.getProperties(); - if (domain.isLegacy()) - return props; - - props.setProperty("ont.dir.url", this.ontdirurl); - if (!domain.getUseSharedTripleStore()) - props.setProperty("ont.dir.map", - "file:" + domain.getDomainDirectory() + File.separator + "ontology"); - - props.setProperty("ont.data.url", this.getDataOntologyUrl()); - props.setProperty("ont.component.url", this.getComponentOntologyUrl()); - props.setProperty("ont.workflow.url", this.getWorkflowOntologyUrl()); - props.setProperty("ont.execution.url", this.getExecutionOntologyUrl()); - if (domain.getUseSharedTripleStore()) - props.setProperty("tdb.repository.dir", this.getTripleStoreDir()); - - ExeEngine pengine = engines.get(domain.getPlanEngine()); - ExeEngine sengine = engines.get(domain.getStepEngine()); - props.putAll(pengine.getProperties()); - props.putAll(sengine.getProperties()); - } else { - props.setProperty("tdb.repository.dir", this.getTripleStoreDir()); - } - props.setProperty("logs.dir", this.getLogsDirectory()); - props.setProperty("dot.path", this.getDotFile()); - - if (this.getResourceOntologyUrl() == null) - this.setResourceOntologyUrl(ontdirurl + "/resource.owl"); - props.setProperty("ont.resource.url", this.getResourceOntologyUrl()); - - props.setProperty("lib.resource.url", - this.getExportCommunityUrl() + "/resource/library.owl"); - - if (domain != null && !domain.getUseSharedTripleStore()) - props.setProperty("lib.resource.map", - "file:" + domain.getDomainDirectory() + File.separator + "ontology" - + File.separator + "resource" + File.separator + "library.owl"); - - props.setProperty("lib.provenance.url", - this.getExportCommunityUrl() + "/provenance/library.owl"); - - if (this.viewerId != null) - props.setProperty("viewer.id", this.viewerId); - if (this.userId != null) - props.setProperty("user.id", this.userId); - - props.setProperty("use_rules", this.getPlannerConfig().useRules() ? "true" : "false"); - return props; - } - - public PlanExecutionEngine getDomainExecutionEngine() { - ExeEngine pengine = engines.get(domain.getPlanEngine()); - ExeEngine sengine = engines.get(domain.getStepEngine()); - try { - pengine.getProperties().putAll(this.getProperties()); - sengine.getProperties().putAll(this.getProperties()); - // TODO: Check if the selected engines are compatible - // and can be used as plan and step engines respectively - PlanExecutionEngine pee = ExecutionFactory.createPlanExecutionEngine( - pengine.getImplementation(), pengine.getProperties()); - StepExecutionEngine see = ExecutionFactory.createStepExecutionEngine( - sengine.getImplementation(), sengine.getProperties()); - ExecutionLoggerAPI logger = ExecutionToolsFactory.createLogger(this.getProperties()); - ExecutionMonitorAPI monitor = ExecutionToolsFactory.createMonitor(this.getProperties()); - ExecutionResourceAPI resource = ExecutionToolsFactory.getResourceAPI(this.getProperties()); - resource.setLocalStorageFolder(this.getStorageDirectory()); - pee.setStepExecutionEngine(see); - pee.setExecutionLogger(logger); - pee.setExecutionMonitor(monitor); - pee.setExecutionResource(resource); - return pee; - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - public StepExecutionEngine getDomainStepEngine() { - return null; - } - - public ExecutionMonitorAPI getDomainExecutionMonitor() { - return ExecutionToolsFactory.createMonitor(this.getProperties()); - } - - // Getters and Setters - public String getWorkflowOntologyUrl() { - return this.workflowOntologyUrl; - } - - public String getComponentOntologyUrl() { - return this.componentOntologyUrl; - } - - public String getDataOntologyUrl() { - return this.dataOntologyUrl; - } - - public void setWorkflowOntologyUrl(String workflowOntologyUrl) { - this.workflowOntologyUrl = workflowOntologyUrl; - } - - public void setDataOntologyUrl(String dataOntologyUrl) { - this.dataOntologyUrl = dataOntologyUrl; - } - - public void setComponentOntologyUrl(String componentOntologyUrl) { - this.componentOntologyUrl = componentOntologyUrl; - } - - public String getExecutionOntologyUrl() { - return executionOntologyUrl; - } - - public void setExecutionOntologyUrl(String executionOntologyUrl) { - this.executionOntologyUrl = executionOntologyUrl; - } - - public String getResourceOntologyUrl() { - return resourceOntologyUrl; - } - - public void setResourceOntologyUrl(String resourceOntologyUrl) { - this.resourceOntologyUrl = resourceOntologyUrl; - } - - public String getConfigFile() { - return configFile; - } - - public void setConfigFile(String configFile) { - this.configFile = configFile; - } - - public String getUsersRelativeDir() { - return usersRelativeDir; - } - - public void setUsersRelativeDir(String usersRelativeDir) { - this.usersRelativeDir = usersRelativeDir; - } - - public String getExportServletPath() { - return exportServletPath; - } - - public void setExportServletPath(String exportServletPath) { - this.exportServletPath = exportServletPath; - } - - public String getUserId() { - return userId; - } - - public void setUserId(String userId) { - if (userDomainUrl != null) - this.userDomainUrl = this.userDomainUrl.replace(this.userId, userId); - if (scriptPath != null) - this.scriptPath = this.scriptPath.replace(this.userId, userId); - if (exportUserUrl != null) - this.exportUserUrl = this.exportUserUrl.replace(this.userId, userId); - if (userPath != null) - this.userPath = this.userPath.replace(this.userId, userId); - if (userDir != null) - this.userDir = this.userDir.replace(this.userId, userId); - this.userId = userId; - } - - public String getUserDir() { - return userDir; - } - - public void setUserDir(String userDir) { - this.userDir = userDir; - } - - public boolean isAdminViewer() { - return isAdminViewer; - } - - public void setAdminViewer(boolean isAdminViewer) { - this.isAdminViewer = isAdminViewer; - } - - public boolean hasMetaWorkflows() { - return hasMetaWorkflows; - } - - public void setMetaWorkflows(boolean hasMetaWorkflows) { - this.hasMetaWorkflows = hasMetaWorkflows; - } - - public void setExportUserUrl(String exportUserUrl) { - this.exportUserUrl = exportUserUrl; - } - - public String getServerUrl() { - return serverUrl; - } - - public void setServerUrl(String serverUrl) { - this.serverUrl = serverUrl; - } - - public String getCommunityPath() { - return communityPath; - } - - public void setCommunityPath(String communityPath) { - this.communityPath = communityPath; - } - - public String getExportCommunityUrl() { - return exportCommunityUrl; - } - - public void setExportCommunityUrl(String exportCommunityUrl) { - this.exportCommunityUrl = exportCommunityUrl; - } - - public String getCommunityDir() { - return communityDir; - } - - public void setCommunityDir(String communityDir) { - this.communityDir = communityDir; - } - - public String getContextRootPath() { - return contextRootPath; - } - - public void setContextRootPath(String root) { - this.contextRootPath = root; - } - - public Domain getDomain() { - return domain; - } - - public void setDomain(Domain domain) { - this.domain = domain; - } - - public String getScriptPath() { - return scriptPath; - } - - public void setScriptPath(String scriptPath) { - this.scriptPath = scriptPath; - } - - public String getExportUserUrl() { - return exportUserUrl; - } - - public String getUserPath() { - return userPath; - } - - public void setUserPath(String userPath) { - this.userPath = userPath; - } - - public String getTripleStoreDir() { - return tdbDirectory; - } - - public void setTripleStoreDir(String tripleStoreDir) { - this.tdbDirectory = tripleStoreDir; - } - - public String getProvenanceDirectory() { - return provenanceDirectory; - } - - public String getLogsDirectory() { - return logsDirectory; - } - - public void setLogsDirectory(String logsDirectory) { - this.logsDirectory = logsDirectory; - } - - public String getSessionId() { - return sessionId; - } - - public void setSessionId(String sessionId) { - this.sessionId = sessionId; - } - - public boolean isSandboxed() { - return isSandboxed; - } - - public void setSandboxed(boolean isSandboxed) { - this.isSandboxed = isSandboxed; - } - - public String getDotFile() { - return dotFile; - } - - public void setDotFile(String dotFile) { - this.dotFile = dotFile; - } - - public boolean isDeleteRunOutputs() { - return deleteRunOutputs; - } - - public void setDeleteRunOutputs(boolean deleteRunOutputs) { - this.deleteRunOutputs = deleteRunOutputs; - } - - public String getClients() { - return clients; - } - - public void setClients(String clients) { - this.clients = clients; - } - - public String getStorageDirectory() { - return storageDirectory; - } - - public void setStorageDirectory(String storageDirectory) { - this.storageDirectory = storageDirectory; - } - - public PlannerConfig getPlannerConfig() { - return plannerConfig; - } - - public void setPlannerConfig(PlannerConfig plannerConfig) { - this.plannerConfig = plannerConfig; - } - - public Set getEnginesList() { - return this.engines.keySet(); - } -} diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java index 5ac2d739..0ea007e2 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java @@ -1,7 +1,5 @@ package edu.isi.wings.portal.classes.config; -import edu.isi.kcap.wings.opmm_deprecated.Main; -import java.io.File; import javax.servlet.http.HttpServletRequest; import org.apache.commons.configuration.plist.PropertyListConfiguration; diff --git a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java index a8230211..17437f6e 100644 --- a/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java +++ b/portal/src/main/java/edu/isi/wings/portal/controllers/RunController.java @@ -22,7 +22,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; import edu.isi.kcap.ontapi.KBTriple; -import edu.isi.kcap.ontapi.KBTriple; import edu.isi.kcap.wings.opmm.Catalog; import edu.isi.kcap.wings.opmm.DataTypes.Links; import edu.isi.kcap.wings.opmm.DataTypes.ProvenanceResponseSchema; @@ -46,17 +45,11 @@ import edu.isi.wings.portal.classes.JsonHandler; import edu.isi.wings.portal.classes.config.ConfigLoader; import edu.isi.wings.portal.classes.config.FileUploadServerConfig; -import edu.isi.wings.portal.classes.config.Publisher; import edu.isi.wings.portal.classes.config.PublisherConfig; -import edu.isi.wings.portal.classes.config.ServerDetails; -import edu.isi.wings.portal.classes.util.ComponentExecutingThread; import edu.isi.wings.portal.classes.util.ComponentExecutingThread; import edu.isi.wings.portal.classes.util.PlanningAPIBindings; -import edu.isi.wings.portal.classes.util.PlanningAPIBindings; -import edu.isi.wings.portal.classes.util.PlanningAndExecutingThread; import edu.isi.wings.portal.classes.util.PlanningAndExecutingThread; import edu.isi.wings.portal.classes.util.TemplateBindings; -import edu.isi.wings.portal.classes.util.TemplateBindings; import edu.isi.wings.workflow.plan.api.ExecutionPlan; import edu.isi.wings.workflow.plan.api.ExecutionStep; import edu.isi.wings.workflow.plan.classes.ExecutionFile; @@ -76,21 +69,13 @@ import javax.servlet.ServletContext; import javax.ws.rs.core.Response; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FileUtils; -import org.apache.http.HttpResponse; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpPut; import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.Cookie; import org.apache.http.entity.StringEntity; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.client.HttpClients; import org.apache.jena.base.Sys; public class RunController { From 8cd8b9d09734b5bc5c1cceaf50bc94a2585461cc Mon Sep 17 00:00:00 2001 From: mosoriob Date: Tue, 5 Sep 2023 13:10:48 +0000 Subject: [PATCH 07/11] Prettified Code! --- .../api/impl/oodt/ComponentCreationFM.java | 18 ++- .../data/api/impl/oodt/DataCreationFM.java | 6 +- .../api/impl/oodt/OODTExecutionEngine.java | 6 +- .../api/impl/oodt/OODTWorkflowAdapter.java | 18 ++- .../wings/util/oodt/CurationServiceAPI.java | 12 +- .../wmservices/client/WmServicesClient.java | 2 +- .../api/impl/kb/ComponentCreationKB.java | 6 +- .../component/api/impl/kb/ComponentKB.java | 6 +- .../api/impl/kb/ComponentReasoningKB.java | 47 +++--- .../classes/requirements/Dependency.java | 4 +- .../data/api/impl/kb/DataCreationKB.java | 6 +- .../data/classes/MetadataProperty.java | 18 +-- .../resource/api/impl/kb/ResourceKB.java | 17 +- .../catalog/resource/classes/Machine.java | 8 +- .../isi/wings/common/CollectionsHelper.java | 16 +- .../isi/wings/common/kb/PropertiesHelper.java | 6 +- .../engine/classes/ExecutionQueue.java | 2 +- .../execution/tools/api/impl/kb/RunKB.java | 89 +++++------ .../api/impl/kb/WorkflowGenerationKB.java | 145 ++++++++++++------ .../java/edu/isi/wings/planner/cli/Wings.java | 21 ++- .../workflow/plan/api/impl/pplan/PPlan.java | 6 +- .../plan/api/impl/pplan/PPlanStep.java | 2 +- .../api/impl/kb/TemplateCreationKB.java | 6 +- .../template/api/impl/kb/TemplateKB.java | 50 +++--- .../template/classes/ConstraintProperty.java | 18 +-- .../workflow/template/classes/Rules.java | 2 +- .../template/classes/sets/SetExpression.java | 24 ++- .../classes/config/ExecutionConfig.java | 5 +- .../util/ComponentExecutingThread.java | 6 +- .../controllers/ComponentController.java | 7 +- .../portal/controllers/DataController.java | 35 +++-- .../portal/controllers/PlanController.java | 32 ++-- .../portal/controllers/RunController.java | 22 +-- .../controllers/TemplateController.java | 5 +- .../filters/servlets/CompressionFilter.java | 8 +- .../portal/resources/TemplateResource.java | 2 +- .../portal/resources/UploadResource.java | 2 +- .../servlets/ProvenanceFileServlet.java | 98 ++++++------ 38 files changed, 439 insertions(+), 344 deletions(-) diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java index bcaae829..e8bd7734 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java @@ -68,10 +68,14 @@ public class ComponentCreationFM implements ComponentCreationAPI { OntFactory ontologyFactory; - private static HashMap compHolders = - new HashMap(); - private static HashMap compMeta = - new HashMap(); + private static HashMap compHolders = new HashMap< + String, + String + >(); + private static HashMap compMeta = new HashMap< + String, + Metadata + >(); public ComponentCreationFM(Properties props) { this.fmurl = props.getProperty("oodt.fmurl"); @@ -102,8 +106,10 @@ public ComponentTree getComponentHierarchy(boolean details) { String rootid = this.absurl + "#Component"; ComponentHolder rootholder = new ComponentHolder(rootid); ComponentTreeNode rootnode = new ComponentTreeNode(rootholder); - HashMap tnmap = - new HashMap(); + HashMap tnmap = new HashMap< + String, + ComponentTreeNode + >(); HashMap pmap = this.curatorApi.getParentTypeMap(); try { for (ProductType ptype : this.fmclient.getProductTypes()) { diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java index 712f3f65..30658317 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java @@ -59,8 +59,10 @@ public class DataCreationFM implements DataCreationAPI { String liburl; String onturl; - private static HashMap prodTypes = - new HashMap(); + private static HashMap prodTypes = new HashMap< + String, + String + >(); public DataCreationFM(Properties props) { this.fmurl = props.getProperty("oodt.fmurl"); diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java index 5d558aee..e59427cf 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java @@ -242,8 +242,10 @@ public ExecutionMonitoringThread( public void run() { planexe.onStart(this.logger); try { - HashMap jobstatus = - new HashMap(); + HashMap jobstatus = new HashMap< + String, + RuntimeInfo.Status + >(); int osleeptime = 1000; int maxsleeptime = 4 * osleeptime; diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java index 080b5236..9158270a 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java @@ -80,8 +80,10 @@ public OODTWorkflowAdapter( public Workflow runWorkflow(RuntimePlan planexe, Metadata meta) throws Exception { // Get List of all Jobs - HashMap tasksById = - new HashMap(); + HashMap tasksById = new HashMap< + String, + WorkflowTask + >(); HashMap opProducers = new HashMap(); HashMap taskJobs = new HashMap(); for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) { @@ -117,8 +119,10 @@ public Workflow runWorkflow(RuntimePlan planexe, Metadata meta) } // Get Parent Child Relationship between jobs - HashMap> parents = - new HashMap>(); + HashMap> parents = new HashMap< + String, + ArrayList + >(); for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) { ExecutionStep step = stepexe.getStep(); ArrayList chparents = parents.get(stepexe.getName()); @@ -132,8 +136,10 @@ public Workflow runWorkflow(RuntimePlan planexe, Metadata meta) } // Arrange Jobs into Job Levels - TreeMap> jobLevels = - new TreeMap>(); + TreeMap> jobLevels = new TreeMap< + Integer, + ArrayList + >(); for (String jobid : tasksById.keySet()) { Integer lvl = new Integer(getJobLevel(jobid, 0, parents)); ArrayList lvljobs; diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java index b4e92b0a..8f37d09d 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java @@ -49,10 +49,10 @@ private String query(String method, String op, Object... args) { String params = "policy=" + URLEncoder.encode(this.policy, "UTF-8"); for (int i = 0; i < args.length; i += 2) { params += - "&" + - args[i] + - "=" + - URLEncoder.encode(args[i + 1].toString(), "UTF-8"); + "&" + + args[i] + + "=" + + URLEncoder.encode(args[i + 1].toString(), "UTF-8"); } URL urlobj = new URL(url); @@ -117,7 +117,7 @@ public boolean addElementsForProductType( ) { String elementIds = ""; for (Element element : elementList) elementIds += - (elementIds != "" ? "," : "") + element.getElementId(); + (elementIds != "" ? "," : "") + element.getElementId(); String result = this.query( "POST", @@ -169,7 +169,7 @@ public boolean removeElementsForProductType( ) { String elementIds = ""; for (Element element : elementList) elementIds += - (elementIds != "" ? "," : "") + element.getElementId(); + (elementIds != "" ? "," : "") + element.getElementId(); String result = this.query( "DELETE", diff --git a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java index 4f46b3e2..5a44cfc8 100644 --- a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java +++ b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java @@ -116,7 +116,7 @@ private String query(String method, String op, Object... args) { for (int i = 0; i < args.length; i += 2) { if (i > 0) params += "&"; params += - args[i] + "=" + URLEncoder.encode(args[i + 1].toString(), "UTF-8"); + args[i] + "=" + URLEncoder.encode(args[i + 1].toString(), "UTF-8"); } if ("GET".equals(method)) { URL urlobj = new URL(url + "?" + params); diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java index e0d75d34..c9355384 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java @@ -306,9 +306,9 @@ protected boolean incrementComponentVersion(String cid, KBAPI writerkb) { KBObject compobj = kb.getIndividual(cid); KBObject versionProp = kb.getProperty(this.pcns + "hasVersion"); KBObject versionVal = kb.getPropertyValue(compobj, versionProp); - int currentVersion = (Integer) ( - versionVal.getValue() != null ? versionVal.getValue() : 0 - ); + int currentVersion = (Integer) (versionVal.getValue() != null + ? versionVal.getValue() + : 0); int newVersion = currentVersion + 1; KBObject cobj = writerkb.getResource(cid); diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java index b2dfd20b..ee2d9ade 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java @@ -820,11 +820,9 @@ public Component getComponent(String cid, boolean details) { protected int getComponentVersion(KBObject compobj) { KBObject versionProp = kb.getProperty(this.pcns + "hasVersion"); KBObject versionVal = kb.getPropertyValue(compobj, versionProp); - return (Integer) ( - (versionVal != null && versionVal.getValue() != null) + return (Integer) ((versionVal != null && versionVal.getValue() != null) ? versionVal.getValue() - : 0 - ); + : 0); } protected ArrayList getComponentInputs(KBObject compobj) { diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java index 59b53864..1b1be789 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java @@ -53,14 +53,22 @@ public class ComponentReasoningKB private boolean useRules = true; private HashMap ccache = new HashMap(); - private HashMap rulescache = - new HashMap(); - private HashMap> abscache = - new HashMap>(); - private HashMap> classcache = - new HashMap>(); - private HashMap> kbcache = - new HashMap>(); + private HashMap rulescache = new HashMap< + String, + KBRuleList + >(); + private HashMap> abscache = new HashMap< + String, + ArrayList + >(); + private HashMap> classcache = new HashMap< + String, + ArrayList + >(); + private HashMap> kbcache = new HashMap< + String, + ArrayList + >(); private ArrayList metricTriples; private ArrayList metricProps; @@ -897,8 +905,10 @@ public ArrayList findOutputDataPredictedDescriptions( ); // Keep a map of variable object to variable name - HashMap variableNameMap = - new HashMap(); + HashMap variableNameMap = new HashMap< + Variable, + String + >(); for (String rolestr : sRoleMap.keySet()) { Variable var = sRoleMap.get(rolestr); @@ -926,8 +936,10 @@ public ArrayList findOutputDataPredictedDescriptions( // Get a mapping of ArgID's to arg for the Component // Also note which roles are inputs - HashMap argMaps = - new HashMap(); + HashMap argMaps = new HashMap< + String, + ComponentRole + >(); HashMap sInputRoles = new HashMap(); for (ComponentRole role : comp.getInputs()) { argMaps.put(role.getRoleName(), role); @@ -1199,8 +1211,9 @@ public ArrayList findOutputDataPredictedDescriptions( // Marking all compatible inputs to be passed through to the output. // Note: If not all outputs have a matching input, then component is still run and not skipped - ArrayList matchingInputs = - new ArrayList(); + ArrayList matchingInputs = new ArrayList< + ComponentRole + >(); for (ComponentRole inrole : comp.getInputs()) { if (!inrole.isParam()) { if (checkTypeCompatibility(varclasses, inrole.getID())) { @@ -1261,11 +1274,9 @@ public ArrayList findOutputDataPredictedDescriptions( for (Variable var : sRoleMap.values()) { if ( var.isParameterVariable() && - ( - noParamBindings.containsKey(var.getID()) || + (noParamBindings.containsKey(var.getID()) || var.getBinding() == null || - var.getBinding().getValue() == null - ) + var.getBinding().getValue() == null) ) { KBObject varobj = tkb.getResource(variableNameMap.get(var)); KBObject origvarobj = tkb.getResource(var.getID()); diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/classes/requirements/Dependency.java b/planner/src/main/java/edu/isi/wings/catalog/component/classes/requirements/Dependency.java index 8be90be8..680e97d1 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/classes/requirements/Dependency.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/classes/requirements/Dependency.java @@ -70,12 +70,12 @@ public boolean checkDependency(KBAPI api, KBObject obj) { if ( this.operator == Operator.GREATER_THAN && Double.parseDouble(obj.getValueAsString()) > - Double.parseDouble(this.desiredValue.toString()) + Double.parseDouble(this.desiredValue.toString()) ) return true; if ( this.operator == Operator.LESS_THAN && Double.parseDouble(obj.getValueAsString()) < - Double.parseDouble(this.desiredValue.toString()) + Double.parseDouble(this.desiredValue.toString()) ) return true; } return false; diff --git a/planner/src/main/java/edu/isi/wings/catalog/data/api/impl/kb/DataCreationKB.java b/planner/src/main/java/edu/isi/wings/catalog/data/api/impl/kb/DataCreationKB.java index c7665dce..f7722c56 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/data/api/impl/kb/DataCreationKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/data/api/impl/kb/DataCreationKB.java @@ -137,8 +137,10 @@ public ArrayList getAllDatatypeIds() { @Override public HashMap> getAllDatatypeDatasets() { - HashMap> typedata = - new HashMap>(); + HashMap> typedata = new HashMap< + String, + ArrayList + >(); String query = "SELECT ?s ?type\n" + "WHERE {\n" + diff --git a/planner/src/main/java/edu/isi/wings/catalog/data/classes/MetadataProperty.java b/planner/src/main/java/edu/isi/wings/catalog/data/classes/MetadataProperty.java index 09d0f97e..0c7a216b 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/data/classes/MetadataProperty.java +++ b/planner/src/main/java/edu/isi/wings/catalog/data/classes/MetadataProperty.java @@ -74,15 +74,15 @@ public void setRange(String id) { public String toString() { String str = ""; str += - "\n" + - getName() + - "(" + - type + - ")\nDomains:" + - domains + - "\nRange:" + - range + - "\n"; + "\n" + + getName() + + "(" + + type + + ")\nDomains:" + + domains + + "\nRange:" + + range + + "\n"; return str; } } diff --git a/planner/src/main/java/edu/isi/wings/catalog/resource/api/impl/kb/ResourceKB.java b/planner/src/main/java/edu/isi/wings/catalog/resource/api/impl/kb/ResourceKB.java index d1303902..b51802ea 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/resource/api/impl/kb/ResourceKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/resource/api/impl/kb/ResourceKB.java @@ -45,8 +45,10 @@ public class ResourceKB extends TransactionsJena implements ResourceAPI { public String onturl, liburl; private String tdbRepository; - private static HashMap initializedLibrary = - new HashMap(); + private static HashMap initializedLibrary = new HashMap< + String, + Boolean + >(); private KBAPI ontkb, libkb; @@ -398,8 +400,9 @@ public ArrayList getAllSoftwareEnvironment() { this.start_read(); boolean batched = this.start_batch_operation(); - ArrayList environment = - new ArrayList(); + ArrayList environment = new ArrayList< + SoftwareEnvironment + >(); for (KBObject swobj : this.getInstancesOfClass( this.cmap.get("SoftwareGroup"), this.libkb @@ -703,8 +706,10 @@ public void setMachineWhitelist(ArrayList whitelist) { } public void copyFrom(ResourceAPI rc, ComponentCreationAPI cc) { - HashMap sws = - new HashMap(); + HashMap sws = new HashMap< + String, + SoftwareVersion + >(); this.start_write(); boolean batched = this.start_batch_operation(); diff --git a/planner/src/main/java/edu/isi/wings/catalog/resource/classes/Machine.java b/planner/src/main/java/edu/isi/wings/catalog/resource/classes/Machine.java index ca333f0e..d5b7f953 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/resource/classes/Machine.java +++ b/planner/src/main/java/edu/isi/wings/catalog/resource/classes/Machine.java @@ -226,14 +226,10 @@ public MachineDetails call() { details.setNumCores(Runtime.getRuntime().availableProcessors()); details.setMaxMemory( - ( - (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean() - ).getTotalPhysicalMemorySize() + ((com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()).getTotalPhysicalMemorySize() ); details.setFreeMemory( - ( - (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean() - ).getFreePhysicalMemorySize() + ((com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()).getFreePhysicalMemorySize() ); File[] roots = File.listRoots(); for (File root : roots) { diff --git a/planner/src/main/java/edu/isi/wings/common/CollectionsHelper.java b/planner/src/main/java/edu/isi/wings/common/CollectionsHelper.java index 72f753c8..f8cff51f 100644 --- a/planner/src/main/java/edu/isi/wings/common/CollectionsHelper.java +++ b/planner/src/main/java/edu/isi/wings/common/CollectionsHelper.java @@ -12,16 +12,18 @@ public class CollectionsHelper { * * @return the combined list of variable object bindings */ - public static ArrayList combineVariableDataObjectMappings( - VariableBindingsListSet listset - ) { - ArrayList combinedList = - new ArrayList(); + public static ArrayList< + VariableBindingsList + > combineVariableDataObjectMappings(VariableBindingsListSet listset) { + ArrayList combinedList = new ArrayList< + VariableBindingsList + >(); for (ArrayList list : listset) { if (combinedList.isEmpty()) combinedList = list; else { - ArrayList templist = - new ArrayList(); + ArrayList templist = new ArrayList< + VariableBindingsList + >(); for (VariableBindingsList map1 : combinedList) { for (VariableBindingsList map2 : list) { VariableBindingsList cmap = new VariableBindingsList(); diff --git a/planner/src/main/java/edu/isi/wings/common/kb/PropertiesHelper.java b/planner/src/main/java/edu/isi/wings/common/kb/PropertiesHelper.java index e017e7db..334ed6ea 100644 --- a/planner/src/main/java/edu/isi/wings/common/kb/PropertiesHelper.java +++ b/planner/src/main/java/edu/isi/wings/common/kb/PropertiesHelper.java @@ -39,8 +39,10 @@ public class PropertiesHelper { private static HashMap pcnsmap = null; private static HashMap dcnsmap = null; - private static HashMap classLoggers = - new HashMap(); + private static HashMap classLoggers = new HashMap< + String, + Logger + >(); private static String ontdir, logdir, resdir, opdir; diff --git a/planner/src/main/java/edu/isi/wings/execution/engine/classes/ExecutionQueue.java b/planner/src/main/java/edu/isi/wings/execution/engine/classes/ExecutionQueue.java index cbef7c6d..ce0b1e0e 100644 --- a/planner/src/main/java/edu/isi/wings/execution/engine/classes/ExecutionQueue.java +++ b/planner/src/main/java/edu/isi/wings/execution/engine/classes/ExecutionQueue.java @@ -66,7 +66,7 @@ public ArrayList getNextStepsToExecute() { if ( parentStep != null && parentStep.getRuntimeInfo().getStatus() != - RuntimeInfo.Status.SUCCESS + RuntimeInfo.Status.SUCCESS ) ok = false; } if (ok) steps.add(step); diff --git a/planner/src/main/java/edu/isi/wings/execution/tools/api/impl/kb/RunKB.java b/planner/src/main/java/edu/isi/wings/execution/tools/api/impl/kb/RunKB.java index 4a5ad612..7ab26270 100644 --- a/planner/src/main/java/edu/isi/wings/execution/tools/api/impl/kb/RunKB.java +++ b/planner/src/main/java/edu/isi/wings/execution/tools/api/impl/kb/RunKB.java @@ -228,25 +228,19 @@ public int getNumberOfRuns( "?run exec:hasExecutionStatus ?status .\n" + "?run exec:hasTemplate ?template .\n" + "?run exec:hasStartTime ?start .\n" + - ( - status != null + (status != null ? "?run exec:hasExecutionStatus '" + status + "') .\n" - : "" - ) + - ( - starttime != null + : "") + + (starttime != null ? "FILTER(?start > '" + starttime + "'^^xsd:dateTime) .\n" - : "" - ) + - ( - pattern != null + : "") + + (pattern != null ? "FILTER REGEX(str(?template), '" + newtplurl + ".*" + pattern + ".*') .\n" - : "" - ) + + : "") + "FILTER REGEX(str(?run), '" + newrunurl + "') .\n" + @@ -289,25 +283,19 @@ public ArrayList getRunListSimple( "?run exec:hasExecutionStatus ?status .\n" + "?run exec:hasTemplate ?template .\n" + "?run exec:hasStartTime ?start .\n" + - ( - status != null + (status != null ? "?run exec:hasExecutionStatus '" + status + "') .\n" - : "" - ) + - ( - starttime != null + : "") + + (starttime != null ? "FILTER(?start > '" + starttime + "'^^xsd:dateTime) .\n" - : "" - ) + - ( - pattern != null + : "") + + (pattern != null ? "FILTER REGEX(str(?template), '" + newtplurl + ".*" + pattern + ".*') .\n" - : "" - ) + + : "") + "FILTER REGEX(str(?run), '" + newrunurl + "') .\n"; @@ -385,41 +373,40 @@ public ArrayList getRunList( if (status == null || status.equals("SUCCESS")) { query += - // Query for Successful runs - "{\n" + - " ?run exec:hasExecutionStatus 'SUCCESS' .\n" + - " ?run exec:hasEndTime ?end .\n" + - "}\n"; + // Query for Successful runs + "{\n" + + " ?run exec:hasExecutionStatus 'SUCCESS' .\n" + + " ?run exec:hasEndTime ?end .\n" + + "}\n"; } if (status == null || status.equals("FAILURE")) { if (status == null) query += "UNION\n"; query += - // Query for Failures - "{\n" + - " ?run exec:hasExecutionStatus 'FAILURE' .\n" + - " ?run exec:hasEndTime ?end .\n" + - // FIXME: Removing this as we could have failures without any steps - /*" ?run exec:hasStep ?step .\n" + + // Query for Failures + "{\n" + + " ?run exec:hasExecutionStatus 'FAILURE' .\n" + + " ?run exec:hasEndTime ?end .\n" + + // FIXME: Removing this as we could have failures without any steps + /*" ?run exec:hasStep ?step .\n" + (fasterQuery ? " ?step exec:hasExecutionStatus 'FAILURE' .\n": "") + " ?step exec:hasExecutionStatus ?stepstatus .\n" +*/ - "}\n"; + "}\n"; } if (status == null || status.equals("RUNNING")) { if (status == null) query += "UNION\n"; query += - // Query for Failures - "{\n" + - " ?run exec:hasExecutionStatus 'RUNNING' .\n" + - " ?run exec:hasStep ?step .\n" + - (fasterQuery ? " ?step exec:hasExecutionStatus 'RUNNING' .\n" : "") + - " ?step exec:hasExecutionStatus ?stepstatus .\n" + - "}\n"; + // Query for Failures + "{\n" + + " ?run exec:hasExecutionStatus 'RUNNING' .\n" + + " ?run exec:hasStep ?step .\n" + + (fasterQuery ? " ?step exec:hasExecutionStatus 'RUNNING' .\n" : "") + + " ?step exec:hasExecutionStatus ?stepstatus .\n" + + "}\n"; } query += "}\n"; query += - "GROUP BY ?run ?status ?template ?start ?end \n" + - "ORDER BY DESC(?start)"; + "GROUP BY ?run ?status ?template ?start ?end \n" + "ORDER BY DESC(?start)"; if (limit >= 0 && start >= 0) { query += " LIMIT " + limit + " OFFSET " + start; @@ -582,10 +569,8 @@ private RuntimePlan getExecutionRun(String runid, boolean details) { RuntimeInfo.Status status = rplan.getRuntimeInfo().getStatus(); if ( details || - ( - status == RuntimeInfo.Status.FAILURE || - status == RuntimeInfo.Status.RUNNING - ) + (status == RuntimeInfo.Status.FAILURE || + status == RuntimeInfo.Status.RUNNING) ) { try { KBAPI tkb = this.ontologyFactory.getKB(rplan.getURL(), OntSpec.PLAIN); @@ -1042,8 +1027,10 @@ public RuntimePlan rePlan(RuntimePlan oldplanexe) { planexe.setPlan(newplan); // Hash steps from current queue - HashMap stepMap = - new HashMap(); + HashMap stepMap = new HashMap< + String, + RuntimeStep + >(); for (RuntimeStep step : planexe.getQueue().getAllSteps()) stepMap.put( step.getID(), step diff --git a/planner/src/main/java/edu/isi/wings/planner/api/impl/kb/WorkflowGenerationKB.java b/planner/src/main/java/edu/isi/wings/planner/api/impl/kb/WorkflowGenerationKB.java index 840b85e7..2415e29e 100644 --- a/planner/src/main/java/edu/isi/wings/planner/api/impl/kb/WorkflowGenerationKB.java +++ b/planner/src/main/java/edu/isi/wings/planner/api/impl/kb/WorkflowGenerationKB.java @@ -208,8 +208,10 @@ public ArrayList