Refresh IG Group file POST addition & messaging improvements

tooling-cli/pom.xml

@@ -16,6 +16,14 @@
     <description>CQF Tooling CLI</description>
 
     <dependencies>
+
+        <!-- Logback Classic -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.4.14</version>
+        </dependency>
+
         <dependency>
             <groupId>org.opencds.cqf</groupId>
             <artifactId>tooling</artifactId>
@@ -32,12 +40,6 @@
             <artifactId>model-jackson</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-simple</artifactId>
-            <scope>compile</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.reflections</groupId>
             <artifactId>reflections</artifactId>

tooling-cli/src/main/java/org/opencds/cqf/tooling/cli/Main.java

@@ -236,6 +236,10 @@
 //import org.opencds.cqf.tooling.operations.ExecutableOperation;
 //import org.opencds.cqf.tooling.operations.Operation;
 //import org.reflections.Reflections;
+
+import ca.uhn.fhir.parser.LenientErrorHandler;
+import ch.qos.logback.classic.Level;
+import org.opencds.cqf.tooling.common.ThreadUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -284,6 +288,17 @@ public class Main {
 //    }
 
     public static void main(String[] args) {
+
+        //ca.uhn.fhir.parser.LenientErrorHandler warning suppression:
+        try {
+            suppressLogsFromLenientErrorHandler();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+
+        //ensure any and all executors are shutdown cleanly when app is shutdown:
+        Runtime.getRuntime().addShutdownHook(new Thread(ThreadUtils::shutdownRunningExecutors));
+
         if (args.length == 0) {
             System.err.println("cqf-tooling version: " + Main.class.getPackage().getImplementationVersion());
             System.err.println("Requests must include which operation to run as a command line argument. See docs for examples on how to use this project.");
@@ -297,4 +312,9 @@ public static void main(String[] args) {
 
         OperationFactory.createOperation(operation.substring(1)).execute(args);
     }
+
+    private static void suppressLogsFromLenientErrorHandler() throws Exception {
+        Logger logger = LoggerFactory.getLogger(LenientErrorHandler.class);
+        logger.getClass().getMethod("setLevel", Level.class).invoke(logger, Level.ERROR);
+    }
 }

tooling/src/main/java/org/opencds/cqf/tooling/common/ThreadUtils.java

@@ -1,6 +1,5 @@
 package org.opencds.cqf.tooling.common;
 
-import org.opencds.cqf.tooling.utilities.LogUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -14,6 +13,9 @@
 
 public class ThreadUtils {
     protected static final Logger logger = LoggerFactory.getLogger(ThreadUtils.class);
+
+    private static List<ExecutorService> runningExecutors = new ArrayList<>();
+
     /**
      * Executes a list of tasks concurrently using a thread pool.
      * <p>
@@ -23,32 +25,59 @@ public class ThreadUtils {
      *
      * @param tasks A list of Callable tasks to execute concurrently.
      */
-    public static void executeTasks(List<Callable<Void>> tasks) {
-        if (tasks == null || tasks.isEmpty()){
+    public static void executeTasks(List<Callable<Void>> tasks, ExecutorService executor) {
+        if (tasks == null || tasks.isEmpty()) {
             return;
         }
 
+        runningExecutors.add(executor);
+
+        List<Callable<Void>> retryTasks = new ArrayList<>();
+
         //let OS handle threading:
-        ExecutorService executorService = Executors.newCachedThreadPool();// Submit tasks and obtain futures
         try {
             List<Future<Void>> futures = new ArrayList<>();
             for (Callable<Void> task : tasks) {
-                futures.add(executorService.submit(task));
+                try {
+                    futures.add(executor.submit(task));
+                } catch (OutOfMemoryError e) {
+                    retryTasks.add(task);
+                }
             }
 
             // Wait for all tasks to complete
             for (Future<Void> future : futures) {
                 future.get();
             }
         } catch (Exception e) {
-            logger.error("ThreadUtils.executeTasks", e);
+            logger.error("ThreadUtils.executeTasks: ", e);
         } finally {
-            executorService.shutdown();
+            if (retryTasks.isEmpty()) {
+                runningExecutors.remove(executor);
+                executor.shutdown();
+            }else{
+                executeTasks(retryTasks, executor);
+            }
         }
     }
 
+    public static void executeTasks(List<Callable<Void>> tasks) {
+        executeTasks(tasks, Executors.newCachedThreadPool());
+    }
+
     public static void executeTasks(Queue<Callable<Void>> callables) {
+        executeTasks(new ArrayList<>(callables), Executors.newCachedThreadPool());
+    }
 
-        executeTasks(new ArrayList<>(callables));
+    public static void shutdownRunningExecutors() {
+        try {
+            if (runningExecutors.isEmpty()) return;
+            for (ExecutorService es : runningExecutors) {
+                es.shutdownNow();
+            }
+            runningExecutors = new ArrayList<>();
+        }catch (Exception e){
+            //fail silently, shutting down anyways
+        }
     }
 }

tooling/src/main/java/org/opencds/cqf/tooling/cql/exception/CqlTranslatorException.java

@@ -0,0 +1,46 @@
+package org.opencds.cqf.tooling.cql.exception;
+
+import org.cqframework.cql.cql2elm.CqlCompilerException;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Custom exception to pass the list of errors returned by the translator to calling methods.
+ */
+public class CqlTranslatorException extends Exception implements Serializable {
+    private static final long serialVersionUID = 20600L;
+
+    /**
+     * Using Set to avoid duplicate entries.
+     */
+    private final transient List<CqlCompilerException>  errors = new ArrayList<>();
+
+    public CqlTranslatorException(Exception e) {
+        super("CQL Translation Error(s): " + e.getMessage());
+    }
+
+    public CqlTranslatorException(List<CqlCompilerException> errors) {
+        super("CQL Translation Error(s)");
+        this.errors.addAll(errors);
+    }
+
+    public CqlTranslatorException(List<String> errorsInput, CqlCompilerException.ErrorSeverity errorSeverity) {
+        super("CQL Translation Error(s)");
+        for (String error : errorsInput){
+            errors.add(new CqlCompilerException(error, errorSeverity));
+        }
+    }
+
+    public CqlTranslatorException(String message) {
+        super("CQL Translation Error(s): " + message);
+    }
+
+    public List<CqlCompilerException> getErrors() {
+        if (errors.isEmpty()) {
+            errors.add(new CqlCompilerException(this.getMessage()));
+        }
+        return errors;
+    }
+}

tooling/src/main/java/org/opencds/cqf/tooling/library/LibraryProcessor.java

@@ -48,21 +48,23 @@
 public class LibraryProcessor extends BaseProcessor {
     private static final Logger logger = LoggerFactory.getLogger(LibraryProcessor.class);
     public static final String ResourcePrefix = "library-";
+
     public static String getId(String baseId) {
         return ResourcePrefix + baseId;
     }
+
     private static Pattern pattern;
 
     private static Pattern getPattern() {
-        if(pattern == null) {
+        if (pattern == null) {
             String regex = "^[a-zA-Z]+[a-zA-Z0-9_\\-\\.]*";
             pattern = Pattern.compile(regex);
         }
         return pattern;
     }
 
     public static void validateIdAlphaNumeric(String id) {
-        if(!getPattern().matcher(id).find()) {
+        if (!getPattern().matcher(id).find()) {
             throw new RuntimeException("The library id format is invalid.");
         }
     }
@@ -76,8 +78,7 @@ public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encodin
     }
 
     public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encoding outputEncoding, String libraryPath, String libraryOutputDirectory, Boolean versioned, FhirContext fhirContext, Boolean shouldApplySoftwareSystemStamp) {
-        System.out.println("Refreshing libraries...");
-        // ArrayList<String> refreshedLibraryNames = new ArrayList<String>();
+        System.out.println("\r\n[Refreshing Libraries]\r\n");
 
         LibraryProcessor libraryProcessor;
         switch (fhirContext.getVersion().getVersion()) {
@@ -93,9 +94,8 @@ public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encodin
         }
 
         if (libraryPath == null) {
-            libraryPath = FilenameUtils.concat(parentContext.getRootDir(), IGProcessor.libraryPathElement);
-        }
-        else if (!Utilities.isAbsoluteFileName(libraryPath)) {
+            libraryPath = FilenameUtils.concat(parentContext.getRootDir(), IGProcessor.LIBRARY_PATH_ELEMENT);
+        } else if (!Utilities.isAbsoluteFileName(libraryPath)) {
             libraryPath = FilenameUtils.concat(parentContext.getRootDir(), libraryPath);
         }
         RefreshLibraryParameters params = new RefreshLibraryParameters();
@@ -117,64 +117,51 @@ else if (!Utilities.isAbsoluteFileName(libraryPath)) {
      * Bundles library dependencies for a given FHIR library file and populates the provided resource map.
      * This method executes asynchronously by invoking the associated task queue.
      *
-     * @param path          The path to the FHIR library file.
-     * @param fhirContext   The FHIR context to use for processing resources.
-     * @param resources     The map to populate with library resources.
-     * @param encoding      The encoding to use for reading and processing resources.
-     * @param versioned     A boolean indicating whether to consider versioned resources.
-     * @return True if the bundling of library dependencies is successful; false otherwise.
+     * @param path        The path to the FHIR library file.
+     * @param fhirContext The FHIR context to use for processing resources.
+     * @param resources   The map to populate with library resources.
+     * @param encoding    The encoding to use for reading and processing resources.
+     * @param versioned   A boolean indicating whether to consider versioned resources.
      */
-    public Boolean bundleLibraryDependencies(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
-                                             Encoding encoding, boolean versioned) {
-        try{
-            Queue<Callable<Void>> bundleLibraryDependenciesTasks = bundleLibraryDependenciesTasks(path, fhirContext, resources, encoding, versioned);
-            ThreadUtils.executeTasks(bundleLibraryDependenciesTasks);
-            return true;
-        }catch (Exception e){
-            return false;
-        }
-
+    public void bundleLibraryDependencies(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
+                                          Encoding encoding, boolean versioned) throws Exception {
+        Queue<Callable<Void>> bundleLibraryDependenciesTasks = bundleLibraryDependenciesTasks(path, fhirContext, resources, encoding, versioned);
+        ThreadUtils.executeTasks(bundleLibraryDependenciesTasks);
     }
 
     /**
      * Recursively bundles library dependencies for a given FHIR library file and populates the provided resource map.
      * Each dependency is added as a Callable task to be executed asynchronously.
      *
-     * @param path          The path to the FHIR library file.
-     * @param fhirContext   The FHIR context to use for processing resources.
-     * @param resources     The map to populate with library resources.
-     * @param encoding      The encoding to use for reading and processing resources.
-     * @param versioned     A boolean indicating whether to consider versioned resources.
+     * @param path        The path to the FHIR library file.
+     * @param fhirContext The FHIR context to use for processing resources.
+     * @param resources   The map to populate with library resources.
+     * @param encoding    The encoding to use for reading and processing resources.
+     * @param versioned   A boolean indicating whether to consider versioned resources.
      * @return A queue of Callable tasks, each representing the bundling of a library dependency.
-     *         The Callable returns null (Void) and is meant for asynchronous execution.
+     * The Callable returns null (Void) and is meant for asynchronous execution.
      */
     public Queue<Callable<Void>> bundleLibraryDependenciesTasks(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
-                                                                Encoding encoding, boolean versioned) {
+                                                                Encoding encoding, boolean versioned) throws Exception {
 
         Queue<Callable<Void>> returnTasks = new ConcurrentLinkedQueue<>();
 
         String fileName = FilenameUtils.getName(path);
         boolean prefixed = fileName.toLowerCase().startsWith("library-");
-        try {
-            Map<String, IBaseResource> dependencies = ResourceUtils.getDepLibraryResources(path, fhirContext, encoding, versioned, logger);
-            // String currentResourceID = IOUtils.getTypeQualifiedResourceId(path, fhirContext);
-            for (IBaseResource resource : dependencies.values()) {
-                returnTasks.add(() -> {
-                    resources.putIfAbsent(resource.getIdElement().getIdPart(), resource);
+        Map<String, IBaseResource> dependencies = ResourceUtils.getDepLibraryResources(path, fhirContext, encoding, versioned, logger);
+        // String currentResourceID = IOUtils.getTypeQualifiedResourceId(path, fhirContext);
+        for (IBaseResource resource : dependencies.values()) {
+            returnTasks.add(() -> {
+                resources.putIfAbsent(resource.getIdElement().getIdPart(), resource);
 
-                    // NOTE: Assuming dependency library will be in directory of dependent.
-                    String dependencyPath = IOUtils.getResourceFileName(IOUtils.getResourceDirectory(path), resource, encoding, fhirContext, versioned, prefixed);
+                // NOTE: Assuming dependency library will be in directory of dependent.
+                String dependencyPath = IOUtils.getResourceFileName(IOUtils.getResourceDirectory(path), resource, encoding, fhirContext, versioned, prefixed);
 
-                    returnTasks.addAll(bundleLibraryDependenciesTasks(dependencyPath, fhirContext, resources, encoding, versioned));
+                returnTasks.addAll(bundleLibraryDependenciesTasks(dependencyPath, fhirContext, resources, encoding, versioned));
 
-                    //return statement needed for Callable<Void>
-                    return null;
-                });
-            }
-        } catch (Exception e) {
-            logger.error(path, e);
-            //purposely break addAll:
-            return null;
+                //return statement needed for Callable<Void>
+                return null;
+            });
         }
         return returnTasks;
     }
@@ -259,7 +246,7 @@ protected void setTranslatorOptions(Library sourceLibrary, CqlTranslatorOptions
             optionsReferenceValue = "#options";
             optionsReference.setReference(optionsReferenceValue);
         }
-        Parameters optionsParameters = (Parameters)sourceLibrary.getContained(optionsReferenceValue);
+        Parameters optionsParameters = (Parameters) sourceLibrary.getContained(optionsReferenceValue);
         if (optionsParameters == null) {
             optionsParameters = new Parameters();
             optionsParameters.setId(optionsReferenceValue.substring(1));