Simple ordering of POST calls by resource type

tooling/src/main/java/org/opencds/cqf/tooling/common/ThreadUtils.java

@@ -16,6 +16,8 @@ public class ThreadUtils {
 
     private static List<ExecutorService> runningExecutors = new ArrayList<>();
 
+    private static final int threadPoolCount = Runtime.getRuntime().availableProcessors() * 2;
+
     /**
      * Executes a list of tasks concurrently using a thread pool.
      * <p>
@@ -62,11 +64,15 @@ public static void executeTasks(List<Callable<Void>> tasks, ExecutorService exec
     }
 
     public static void executeTasks(List<Callable<Void>> tasks) {
-        executeTasks(tasks, Executors.newCachedThreadPool());
+        ExecutorService executor = Executors.newFixedThreadPool(threadPoolCount);
+
+        executeTasks(tasks, executor);
     }
 
     public static void executeTasks(Queue<Callable<Void>> callables) {
-        executeTasks(new ArrayList<>(callables), Executors.newCachedThreadPool());
+        ExecutorService executor = Executors.newFixedThreadPool(threadPoolCount);
+
+        executeTasks(new ArrayList<>(callables), executor);
     }
 
     public static void shutdownRunningExecutors() {
@@ -77,7 +83,7 @@ public static void shutdownRunningExecutors() {
             }
             runningExecutors = new ArrayList<>();
         }catch (Exception e){
-            //fail silently, shutting down anyways
+            //fail silently, shutting down anyway
         }
     }
 }

tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java

@@ -16,11 +16,17 @@
 
 public class MeasureBundler extends AbstractBundler {
     public static final String ResourcePrefix = "measure-";
+    public static final String LIBRARY_DEPS = "library-deps-";
+    public static final String VALUESETS = "valuesets-";
+    public static final String TESTS = "tests-";
+    public static final String GROUP = "group-";
+
     protected CopyOnWriteArrayList<Object> identifiers;
 
     public static String getId(String baseId) {
         return ResourcePrefix + baseId;
     }
+
     @Override
     protected String getSourcePath(FhirContext fhirContext, Map.Entry<String, IBaseResource> resourceEntry) {
         return IOUtils.getMeasurePathMap(fhirContext).get(resourceEntry.getKey());
@@ -42,66 +48,60 @@ protected Set<String> getPaths(FhirContext fhirContext) {
     }
 
     //so far only the Measure Bundle process needs to persist extra files:
+    /**
+     * This method will group files of similar naming to attempt to post the resources in an order that the fhir endpoint
+     * can understand. For instance, attempting to post a Group file before the Patient resource is posted can result
+     * in a reference error.
+     *
+     * @param bundleDestPath
+     * @param libraryName
+     * @param encoding
+     * @param fhirContext
+     * @param fhirUri
+     * @return
+     */
     @Override
     protected int persistFilesFolder(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) {
-        //persist tests-* before group-* files and make a record of which files were tracked:
-        List<String> persistedFiles = persistTestFilesWithPriority(bundleDestPath, libraryName, encoding, fhirContext, fhirUri);
-        persistedFiles.addAll(persistEverythingElse(bundleDestPath, libraryName, encoding, fhirContext, fhirUri, persistedFiles));
-
-        return persistedFiles.size();
-    }
-
-    private List<String> persistTestFilesWithPriority(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) {
         List<String> persistedResources = new ArrayList<>();
         String filesLoc = bundleDestPath + File.separator + libraryName + "-files";
         File directory = new File(filesLoc);
         if (directory.exists()) {
             File[] filesInDir = directory.listFiles();
             if (!(filesInDir == null || filesInDir.length == 0)) {
                 for (File file : filesInDir) {
-                    if (file.getName().toLowerCase().startsWith("tests-")) {
-                        try {
-                            IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true);
-                            HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), true);
-                            persistedResources.add(file.getAbsolutePath());
-                        } catch (Exception e) {
-                            //resource is likely not IBaseResource
-                            logger.error("MeasureBundler.persistTestFilesWithPriority", e);
-                        }
+
+                    if (!file.getName().toLowerCase().endsWith(".json") && !file.getName().toLowerCase().endsWith(".xml")){
+                        continue;
+                    }
+
+                    HttpClientUtils.HttpPOSTResourceType type = getHttpPOSTResourceType(file);
+                    try {
+                        IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true);
+                        HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), type);
+                        persistedResources.add(file.getAbsolutePath());
+                    } catch (Exception e) {
+                        //resource is likely not IBaseResource
+                        logger.error("MeasureBundler.persistFilesFolder", e);
                     }
                 }
             }
         }
-        return persistedResources;
+        return persistedResources.size();
     }
 
-    private List<String> persistEverythingElse(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri, List<String> alreadyPersisted) {
-        List<String> persistedResources = new ArrayList<>();
-        String filesLoc = bundleDestPath + File.separator + libraryName + "-files";
-        File directory = new File(filesLoc);
-        if (directory.exists()) {
-
-            File[] filesInDir = directory.listFiles();
+    private static HttpClientUtils.HttpPOSTResourceType getHttpPOSTResourceType(File file) {
+        HttpClientUtils.HttpPOSTResourceType type = HttpClientUtils.HttpPOSTResourceType.OTHER;
 
-            if (!(filesInDir == null || filesInDir.length == 0)) {
-                for (File file : filesInDir) {
-                    //don't post what has already been processed
-                    if (alreadyPersisted.contains(file.getAbsolutePath())) {
-                        continue;
-                    }
-                    if (file.getName().toLowerCase().endsWith(".json") || file.getName().toLowerCase().endsWith(".xml")) {
-                        try {
-                            IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true);
-                            HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), false);
-                            persistedResources.add(file.getAbsolutePath());
-                        } catch (Exception e) {
-                            //resource is likely not IBaseResource
-                            logger.error("persistEverythingElse", e);
-                        }
-                    }
-                }
-            }
+        if (file.getName().toLowerCase().startsWith(LIBRARY_DEPS)) {
+            type = HttpClientUtils.HttpPOSTResourceType.LIBRARY_DEPS;
+        } else if (file.getName().toLowerCase().startsWith(VALUESETS)) {
+            type = HttpClientUtils.HttpPOSTResourceType.VALUESETS;
+        } else if (file.getName().toLowerCase().startsWith(TESTS)) {
+            type = HttpClientUtils.HttpPOSTResourceType.TESTS;
+        } else if (file.getName().toLowerCase().startsWith(GROUP)) {
+            type = HttpClientUtils.HttpPOSTResourceType.GROUP;
         }
-        return persistedResources;
+        return type;
     }
+
 }

tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java

@@ -142,7 +142,7 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
             final Map<String, String> libraryPathMap = new ConcurrentHashMap<>(IOUtils.getLibraryPathMap(fhirContext));
 
             if (resourcesMap.isEmpty()) {
-                logger.info("[INFO] No " + getResourceBundlerType() + "s found. Continuing...");
+                logger.info("\n\r" + "[INFO] No " + getResourceBundlerType() + "s found. Continuing...\n\r");
                 return;
             }
 
@@ -169,7 +169,7 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
                 tasks.add(() -> {
                     //check if resourceSourcePath has been processed before:
                     if (processedResources.contains(resourceSourcePath)) {
-                        logger.info(getResourceBundlerType() + " processed already: " + resourceSourcePath);
+                        logger.info("\n\r" + getResourceBundlerType() + " processed already: " + resourceSourcePath);
                         return null;
                     }
                     String resourceName = FilenameUtils.getBaseName(resourceSourcePath).replace(getResourcePrefix(), "");
@@ -296,7 +296,7 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
         //Output final report:
         String summaryOutput = generateBundleProcessSummary(refreshedLibraryNames, fhirContext, fhirUri, verboseMessaging,
                 persistedFileReport, bundledResources, failedExceptionMessages, cqlTranslatorErrorMessages).toString();
-        logger.info(summaryOutput);
+        logger.info("\n\r" + summaryOutput);
     }
 
     /**
@@ -458,7 +458,8 @@ private void persistBundle(String bundleDestPath, String libraryName,
 
         if (fhirUri != null && !fhirUri.isEmpty()) {
             String resourceWriteLocation = bundleDestPath + separator + libraryName + "-bundle." + encoding;
-            HttpClientUtils.post(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation, true);
+            //give resource the highest priority (0):
+            HttpClientUtils.post(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation, HttpClientUtils.HttpPOSTResourceType.BUNDLE);
         }
     }
 
@@ -476,7 +477,7 @@ private void bundleFiles(String igPath, String bundleDestPath, String primaryLib
         IOUtils.copyFile(librarySourcePath, FilenameUtils.concat(bundleDestFilesPath, FilenameUtils.getName(librarySourcePath)));
 
         String cqlFileName = IOUtils.formatFileName(FilenameUtils.getBaseName(librarySourcePath), IOUtils.Encoding.CQL, fhirContext);
-        if (cqlFileName.toLowerCase().startsWith("library-")) {
+        if (cqlFileName.toLowerCase().startsWith("library-") && !cqlFileName.toLowerCase().startsWith("library-deps-")) {
             cqlFileName = cqlFileName.substring(8);
         }
         String cqlLibrarySourcePath = IOUtils.getCqlLibrarySourcePath(primaryLibraryName, cqlFileName, binaryPaths);

tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java

@@ -49,7 +49,7 @@ public void bundleIg(ArrayList<String> refreshedLibraryNames, String igPath, Lis
 
         //run collected post calls last:
         if (HttpClientUtils.hasPostTasksInQueue()) {
-            logger.info("[Persisting Files to " + fhirUri + "]");
+            logger.info("\n\r[Persisting Files to " + fhirUri + "]\n\r");
             HttpClientUtils.postTaskCollection();
         }