2.SpeedingUpTheSlowOption

catalogbuilder/intakebuilder/getinfo.py

@@ -213,6 +213,11 @@ def getInfoFromVarAtts(fname,variable_id,dictInfo,att="standard_name",filexra=No
           cfname = filexr[variable_id].attrs["standard_name"]
       except KeyError:
           cfname = "NA"
+          try:
+              long_name = filexr[variable_id].attrs["long_name"]
+          except KeyError:
+              long_name = "NA"
+          cfname = long_name.replace(" ", "_")
       dictInfo["standard_name"] = cfname 
       print("standard_name found",dictInfo["standard_name"])
     return dictInfo

catalogbuilder/intakebuilder/gfdlcrawler.py

@@ -33,7 +33,7 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
     set_ptemplate = set()
     set_ftemplate = set()
 
-    if( configyaml is not None):
+    if(configyaml is not None):
         if (configyaml.output_path_template is not None) & (configyaml.output_file_template is not None) :
           list_ptemplate = configyaml.output_path_template
           list_ftemplate = configyaml.output_file_template
@@ -59,7 +59,10 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
       missingcols.remove("path") #because we get this anyway
       logger.debug("Missing cols from metadata sources:"+ (str)(missingcols))
 
-
+    #Creating a dictionary to track the unique datasets we come across when using slow mode
+    #The keys are the standard names and the values are lists tracking var_id,realm,etc..
+    unique_datasets = {'':''}
+
     #TODO INCLUDE filter in traversing through directories at the top
     for dirpath, dirs, files in os.walk(projectdir):
         searchpath = dirpath
@@ -114,12 +117,26 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
                # todo do the reverse if slow is on. Open file no matter what and populate dictionary values and if there is something missed out
                # we can scan filenames or config etc 
                #here, we will see if there are missing header values and compare with file attributes if slow option is turned on
-               if (slow == True) & (bool(dictInfo) == True) :
-                    print("Slow option turned on.. lets open some files using xarray and lookup atts",filename)
-                    #todo we could look at var attributes, but right now we stick to those that are necessary. scope to extend this easily to missngcols or if header info is not in config yaml 
-                    if "standard_name" in missingcols: 
+               if (slow == True) & (bool(dictInfo) == True):
+                    #print("Slow option turned on.. lets open some files using xarray and lookup atts")
+
+                    #todo we could look at var attributes, but right now we stick to those that are necessary. scope to extend this easily to missngcols or if header info is not in config yaml
+                    if "standard_name" in missingcols:
+
+                        # Set standard_name as na to avoid error from getInfoFromVarAtts
                         dictInfo["standard_name"] = "na"
-                        getinfo.getInfoFromVarAtts(dictInfo["path"],dictInfo["variable_id"],dictInfo)
+
+                        # qualities define the uniqueness and help us determine when to open files. here, we define uniqueness by realm and var_id combinations. we store the realm/var_id pairs + their standard_names in unique_datasets{} and the current pair being checked as a tuple list called 'qualities'. if a pair stored in unique_datasets aligns with the current pair being checked, we won't open the file and will instead use the standard_name already found
+                        qualities=(dictInfo["variable_id"],dictInfo["realm"])
+                        if qualities in unique_datasets.keys():
+                            standard_name=unique_datasets[qualities]
+                            dictInfo["standard_name"]=standard_name
+
+                        else:
+                            print("Retrieving standard_name from ", filename)
+                            getinfo.getInfoFromVarAtts(dictInfo["path"],dictInfo["variable_id"],dictInfo)
+                            unique_datasets.update({ qualities : dictInfo["standard_name"] })
+
                #replace frequency as needed 
                if 'frequency' in dictInfo.keys():
                       package_dir = os.path.dirname(os.path.abspath(__file__))