Merge pull request #9 from NCAR-CISL-ASAP/devel

Merging in bugfix and small changes for minor release

CHANGES.rst

@@ -95,4 +95,12 @@ VERSION 0.9.4
  - Performance improvement (no explicit loop over time-steps in a time-slice
    file needed)
  - Enabling user-defined compression level for NetCDF4 output
+
+VERSION 0.9.5
+-------------
+
+6 October 2015:
+ - Fix bug in the 'slice2series' script
+ - Adds a write to file option for the Specifiers
+ - Modifying output message verbosity settings
 

bin/slice2series

@@ -102,13 +102,13 @@ def main(options, input_file_list):
         # Create the input object for the Reshaper
         spec = specification.create_specifier()
 
-    # Generate the input file list from (potentially) globs/wildcards
-    full_input_file_list = []
-    for arg in input_file_list:
-        full_input_file_list.extend(glob.glob(arg))
-    
+        # Generate the input file list from (potentially) globs/wildcards
+        full_input_file_list = []
+        for arg in input_file_list:
+            full_input_file_list.extend(glob.glob(arg))
+
         # Add input to the specifier
-        spec.input_file_list = input_file_list
+        spec.input_file_list = full_input_file_list
         spec.netcdf_format = options.netcdf_format
         spec.output_file_prefix = options.output_prefix
         spec.output_file_suffix = options.output_suffix

source/pyreshaper/reshaper.py

@@ -281,7 +281,7 @@ def __init__(self, specifier, serial=False, verbosity=1,
 
         # Debug output starting
         if self._simplecomm.is_manager():
-            self._vprint('Initializing Reshaper', verbosity=1)
+            self._vprint('Initializing Reshaper...', verbosity=0)
 
         # Validate the user input data
         self._timer.start('Specifier Validation')
@@ -312,29 +312,29 @@ def __init__(self, specifier, serial=False, verbosity=1,
             opt.CompressionLevel = specifier.compression_level
         self._nio_options = opt
         if self._simplecomm.is_manager():
-            self._vprint('PyNIO options set', verbosity=2)
+            self._vprint('PyNIO options set', verbosity=1)
 
         # Validate the input files themselves
         if self._simplecomm.is_manager():
-            self._vprint('Inspecting input files...', verbosity=1)
+            self._vprint('Inspecting input files...', verbosity=0)
         self._timer.start('Inspect Input Files')
         self._inspect_input_files(specifier.time_variant_metadata)
         self._timer.stop('Inspect Input Files')
         if self._simplecomm.is_manager():
-            self._vprint('Input files inspected.', verbosity=1)
+            self._vprint('Input files inspected.', verbosity=0)
 
         # Validate the output files
         if self._simplecomm.is_manager():
-            self._vprint('Checking output file status...', verbosity=1)
+            self._vprint('Checking output file status...', verbosity=0)
         self._timer.start('Check Output File Status')
         self._check_output_file_status(specifier, skip_existing, overwrite)
         self._timer.stop('Check Output File Status')
         if self._simplecomm.is_manager():
-            self._vprint('Output file status checked.', verbosity=1)
+            self._vprint('Output file status checked.', verbosity=0)
 
         # Helpful debugging message
         if self._simplecomm.is_manager():
-            self._vprint('Reshaper initialized.', verbosity=1)
+            self._vprint('Reshaper initialized.', verbosity=0)
 
         # Sync before continuing..
         self._simplecomm.sync()
@@ -428,7 +428,7 @@ def _inspect_input_files(self, metadata_names=[]):
                        "not in all input files:{}{}").format(linesep, '   ')
             for var in missing_vars:
                 warning += ' {}'.format(var)
-            self._vprint(warning, header=True, verbosity=1)
+            self._vprint(warning, header=True, verbosity=0)
 
         #===== SORT INPUT FILES BY TIME =====
 
@@ -459,11 +459,11 @@ def _inspect_input_files(self, metadata_names=[]):
         # Debug output
         if self._simplecomm.is_manager():
             self._vprint('Time-Invariant Metadata: ' +
-                         str(self._time_invariant_metadata.keys()), verbosity=2)
+                         str(self._time_invariant_metadata.keys()), verbosity=1)
             self._vprint('Time-Variant Metadata: ' +
-                         str(self._time_variant_metadata.keys()), verbosity=2)
+                         str(self._time_variant_metadata.keys()), verbosity=1)
             self._vprint('Time-Series Variables: ' +
-                         str(self._time_series_variables.keys()), verbosity=2)
+                         str(self._time_series_variables.keys()), verbosity=1)
 
         # Add 'once' variable if writing to a once file
         # NOTE: This is a "cheat"!  There is no 'once' variable.  It's just
@@ -513,7 +513,7 @@ def _check_output_file_status(self, specifier,
             if self._simplecomm.is_manager():
                 self._vprint('WARNING: Deleting existing output files for '
                              'time-series variables: {}'.format(existing),
-                             verbosity=1)
+                             verbosity=0)
             for variable in existing:
                 remove(self._time_series_filenames[variable])
 
@@ -523,7 +523,7 @@ def _check_output_file_status(self, specifier,
             if self._simplecomm.is_manager():
                 self._vprint('WARNING: Skipping time-series variables with '
                              'existing output files: {}'.format(existing),
-                             verbosity=1)
+                             verbosity=0)
             for variable in existing:
                 self._time_series_variables.pop(variable)
 
@@ -557,7 +557,7 @@ def convert(self, output_limit=0):
 
         # Debugging output
         if self._simplecomm.is_manager():
-            self._vprint('Converting time-slices to time-series', verbosity=1)
+            self._vprint('Converting time-slices to time-series...', verbosity=0)
 
         # Partition the time-series variables across all processors
         tsv_names_loc = self._simplecomm.partition(
@@ -568,7 +568,7 @@ def convert(self, output_limit=0):
 
         # Print partitions for all ranks
         dbg_msg = 'Local time-series variables are {}'.format(tsv_names_loc)
-        self._vprint(dbg_msg, header=True, verbosity=2)
+        self._vprint(dbg_msg, header=True, verbosity=1)
 
         # Reset all of the timer values (as it is possible that there are no
         # time-series variables in the local list procuded above)
@@ -781,7 +781,7 @@ def _get_bytesize(data):
         self._simplecomm.sync()
         if self._simplecomm.is_manager():
             self._vprint(('Finished converting time-slices '
-                          'to time-series.'), verbosity=1)
+                          'to time-series.'), verbosity=0)
 
         # Finish clocking the entire convert procedure
         self._timer.stop('Complete Conversion Process')

source/pyreshaper/specification.py

@@ -11,6 +11,7 @@
 """
 
 # Built-in imports
+import cPickle as pickle
 from os import path as ospath
 
 
@@ -21,7 +22,7 @@ def create_specifier(**kwargs):
     """
     Factory function for Specifier class objects.  Defined for convenience.
 
-    Keyword Arguments:
+    Parameters:
         kwargs (dict): Optional arguments to be passed to the newly created
             Specifier object's constructor.
 
@@ -61,7 +62,7 @@ def __init__(self,
 
         The output_file_name should be a full-path filename.
 
-        Keyword Arguments:
+        Parameters:
             infiles (list): List of full-path input filenames
             ncfmt (str): String specifying the NetCDF
                 data format ('netcdf','netcdf4','netcdf4c')
@@ -223,6 +224,21 @@ def validate_values(self):
         if (self.output_file_suffix[-3:] != '.nc'):
             self.output_file_suffix += '.nc'
 
+    def write(self, fname):
+        """
+        Write the specifier to a file
+
+        Parameters:
+            fname (str): Name of file to write
+        """
+        try:
+            fobj = open(fname, 'w')
+            pickle.dump(self, fobj)
+            fobj.close()
+        except:
+            err_msg = "Failed to write Specifier to file '{}'".format(fname)
+            raise OSError(err_msg)
+
 
 #==============================================================================
 # Command-line Operation

source/pyreshaper/version.py

@@ -1,2 +1,2 @@
 # Single place for version information
-__version__ = '0.9.4'
+__version__ = '0.9.5'

tests/yellowstone/testinfo.json

@@ -187,6 +187,7 @@
          "time",
          "time_bnds",
          "time_written",
+         "date_written",
          "gw",
          "hyai",
          "hyam",
@@ -236,6 +237,7 @@
          "time",
          "time_bnds",
          "time_written",
+         "date_written",
          "gw",
          "hyai",
          "hyam",
@@ -285,6 +287,7 @@
          "time",
          "time_bnds",
          "time_written",
+         "date_written",
          "gw",
          "hyai",
          "hyam",