Release 0.4.3

README.md

@@ -84,6 +84,14 @@ all existing files. We apologize for the inconveniences it incurs.
 
 ## Change Log
 
+### 0.4.3
+
+- Issue #94 revisited.
+- [minor] Fix syntax incompatibility with earlier versions of `msgpack`.
+- [minor] Fix a bug of not updating meta-database when `-s existing` is used.
+
+**A file signature clean up is required after this upgrade.**
+
 ### 0.4.2
 
 - Issue #214 for container support.

dscrutils/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: dscrutils
 Encoding: UTF-8
 Type: Package
-Version: 0.4.2
+Version: 0.4.3
 Date: 2019-12-03
 Title: Dynamic Statistical Comparisons R Interface
 Authors@R: c(person("Gao","Wang",role=c("aut","cre"),

release.sos

@@ -15,7 +15,7 @@ sh: workdir = '.'
 	pandoc --from=markdown --to=rst --output=README.rst README.md
 
 [pip]
-depends: "README.rst"
+depends: "README.rst", Py_Module('twine')
 parameter: version = str
 # check the version of the latest version
 cur_ver = get_output(f"pip{py} show dsc | grep Version | cut -d' ' -f2").strip()
@@ -25,13 +25,7 @@ stop_if(cur_ver == version)
 
 sh:	workdir = '.', expand = True 
 	python{py} setup.py sdist && \
-	rm -rf /tmp/release_dsc && \
-	mkdir /tmp/release_dsc && \
-	cp dist/dsc-{version}.tar.gz /tmp/release_dsc && \
-	cd /tmp/release_dsc && \
-	tar zxf dsc-{version}.tar.gz && \
-	cd dsc-{version} && \
-	python{py} setup.py sdist bdist_wheel upload && \
+	twine upload dist/dsc-{version}.tar.gz && \
 	pip{py} install -U --upgrade-strategy only-if-needed .
 
 [upgrade]

setup.py

@@ -53,7 +53,7 @@ def run(self):
       cmdclass    = cmdclass,
       package_dir = {'dsc': 'src'},
       install_requires = ['numpy', 'pandas>=0.24.1', 'sympy', 'numexpr',
-                          'sos>=0.21.5', 'sos-pbs>=0.20.3', 'h5py', 'PTable',
+                          'sos>=0.21.6', 'sos-pbs>=0.20.3', 'h5py', 'PTable',
                           'pyarrow>=0.5.0', 'sqlalchemy', 'tzlocal',
                           'msgpack-python']
       )

src/utils.py

@@ -8,7 +8,7 @@
 from itertools import cycle, chain, islice
 from fnmatch import fnmatch
 from difflib import SequenceMatcher
-from collections.abc import MutableMapping
+from collections.abc import Mapping, MutableMapping
 try:
     from xxhash import xxh32 as xxh
 except ImportError:
@@ -151,7 +151,7 @@ def lower_keys(x, level_start=0, level_end=2, mapping=dict):
         return x
     if isinstance(x, list):
         return [lower_keys(v, level_start, level_end) for v in x]
-    elif isinstance(x, collections.Mapping):
+    elif isinstance(x, Mapping):
         return mapping((k.lower(), lower_keys(v, level_start, level_end))
                        for k, v in x.items())
     else:
@@ -164,7 +164,7 @@ def is_null(var):
     if isinstance(var, str):
         if var.lower() in ['na', 'nan', 'null', 'none', '']:
             return True
-    if isinstance(var, (list, tuple, collections.Mapping)):
+    if isinstance(var, (list, tuple, Mapping)):
         return True if len(var) == 0 else False
     return False
 
@@ -217,11 +217,11 @@ def flatten_list(lst):
 
 
 def flatten_dict(d, mapping=dict):
-    if not isinstance(d, collections.Mapping):
+    if not isinstance(d, Mapping):
         return d
     items = []
     for k, v in d.items():
-        if isinstance(v, collections.Mapping):
+        if isinstance(v, Mapping):
             items.extend(flatten_dict(v).items())
         else:
             items.append((k, v))
@@ -371,19 +371,19 @@ def find_nested_key(key, dictionary):
     for k, v in dictionary.items():
         if k == key:
             yield [k]
-        elif isinstance(v, collections.Mapping):
+        elif isinstance(v, Mapping):
             for result in find_nested_key(key, v):
                 yield [k] + result
         elif isinstance(v, list):
             for d in v:
-                if isinstance(d, collections.Mapping):
+                if isinstance(d, Mapping):
                     for result in find_nested_key(key, d):
                         yield [k] + result
 
 
 def recursive_items(dictionary):
     for key, value in dictionary.items():
-        if isinstance(value, collections.Mapping):
+        if isinstance(value, Mapping):
             yield (key, value)
             yield from recursive_items(value)
         else:
@@ -402,7 +402,7 @@ def dict2str(value):
 
 def update_nested_dict(d, u, mapping=dict):
     for k, v in u.items():
-        if isinstance(v, collections.Mapping):
+        if isinstance(v, Mapping):
             r = update_nested_dict(d.get(k, mapping()), v)
             d[k] = r
         else:
@@ -411,7 +411,7 @@ def update_nested_dict(d, u, mapping=dict):
 
 
 def strip_dict(data, mapping=dict, into_list=False, skip_keys=None):
-    if not isinstance(data, collections.Mapping):
+    if not isinstance(data, Mapping):
         return data
     skip_keys = skip_keys or []
     mapping_null = [dict()]
@@ -420,7 +420,7 @@ def strip_dict(data, mapping=dict, into_list=False, skip_keys=None):
         if k in skip_keys:
             new_data[k] = v
             continue
-        if isinstance(v, collections.Mapping):
+        if isinstance(v, Mapping):
             v = strip_dict(v, mapping, into_list)
         if isinstance(v, list) and into_list:
             v = [strip_dict(x, mapping, into_list) for x in v]

src/version.py

@@ -1 +1 @@
-__version__ = '0.4.2'
+__version__ = '0.4.3'

test/test_interface.sos

@@ -13,5 +13,6 @@ bash: expand = True, workdir = f'{_input:d}'
   fi
 
 [default_2]
+input: group_by = 'all'
 R: workdir = '../dscrutils'
   devtools::test('tests')

test/test_parser.py

@@ -55,7 +55,7 @@
     @simulate:
         x: 1
         y: 2
-    @ALIAS: 
+    @ALIAS:
         simulate: x_1 = x
     $out: x
 '''
@@ -72,7 +72,7 @@
     @simulate:
         x: 1
         y: 2
-        @ALIAS: 
+        @ALIAS:
             *: x_1 = x
     $out: x
 '''
@@ -147,9 +147,9 @@ def testBasicSyntaxPass(self):
         self.assertEqual(list(res.modules['simulate'].plugin.alias_map.items()), [('x', 'x_1')])
         # use global variable
         res = DSC_Script(text11)
-        self.assertEqual(list(res.modules['simulate'].dump()['input'].items()), [('x', [2]), ('y', [2])])
+        self.assertEqual(list(res.modules['simulate'].dump()['input'].items()), [('x', [1]), ('y', [2])])
         res = DSC_Script(text12)
-        self.assertEqual(list(res.modules['simulate'].dump()['input'].items()), [('x', [4,3,2,1]), ('y', [2])])
+        self.assertEqual(list(res.modules['simulate'].dump()['input'].items()), [('x', [1]), ('y', [2])])
         # alias partial list / dict
         res = DSC_Script(text13)
         self.assertEqual(list(res.modules['simulate'].plugin.dump()['container_variables'].items()), [('x', [None, 'xvar']), ('y', [None, 'yy'])])
@@ -188,7 +188,7 @@ def testLegalNamesFail(self):
         text = '''
 simulate: R()
     .x: 1
-DSC: 
+DSC:
     run: simulate
 '''
         self.assertRaises(FormatError, DSC_Script, text)
@@ -244,7 +244,7 @@ def testModuleDerivationPass(self):
 base: R(base=1)
     x: 2
     $out: x
-simulate(base): 
+simulate(base):
     x: R(1:5)
 '''
         res = DSC_Script(text)
@@ -266,7 +266,7 @@ def testModuleDerivationPass(self):
         y: 5
         n: 6
     $x: x
-    
+
 simulate(normal):
     mu: 1
 '''
@@ -277,7 +277,7 @@ def testModuleDerivationPass(self):
     def testModuleDerivationFail(self):
         # missing executable
         text = text0 + '''
-simulate: 
+simulate:
     x: R{1:5}
     $out: x
 '''
@@ -290,8 +290,8 @@ def testModuleDerivationFail(self):
         y: 5
         n: 6
     $x: x
-    
-simulate(normal, t): 
+
+simulate(normal, t):
     mu: 1
 DSC:
     run: test
@@ -305,17 +305,17 @@ def testModuleDerivationFail(self):
         y: 5
         n: 6
     $x: x
-    
+
 shifted_normal(normal):
     mu: 1
-'''        
+'''
         self.assertRaises(FormatError, DSC_Script, text)
         # non-existing base
         text = text0 + '''
 base: R()
     x: 2
     $out: x
-simulate(base1): 
+simulate(base1):
     x: R(1:5)
 '''
         self.assertRaises(FormatError, DSC_Script, text)
@@ -409,7 +409,7 @@ def testDuplicatesFail(self):
 
 simulate: R()
     $x: 7
-''' 
+'''
         self.assertWarns(UserWarning, DSC_Script, text)
         text = text0 + '''
 simulate: R()
@@ -472,7 +472,7 @@ def testFilterPass(self):
 simulate, t: R(), R()
     n: 100, 200, 300, 400, 500
     k: 0, 1
-    @FILTER: 
+    @FILTER:
         *: n in [100,200,300]
         t: n = 300
     $x: x