Overall code improvements (#555)

* fix: use six.moves instead of future.moves to fix annoying import of top level modules

* fix: missing future import + clean up dependencies

* fix: __package__ does not exist in Python 2.7

* fix: string decoding issue in Python 2.7

* update tox.ini

* fix: install all extras in tox.init

* update travis and appveyor config

* remove unused requirements files

* update tox.ini

* remove unwanted line in test.sh

* fix: Python version 3.9 not available in AppVeyor yet

* fix: only keep minimum requirements for test

.travis.yml

@@ -21,14 +21,14 @@ matrix:
       env: TOXENV=old_tests
     - python: 3.6
       env: TOXENV=mypy
+    - python: 3.9
 
   allow_failures:
     - python: 3.6  # until we solve all typing issues
       env: TOXENV=mypy
 
 install:
-  - pip install tox-travis
-  - pip install -r requirements.tox.txt
+  - pip install tox tox-travis
 
 script:
   - tox
@@ -41,4 +41,4 @@ deploy:
   skip_cleanup: true
   on:
     tags: true
-    branch: master 
+    branch: master

appveyor.yml

@@ -29,7 +29,7 @@ install:
   - py -3.6 -m venv "%VENV%"
   - "%VENV%\\Scripts\\activate"
   - python -c "import sys; print(sys.version)"
-  - pip install -r requirements.tox.txt
+  - pip install tox
   - ps: Update-AppveyorBuild -Version "v$(python get_version.py) b$Env:APPVEYOR_BUILD_NUMBER"
 
 build_script:

requirements.test.txt

@@ -0,0 +1,2 @@
+pathlib2; python_version < '3.4'
+pytest

requirements.txt

@@ -1,11 +1,11 @@
+attrs>=18.1.0
 click
+enum34; python_version < '3.4'
+future
 lxml
-xlwt
-xlrd==1.2.0
-xlsxwriter
 pyyaml
-future
-attrs>=18.1.0
+six
 typing; python_version < '3.5'
-pathlib2
-enum34; python_version <= '2.7'
+xlrd
+xlsxwriter
+xlwt

setup.py

@@ -57,8 +57,8 @@
 Programming Language :: Python :: 3.7
 """
 
-import sys
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
+
 import versioneer
 
 doclines = __doc__.split("\n")
@@ -76,26 +76,26 @@
     long_description = "\n".join(doclines[2:]),
     license = "BSD",
     platforms = ["any"],
-      install_requires = [
+    install_requires = [
         "attrs>=18.1.0",
-        "bitstruct",
         "click",
+        "enum34; python_version < '3.4'",
         "future",
-        "pathlib2",
+        "six",
         "typing; python_version < '3.5'",
     ],
     extras_require = {
         "arxml": ["lxml"],
-        "kcd": ["lxml"],
-        "fibex": ["lxml"],
-        "xls": ["xlrd", "xlwt"],
-        "xlsx": ["xlsxwriter"],
-        "yaml": ["pyyaml"],
         "dbc": [],
         "dbf": [],
+        "fibex": ["lxml"],
         "json": [],
+        "kcd": ["lxml"],
         "sym": [],
-        "test": ["coverage", "pytest", "pytest-cov", "tox"],
+        "test": ["pathlib2; python_version < '3.4'", "pytest"],
+        "xls": ["xlrd", "xlwt"],
+        "xlsx": ["xlsxwriter"],
+        "yaml": ["pyyaml"],
     },
 
     packages = find_packages("src"),
@@ -104,4 +104,3 @@
     entry_points={'console_scripts': ['cancompare = canmatrix.cli.compare:cli_compare',
                                       'canconvert = canmatrix.cli.convert:cli_convert']}
 )
-

src/canmatrix/canmatrix.py

@@ -39,8 +39,8 @@
 from builtins import *
 
 import attr
-from future.moves.itertools import zip_longest
 from past.builtins import basestring
+from six.moves import zip_longest
 
 import canmatrix.copy
 import canmatrix.types
@@ -1513,6 +1513,7 @@ def update(self):  # type: () -> None
 
 import enum
 
+
 class matrix_class(enum.Enum):
     CAN = 1
     FLEXRAY = 2

src/canmatrix/formats/__init__.py

@@ -6,15 +6,12 @@
 import os
 import sys
 import typing
+from builtins import str
+from io import BytesIO
 
 import canmatrix
 import canmatrix.cancluster
 
-if sys.version_info > (3, 0):
-    import io
-else:
-    import StringIO
-
 logger = logging.getLogger(__name__)
 moduleList = ["arxml", "csv", "dbc", "dbf", "json",
               "kcd", "fibex", "sym", "xls", "xlsx", "yaml", "scapy", "wireshark"]
@@ -49,12 +46,8 @@
 
 def loads(string, import_type=None, key="", encoding="utf-8", **options):
     # type: (typing.Union[bytes,str], str, str, str, **str) -> typing.Union[typing.Dict[str, canmatrix.CanMatrix], None]
-    if sys.version_info > (3, 0):
-        byte_str = bytes(string, encoding) if isinstance(string, str) else string
-        file_object = io.BytesIO(byte_str)
-    else:
-        string = string.encode(encoding)
-        file_object = StringIO.StringIO(string)
+    bytes_str = string.encode(encoding=encoding) if isinstance(string, str) else string
+    file_object = BytesIO(bytes_str)
     return load(file_object, import_type, key, **options)
 
 

src/canmatrix/j1939_decoder.py

@@ -4,16 +4,21 @@
 from builtins import *
 
 import attr
-import pathlib2
 
 import canmatrix.formats
 
+try:
+    from importlib.resources import read_binary
+except ImportError:
+    from pkgutil import get_data as read_binary
+
 
 @attr.s
 class j1939_decoder(object):
-    here = pathlib2.Path(__file__).parent
-
-    j1939_db = canmatrix.formats.loadp_flat(str(here / "j1939.dbc"), dbcImportEncoding = "utf8")
+    string = read_binary(__name__.rpartition('.')[0], "j1939.dbc")
+    j1939_db = canmatrix.formats.loads_flat(
+        string, import_type="dbc", dbcImportEncoding="utf8"
+    )
     length = attr.ib(default=0)  # type: int
     count_succesive_frames = attr.ib(default=0)  # type: int
     transfered_pgn = attr.ib(default=0)  # type: int

src/canmatrix/tests/test_arxml.py

@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 import canmatrix.formats.arxml
-import pathlib2
-import io
-import textwrap
+
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
+
 
 def test_ecu_extract():
-    here = pathlib2.Path(__file__).parent
+    here = Path(__file__).parent
 
     db = canmatrix.formats.arxml.load(str(here / "MyECU.ecuc.arxml"))['']
     assert db.frames is not None
@@ -15,20 +18,20 @@ def test_ecu_extract():
 
 
 def test_get_signals_from_container_i_pdu():
-    here = pathlib2.Path(__file__).parent
+    here = Path(__file__).parent
     matrix = canmatrix.formats.arxml.load(str(here / "ARXMLContainerTest.arxml"))
     assert matrix["New_CanCluster"].frames[0].signals[0].name == 'Header_ID'
     assert matrix["New_CanCluster"].frames[0].signals[1].name == 'Header_DLC'
     assert matrix["New_CanCluster"].frames[0].signals[2].name == 'PDU_Contained_1_Signal1_905db81da40081cb'
     assert matrix["New_CanCluster"].frames[0].signalGroups[0].signals[0].name == 'PDU_Contained_1_Signal1_905db81da40081cb'
 
 def test_get_signals_from_secured_pdu():
-    here = pathlib2.Path(__file__).parent
+    here = Path(__file__).parent
     matrix = canmatrix.formats.arxml.load(str(here / "ARXMLSecuredPDUTest.arxml"))
     assert matrix["CAN"].frames[0].signals[0].name == 'someTestSignal'
     assert matrix["CAN"].frames[0].signals[1].name == 'Signal'
 
 def test_min_max():
-    here = pathlib2.Path(__file__).parent
+    here = Path(__file__).parent
     matrix = canmatrix.formats.arxml.load(str(here / "ARXML_min_max.arxml"))
-    assert matrix["New_CanCluster"].frames[0].signals[0].is_signed == False
+    assert matrix["New_CanCluster"].frames[0].signals[0].is_signed == False

src/canmatrix/tests/test_cli_compare.py

@@ -1,11 +1,16 @@
-import pytest
-import pathlib2
 import sys
+
 import canmatrix.formats
-pytest_plugins = ["pytester"]
+import pytest
+
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
 
+pytest_plugins = ["pytester"]
+here = Path(__file__).parent
 
-here = pathlib2.Path(__file__).parent
 
 @pytest.fixture
 def run(testdir):

src/canmatrix/tests/test_cli_convert.py

@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 import sys
 
-import pathlib2
+import canmatrix.formats
 import pytest
 
-import canmatrix.formats
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
 
 pytest_plugins = ["pytester"]
+here = Path(__file__).parent
 
 
-here = pathlib2.Path(__file__).parent
-
 @pytest.fixture
 def run(testdir):
     def do_run(*args):

test.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
-pip install -e .[arxml,kcd,fibex,xls,xlsx,yaml]
+
 cd test
 python ./test.py

tox.ini

@@ -1,10 +1,12 @@
 [tox]
-envlist = py{27,35,36,37,38}, pypy{,3}, mypy
+envlist = py{27,35,36,37,38,39}, pypy{,3}, mypy
 
 [testenv]
 deps=
-    py27,pypy: -r {toxinidir}/requirements.test.py2.txt
-    py3{5,6,7,8},pypy3: -r {toxinidir}/requirements.test.py3.txt
+    -r requirements.test.txt
+    -r requirements.txt
+    coverage
+    pytest-cov
 passenv=
     TOXENV
     CI