diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d60f7d8..6a34142 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: - name: Run tests run: | + tox -e setup tox env: PLATFORM: ${{ matrix.os }} diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..0acc386 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,111 @@ +name: Pylint + +on: + workflow_dispatch: + pull_request: + push: + branches: [ main ] + +jobs: + pylint: + + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + outputs: + branch: ${{ steps.extract_branch.outputs.branch }} + rating: ${{ steps.analyze.outputs.rating }} + path: ${{ steps.analyze.outputs.path }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Extract base branch name + id: extract_branch + shell: bash + run: | + TMP_PULL_BASE_REF="${{ github.base_ref }}" + TMP_GITHUB_REF="${GITHUB_REF#refs/heads/}" + EXPORT_VALUE="" + if [ "${TMP_PULL_BASE_REF}" != "" ] + then + EXPORT_VALUE="${TMP_PULL_BASE_REF}" + else + EXPORT_VALUE="${TMP_GITHUB_REF}" + fi + echo "##[set-output name=branch;]${EXPORT_VALUE}" + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install tox + run: | + python -m pip install --upgrade pip wheel + pip install tox tox-gh-actions + + - name: Run pylint + id: analyze + env: + BADGE_PATH: badges/pylint-score.svg + run: | + rating=$(bash -c 'tox -e lint' | grep 'Your code has been rated at' | cut -f7 -d " ") + echo "Pylint score: ${rating}" + echo "##[set-output name=rating;]${rating}" + echo "##[set-output name=path;]${BADGE_PATH}" + + badge: + # Only generate and publish if these conditions are met: + # - The previous job/analyze step ended successfully + # - At least one of these is true: + # - This is a push event and the push event is on branch 'master' or 'develop' + # Note: if this repo is personal (ie, not an org repo) then you can + # use the following to change the scope of the next 2 jobs + # instead of running on branch push as shown below: + # - This is a pull request event and the pull actor is the same as the repo owner + # if: ${{ ( github.event_name == 'pull_request' && github.actor == github.repository_owner ) || github.ref == 'refs/heads/master' }} + name: Generate badge image with pylint score + runs-on: ubuntu-20.04 + needs: [pylint] + if: ${{ github.event_name == 'push' }} + + steps: + - uses: actions/checkout@v2 + with: + ref: badges + path: badges + + # Use the output from the `analyze` step + - name: Create pylint badge + uses: emibcn/badge-action@v1 + id: badge + with: + label: 'Pylint score' + status: ${{ needs.pylint.outputs.rating }} + color: 'green' + path: ${{ needs.pylint.outputs.path }} + + - name: Commit badge + env: + BRANCH: ${{ needs.pylint.outputs.branch }} + FILE: 'pylint-score.svg' + working-directory: ./badges + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + mkdir -p "${BRANCH}" + mv "${FILE}" "${BRANCH}" + git add "${BRANCH}/${FILE}" + # Will give error if badge has not changed + git commit -m "Add/Update badge" || true + + - name: Push badge commit + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: badges + directory: badges diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f9a42ed --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,91 @@ +name: Release + +on: + push: + # release on tag push + tags: + - '*' + +jobs: + wheels: + + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + env: + PYTHONIOENCODING: utf-8 + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: [3.7, 3.8, 3.9, '3.10'] + + steps: + - name: Set git crlf/eol + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install tox tox-gh-actions + + - name: Build dist pkgs + run: | + tox -e deploy + + - name: Upload artifacts + if: matrix.python-version == 3.8 && runner.os != 'Windows' + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./dist/*.whl + + create_release: + name: Create Release + needs: [wheels] + runs-on: ubuntu-20.04 + + steps: + - name: Get version + id: get_version + run: | + echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV + echo ${{ env.VERSION }} + + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # download all artifacts to project dir + - uses: actions/download-artifact@v2 + + - name: Generate changes file + uses: sarnold/gitchangelog-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN}} + + - name: Create release + id: create_release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.VERSION }} + name: Release v${{ env.VERSION }} + body_path: CHANGES.md + draft: false + prerelease: false + files: | + wheels/yml*.whl diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..d27bd81 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,53 @@ +name: Wheels + +on: + workflow_dispatch: + pull_request: + push: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + env: + PYTHONIOENCODING: utf-8 + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: [3.7, 3.8, 3.9, '3.10'] + + steps: + - name: Set git crlf/eol + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install tox tox-gh-actions + + - name: Build dist pkgs + run: | + tox -e deploy,check + + - name: Upload artifacts + if: matrix.python-version == 3.8 && runner.os == 'Linux' + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./dist/*.whl diff --git a/.gitignore b/.gitignore index b900fa0..1d901fa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,13 @@ __pycache__/ *.py[cod] *$py.class -# generated test files +# generated/user files +src/ymltoxml/_version.py in.* out.* +munch/ +.*.yaml +*.xml # C extensions *.so diff --git a/README.rst b/README.rst index b871a04..fbc6268 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,8 @@ Some scripts, and eventually a package, to convert real-world XML_ files to YAML_ and back again, preserving attributes and comments (with minor -corrections). +corrections). The default file encoding for both types is UTF-8 without +a Byte-Order Marker. Developer workflow ================== @@ -17,6 +18,10 @@ Both mavlink and pymavlink require a (host) GCC toolchain for full builds, however, the basic workflow to generate the library headers requires only Git, Python, and Tox. +.. _Tox: https://github.com/tox-dev/tox +.. _XML: https://en.wikipedia.org/wiki/Extensible_Markup_Language +.. _YAML: https://en.wikipedia.org/wiki/YAML + In-repo workflow with Tox ------------------------- @@ -28,7 +33,6 @@ package manager, eg:: $ sudo apt-get update $ sudo apt-get install tox -.. _Tox: https://github.com/tox-dev/tox After cloning the repository, you can run the repo checks with the ``tox`` command. It will build a virtual python environment with diff --git a/pyproject.toml b/pyproject.toml index 1c501ae..66552f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ fail_under = 50 show_missing = true [tool.black] -line-length = 85 +line-length = 90 [tool.pycln] all = true @@ -45,4 +45,4 @@ dirty = "{version}+d{build_date:%Y%m%d}" distance-dirty = "{next_version}.dev{distance}" [tool.versioningit.write] -file = "_version.py" +file = "src/ymltoxml/_version.py" diff --git a/requirements-dev.txt b/requirements.txt similarity index 82% rename from requirements-dev.txt rename to requirements.txt index f38c421..1629e3e 100644 --- a/requirements-dev.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ xmltodict +munch ruamel.yaml PyYAML diff --git a/scripts/genyaml.py b/scripts/genyaml.py index 83ae231..3ffe607 100644 --- a/scripts/genyaml.py +++ b/scripts/genyaml.py @@ -1,14 +1,32 @@ from pathlib import Path -import ruamel.yaml import xmltodict +from ruamel.yaml import YAML +from ruamel.yaml.compat import StringIO + + +class StrYAML(YAML): + """ + New API likes dumping straight to file/stdout, so we subclass and + create 'inefficient' custom string dumper. + """ + def dump(self, data, stream=None, **kw): + inefficient = False + if stream is None: + inefficient = True + stream = StringIO() + YAML.dump(self, data, stream, **kw) + if inefficient: + return stream.getvalue() + with open('in.xml', 'r+b') as xfile: payload = xmltodict.parse(xfile, process_comments=True) -yaml = ruamel.yaml.YAML() +yaml = StrYAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.preserve_quotes = True # type: ignore -pfile = Path('out.yaml') -yaml.dump(payload, pfile) +res = yaml.dump(payload) + +Path('out.yaml').write_text(res) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..66c0ea9 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,78 @@ +[metadata] +name = ymltoxml +version = attr: ymltoxml.__version__ +description = attr: ymltoxml.__description__ +url = https://github.com/sarnold/ymltoxml +author = Stephen Arnold +author_email = nerdboy@gentoo.org +long_description = file: README.rst +long_description_content_type = text/rst; charset=UTF-8 +license_expression = LGPL-2.1-or-later +license_files = LICENSE +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + Programming Language :: Python + Environment :: Console + Topic :: Software Development + Topic :: Software Development :: Testing + +[options] +python_requires = >= 3.6 +install_requires = + importlib-metadata; python_version < '3.8' + importlib_resources; python_version < '3.10' + xmltodict + munch + ruamel.yaml + PyYAML + +packages = find_namespace: +package_dir = + =src + +[options.packages.find] +where = src + +[options.package_data] +ymltoxml.data = + *.yaml + +[options.entry_points] +console_scripts = + ymltoxml = ymltoxml.ymltoxml:main + +# extra deps are included here mainly for local/venv installs using pip +# otherwise deps are handled via tox, ci config files or pkg managers +[options.extras_require] +test = + pytest +cov = + pytest-cov + coverage[toml] + coverage_python_version +all = + %(cov)s + %(test)s + +[check] +metadata = true +restructuredtext = true +strict = false + +[check-manifest] +ignore = + .gitattributes + .gitignore + .pre-commit-config.yaml + +[flake8] +exclude = + .git, + __pycache__, + build, + dist, + docs, + tests + +max-line-length = 90 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..bac24a4 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import setuptools + +if __name__ == "__main__": + setuptools.setup() diff --git a/src/ymltoxml/__init__.py b/src/ymltoxml/__init__.py new file mode 100644 index 0000000..dcd8b2d --- /dev/null +++ b/src/ymltoxml/__init__.py @@ -0,0 +1,6 @@ +from ._version import __version__ + +version = __version__ +__description__ = "Console tool for bidirectional transformation of YAML and XML files." + +__all__ = ["__description__", "__version__", "version"] diff --git a/src/ymltoxml/data/ymltoxml.yaml b/src/ymltoxml/data/ymltoxml.yaml new file mode 100644 index 0000000..a6d74ae --- /dev/null +++ b/src/ymltoxml/data/ymltoxml.yaml @@ -0,0 +1,8 @@ +file_encoding: 'utf-8' +process_comments: True +mapping: 2 +sequence: 4 +offset: 2 +short_empty_elements: False +pretty: True +indent: ' ' diff --git a/src/ymltoxml/ymltoxml.py b/src/ymltoxml/ymltoxml.py new file mode 100644 index 0000000..3b95d58 --- /dev/null +++ b/src/ymltoxml/ymltoxml.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python + +# Copyright 2022 Stephen L Arnold +# +# This is free software, licensed under the LGPL-2.1 license +# available in the accompanying LICENSE file. + +""" +Transform YAML to XML and XML to YAML. +""" + +import os +import sys +from pathlib import Path + +try: + from importlib_metadata import version +except ImportError: + from importlib.metadata import version +try: + from importlib_resources import files +except ImportError: + from importlib.resources import files # type: ignore + +import xmltodict +import yaml as yaml_loader +from ruamel.yaml import YAML +from ruamel.yaml.compat import StringIO + +from munch import Munch + + +class FileTypeError(Exception): + """Raise when the file extension is not '.xml', '.yml', or '.yaml'""" + __module__ = Exception.__module__ + + +class StrYAML(YAML): + """ + New API likes dumping straight to file/stdout, so we subclass and + create 'inefficient' custom string dumper. + """ + def dump(self, data, stream=None, **kw): + stream = StringIO() + YAML.dump(self, data, stream, **kw) + return stream.getvalue() + + +def load_config(file_encoding='utf-8'): + """ + Load yaml configuration file and munchify the data. If local file is + not found in current directory, the default will be loaded. + :param str file_encoding: cfg file encoding + :return tuple: Munch cfg obj and cfg file as Path obj + """ + cfgfile = Path('.ymltoxml.yaml') + if not cfgfile.exists(): + cfgfile = files('ymltoxml.data').joinpath('ymltoxml.yaml') + cfgobj = Munch.fromYAML(cfgfile.read_text(encoding=file_encoding)) + + return cfgobj, cfgfile + + +def get_input_type(filepath, prog_opts): + """ + Check filename extension, open and process by file type, return type + flag and data from appropriate loader. + :param Path filepath: filename as Path obj + :return tuple: destination type flag and file data + """ + to_xml = False + data_in = None + + if filepath.name.lower().endswith(('.yml', '.yaml')): + with filepath.open() as infile: + data_in = yaml_loader.load(infile, Loader=yaml_loader.Loader) + to_xml = True + elif filepath.name.lower().endswith('.xml'): + with filepath.open('r+b') as infile: + data_in = xmltodict.parse(infile, + process_comments=prog_opts['process_comments']) + else: + raise FileTypeError("FileTypeError: unknown input file extension") + return to_xml, data_in + + +def restore_xml_comments(xmls): + """ + Turn tagged comment elements back into xml comments. + :param str xmls: xml (file) output from ``unparse`` + :return str xmls: processed xml string + """ + for rep in (("<#comment>", "")): + xmls = xmls.replace(*rep) + return xmls + + +def transform_data(payload, prog_opts, to_xml=True): + """ + Produce output data from dict-ish object using ``direction``. + :param payload: input from xmltodict or yaml loader. + :param dict prog_opts: configuration options + :param bool to_xml: output direction, ie, if to_xml is True then output + data is XML format. + :return str res: output file data in specified format. + """ + res = '' + if to_xml: + xml = xmltodict.unparse(payload, + short_empty_elements=prog_opts['short_empty_elements'], + pretty=prog_opts['pretty'], + indent=prog_opts['indent']) + + if prog_opts['process_comments']: + res = restore_xml_comments(xml) + + else: + yaml = StrYAML() + yaml.indent(mapping=prog_opts['mapping'], + sequence=prog_opts['sequence'], + offset=prog_opts['offset']) + + yaml.preserve_quotes = True # type: ignore + res = yaml.dump(payload) + + return res + + +def main(argv=None): + """ + Transform mavlink-style xml files to/from xml and yaml. Note yaml format uses + custom markup for attributes and comments. See xmltodict docs for details. + """ + + debug = False + + if os.getenv('VERBOSE') and os.getenv('VERBOSE') == '1': + debug = True + + cfg, pfile = load_config() + popts = Munch.toDict(cfg) + + if argv is None: + argv = sys.argv + args = argv[1:] + + if args == ['--version']: + print(f'[ymltoxml {VERSION}]') + sys.exit(0) + elif args == ['--dump-config']: + sys.stdout.write(pfile.read_text(encoding=popts['file_encoding'])) + sys.exit(0) + + for filearg in args: + fpath = Path(filearg) + if not fpath.exists(): + print(f'Input file {fpath} not found! Skipping...') + else: + if debug: + print(f'Processing data from {filearg}') + + try: + from_yml, indata = get_input_type(fpath, popts) + except FileTypeError as exc: + print(f'{exc} => {fpath}') + break + + outdata = transform_data(indata, popts, to_xml=from_yml) + + if from_yml: + fpath.with_suffix('.xml').write_text(outdata + '\n', + encoding=popts['file_encoding']) + else: + fpath.with_suffix('.yaml').write_text(outdata, + encoding=popts['file_encoding']) + + +VERSION = version("ymltoxml") + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index d69434e..ffe6915 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py3{6,7,8,9,10}-dev +envlist = py3{6,7,8,9,10}-{linux,macos,windows} skip_missing_interpreters = true isolated_build = true skipsdist = true @@ -22,11 +22,19 @@ deps = pip>=21.1 versioningit +[build] +deps = + pip>=21.1 + build + twine + [testenv] skip_install = true setenv = - MDEF = {envsitepackagesdir}/pymavlink/message_definitions/v1.0 + PYTHONPATH = {toxinidir}/src + DISABLE_MAVNATIVE = True + MDEF_PATH = {envsitepackagesdir}/pymavlink/message_definitions/v1.0 passenv = CC @@ -35,7 +43,6 @@ passenv = AR NM RANLIB - PYTHON DISPLAY XAUTHORITY HOME @@ -43,37 +50,80 @@ passenv = USER XDG_* CI - PYTHON + OS PYTHONIOENCODING - GITHUB* PIP_DOWNLOAD_CACHE -whitelist_externals = +allowlist_externals = bash deps = {[base]deps} + #-r requirements.txt pymavlink - xmltodict - PyYAML - ruamel.yaml + . commands = - bash -c 'cp $MDEF/paparazzi.xml in.xml' - python scripts/genyaml.py - bash -c 'cp out.yaml in.yaml' - python scripts/genxml.py + bash -c 'cp $MDEF_PATH/{posargs:paparazzi}.xml in.xml' + ymltoxml in.xml + bash -c 'cp in.yaml out.yaml' + ymltoxml out.yaml bash -c 'diff -u in.xml out.xml || true' + ymltoxml --version + ymltoxml --dump-config + ymltoxml out.txt [testenv:clean] -whitelist_externals = +allowlist_externals = bash deps = pip>=21.1 commands = - bash -c 'rm -f in.* out.* paparazzi.xml' + bash -c 'rm -rf in.* out.* paparazzi.xml munch/' + +[testenv:setup] +passenv = + CI + PYTHONIOENCODING + +setenv = PYTHONPATH = {toxinidir} + +deps = + {[base]deps} + #-r requirements.txt + +commands = + python setup.py egg_info + +[testenv:deploy] +skip_install = true + +passenv = + pythonLocation + CI + PYTHONIOENCODING + PIP_DOWNLOAD_CACHE + +deps = + {[build]deps} + +commands = + python -m build . + twine check dist/* + +[testenv:check] +skip_install = true +passenv = CI + +deps = + #{[base]deps} + pip>=21.1 + +commands = + pip install ymltoxml --force-reinstall --pre --prefer-binary -f dist/ + ymltoxml --version [testenv:lint] passenv = @@ -85,10 +135,10 @@ setenv = PYTHONPATH = {toxinidir} deps = {[base]deps} pylint - -r requirements-dev.txt + -r requirements.txt commands = - pylint --fail-under=7 scripts/ + pylint --fail-under=9.90 src/ymltoxml/ymltoxml.py [testenv:style] passenv = @@ -102,7 +152,7 @@ deps = flake8-bugbear commands = - flake8 scripts/ + flake8 scripts/ src/ [testenv:mypy] skip_install = true @@ -112,10 +162,12 @@ setenv = PYTHONPATH = {toxinidir} deps = {[base]deps} mypy - -r requirements-dev.txt + importlib_resources + -r requirements.txt commands = - python -m mypy --follow-imports=normal --install-types --non-interactive scripts/ + stubgen -m munch --export-less -o {toxinidir} + python -m mypy --follow-imports=normal --install-types --non-interactive src/ [testenv:isort] skip_install = true @@ -125,7 +177,7 @@ setenv = PYTHONPATH = {toxinidir} deps = {[base]deps} isort - -r requirements-dev.txt + -r requirements.txt commands = - python -m isort scripts/ + python -m isort scripts/ src/