From 1bb12f8c225b334fbed92dc771301e5979accb60 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 21 Jun 2022 13:34:21 -0700 Subject: [PATCH 1/2] new: dev: add pre-commit and pep8speaks configs, apply some cleanup Signed-off-by: Stephen L Arnold --- .gitignore | 3 +- .pep8speaks.yml | 15 +++++ .pre-commit-config.yaml | 130 +++++++++++++++++++++++++++++++++++++++ README.rst | 65 ++++++++++++++++++-- pyproject.toml | 17 +++++ requirements.txt | 4 +- scripts/genxml.py | 1 + scripts/genyaml.py | 1 + src/ymltoxml/__init__.py | 1 + src/ymltoxml/ymltoxml.py | 41 +++++++----- 10 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 .pep8speaks.yml create mode 100644 .pre-commit-config.yaml diff --git a/.gitignore b/.gitignore index 2190c4c..5adfc06 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,10 @@ __pycache__/ # generated/user files src/ymltoxml/_version.py +.ymltoxml.yaml in.* out.* munch/ -.*.yaml -*.xml # C extensions *.so diff --git a/.pep8speaks.yml b/.pep8speaks.yml new file mode 100644 index 0000000..ad5d8b9 --- /dev/null +++ b/.pep8speaks.yml @@ -0,0 +1,15 @@ +scanner: + diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned. + linter: flake8 # Other option is pycodestyle + +no_blank_comment: False # If True, no comment is made on PR without any errors. +descending_issues_order: True # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file + +pycodestyle: # Same as scanner.linter value. Other option is flake8 + max-line-length: 110 # Default is 79 in PEP 8 + +flake8: + max-line-length: 90 # Default is 79 in PEP 8 + exclude: + - tests + - docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..aa2b5af --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,130 @@ +# To install the git pre-commit hook run: +# pre-commit install +# To update the pre-commit hooks run: +# pre-commit install-hooks +exclude: '^(.tox/|docs/|^setup.py$)' +repos: + - repo: meta + hooks: + - id: check-useless-excludes + - id: check-hooks-apply + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + # - id: check-symlinks + - id: debug-statements + - id: requirements-txt-fixer + - id: fix-encoding-pragma + - id: trailing-whitespace + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: check-toml + - id: check-yaml + exclude: '(conda/meta.yaml|.pep8speaks.yml)' + + # use ffffff (black fork) for single quote normalization + # (otherwise switch to black for double quotes) + - repo: https://github.com/grktsh/ffffff + rev: v2020.8.31 + hooks: + - id: ffffff + name: "Format code (ffffff)" + files: src/ymltoxml/ymltoxml.py + language_version: python3 + + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + args: [--settings-path=pyproject.toml] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.961 + hooks: + - id: mypy + additional_dependencies: + - importlib_metadata + - importlib_resources + - munch + - ruamel.yaml + args: + - --follow-imports=normal + - --install-types + - --non-interactive + - --ignore-missing-imports + files: src/ymltoxml/ymltoxml.py + + - repo: "https://github.com/asottile/blacken-docs" + rev: "v1.12.1" + hooks: + - id: "blacken-docs" + name: "Format docs (blacken-docs)" + args: ["-l", "64"] + additional_dependencies: + - "black==21.9b0" + + - repo: https://github.com/PyCQA/doc8 + rev: 0.11.2 + hooks: + - id: doc8 + args: + - '--max-line-length=90' + - '--ignore=D001' + + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: rst-backticks + # exclude: ChangeLog\.rst$ + - id: rst-directive-colons + - id: rst-inline-touching-normal + + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + files: src/ymltoxml/ymltoxml.py + args: + - --in-place + - --remove-all-unused-imports + - --remove-duplicate-keys + - --remove-unused-variables + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + files: src/ymltoxml/ymltoxml.py + additional_dependencies: ["flake8-bugbear"] + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + args: ["-ll", "-q"] + files: src/ymltoxml/ymltoxml.py + +# - repo: https://github.com/lovesegfault/beautysh +# rev: v6.2.1 +# hooks: +# - id: beautysh + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: false + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false + +# re-running a pull request: you can trigger a re-run on a pull request by +# commenting pre-commit.ci run (must appear on a line by itself). +# skipping push runs: skip a run by putting [skip ci], [ci skip], +# [skip pre-commit.ci], or [pre-commit.ci skip] in the commit message. diff --git a/README.rst b/README.rst index bac13b0..f9b1bb0 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,10 @@ -========================================= - ymltoxml (and xmltoyml) for mavlink xml -========================================= +========================= + ymltoxml (and xmltoyml) +========================= |ci| |wheels| |release| |badge| -|pylint| +|pre| |pylint| |tag| |license| |python| @@ -78,6 +78,59 @@ To build/lint the api docs, use the following tox commands: * ``tox -e docs`` build the documentation using sphinx and the api-doc plugin * ``tox -e docs-lint`` build the docs and run the sphinx link checking +Pre-commit +---------- + +This repo is now pre-commit_ enabled for python/rst source and file-type +linting. The checks run automatically on commit and will fail the commit +(if not clean) and perform simple file corrections. For example, if the +mypy check fails on commit, you must first fix any fatal errors for the +commit to succeed. That said, pre-commit does nothing if you don't install +it first (both the program itself and the hooks in your local repository +copy). + +You will need to install pre-commit before contributing any changes; +installing it using your system's package manager is recommended, +otherwise install with pip into your usual virtual environment using +something like:: + + $ sudo emerge pre-commit --or-- + $ pip install pre-commit + +then install it into the repo you just cloned:: + + $ git clone https://github.com/sarnold/ymltoxml + $ cd ymltoxml/ + $ pre-commit install + +It's usually a good idea to update the hooks to the latest version:: + + $ pre-commit autoupdate + +Most (but not all) of the pre-commit checks will make corrections for you, +however, some will only report errors, so these you will need to correct +manually. + +Automatic-fix checks include ffffff, isort, autoflake, the yaml/xml format +checks, and the miscellaneous file fixers. If any of these fail, you can +review the changes with ``git diff`` and just add them to your commit and +continue. + +If any of the mypy, bandit, or rst source checks fail, you will get a report, +and you must fix any errors before you can continue adding/committing. + +To see a "replay" of any ``rst`` check errors, run:: + + $ pre-commit run rst-backticks -a + $ pre-commit run rst-directive-colons -a + $ pre-commit run rst-inline-touching-normal -a + +To run all ``pre-commit`` checks manually, try:: + + $ pre-commit run -a + +.. _pre-commit: https://pre-commit.com/index.html + .. |ci| image:: https://github.com/sarnold/ymltoxml/actions/workflows/ci.yml/badge.svg :target: https://github.com/sarnold/ymltoxml/actions/workflows/ci.yml @@ -110,3 +163,7 @@ To build/lint the api docs, use the following tox commands: .. |python| image:: https://img.shields.io/badge/python-3.6+-blue.svg :target: https://www.python.org/downloads/ :alt: Python + +.. |pre| image:: https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white + :target: https://github.com/pre-commit/pre-commit + :alt: pre-commit diff --git a/pyproject.toml b/pyproject.toml index 66552f3..bbab6f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,23 @@ show_missing = true [tool.black] line-length = 90 +skip-string-normalization = true +include = '\.py$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | docs + | dist + | tests +)/ +''' [tool.pycln] all = true diff --git a/requirements.txt b/requirements.txt index 1629e3e..bbbed12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -xmltodict munch -ruamel.yaml PyYAML +ruamel.yaml +xmltodict diff --git a/scripts/genxml.py b/scripts/genxml.py index 0a27b71..2d9e2f9 100644 --- a/scripts/genxml.py +++ b/scripts/genxml.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from pathlib import Path import xmltodict diff --git a/scripts/genyaml.py b/scripts/genyaml.py index 3ffe607..00972e6 100644 --- a/scripts/genyaml.py +++ b/scripts/genyaml.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from pathlib import Path import xmltodict diff --git a/src/ymltoxml/__init__.py b/src/ymltoxml/__init__.py index dcd8b2d..ebb7bd0 100644 --- a/src/ymltoxml/__init__.py +++ b/src/ymltoxml/__init__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from ._version import __version__ version = __version__ diff --git a/src/ymltoxml/ymltoxml.py b/src/ymltoxml/ymltoxml.py index 6893306..427622a 100644 --- a/src/ymltoxml/ymltoxml.py +++ b/src/ymltoxml/ymltoxml.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Copyright 2022 Stephen L Arnold # @@ -26,14 +27,14 @@ import xmltodict import yaml as yaml_loader +from munch import Munch from ruamel.yaml import YAML from ruamel.yaml.compat import StringIO -from munch import Munch - class FileTypeError(Exception): """Raise when the file extension is not '.xml', '.yml', or '.yaml'""" + __module__ = Exception.__module__ @@ -42,6 +43,7 @@ class StrYAML(YAML): New API likes dumping straight to file/stdout, so we subclass and create 'inefficient' custom string dumper. """ + def dump(self, data, stream=None, **kw): stream = StringIO() YAML.dump(self, data, stream, **kw) @@ -82,12 +84,13 @@ def get_input_type(filepath, prog_opts): if filepath.name.lower().endswith(('.yml', '.yaml')): with filepath.open() as infile: - data_in = yaml_loader.load(infile, Loader=yaml_loader.Loader) + data_in = yaml_loader.safe_load(infile) to_xml = True elif filepath.name.lower().endswith('.xml'): with filepath.open('r+b') as infile: - data_in = xmltodict.parse(infile, - process_comments=prog_opts['process_comments']) + data_in = xmltodict.parse( + infile, process_comments=prog_opts['process_comments'] + ) else: raise FileTypeError("FileTypeError: unknown input file extension") return to_xml, data_in @@ -120,19 +123,23 @@ def transform_data(payload, prog_opts, to_xml=True): """ res = '' if to_xml: - xml = xmltodict.unparse(payload, - short_empty_elements=prog_opts['short_empty_elements'], - pretty=prog_opts['pretty'], - indent=prog_opts['indent']) + xml = xmltodict.unparse( + payload, + short_empty_elements=prog_opts['short_empty_elements'], + pretty=prog_opts['pretty'], + indent=prog_opts['indent'], + ) if prog_opts['process_comments']: res = restore_xml_comments(xml) else: yaml = StrYAML() - yaml.indent(mapping=prog_opts['mapping'], - sequence=prog_opts['sequence'], - offset=prog_opts['offset']) + yaml.indent( + mapping=prog_opts['mapping'], + sequence=prog_opts['sequence'], + offset=prog_opts['offset'], + ) yaml.preserve_quotes = True # type: ignore res = yaml.dump(payload) @@ -191,11 +198,13 @@ def main(argv=None): outdata = transform_data(indata, popts, to_xml=from_yml) if from_yml: - fpath.with_suffix('.xml').write_text(outdata + '\n', - encoding=popts['file_encoding']) + fpath.with_suffix('.xml').write_text( + outdata + '\n', encoding=popts['file_encoding'] + ) else: - fpath.with_suffix('.yaml').write_text(outdata, - encoding=popts['file_encoding']) + fpath.with_suffix('.yaml').write_text( + outdata, encoding=popts['file_encoding'] + ) VERSION = version("ymltoxml") From 07e6d7a3cd77c509545e1a9ce193561d99a55238 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Wed, 22 Jun 2022 09:31:14 -0700 Subject: [PATCH 2/2] chg: doc: update readme and usage output Signed-off-by: Stephen L Arnold --- README.rst | 82 +++++++++++++++++++++++++++++++++++++++- src/ymltoxml/ymltoxml.py | 2 + 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f9b1bb0..d6f198c 100644 --- a/README.rst +++ b/README.rst @@ -13,12 +13,90 @@ preserving attributes and comments (with minor corrections). The default file encoding for both types is UTF-8 without a BOM. The main intent is to support YAML-based development of custom mavlink_ dialects. +Quick Start +=========== + +Install with pip +---------------- + +This package is *not* yet published on PyPI, thus use one of the +following to install the latest ymltoxml on any platform:: + + $ pip install -U -f https://github.com/sarnold/ymltoxml/releases/ ymltoxml + +or use this command to install a specific version:: + + $ pip install git+https://github.com/sarnold/ymltoxml.git@0.1.0 + +The full package provides the ``ymltoxml.py`` executable as well as +a reference configuration file that provides defaults for all values. + +If you'd rather work from the source repository, it supports the common +idiom to install it on your system in a virtual env after cloning:: + + $ python3 -m venv env + $ source env/bin/activate + $ pip install . + $ ymltoxml --version + $ ymltoxml --dump-config + $ deactivate + +The alternative to python venv is the ``tox`` test driver. If you have it +installed already, see the example tox commands below. + +Usage +----- + +The current version reads very minimal command options, and the only +required command arguments are one or more files of a single type:: + + $ ymltoxml + Transform YAML to XML and XML to YAML. + + Usage: + ymltoxml file1.yaml file2.yaml ... + ymltoxml file1.xml file2.xml ... + + Each output file is named for the corresponding input file using + the output extension (more options coming soon). + +The main processing tweaks for yml/xml output formatting are specified +in the default configuration file; if you need to change something, you +can use your own config file in the working directory; note the local +copy must be named ``.ymltoxml.yaml``. To get a copy of the default +configuration file, do:: + + $ cd path/to/work/dir/ + $ ymltoxml --dump-config > .ymltoxml.yaml + $ $EDITOR .ymltoxml.yaml + + +Features and limitations +------------------------ + +We only test on mavlink XML message definitions, so it probably *will not* +work at all on arbitrarily complex XML files with namespaces, etc. The +current round-trip is not exact, due to the following: + +* missing encoding is added to version tag +* leading/trailing whitespace in text elements and comments is not preserved +* elements with self-closing tags are converted to full closing tags +* empty elements on more than one line are not preserved + +For the files tested (eg, mavlink) the end result is cleaner/shinier XML. + Local workflow =============== -Tool requirements: +This tool is intended to be part of larger workflow, ie, developing a +custom mavlink message dialect and generating/deploying the resulting +mavlink language interfaces. To be more specific, for this example we +use a mavlink-compatible component running on a micro-controller, thus +the target language bindings are C and C++. + +Tool requirements for the full mavlink workflow: -* initially just recent Python and Tox_ +* initially just recent pymavlink, Python, and Tox_ Both mavlink and pymavlink require a (host) GCC toolchain for full builds, however, the basic workflow to generate mavlink library headers requires diff --git a/src/ymltoxml/ymltoxml.py b/src/ymltoxml/ymltoxml.py index 427622a..cd40df4 100644 --- a/src/ymltoxml/ymltoxml.py +++ b/src/ymltoxml/ymltoxml.py @@ -157,6 +157,8 @@ def main(argv=None): Each output file is named for the corresponding input file using the output extension (more options coming soon). + Create the config with: + ymltoxml --dump-config > .ymltoxml.yaml """ debug = False