diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index b6a5faa6f..6329ff04a 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.10" ] + python-version: [ "3.7" ] steps: - name: Checkout code uses: actions/checkout@v2 @@ -34,10 +34,19 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install and test panoptes-utils on ${{ matrix.python-version }} + - name: Install dependencies + run: | + # TODO don't get all the indexes. + sudo apt-get install -y exiftool fonts-freefont-ttf libcfitsio-bin astrometry.net astrometry-data-tycho2-10-19 + - name: Download CR2 file + run: | + curl https://storage.googleapis.com/panoptes-resources/test-data/canon.cr2 --output ./tests/data/canon.cr2 + - name: Install panoptes-utils on ${{ matrix.python-version }} run: | pip install -e ".[config,images,testing,social]" - pytest + - name: Test panoptes-utils on ${{ matrix.python-version }} + run: | + pytest --test-solve --test-databases=all - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 if: success() diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cf31fe6e8..d7b4e5a51 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,16 +9,23 @@ version: 2 sphinx: configuration: docs/conf.py -formats: htmlzip +formats: + - htmlzip + +build: + os: ubuntu-22.04 + tools: + python: "3.9" python: - version: 3.7 install: + - requirements: docs/requirements.txt - method: pip path: . - extras_requirements: + extra_requirements: - config - docs - images - social + - testing system_packages: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b5ebac0d6..1bfe7f7a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,32 @@ Changelog ========= +0.2.37 - 2022-08-09 +------------------- + +Added +^^^^^ + +* The ``panoptes-utils image watch `` command with default processing that will convert ``CR2`` files to ``JPG`` and ``FITS`` and then plate-solve the ``FITS`` files. +* GHA downloads a ``CR2`` file for testing. +* Plot directives for documentation. + +Changed +^^^^^^^ + +* Testing now includes ``--test-solve`` for plate-solving in GHA. +* Local tests only uses ``memory`` database. +* Rearranged some functions in the ``panoptes.utils.images`` namespace. + +Removed +^^^^^^^ + +* Unused stamp plotting functions. +* Testing of config servers on GHA. +* `CountdownTimer.is_non_blocking` predicate that wasn't being used. +* Extra serial protocol handlers. + + 0.2.36 ------ diff --git a/README.md b/README.md index f2efead84..bc8fd89e3 100644 --- a/README.md +++ b/README.md @@ -25,22 +25,11 @@ pip install panoptes-utils Full options for install: ```bash -pip install -e ".[config,docs,images,testing,social]" +pip install "panoptes-utils[config,docs,images,testing,social]" ``` See the full documentation at: https://panoptes-utils.readthedocs.io -Config Server -------------- - -There is a simple key-value configuration server available as part of the module. - -After installing with the `config` option as above, type: - -```bash -panoptes-config-server run --config-file -``` - Dependencies ------------ @@ -48,7 +37,7 @@ There are a few system dependencies depending on what functionality you will be In particular, the plate solving requires `astrometry.net` and the appropriate index files. -Use the following on a debian-based system (e.g. Ubuntu) to install all dependencies: +Use the following on a debian-based system (e.g. Ubuntu) to easily install all dependencies: ```bash apt-get update && apt-get install --no-install-recommends --yes \ @@ -58,6 +47,31 @@ apt-get update && apt-get install --no-install-recommends --yes \ libfreetype6-dev libpng-dev libjpeg-dev libffi-dev ``` +Command Line +------------ + +The `panoptes-utils` command line tool is available for use with subcommands +corresponding to the modules in this library. Currently, the only implemented +subcommand is `image`, which includes commands for converting `cr2` files into +`jpg` and/or `fits` files as well as for plate-solving `fits` images. + +The `panoptes-utils image watch ` command will watch the given path for +new files and convert them to `jpg` and/or `fits` files as they are added. + +See `panoptes-utils --help` and `panoptes-utils image --help` for details. + + +Config Server +------------- + +There is a simple key-value configuration server available as part of the module. + +After installing with the `config` option as above, type: + +```bash +panoptes-config-server run --config-file +``` + Developing ---------- diff --git a/bin/cr2-to-jpg b/bin/cr2-to-jpg deleted file mode 100755 index 245f77173..000000000 --- a/bin/cr2-to-jpg +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -e - -usage() { - echo -n "################################################## -# Make a jpeg from the Canon Raw v2 (.CR2) file. -# -# If exiftool is present this merely extracts the thumbnail from -# the CR2 file, otherwise use dcraw to create a jpeg. -# -# If present the TITLE is added as a title to the jpeg. -################################################## - $ $(basename $0) FILENAME [TITLE] - - Options: - FILENAME Name of CR2 file that holds jpeg. - TITLE Optional title to be placed on jpeg. - - Example: - cr2-to-jpg /var/panoptes/images/temp.cr2 \"M42 (Orion's Nebula)\" -" -} - -if [[ $# -eq 0 ]]; then - usage - exit 1 -fi - -FNAME=$1 -TITLE="${2}" - -JPG="${FNAME%.cr2}.jpg" - -echo "Converting CR2 to ${JPG}." - -function command_exists() { - # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 - # this should be a very portable way of checking if something is on the path - # usage: "if command_exists foo; then echo it exists; fi" - type "$1" &>/dev/null -} - -# Use exiftool to extract preview if it exists -if command_exists exiftool; then - echo "Using exiftool to extract JPG." - exiftool -b -PreviewImage "${FNAME}" >"${JPG}" -else - if command_exists dcraw; then - # Convert CR2 to JPG - echo "Using dcraw to convert to JPG." - dcraw -c -q 3 -a -w -H 5 -b 5 "${FNAME}" | cjpeg -quality 90 >"${JPG}" - else - echo "Can't find either exiftool or dcraw, cannot proceed" - exit 1 - fi -fi - -# Test for file -if [[ ! -s "${JPG}" ]]; then - echo "JPG was not extracted successfully." - exit 1 -fi - -if [[ -n "$TITLE" ]]; then - if command_exists convert; then - echo "Adding title \"${TITLE}\"" - # Make thumbnail from jpg. - convert "${JPG}" -background black -fill red \ - -font 'Lato-Regular' -pointsize 60 label:"${TITLE}" \ - -gravity South -append "${JPG}" - fi -fi - -echo "${JPG}" diff --git a/conftest.py b/conftest.py index 20fcc4e03..12cf8afa4 100644 --- a/conftest.py +++ b/conftest.py @@ -176,12 +176,12 @@ def add_doctest_dependencies(doctest_namespace): @pytest.fixture def caplog(_caplog): - class PropogateHandler(logging.Handler): + class PropagateHandler(logging.Handler): def emit(self, record): logging.getLogger(record.name).handle(record) logger.enable('panoptes') - handler_id = logger.add(PropogateHandler(), format="{message}") + handler_id = logger.add(PropagateHandler(), format="{message}") yield _caplog with suppress(ValueError): logger.remove(handler_id) diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 000000000..1b50bae7b --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,111 @@ +.. _cli: + +================== +Command Line Utils +================== + +``panoptes-utils`` provides a command line interface for some of the common functions +in the module. + +Commands +-------- + +The main command is called ``panoptes-utils`` and includes subcommands for specific +tasks. The subcommands are available via the help menu: + +.. code-block:: bash + + $ panoptes-utils --help + Usage: panoptes-utils [options] [] + Options: + --help Show this message and exit. + Commands: + image Process an image. + +image +===== + +The ``image`` subcommand provides access to image conversion and plate-solving as +well as a generic tool for watching a directory and performing any of the other +image subcommands. + +.. code-block:: bash + + $ panoptes-utils image --help + Usage: panoptes-utils image [OPTIONS] COMMAND [ARGS]... + + Process an image. + + Options: + --help Show this message and exit. + + Commands: + cr2 + fits + watch Watch a directory for changes and process any new files. + +image watch +~~~~~~~~~~~ + +A tool for watching a directory and performing subcommands on all incoming files. +This command will block until cancelled by the user via ``Ctrl-c``. + +.. code-block:: bash + + Usage: panoptes-utils image watch [OPTIONS] PATH + + Watch a directory for changes and process any new files. + + The files will be processed according to the boolean flags, with the flag + names corresponding to other image commands. + + By default, all the flags are enabled, which will: + + * Extract JPG files from a CR2. + * Convert CR2 files to FITS. + * Plate-solve FITS files. + + Arguments: + PATH [required] + + Options: + --to-jpg / --no-to-jpg [default: to-jpg] + --to-fits / --no-to-fits [default: to-fits] + --solve / --no-solve [default: solve] + --overwrite / --no-overwrite [default: no-overwrite] + --remove-cr2 / --no-remove-cr2 [default: no-remove-cr2] + --help Show this message and exit. + + +image cr2 +~~~~~~~~~ + +Canon ``CR2`` can have a JPG extracted and be converted to FITS files. See the ``--help`` +command for each of the specific subcommands for more details. + +.. code-block:: bash + + Usage: panoptes-utils image cr2 [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Commands: + to-fits Convert a CR2 image to a FITS, return the new path name. + to-jpg Extract a JPG image from a CR2, return the new path name. + + +image fits +~~~~~~~~~~ + +FITS files can be easily plate-solved. + +.. code-block:: bash + + Usage: panoptes-utils image fits [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Commands: + solve Plate-solve a FITS file. diff --git a/docs/conf.py b/docs/conf.py index 8b9c35946..a6d3ccde2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,6 +62,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'matplotlib.sphinxext.plot_directive', "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.todo", @@ -72,13 +73,14 @@ "sphinx.ext.ifconfig", "sphinx.ext.mathjax", "sphinx.ext.napoleon", + "myst_parser" ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = ".rst" +source_suffix = [".rst", ".md"] # The encoding of source files. # source_encoding = 'utf-8-sig' @@ -137,7 +139,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" +pygments_style = "friendly" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -152,15 +154,15 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "alabaster" +html_theme = "piccolo_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { - "sidebar_width": "300px", - "page_width": "1200px" -} +# html_theme_options = { +# "sidebar_width": "300px", +# "page_width": "1200px" +# } # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index e582053ea..000000000 --- a/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/docs/docker.rst b/docs/docker.rst deleted file mode 100644 index 678d0b023..000000000 --- a/docs/docker.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _docker: - -====== -Docker -====== - -The PANOPTES utilities are available as a docker image that can be built -locally for testing purposes. We also use containers based off -``latest`` in the Google Cloud Registry (GCR): - -Image name: ``panoptes-utils`` - -Tags: ``latest`` and ``develop``. - -Tags -~~~~ - -The ``panoptes-utils`` image comes in two separate flavors, or tags, -that serve different purposes. - -latest -^^^^^^ - -The ``latest`` image is typically used to run services or to serve as a -foundational layer for other docker images. It includes all the tools -required to run the various functions with the ``panoptes-utils`` -module, including a plate-solver (astrometry.net), ``sextractor``, etc. - -The ``latest`` image is also used as a base image for the -`POCS `__ images. - -develop -^^^^^^^ - -The ``develop`` image is used for running the automated tests against -the ``develop`` branch. These are run automatically on both GitHub and -Travis for all code pushes but can also be run locally while doing -development. - -Building -~~~~~~~~ - -To build the test image: - -.. code:: bash - - docker/setup-local-environment.sh - -Running -~~~~~~~ - -To run the test suite locally: - -.. code:: bash - - scripts/testing/test-software.sh - diff --git a/docs/index.rst b/docs/index.rst index a0a71698e..04254a326 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,144 +2,22 @@ PANOPTES Utilities ================== -|PyPI version| |Build Status| |codecov| |Documentation Status| +.. include:: ../README.md + :parser: myst_parser.sphinx_ -Utility functions for use within the PANOPTES ecosystem and for general astronomical processing. - -This library defines a number of modules that contain useful functions as well as a few services. - -Getting -======= - -See :ref:`docker` for ways to run ``panoptes-utils`` services without installing to your host computer. - -pip ---- - -To install type: - -.. code:: bash - - pip install panoptes-utils - -Docker ------- - -Docker containers are available for running the ``panoptes-utils`` -module and associated services, which also serve as the base container -for all other PANOPTES related containers. - -See the :ref:`docker` page for details. - -Using -===== - -Modules -------- - -The modules can be used as helper utilities anywhere you would like. - -Services --------- - -The services can be run either from a :ref:`docker` image or from the -installed script, as described below. - -Config Server -~~~~~~~~~~~~~ - -A simple config param server. Runs as a Flask microservice that delivers -JSON documents in response to requests for config key items. - -Can be run from the installed script (defaults to ``http://localhost:6563/get-config``): - -.. code:: - bash - - $ panoptes-config-server -h - usage: panoptes-config-server [-h] [--host HOST] [--port PORT] [--public] [--config-file CONFIG_FILE] [--no-save] [--ignore-local] [--debug] - - Start the config server for PANOPTES - - optional arguments: - -h, --help show this help message and exit - --host HOST Host name, defaults to local interface. - --port PORT Local port, default 6563 - --public If server should be public, default False. Note: inside a docker container set this to True to expose to host. - --config-file CONFIG_FILE - Config file, default $PANDIR/conf_files/pocs.yaml - --no-save Prevent auto saving of any new values. - --ignore-local Ignore the local config files, default False. Mostly for testing. - --debug Debug - - -Or inside a python process: - -.. code:: - python - - >>> from panoptes.utils.config.server import config_server - >>> from panoptes.utils.config import client - - >>> server_process=config_server() - - >>> client.get_config('location.horizon') - 30.0 - - >>> server_process.terminate() # Or just exit notebook/console - -For more details and usage examples, see the :ref:`config-server`. - -Development -=========== - -Environment ------------ - -Most users of ``panoptes-utils`` who need the full environment will also -want the fulle `POCS Environment`_. - -Logging -------- - -The ``panoptes-utils`` module uses `loguru`_ for logging, which also -serves as the basis for the POCS logger (see `Logger`_). - -To access the logs for the module, you can import directly from the -``logger`` module, i.e., ``from panoptes.utils.logger import logger``. -This is a simple wrapper around ``luguru`` with no extra configuration: - -.. code-block:: - python - - >>> from panoptes.utils import CountdownTimer - >>> # No logs by default - >>> t0 = CountdownTimer(5) - >>> t0.sleep() - False - - >>> # Enable the logs - >>> from panoptes.utils.logger import logger - >>> logger.enable('panoptes') - - >>> t1 = CountdownTimer(5) - 2020-03-04 06:42:50 | DEBUG | panoptes.utils.time:restart:162 - Restarting Timer (blocking) 5.00/5.00 - >>> t1.sleep() - 2020-03-04 06:42:53 | DEBUG | panoptes.utils.time:sleep:183 - Sleeping for 2.43 seconds - False Contents ======== .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + CLI Config Server - Docker - License - Authors - Changelog Module Reference + Changelog + Authors + License Indices and tables ================== diff --git a/docs/requirements.txt b/docs/requirements.txt index 483a4e960..f4db34ab9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,7 @@ -sphinx_rtd_theme +docutils>=0.17 +# jinja2==3.0.0 +mkdocs==1.2.3 +myst-parser +piccolo-theme +pytest_mpl +# sphinx==4.2.0 diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 1828e4579..000000000 --- a/environment.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: panoptes-utils -channels: - - conda-forge -dependencies: - - astroplan - - astropy - - flask - - libffi - - loguru - - matplotlib-base - - numpy>=1.19 - - pandas - - photutils - - pillow>=9.1.1 - - pip - - pyserial - - pytest - - python>=3.7 - - readline - - scipy - - seaborn - - streamz - - tox - - tweepy - - typer - - pip: - - -e . - - scalpl diff --git a/setup.cfg b/setup.cfg index ac55f0fc3..a54a97f11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,6 @@ include_package_data = True package_dir = =src scripts = - bin/cr2-to-jpg # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD! # Add here dependencies of your project (semicolon/line-separated), e.g. @@ -51,7 +50,7 @@ install_requires = python-dateutil requests ruamel.yaml - typer + typer[all] # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov # Require a specific Python version, e.g. Python 2.7 or >= 3.4 @@ -72,7 +71,10 @@ config = requests scalpl docs = - sphinx_rtd_theme + myst-parser + piccolo-theme + pytest_mpl + sphinx examples = matplotlib pandas @@ -83,6 +85,7 @@ images = photutils pillow>=9.1.1 scipy + watchfiles testing = coverage pycodestyle @@ -90,6 +93,7 @@ testing = pytest-cov pytest-doctestplus pytest-remotedata>=0.3.1 + pytest_mpl python-dotenv social = requests @@ -113,10 +117,13 @@ addopts = --cov-report xml:build/coverage.xml --strict-markers --doctest-modules - --test-databases all + --test-databases memory --strict-markers -vv -ra + --ignore=tests/config/test_config_cli.py + --ignore=tests/config/test_config_server.py + --ignore=src/panoptes/utils/config norecursedirs = dist build @@ -128,7 +135,7 @@ filterwarnings = ignore::pytest.PytestDeprecationWarning doctest_plus = enabled markers = - astrometry: tests that require astrometry.net (i.e. solve-field). + plate_solve: tests that require astrometry.net (i.e. solve-field). slow: marks tests as slow (deselect with '-m "not slow"'). [aliases] diff --git a/src/panoptes/utils/cli/image.py b/src/panoptes/utils/cli/image.py index 25651b173..9cff060ac 100644 --- a/src/panoptes/utils/cli/image.py +++ b/src/panoptes/utils/cli/image.py @@ -1,19 +1,105 @@ from pathlib import Path +from typing import Optional import typer +from watchfiles import watch, Change from panoptes.utils import error from panoptes.utils.images import cr2 from panoptes.utils.images import fits as fits_utils app = typer.Typer() + cr2_app = typer.Typer() -fits_app = typer.Typer() app.add_typer(cr2_app, name='cr2') + +fits_app = typer.Typer() app.add_typer(fits_app, name='fits') -@cr2_app.command('convert') +@app.command('watch') +def watch_directory(path: Path, + to_jpg: bool = True, + to_fits: bool = True, + solve: bool = True, + overwrite: bool = False, + remove_cr2: bool = False, + ) -> None: + """ Watch a directory for changes and process any new files. + + The files will be processed according to the boolean flags, with the flag + names corresponding to other image commands. + + By default, all the flags are enabled, which will: + + * Extract JPG files from a CR2. + * Convert CR2 files to FITS. + * Plate-solve FITS files. + + """ + typer.secho(f'Watching {path}', fg='green') + for changes in watch(path): + for change in changes: + change_type = change[0] + change_path = Path(change[1]) + + if change_type == Change.added: + if change_path.suffix == '.cr2': + if to_jpg: + typer.secho(f'Converting {change_path} to JPG') + try: + cr2_to_jpg(change_path, overwrite=overwrite, + remove_cr2=remove_cr2 and not to_fits) + except Exception as e: + typer.secho(f'Error converting {change_path} to JPG: {e}', fg='red') + if to_fits: + typer.secho(f'Converting {change_path} to FITS') + try: + cr2_to_fits(change_path, remove_cr2=remove_cr2, overwrite=overwrite) + except Exception as e: + typer.secho(f'Error converting {change_path} to FITS: {e}', fg='red') + if change_path.suffix == '.fits': + if solve: + typer.secho(f'Solving {change_path}') + try: + solve_fits(change_path) + except Exception as e: + typer.secho(f'Error solving {change_path}: {e}', fg='red') + + +@cr2_app.command('to-jpg') +def cr2_to_jpg( + cr2_fname: Path, + jpg_fname: str = None, + title: str = '', + overwrite: bool = False, + remove_cr2: bool = False, +) -> Optional[Path]: + """Extract a JPG image from a CR2, return the new path name. + + Args: + cr2_fname (Path): Path to the CR2 file. + jpg_fname (str): Path to the JPG file. + title (str): Title to use for the JPG file. + overwrite (bool): Overwrite existing JPG file. + remove_cr2 (bool): Remove the CR2 file after conversion. + """ + typer.secho(f'Converting {cr2_fname} to JPG', fg='green') + jpg_fname = cr2.cr2_to_jpg( + cr2_fname, + jpg_fname=jpg_fname, + title=title, + overwrite=overwrite, + remove_cr2=remove_cr2, + ) + + if jpg_fname.exists(): + typer.secho(f'Wrote {jpg_fname}', fg='green') + + return jpg_fname + + +@cr2_app.command('to-fits') def cr2_to_fits( cr2_fname: Path, fits_fname: str = None, @@ -21,29 +107,30 @@ def cr2_to_fits( remove_cr2: bool = False, ) -> Path: """Convert a CR2 image to a FITS, return the new path name.""" - print(f'Converting {cr2_fname} to FITS') + typer.secho(f'Converting {cr2_fname} to FITS', fg='green') fits_fn = cr2.cr2_to_fits(cr2_fname, fits_fname=fits_fname, overwrite=overwrite, remove_cr2=remove_cr2) if fits_fname is not None: - print(f'FITS file available at {fits_fn}') + typer.secho(f'FITS file available at {fits_fn}', fg='green') return Path(fits_fn) @fits_app.command('solve') -def solve_fits(fits_fname: Path) -> Path: +def solve_fits(fits_fname: Path, **kwargs) -> Optional[Path]: """Plate-solve a FITS file.""" - print(f'Solving {str(fits_fname)}') + typer.secho(f'Solving {fits_fname}', fg='green') try: - solve_info = fits_utils.get_solve_field(fits_fname) + solve_info = fits_utils.get_solve_field(fits_fname, **kwargs) except error.InvalidSystemCommand as e: - return + typer.secho(f'Error while trying to solve {fits_fname}: {e!r}', fg='red') + return None solve_fn = solve_info['solved_fits_file'] - print(f'Plate-solved file available at {solve_fn}') + typer.secho(f'Plate-solved file available at {solve_fn}', fg='green') return Path(solve_fn) diff --git a/src/panoptes/utils/config/helpers.py b/src/panoptes/utils/config/helpers.py index 606aa2ece..4ec243046 100644 --- a/src/panoptes/utils/config/helpers.py +++ b/src/panoptes/utils/config/helpers.py @@ -1,9 +1,9 @@ -import os from contextlib import suppress from pathlib import Path from typing import Dict, List, Union from loguru import logger + from panoptes.utils import error from panoptes.utils.serializers import from_yaml from panoptes.utils.serializers import to_yaml @@ -67,6 +67,7 @@ def load_config(config_files: Union[Path, List] = None, parse: bool = True, config_files = listify(config_files) logger.debug(f'Loading config files: config_files={config_files!r}') for config_file in config_files: + config_file = Path(config_file) try: logger.debug(f'Adding config_file={config_file!r} to config dict') _add_to_conf(config, config_file, parse=parse) @@ -75,8 +76,8 @@ def load_config(config_files: Union[Path, List] = None, parse: bool = True, # Load local version of config if load_local: - local_version = config_file.replace('.', '_local.') - if os.path.exists(local_version): + local_version = config_file.parent / Path(config_file.stem + '_local.yaml') + if local_version.exists(): try: _add_to_conf(config, local_version, parse=parse) except Exception as e: # pragma: no cover @@ -92,11 +93,12 @@ def load_config(config_files: Union[Path, List] = None, parse: bool = True, return config -def save_config(path: Path, config: dict, overwrite: bool = True): +def save_config(save_path: Path, config: dict, overwrite: bool = True): """Save config to local yaml file. Args: - path (str): Path to save, can be relative or absolute. See Notes in ``load_config``. + save_path (str): Path to save, can be relative or absolute. See Notes in + ``load_config``. config (dict): Config to save. overwrite (bool, optional): True if file should be updated, False to generate a warning for existing config. Defaults to True @@ -108,27 +110,19 @@ def save_config(path: Path, config: dict, overwrite: bool = True): Raises: FileExistsError: If the local path already exists and ``overwrite=False``. """ - # Make sure ends with '_local.yaml' - base, ext = os.path.splitext(path) - - # Always want .yaml (although not actually used). - ext = '.yaml' - - # Check for _local name. - if not base.endswith('_local'): - base = f'{base}_local' - - full_path = f'{base}{ext}' + # Make sure ends with '_local.yaml'. + if save_path.stem.endswith('_local') is False: + save_path = save_path.with_name(save_path.stem + '_local.yaml') - if os.path.exists(full_path) and overwrite is False: - raise FileExistsError(f"Path exists and overwrite=False: {full_path}") + if save_path.exists() and overwrite is False: + raise FileExistsError(f"Path exists and overwrite=False: {save_path}") else: - # Create directory if does not exist - os.makedirs(os.path.dirname(full_path), exist_ok=True) - logger.info(f'Saving config to {full_path}') - with open(full_path, 'w') as f: - to_yaml(config, stream=f) - logger.success(f'Config info saved to {full_path}') + # Create directory if it does not exist. + save_path.parent.mkdir(parents=True, exist_ok=True) + logger.info(f'Saving config to {save_path}') + with save_path.open('w') as fn: + to_yaml(config, stream=fn) + logger.success(f'Config info saved to {save_path}') return True @@ -195,5 +189,5 @@ def parse_config_directories(directories: Dict[str, str]): def _add_to_conf(config: dict, conf_fn: Path, parse: bool = False): with suppress(IOError, TypeError): - with open(conf_fn, 'r') as fn: + with conf_fn.open('r') as fn: config.update(from_yaml(fn.read(), parse=parse)) diff --git a/src/panoptes/utils/config/server.py b/src/panoptes/utils/config/server.py index c5a9fd367..1739f79f1 100644 --- a/src/panoptes/utils/config/server.py +++ b/src/panoptes/utils/config/server.py @@ -36,7 +36,7 @@ def default(self, obj): return serialize_object(obj) -app.json_encoder = CustomJSONEncoder +app.json_provider_class = CustomJSONEncoder def config_server(config_file, @@ -81,41 +81,44 @@ def config_server(config_file, logger.success(f'{config!r}') cut_config = Cut(config) - app.config['config_file'] = config_file - app.config['save_local'] = save_local - app.config['load_local'] = load_local - app.config['POCS'] = config - app.config['POCS_cut'] = cut_config - logger.info(f'Config items saved to flask config-server') + with app.app_context(): + app.config['config_file'] = config_file + app.config['save_local'] = save_local + app.config['load_local'] = load_local + app.config['POCS'] = config + app.config['POCS_cut'] = cut_config + logger.info(f'Config items saved to flask config-server') - # Set up access and error logs for server. - access_logs = logger if access_logs == 'logger' else access_logs - error_logs = logger if error_logs == 'logger' else error_logs + # Set up access and error logs for server. + access_logs = logger if access_logs == 'logger' else access_logs + error_logs = logger if error_logs == 'logger' else error_logs - def start_server(host='localhost', port=6563): - try: - logger.info(f'Starting panoptes config server with {host}:{port}') - http_server = WSGIServer((host, int(port)), app, log=access_logs, error_log=error_logs) - http_server.serve_forever() - except OSError: - logger.warning(f'Problem starting config server, is another config server already running?') - return None - except Exception as e: - logger.warning(f'Problem starting config server: {e!r}') - return None - - host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost') - port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563) - cmd_kwargs = dict(host=host, port=port) - logger.debug(f'Setting up config server process with cmd_kwargs={cmd_kwargs!r}') - server_process = Process(target=start_server, - daemon=True, - kwargs=cmd_kwargs) - - if auto_start: - server_process.start() - - return server_process + def start_server(host='localhost', port=6563): + try: + logger.info(f'Starting panoptes config server with {host}:{port}') + http_server = WSGIServer((host, int(port)), app, log=access_logs, + error_log=error_logs) + http_server.serve_forever() + except OSError: + logger.warning( + f'Problem starting config server, is another config server already running?') + return None + except Exception as e: + logger.warning(f'Problem starting config server: {e!r}') + return None + + host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost') + port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563) + cmd_kwargs = dict(host=host, port=port) + logger.debug(f'Setting up config server process with cmd_kwargs={cmd_kwargs!r}') + server_process = Process(target=start_server, + daemon=True, + kwargs=cmd_kwargs) + + if auto_start: + server_process.start() + + return server_process @app.route('/heartbeat', methods=['GET', 'POST']) diff --git a/src/panoptes/utils/error.py b/src/panoptes/utils/error.py index 8388daebe..fe7099ab9 100644 --- a/src/panoptes/utils/error.py +++ b/src/panoptes/utils/error.py @@ -73,6 +73,11 @@ class NotFound(PanError): pass +class AlreadyExists(PanError): + """ Generic already exists class """ + pass + + class InvalidConfig(PanError): """ PanError raised if config file is invalid """ pass diff --git a/src/panoptes/utils/images/__init__.py b/src/panoptes/utils/images/__init__.py index 095b49fa2..16a915c59 100644 --- a/src/panoptes/utils/images/__init__.py +++ b/src/panoptes/utils/images/__init__.py @@ -1,87 +1,25 @@ import os -import re -import shutil -import subprocess +from _warnings import warn from contextlib import suppress -from warnings import warn +from pathlib import Path +from typing import Optional -import numpy as np -from astropy import units as u -from astropy.nddata import Cutout2D -from astropy.visualization import ImageNormalize -from astropy.visualization import LogStretch -from astropy.visualization import PercentileInterval -from astropy.wcs import WCS -from dateutil.parser import parse as date_parse -from loguru import logger -from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas -from matplotlib.figure import Figure - -from panoptes.utils import error -from panoptes.utils.images import fits as fits_utils -from panoptes.utils.images.plot import add_colorbar -from panoptes.utils.images.plot import get_palette -from panoptes.utils.time import current_time - - -def crop_data(data, box_width=200, center=None, data_only=True, wcs=None, **kwargs): - """Return a cropped portion of the image - - Shape is a box centered around the middle of the data - - Args: - data (`numpy.array`): Array of data. - box_width (int, optional): Size of box width in pixels, defaults to 200px. - center (tuple(int, int), optional): Crop around set of coords, default to image center. - data_only (bool, optional): If True (default), return only data. If False - return the `Cutout2D` object. - wcs (None|`astropy.wcs.WCS`, optional): A valid World Coordinate System (WCS) that will - be cropped along with the data if provided. - - Returns: - np.array: A clipped (thumbnailed) version of the data if `data_only=True`, otherwise - a `astropy.nddata.Cutout2D` object. - - """ - assert data.shape[ - 0] >= box_width, f"Can't clip data, it's smaller than {box_width} ({data.shape})" - # Get the center - if center is None: - x_len, y_len = data.shape - x_center = int(x_len / 2) - y_center = int(y_len / 2) - else: - y_center = int(center[0]) - x_center = int(center[1]) - - logger.debug(f"Using center: {x_center} {y_center}") - logger.debug(f"Box width: {box_width}") - - cutout = Cutout2D(data, (y_center, x_center), box_width, wcs=wcs) - - if data_only: - return cutout.data - - return cutout +from panoptes.utils.images.cr2 import cr2_to_jpg +from panoptes.utils.images.fits import fits_to_jpg def make_pretty_image(fname, title=None, - timeout=15, img_type=None, link_path=None, - **kwargs): + **kwargs) -> Optional[Path]: """Make a pretty image. This will create a jpg file from either a CR2 (Canon) or FITS file. - Notes: - See ``scripts/cr2_to_jpg.sh`` for CR2 process. - Arguments: fname (str): The path to the raw image. title (None|str, optional): Title to be placed on image, default None. - timeout (int, optional): Timeout for conversion, default 15 seconds. img_type (None|str, optional): Image type of fname, one of '.cr2' or '.fits'. The default is `None`, in which case the file extension of fname is used. link_path (None|str, optional): Path to location that image should be symlinked. @@ -91,9 +29,6 @@ def make_pretty_image(fname, Returns: str -- Filename of image that was created. - Deleted Parameters: - link_latest (bool, optional): If the pretty picture should be linked to - ``link_path``, default False. """ if img_type is None: img_type = os.path.splitext(fname)[-1] @@ -102,15 +37,15 @@ def make_pretty_image(fname, warn(f"File doesn't exist, can't make pretty: {fname}") return None elif img_type == '.cr2': - pretty_path = _make_pretty_from_cr2(fname, title=title, timeout=timeout, **kwargs) + pretty_path = cr2_to_jpg(Path(fname), title=title, **kwargs) elif img_type in ['.fits', '.fz']: - pretty_path = _make_pretty_from_fits(fname, title=title, **kwargs) + pretty_path = fits_to_jpg(fname, title=title, **kwargs) else: warn("File must be a Canon CR2 or FITS file.") return None if link_path is None or not os.path.exists(os.path.dirname(link_path)): - return pretty_path + return Path(pretty_path) # Remove existing symlink with suppress(FileNotFoundError): @@ -121,247 +56,4 @@ def make_pretty_image(fname, except Exception as e: # pragma: no cover warn(f"Can't link latest image: {e!r}") - return link_path - - -def _make_pretty_from_fits(fname=None, - title=None, - figsize=(10, 10 / 1.325), - dpi=150, - alpha=0.2, - number_ticks=7, - clip_percent=99.9, - **kwargs): - data = mask_saturated(fits_utils.getdata(fname)) - header = fits_utils.getheader(fname) - wcs = WCS(header) - - if not title: - field = header.get('FIELD', 'Unknown field') - exptime = header.get('EXPTIME', 'Unknown exptime') - filter_type = header.get('FILTER', 'Unknown filter') - - try: - date_time = header['DATE-OBS'] - except KeyError: - # If we don't have DATE-OBS, check filename for date - try: - basename = os.path.splitext(os.path.basename(fname))[0] - date_time = date_parse(basename).isoformat() - except Exception: # pragma: no cover - # Otherwise use now - date_time = current_time(pretty=True) - - date_time = date_time.replace('T', ' ', 1) - - title = f'{field} ({exptime}s {filter_type}) {date_time}' - - norm = ImageNormalize(interval=PercentileInterval(clip_percent), stretch=LogStretch()) - - fig = Figure() - FigureCanvas(fig) - fig.set_size_inches(*figsize) - fig.dpi = dpi - - if wcs.is_celestial: - ax = fig.add_subplot(1, 1, 1, projection=wcs) - ax.coords.grid(True, color='white', ls='-', alpha=alpha) - - ra_axis = ax.coords['ra'] - ra_axis.set_axislabel('Right Ascension') - ra_axis.set_major_formatter('hh:mm') - ra_axis.set_ticks( - number=number_ticks, - color='white', - exclude_overlapping=True - ) - - dec_axis = ax.coords['dec'] - dec_axis.set_axislabel('Declination') - dec_axis.set_major_formatter('dd:mm') - dec_axis.set_ticks( - number=number_ticks, - color='white', - exclude_overlapping=True - ) - else: - ax = fig.add_subplot(111) - ax.grid(True, color='white', ls='-', alpha=alpha) - - ax.set_xlabel('X / pixels') - ax.set_ylabel('Y / pixels') - - im = ax.imshow(data, norm=norm, cmap=get_palette(), origin='lower') - add_colorbar(im) - fig.suptitle(title) - - new_filename = re.sub(r'.fits(.fz)?', '.jpg', fname) - fig.savefig(new_filename, bbox_inches='tight') - - # explicitly close and delete figure - fig.clf() - del fig - - return new_filename - - -def _make_pretty_from_cr2(fname, title=None, **kwargs): - script_name = shutil.which('cr2-to-jpg') - cmd = [script_name, fname] - - if title: - cmd.append(title) - - logger.debug(f'Pretty cr2 command: {cmd!r}') - - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - logger.debug(f'Pretty CR2 output={output!r}') - except subprocess.CalledProcessError as e: - raise error.InvalidCommand(f"Error executing {script_name}: {e.output!r}\nCommand: {cmd}") - - return fname.replace('cr2', 'jpg') - - -def mask_saturated(data, saturation_level=None, threshold=0.9, bit_depth=None, dtype=None): - """Convert data to a masked array with saturated values masked. - - Args: - data (array_like): The numpy data array. - saturation_level (scalar, optional): The saturation level. If not given then the - saturation level will be set to threshold times the maximum pixel value. - threshold (float, optional): The fraction of the maximum pixel value to use as - the saturation level, default 0.9. - bit_depth (astropy.units.Quantity or int, optional): The effective bit depth of the - data. If given the maximum pixel value will be assumed to be 2**bit_depth, - otherwise an attempt will be made to infer the maximum pixel value from the - data type of the data. If data is not an integer type the maximum pixel value - cannot be inferred and an IllegalValue exception will be raised. - dtype (numpy.dtype, optional): The requested dtype for the masked array. If not given - the dtype of the masked array will be same as data. - - Returns: - numpy.ma.array: The masked numpy array. - - Raises: - error.IllegalValue: Raised if bit_depth is an astropy.units.Quantity object but the - units are not compatible with either bits or bits/pixel. - error.IllegalValue: Raised if neither saturation level or bit_depth are given, and data - has a non integer data type. - """ - if not saturation_level: - if bit_depth is not None: - try: - with suppress(AttributeError): - bit_depth = bit_depth.to_value(unit=u.bit) - except u.UnitConversionError: - try: - bit_depth = bit_depth.to_value(unit=u.bit / u.pixel) - except u.UnitConversionError: - raise error.IllegalValue("bit_depth must have units of bits or bits/pixel, " + - f"got {bit_depth!r}") - - bit_depth = int(bit_depth) - logger.trace(f"Using bit depth {bit_depth!r}") - saturation_level = threshold * (2 ** bit_depth - 1) - else: - # No bit depth specified, try to guess. - logger.trace(f"Inferring bit_depth from data type, {data.dtype!r}") - try: - # Try to use np.iinfo to compute machine limits. Will work for integer types. - saturation_level = threshold * np.iinfo(data.dtype).max - except ValueError: - # ValueError from np.iinfo means not an integer type. - raise error.IllegalValue("Neither saturation_level or bit_depth given, and data " + - "is not an integer type. Cannot determine correct " + - "saturation level.") - logger.debug(f"Masking image using saturation level {saturation_level!r}") - # Convert data to masked array of requested dtype, mask values above saturation level. - return np.ma.array(data, mask=(data > saturation_level), dtype=dtype) - - -def make_timelapse( - directory, - fn_out=None, - glob_pattern='20[1-9][0-9]*T[0-9]*.jpg', - overwrite=False, - timeout=60, - **kwargs): - """Create a timelapse. - - A timelapse is created from all the images in given ``directory`` - - Args: - directory (str): Directory containing image files. - fn_out (str, optional): Full path to output file name, if not provided, - defaults to `directory` basename. - glob_pattern (str, optional): A glob file pattern of images to include, - default '20[1-9][0-9]*T[0-9]*.jpg', which corresponds to the observation - images but excludes any pointing images. The pattern should be relative - to the local directory. - overwrite (bool, optional): Overwrite timelapse if exists, default False. - timeout (int): Timeout for making movie, default 60 seconds. - **kwargs (dict): - - Returns: - str: Name of output file - - Raises: - error.InvalidSystemCommand: Raised if ffmpeg command is not found. - FileExistsError: Raised if fn_out already exists and overwrite=False. - """ - if fn_out is None: - head, tail = os.path.split(directory) - if tail == '': - head, tail = os.path.split(head) - - field_name = head.split('/')[-2] - cam_name = head.split('/')[-1] - fname = f'{field_name}_{cam_name}_{tail}.mp4' - fn_out = os.path.normpath(os.path.join(directory, fname)) - - if os.path.exists(fn_out) and not overwrite: - raise FileExistsError("Timelapse exists. Set overwrite=True if needed") - - ffmpeg = shutil.which('ffmpeg') - if ffmpeg is None: - raise error.InvalidSystemCommand("ffmpeg not found, can't make timelapse") - - inputs_glob = os.path.join(directory, glob_pattern) - - try: - ffmpeg_cmd = [ - ffmpeg, - '-r', '3', - '-pattern_type', 'glob', - '-i', inputs_glob, - '-s', 'hd1080', - '-vcodec', 'libx264', - ] - - if overwrite: - ffmpeg_cmd.append('-y') - - ffmpeg_cmd.append(fn_out) - - logger.debug(ffmpeg_cmd) - - proc = subprocess.Popen(ffmpeg_cmd, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - # Don't wait forever - outs, errs = proc.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - proc.kill() - outs, errs = proc.communicate() - finally: - logger.debug(f"Output: {outs}") - logger.debug(f"Errors: {errs}") - - # Double-check for file existence - if not os.path.exists(fn_out): - fn_out = None - except Exception as e: - raise error.PanError(f"Problem creating timelapse in {fn_out}: {e!r}") - - return fn_out + return Path(link_path) diff --git a/src/panoptes/utils/images/bayer.py b/src/panoptes/utils/images/bayer.py index f7c94bfab..d60e2b6d9 100644 --- a/src/panoptes/utils/images/bayer.py +++ b/src/panoptes/utils/images/bayer.py @@ -5,14 +5,15 @@ from astropy.io import fits from astropy.stats import SigmaClip from loguru import logger -from panoptes.utils.images import fits as fits_utils from photutils import Background2D from photutils import BkgZoomInterpolator +from photutils import MMMBackground from photutils import MeanBackground from photutils import MedianBackground -from photutils import MMMBackground from photutils import SExtractorBackground +from panoptes.utils.images import fits as fits_utils + class RGB(IntEnum): """Helper class for array index access.""" @@ -366,7 +367,7 @@ def get_stamp_slice(x, y, stamp_size=(14, 14), ignore_superpixel=False, as_slice def get_rgb_background(data, box_size=(79, 84), - filter_size=(11, 12), + filter_size=(11, 11), estimator='mmm', interpolator='zoom', sigma=5, diff --git a/src/panoptes/utils/images/cr2.py b/src/panoptes/utils/images/cr2.py index 5c2540ad2..207496e0a 100644 --- a/src/panoptes/utils/images/cr2.py +++ b/src/panoptes/utils/images/cr2.py @@ -3,10 +3,11 @@ import subprocess from json import loads from pathlib import Path -from typing import Union +from typing import Union, Optional from warnings import warn import numpy as np +from PIL import Image, ImageDraw, ImageFont from astropy.io import fits from dateutil.parser import parse as date_parse from loguru import logger @@ -252,3 +253,53 @@ def read_pgm(fname, byteorder='>', remove_after=False): # pragma: no cover os.remove(fname) return data + + +def cr2_to_jpg( + cr2_fname: Path, + jpg_fname: str = None, + title: str = '', + overwrite: bool = False, + remove_cr2: bool = False, +) -> Optional[Path]: + """Extract a JPG image from a CR2, return the new path name.""" + exiftool = shutil.which('exiftool') + if not exiftool: # pragma: no cover + raise error.InvalidSystemCommand('exiftool not found') + + jpg_fname = Path(jpg_fname) if jpg_fname else cr2_fname.with_suffix('.jpg') + + if jpg_fname.exists() and overwrite is False: + raise error.AlreadyExists(f'{jpg_fname} already exists and overwrite is False') + + cmd = [exiftool, '-b', '-PreviewImage', cr2_fname.as_posix()] + comp_proc = subprocess.run(cmd, check=True, stdout=jpg_fname.open('wb')) + + if comp_proc.returncode != 0: # pragma: no cover + raise error.InvalidSystemCommand(f'{comp_proc.returncode}') + + if title and title > '': + try: + im = Image.open(jpg_fname) + id = ImageDraw.Draw(im) + + im.info['title'] = title + + try: + fnt = ImageFont.truetype('FreeMono.ttf', 120) + except Exception: # pragma: no cover + fnt = ImageFont.load_default() + bottom_padding = 25 + position = (im.size[0] / 2, im.size[1] - bottom_padding) + id.text(position, title, font=fnt, fill=(255, 0, 0), anchor='ms') + + print(f'Adding title={title} to {jpg_fname.as_posix()}') + im.save(jpg_fname) + except Exception: + raise error.InvalidSystemCommand(f'Error adding title to {jpg_fname.as_posix()}') + + if remove_cr2: + print(f'Removing {cr2_fname}') + cr2_fname.unlink() + + return jpg_fname diff --git a/src/panoptes/utils/images/fits.py b/src/panoptes/utils/images/fits.py index c8c0ad2fb..394a7cb7c 100644 --- a/src/panoptes/utils/images/fits.py +++ b/src/panoptes/utils/images/fits.py @@ -11,12 +11,17 @@ from astropy import units as u from astropy.io import fits from astropy.time import Time +from astropy.visualization import ImageNormalize, PercentileInterval, LogStretch from astropy.wcs import WCS from dateutil.parser import parse as parse_date from dateutil.tz import UTC from loguru import logger +from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas +from matplotlib.figure import Figure from panoptes.utils import error +from panoptes.utils.images.misc import mask_saturated +from panoptes.utils.images.plot import get_palette, add_colorbar from panoptes.utils.time import flatten_time PATH_MATCHER: Pattern[str] = re.compile(r"""^ @@ -810,3 +815,74 @@ def getval(fn, *args, **kwargs): if fn.endswith('.fz'): ext = 1 return fits.getval(fn, *args, ext=ext, **kwargs) + + +def fits_to_jpg(fname=None, + title=None, + figsize=(10, 10 / 1.325), + dpi=150, + alpha=0.2, + number_ticks=7, + clip_percent=99.9, + **kwargs): + data = mask_saturated(getdata(fname)) + header = getheader(fname) + wcs = WCS(header) + + if not title: + field = header.get('FIELD', 'Unknown field') + exptime = header.get('EXPTIME', 'Unknown exptime') + filter_type = header.get('FILTER', 'Unknown filter') + + try: + date_time = header['DATE-OBS'] + except KeyError: + # If we don't have DATE-OBS, check filename for date. + basename = os.path.splitext(os.path.basename(fname))[0] + date_time = parse_date(basename).isoformat() + + date_time = date_time.replace('T', ' ', 1) + + title = f'{field} ({exptime}s {filter_type}) {date_time}' + + norm = ImageNormalize(interval=PercentileInterval(clip_percent), stretch=LogStretch()) + + fig = Figure() + FigureCanvas(fig) + fig.set_size_inches(*figsize) + fig.dpi = dpi + + if wcs.is_celestial: + ax = fig.add_subplot(1, 1, 1, projection=wcs) + ax.coords.grid(True, color='white', ls='-', alpha=alpha) + + ra_axis = ax.coords['ra'] + ra_axis.set_axislabel('Right Ascension') + ra_axis.set_major_formatter('hh:mm') + ra_axis.set_ticks(number=number_ticks, color='white') + ra_axis.set_ticklabel(color='white', exclude_overlapping=True) + + dec_axis = ax.coords['dec'] + dec_axis.set_axislabel('Declination') + dec_axis.set_major_formatter('dd:mm') + dec_axis.set_ticks(number=number_ticks, color='white') + dec_axis.set_ticklabel(color='white', exclude_overlapping=True) + else: + ax = fig.add_subplot(111) + ax.grid(True, color='white', ls='-', alpha=alpha) + + ax.set_xlabel('X / pixels') + ax.set_ylabel('Y / pixels') + + im = ax.imshow(data, norm=norm, cmap=get_palette(), origin='lower') + add_colorbar(im) + fig.suptitle(title) + + new_filename = re.sub(r'.fits(.fz)?', '.jpg', fname) + fig.savefig(new_filename, bbox_inches='tight') + + # explicitly close and delete figure + fig.clf() + del fig + + return new_filename diff --git a/src/panoptes/utils/images/focus.py b/src/panoptes/utils/images/focus.py index 66f2e6296..8b9e3e6c5 100644 --- a/src/panoptes/utils/images/focus.py +++ b/src/panoptes/utils/images/focus.py @@ -21,8 +21,7 @@ def focus_metric(data, merit_function='vollath_F4', **kwargs): try: merit_function = globals()[merit_function] except KeyError: - raise KeyError( - "Focus merit function '{}' not found in panoptes.utils.images!".format(merit_function)) + raise KeyError(f'Focus merit function {merit_function} not found.') return merit_function(data, **kwargs) diff --git a/src/panoptes/utils/images/misc.py b/src/panoptes/utils/images/misc.py new file mode 100644 index 000000000..fcb3ff21f --- /dev/null +++ b/src/panoptes/utils/images/misc.py @@ -0,0 +1,237 @@ +import os +import shutil +import subprocess +from contextlib import suppress + +import numpy as np +from astropy import units as u +from astropy.nddata import Cutout2D +from loguru import logger + +from panoptes.utils import error + + +def make_timelapse( + directory, + fn_out=None, + glob_pattern='20[1-9][0-9]*T[0-9]*.jpg', + overwrite=False, + timeout=60, + **kwargs): # pragma: no cover + """Create a timelapse. + + A timelapse is created from all the images in given ``directory`` + + Args: + directory (str): Directory containing image files. + fn_out (str, optional): Full path to output file name, if not provided, + defaults to `directory` basename. + glob_pattern (str, optional): A glob file pattern of images to include, + default '20[1-9][0-9]*T[0-9]*.jpg', which corresponds to the observation + images but excludes any pointing images. The pattern should be relative + to the local directory. + overwrite (bool, optional): Overwrite timelapse if exists, default False. + timeout (int): Timeout for making movie, default 60 seconds. + **kwargs (dict): + + Returns: + str: Name of output file + + Raises: + error.InvalidSystemCommand: Raised if ffmpeg command is not found. + FileExistsError: Raised if fn_out already exists and overwrite=False. + """ + if fn_out is None: + head, tail = os.path.split(directory) + if tail == '': + head, tail = os.path.split(head) + + field_name = head.split('/')[-2] + cam_name = head.split('/')[-1] + fname = f'{field_name}_{cam_name}_{tail}.mp4' + fn_out = os.path.normpath(os.path.join(directory, fname)) + + if os.path.exists(fn_out) and not overwrite: + raise FileExistsError("Timelapse exists. Set overwrite=True if needed") + + ffmpeg = shutil.which('ffmpeg') + if ffmpeg is None: + raise error.InvalidSystemCommand("ffmpeg not found, can't make timelapse") + + inputs_glob = os.path.join(directory, glob_pattern) + + try: + ffmpeg_cmd = [ + ffmpeg, + '-r', '3', + '-pattern_type', 'glob', + '-i', inputs_glob, + '-s', 'hd1080', + '-vcodec', 'libx264', + ] + + if overwrite: + ffmpeg_cmd.append('-y') + + ffmpeg_cmd.append(fn_out) + + logger.debug(ffmpeg_cmd) + + proc = subprocess.Popen(ffmpeg_cmd, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + # Don't wait forever + outs, errs = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + outs, errs = proc.communicate() + finally: + logger.debug(f"Output: {outs}") + logger.debug(f"Errors: {errs}") + + # Double-check for file existence + if not os.path.exists(fn_out): + fn_out = None + except Exception as e: + raise error.PanError(f"Problem creating timelapse in {fn_out}: {e!r}") + + return fn_out + + +def crop_data(data, box_width=200, center=None, data_only=True, wcs=None, **kwargs): + """Return a cropped portion of the image. + + Shape is a box centered around the middle of the data + + .. plot:: + :include-source: + + >>> from matplotlib import pyplot as plt + >>> from astropy.wcs import WCS + >>> from panoptes.utils.images.misc import crop_data + >>> from panoptes.utils.images.plot import add_colorbar, get_palette + >>> from panoptes.utils.images.fits import getdata + >>> + >>> fits_url = 'https://github.com/panoptes/panoptes-utils/raw/develop/tests/data/solved.fits.fz' + >>> data, header = getdata(fits_url, header=True) + >>> wcs = WCS(header) + >>> # Crop a portion of the image by WCS and get Cutout2d object. + >>> cropped = crop_data(data, center=(600, 400), box_width=100, wcs=wcs, data_only=False) + >>> fig, ax = plt.subplots() + >>> im = ax.imshow(cropped.data, origin='lower', cmap=get_palette()) + >>> add_colorbar(im) + >>> plt.show() + + + Args: + data (`numpy.array`): Array of data. + box_width (int, optional): Size of box width in pixels, defaults to 200px. + center (tuple(int, int), optional): Crop around set of coords, default to image center. + data_only (bool, optional): If True (default), return only data. If False + return the `Cutout2D` object. + wcs (None|`astropy.wcs.WCS`, optional): A valid World Coordinate System (WCS) that will + be cropped along with the data if provided. + + Returns: + np.array: A clipped (thumbnailed) version of the data if `data_only=True`, otherwise + a `astropy.nddata.Cutout2D` object. + + """ + assert data.shape[ + 0] >= box_width, f"Can't clip data, it's smaller than {box_width} ({data.shape})" + # Get the center + if center is None: + x_len, y_len = data.shape + x_center = int(x_len / 2) + y_center = int(y_len / 2) + else: + y_center = int(center[0]) + x_center = int(center[1]) + + logger.debug(f"Using center: {x_center} {y_center}") + logger.debug(f"Box width: {box_width}") + + cutout = Cutout2D(data, (y_center, x_center), box_width, wcs=wcs) + + if data_only: + return cutout.data + + return cutout + + +def mask_saturated(data, saturation_level=None, threshold=0.9, bit_depth=None, dtype=None): + """Convert data to a masked array with saturated values masked. + + .. plot:: + :include-source: + + >>> from matplotlib import pyplot as plt + >>> from astropy.wcs import WCS + >>> from panoptes.utils.images.misc import crop_data, mask_saturated + >>> from panoptes.utils.images.plot import add_colorbar, get_palette + >>> from panoptes.utils.images.fits import getdata + >>> + >>> fits_url = 'https://github.com/panoptes/panoptes-utils/raw/develop/tests/data/solved.fits.fz' + >>> data, header = getdata(fits_url, header=True) + >>> wcs = WCS(header) + >>> # Crop a portion of the image by WCS and get Cutout2d object. + >>> cropped = crop_data(data, center=(600, 400), box_width=100, wcs=wcs, data_only=False) + >>> masked = mask_saturated(cropped.data, saturation_level=11535) + >>> fig, ax = plt.subplots() + >>> im = ax.imshow(masked, origin='lower', cmap=get_palette()) + >>> add_colorbar(im) + >>> fig.show() + + + Args: + data (array_like): The numpy data array. + saturation_level (scalar, optional): The saturation level. If not given then the + saturation level will be set to threshold times the maximum pixel value. + threshold (float, optional): The fraction of the maximum pixel value to use as + the saturation level, default 0.9. + bit_depth (astropy.units.Quantity or int, optional): The effective bit depth of the + data. If given the maximum pixel value will be assumed to be 2**bit_depth, + otherwise an attempt will be made to infer the maximum pixel value from the + data type of the data. If data is not an integer type the maximum pixel value + cannot be inferred and an IllegalValue exception will be raised. + dtype (numpy.dtype, optional): The requested dtype for the masked array. If not given + the dtype of the masked array will be same as data. + + Returns: + numpy.ma.array: The masked numpy array. + + Raises: + error.IllegalValue: Raised if bit_depth is an astropy.units.Quantity object but the + units are not compatible with either bits or bits/pixel. + error.IllegalValue: Raised if neither saturation level or bit_depth are given, and data + has a non integer data type. + """ + if not saturation_level: + if bit_depth is not None: + try: + with suppress(AttributeError): + bit_depth = bit_depth.to_value(unit=u.bit) + except u.UnitConversionError: + try: + bit_depth = bit_depth.to_value(unit=u.bit / u.pixel) + except u.UnitConversionError: + raise error.IllegalValue("bit_depth must have units of bits or bits/pixel, " + + f"got {bit_depth!r}") + + bit_depth = int(bit_depth) + logger.trace(f"Using bit depth {bit_depth!r}") + saturation_level = threshold * (2 ** bit_depth - 1) + else: + # No bit depth specified, try to guess. + logger.trace(f"Inferring bit_depth from data type, {data.dtype!r}") + try: + # Try to use np.iinfo to compute machine limits. Will work for integer types. + saturation_level = threshold * np.iinfo(data.dtype).max + except ValueError: + # ValueError from np.iinfo means not an integer type. + raise error.IllegalValue("Neither saturation_level or bit_depth given, and data " + + "is not an integer type. Cannot determine correct " + + "saturation level.") + logger.debug(f"Masking image using saturation level {saturation_level!r}") + # Convert data to masked array of requested dtype, mask values above saturation level. + return np.ma.array(data, mask=(data > saturation_level), dtype=dtype) diff --git a/src/panoptes/utils/images/plot.py b/src/panoptes/utils/images/plot.py index 07ec4ebac..017c88530 100644 --- a/src/panoptes/utils/images/plot.py +++ b/src/panoptes/utils/images/plot.py @@ -1,16 +1,8 @@ from copy import copy -from warnings import warn -from matplotlib import rc -from matplotlib import animation -from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas -from matplotlib.figure import Figure +import numpy as np from matplotlib import cm from mpl_toolkits.axes_grid1 import make_axes_locatable -import numpy as np -from astropy.visualization import ImageNormalize, LinearStretch, LogStretch, MinMaxInterval - -rc('animation', html='html5') def get_palette(cmap='inferno'): @@ -42,6 +34,8 @@ def add_colorbar(axes_image, size='5%', pad=0.05, orientation='vertical'): `matplotlib.pyplot.imshow`. .. plot:: + :include-source: + :caption: A colorbar with sane settings. >>> from matplotlib import pyplot as plt >>> import numpy as np @@ -52,12 +46,14 @@ def add_colorbar(axes_image, size='5%', pad=0.05, orientation='vertical'): >>> X, Y = np.meshgrid(x, y) >>> >>> func = lambda x, y: x**2 + y**2 - >>> >>> z = func(X, Y) >>> >>> fig, ax = plt.subplots() >>> im1 = ax.imshow(z, origin='lower') + >>> + >>> # Add the colorbar to the Image object (not the Axes). >>> add_colorbar(im1) + >>> >>> fig.show() @@ -71,6 +67,40 @@ def add_colorbar(axes_image, size='5%', pad=0.05, orientation='vertical'): def add_pixel_grid(ax1, grid_height, grid_width, show_axis_labels=True, show_superpixel=False, major_alpha=0.5, minor_alpha=0.25): + """ Adds a pixel grid to a plot, including features for the Bayer array superpixel. + + .. plot:: + :include-source: + :caption: The Bayer array superpixel pattern. Grid height and size must be even. + + >>> from matplotlib import pyplot as plt + >>> import numpy as np + >>> from panoptes.utils.images.plot import add_pixel_grid + >>> + >>> x = np.arange(-5, 5) + >>> y = np.arange(-5, 5) + >>> X, Y = np.meshgrid(x, y) + >>> func = lambda x, y: x**2 - y**2 + >>> + >>> fig, ax = plt.subplots() + >>> im1 = ax.imshow(func(X, Y), origin='lower', cmap='Greys') + >>> + >>> # Add the grid to the Axes object. + >>> add_pixel_grid(ax, grid_height=10, grid_width=10, show_superpixel=True, show_axis_labels=False) + >>> + >>> fig.show() + + Args: + ax1 (`matplotlib.axes.Axes`): The axes to add the grid to. + grid_height (int): The height of the grid in pixels. + grid_width (int): The width of the grid in pixels. + show_axis_labels (bool, optional): Whether to show the axis labels. Default True. + show_superpixel (bool, optional): Whether to show the superpixel pattern. Default False. + major_alpha (float, optional): The alpha value for the major grid lines. Default 0.5. + minor_alpha (float, optional): The alpha value for the minor grid lines. Default 0.25. + """ + ax1.set_xticks([]) + ax1.set_yticks([]) # major ticks every 2, minor ticks every 1 if show_superpixel: x_major_ticks = np.arange(-0.5, grid_width, 2) @@ -80,9 +110,6 @@ def add_pixel_grid(ax1, grid_height, grid_width, show_axis_labels=True, show_sup ax1.set_yticks(y_major_ticks) ax1.grid(which='major', color='r', linestyle='--', lw=3, alpha=major_alpha) - else: - ax1.set_xticks([]) - ax1.set_yticks([]) x_minor_ticks = np.arange(-0.5, grid_width, 1) y_minor_ticks = np.arange(-0.5, grid_height, 1) @@ -95,129 +122,3 @@ def add_pixel_grid(ax1, grid_height, grid_width, show_axis_labels=True, show_sup if show_axis_labels is False: ax1.set_xticklabels([]) ax1.set_yticklabels([]) - - -def animate_stamp(d0): - fig = Figure() - FigureCanvas(fig) - - ax = fig.add_subplot(111) - ax.set_xticks([]) - ax.set_yticks([]) - - line = ax.imshow(d0[0]) - ax.set_title(f'Frame 0') - - def animate(i): - line.set_data(d0[i]) # update the data - ax.set_title(f'Frame {i:03d}') - return line, - - # Init only required for blitting to give a clean slate. - def init(): - line.set_data(d0[0]) - return line, - - ani = animation.FuncAnimation(fig, animate, np.arange(0, len(d0)), init_func=init, - interval=500, blit=True) - - return ani - - -def show_stamps(pscs, - frame_idx=None, - stamp_size=11, - aperture_position=None, - show_residual=False, - stretch=None, - save_name=None, - show_max=False, - show_pixel_grid=False, - **kwargs): - if aperture_position is None: - midpoint = (stamp_size - 1) / 2 - aperture_position = (midpoint, midpoint) - - ncols = len(pscs) - - if show_residual: - ncols += 1 - - nrows = 1 - - fig = Figure() - FigureCanvas(fig) - fig.set_figheight(4) - fig.set_figwidth(8) - - if frame_idx is not None: - s0 = pscs[0][frame_idx] - s1 = pscs[1][frame_idx] - else: - s0 = pscs[0] - s1 = pscs[1] - - if stretch == 'log': - stretch = LogStretch() - else: - stretch = LinearStretch() - - norm = ImageNormalize(s0, interval=MinMaxInterval(), stretch=stretch) - - ax1 = fig.add_subplot(nrows, ncols, 1) - - im = ax1.imshow(s0, cmap=get_palette(), norm=norm) - - # create an axes on the right side of ax. The width of cax will be 5% - # of ax and the padding between cax and ax will be fixed at 0.05 inch. - # https://stackoverflow.com/questions/18195758/set-matplotlib-colorbar-size-to-match-graph - divider = make_axes_locatable(ax1) - cax = divider.append_axes("right", size="5%", pad=0.05) - fig.colorbar(im, cax=cax) - ax1.set_title('Target') - - # Comparison - ax2 = fig.add_subplot(nrows, ncols, 2) - im = ax2.imshow(s1, cmap=get_palette(), norm=norm) - - divider = make_axes_locatable(ax2) - cax = divider.append_axes("right", size="5%", pad=0.05) - fig.colorbar(im, cax=cax) - ax2.set_title('Comparison') - - if show_pixel_grid: - add_pixel_grid(ax1, stamp_size, stamp_size, show_superpixel=False) - add_pixel_grid(ax2, stamp_size, stamp_size, show_superpixel=False) - - if show_residual: - ax3 = fig.add_subplot(nrows, ncols, 3) - - # Residual - residual = s0 - s1 - im = ax3.imshow(residual, cmap=get_palette(), norm=ImageNormalize( - residual, interval=MinMaxInterval(), stretch=LinearStretch())) - - divider = make_axes_locatable(ax3) - cax = divider.append_axes("right", size="5%", pad=0.05) - fig.colorbar(im, cax=cax) - ax3.set_title('Noise Residual') - ax3.set_title('Residual RMS: {:.01%}'.format(residual.std())) - ax3.set_yticklabels([]) - ax3.set_xticklabels([]) - - if show_pixel_grid: - add_pixel_grid(ax1, stamp_size, stamp_size, show_superpixel=False) - - # Turn off tick labels - ax1.set_yticklabels([]) - ax1.set_xticklabels([]) - ax2.set_yticklabels([]) - ax2.set_xticklabels([]) - - if save_name: - try: - fig.savefig(save_name) - except Exception as e: - warn("Can't save figure: {}".format(e)) - - return fig diff --git a/src/panoptes/utils/rs232.py b/src/panoptes/utils/rs232.py index edf506812..68eb1616c 100644 --- a/src/panoptes/utils/rs232.py +++ b/src/panoptes/utils/rs232.py @@ -5,9 +5,10 @@ import serial from deprecated import deprecated from loguru import logger +from serial.tools.list_ports import comports as get_comports + from panoptes.utils import error from panoptes.utils import serializers -from serial.tools.list_ports import comports as get_comports @deprecated(reason='Use panoptes.utils.serial.device') @@ -71,39 +72,17 @@ class SerialData(object): .. doctest:: - # Register our serial simulators by importing protocol. - >>> from panoptes.utils.serial.handlers import protocol_buffers - - # Import our serial utils. >>> from panoptes.utils.rs232 import SerialData - - # Connect to our fake buffered device - >>> device_listener = SerialData(port='buffers://') - - # Note: A manual reset is currently required because implementation is not complete. - # See https://github.com/panoptes/POCS/issues/758 for details. - >>> protocol_buffers.reset_serial_buffers() + >>> # Connect to our fake buffered device + >>> device_listener = SerialData(port='loop://') >>> device_listener.is_connected True - >>> device_listener.port - 'buffers://' - - # Device sends event - >>> protocol_buffers.set_serial_read_buffer(b'emit event') - - # Listen for event - >>> device_listener.read() - 'emit event' - - >>> device_listener.write('ack event') - 9 - >>> protocol_buffers.get_serial_write_buffer() - b'ack event' - - # Remove custom handlers - >>> import serial - >>> serial.protocol_handler_packages.remove('panoptes.utils.serial.handlers') + 'loop://' + >>> # Device sends event + >>> bytes = device_listener.write('Hello World') + >>> device_listener.read(bytes) + 'Hello World' """ def __init__(self, @@ -159,21 +138,21 @@ def __init__(self, self.ser.rtscts = False self.ser.dsrdtr = False - self.logger.debug('SerialData for {} created', self.name) + self.logger.debug(f'SerialData for {self.name} created') # Properties have been set to reasonable values, ready to open the port. try: - self.ser.open() - except serial.serialutil.SerialException as err: - self.logger.debug('Unable to open {}. Error: {}', self.name, err) + self.connect() + except serial.serialutil.SerialException as err: # pragma: no cover + self.logger.debug(f'Unable to open {self.name}. Error: {err}') return open_delay = max(0.0, float(open_delay)) if open_delay > 0.0: - self.logger.debug('Opened {}, sleeping for {} seconds', self.name, open_delay) + self.logger.debug(f'Opened {self.name}, sleeping for {open_delay} seconds') time.sleep(open_delay) else: - self.logger.debug('Opened {}', self.name) + self.logger.debug(f'Opened {self.name}') @property def port(self): @@ -192,19 +171,18 @@ def connect(self): error.BadSerialConnection if unable to open the connection. """ if self.is_connected: - self.logger.debug('Connection already open to {}', self.name) + self.logger.debug(f'Connection already open to {self.name}') return - self.logger.debug('SerialData.connect called for {}', self.name) + self.logger.debug(f'SerialData.connect called for {self.name}') try: # Note: we must not call open when it is already open, else an exception is thrown of # the same type thrown when open fails to actually open the device. self.ser.open() - if not self.is_connected: - raise error.BadSerialConnection( - msg="Serial connection {} is not open".format(self.name)) + if not self.is_connected: # pragma: no cover + raise error.BadSerialConnection(msg=f'Serial connection {self.name} is not open') except serial.serialutil.SerialException as err: raise error.BadSerialConnection(msg=err) - self.logger.debug('Serial connection established to {}', self.name) + self.logger.debug(f'Serial connection established to {self.name}') def disconnect(self): """Closes the serial connection. @@ -213,21 +191,16 @@ def disconnect(self): error.BadSerialConnection if unable to close the connection. """ # Fortunately, close() doesn't throw an exception if already closed. - self.logger.debug('SerialData.disconnect called for {}', self.name) + self.logger.debug(f'SerialData.disconnect called for {self.name}') try: self.ser.close() - except Exception as err: - raise error.BadSerialConnection( - msg="SerialData.disconnect failed for {}; underlying error: {}".format( - self.name, err)) - if self.is_connected: - raise error.BadSerialConnection( - msg="SerialData.disconnect failed for {}".format(self.name)) + except Exception as e: # pragma: no cover + raise error.BadSerialConnection(msg=f'disconnect failed for {self.name}; {e!r}') + if self.is_connected: # pragma: no cover + raise error.BadSerialConnection(msg=f'SerialData.disconnect failed for {self.name}') def write_bytes(self, data): """Write data of type bytes.""" - assert self.ser - assert self.ser.isOpen() return self.ser.write(data) def write(self, value): @@ -245,8 +218,6 @@ def read_bytes(self, size=1): Returns: Bytes read from the port. """ - assert self.ser - assert self.ser.isOpen() return self.ser.read(size=size) def read(self, retry_limit=None, retry_delay=None): @@ -255,20 +226,20 @@ def read(self, retry_limit=None, retry_delay=None): If no response is given, delay for retry_delay and then try to read again. Fail after retry_limit attempts. """ - assert self.ser - assert self.ser.isOpen() - if retry_limit is None: retry_limit = self.retry_limit if retry_delay is None: retry_delay = self.retry_delay + data = '' for _ in range(retry_limit): - data = self.ser.readline() - if data: - return data.decode(encoding='ascii') + line = self.ser.readline() + if line: + data = line.decode(encoding='ascii') + break time.sleep(retry_delay) - return '' + + return data def get_reading(self): """Reads and returns a line, along with the timestamp of the read. @@ -294,16 +265,17 @@ def get_and_parse_reading(self, retry_limit=5): A pair (tuple) of (timestamp, decoded JSON line). The timestamp is the time of completion of the readline operation. """ + reading = None for _ in range(max(1, retry_limit)): (ts, line) = self.get_reading() - if not line: - continue - with suppress(error.InvalidDeserialization): + with suppress(error.InvalidDeserialization, TypeError): data = serializers.from_json(line) if data: - return (ts, data) - return None + reading = (ts, data) + break + + return reading def reset_input_buffer(self): """Clear buffered data from connected port/device. diff --git a/src/panoptes/utils/serial/device.py b/src/panoptes/utils/serial/device.py index 3508d50a7..66b064497 100644 --- a/src/panoptes/utils/serial/device.py +++ b/src/panoptes/utils/serial/device.py @@ -6,10 +6,11 @@ import serial from loguru import logger -from panoptes.utils import error from serial.threaded import LineReader, ReaderThread from serial.tools.list_ports import comports as get_comports +from panoptes.utils import error + @dataclass class SerialDeviceDefaults: @@ -231,7 +232,7 @@ def connection_made(this, transport): super(LineReader, this).connection_made(transport) def connection_lost(this, exc): - logger.warning(f'Disconnected from {self}') + logger.trace(f'Disconnected from {self}') def handle_line(this, data): try: diff --git a/src/panoptes/utils/serial/handlers/__init__.py b/src/panoptes/utils/serial/handlers/__init__.py deleted file mode 100644 index 7fdb9b2ad..000000000 --- a/src/panoptes/utils/serial/handlers/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""The protocol_*.py files in this package are based on PySerial's file -test/handlers/protocol_test.py, modified for different behaviors. -The call serial.serial_for_url("XYZ://") looks for a class Serial in a -file named protocol_XYZ.py in this package (i.e. directory). -""" - -import serial - -# Import this namespace automatically. -serial.protocol_handler_packages.append('panoptes.utils.serial.handlers') diff --git a/src/panoptes/utils/serial/handlers/protocol_arduinosimulator.py b/src/panoptes/utils/serial/handlers/protocol_arduinosimulator.py deleted file mode 100644 index ffb9f58d8..000000000 --- a/src/panoptes/utils/serial/handlers/protocol_arduinosimulator.py +++ /dev/null @@ -1,520 +0,0 @@ -"""Provides a simple simulator for telemetry_board.ino or camera_board.ino. - -We use the pragma "no cover" in several places that happen to never be -reached or that would only be reached if the code was called directly, -i.e. not in the way it is intended to be used. -""" - -import copy -import datetime -import queue -import random -import threading -import time -import urllib.parse - -import panoptes.utils.serial.handlers.protocol_no_op -from loguru import logger -from panoptes.utils.serializers import from_json -from panoptes.utils.serializers import to_json -from serial import serialutil -from serial.serialutil import PortNotOpenError - - -def _drain_queue(q): - cmd = None - while not q.empty(): - cmd = q.get_nowait() - return cmd # Present just for debugging. - - -class ArduinoSimulator: - """Simulates the serial behavior of the PANOPTES Arduino sketches. - - The RS-232 connection is simulated with an input and output queue of bytes. This class provides - a run function which can be called from a Thread to execute. Every two seconds while running it - will generate another json output line, and then send that to the json_queue in small chunks - at a rate similar to 9600 baud, the rate used by our Arduino sketches. - """ - - def __init__(self, message, relay_queue, json_queue, chunk_size, stop): - """ - Args: - message: The message to be sent (millis and report_num will be added). - relay_queue: The queue.Queue instance from which relay command - bytes are read and acted upon. Elements are of type bytes. - json_queue: The queue.Queue instance to which json messages - (serialized to bytes) are written at ~9600 baud. Elements - are of type bytes (i.e. each element is a sequence of bytes of - length up to chunk_size). - chunk_size: The number of bytes to write to json_queue at a time. - stop: a threading.Event which is checked to see if run should stop executing. - """ - self.message = copy.deepcopy(message) - self.logger = logger - self.logger.critical(f'message: {message}') - self.relay_queue = relay_queue - self.json_queue = json_queue - self.stop = stop - # Time between producing messages. - self.message_delta = datetime.timedelta(seconds=2) - self.next_message_time = None - # Size of a chunk of bytes. - self.chunk_size = chunk_size - # Interval between outputing chunks of bytes. - chunks_per_second = 1000.0 / self.chunk_size - chunk_interval = 1.0 / chunks_per_second - self.logger.debug(f'chunks_per_second={chunks_per_second} chunk_interval={chunk_interval}') - self.chunk_delta = datetime.timedelta(seconds=chunk_interval) - self.next_chunk_time = None - self.pending_json_bytes = bytearray() - self.pending_relay_bytes = bytearray() - self.command_lines = [] - self.start_time = datetime.datetime.now() - self.report_num = 0 - self.logger.info('ArduinoSimulator created') - - def __del__(self): - if not self.stop.is_set(): # pragma: no cover - self.logger.critical('ArduinoSimulator.__del__ stop is NOT set') - - def run(self): - """Produce messages periodically and emit their bytes at a limited rate.""" - self.logger.info('ArduinoSimulator.run ENTER') - # Produce a message right away, but remove a random number of bytes at the start to reflect - # what happens when we connect at a random time to the Arduino. - now = datetime.datetime.now() - self.next_chunk_time = now - self.next_message_time = now + self.message_delta - b = self.generate_next_message_bytes(now) - cut = random.randrange(len(b)) - if cut > 0: - self.logger.info(f'Cutting off the leading {cut} bytes of the first message') - b = b[cut:] - self.pending_json_bytes.extend(b) - # Now two interleaved loops: - # 1) Generate messages every self.message_delta - # 2) Emit a chunk of bytes from pending_json_bytes every self.chunk_delta. - # Clearly we need to emit all the bytes from pending_json_bytes at least - # as fast as we append new messages to it, else we'll have a problem - # (i.e. the simulated baud rate will be too slow for the output rate). - while True: - if self.stop.is_set(): - self.logger.info('Returning from ArduinoSimulator.run EXIT') - return - now = datetime.datetime.now() - if now >= self.next_chunk_time: - self.output_next_chunk(now) - if now >= self.next_message_time: - self.generate_next_message(now) - if self.pending_json_bytes and self.next_chunk_time < self.next_message_time: - next_time = self.next_chunk_time - else: - next_time = self.next_message_time - self.read_relay_queue_until(next_time) - - def handle_pending_relay_bytes(self): - """Process complete relay commands.""" - newline = b'\n' - while True: - index = self.pending_relay_bytes.find(newline) - if index < 0: - break - line = str(self.pending_relay_bytes[0:index], 'ascii') - self.logger.info(f'Received command: {line}') - del self.pending_relay_bytes[0:index + 1] - self.command_lines.append(line) - if self.pending_relay_bytes: - self.logger.info(f'Accumulated {len(self.pending_relay_bytes)} bytes.') - - def read_relay_queue_until(self, next_time): - """Read and process relay queue bytes until time for the next action.""" - while True: - now = datetime.datetime.now() - if now >= next_time: - # Already reached the time for the next main loop event, - # so return to repeat the main loop. - return - remaining = (next_time - now).total_seconds() - assert remaining > 0 - self.logger.info(f'ArduinoSimulator.read_relay_queue_until remaining={remaining}') - try: - b = self.relay_queue.get(block=True, timeout=remaining) - assert isinstance(b, (bytes, bytearray)) - self.pending_relay_bytes.extend(b) - self.handle_pending_relay_bytes() - # Fake a baud rate for reading by waiting based on the - # number of bytes we just read. - time.sleep(1.0 / 1000 * len(b)) - except queue.Empty: - # Not returning here so that the return above is will be - # hit every time this method executes. - pass - - def output_next_chunk(self, now): - """Output one chunk of pending json bytes.""" - self.next_chunk_time = now + self.chunk_delta - if len(self.pending_json_bytes) == 0: - return - last = min(self.chunk_size, len(self.pending_json_bytes)) - chunk = bytes(self.pending_json_bytes[0:last]) - del self.pending_json_bytes[0:last] - if self.json_queue.full(): - self.logger.info('Dropping chunk because the queue is full') - return - self.json_queue.put_nowait(chunk) - self.logger.debug('output_next_chunk -> {}', chunk) - - def generate_next_message(self, now): - """Append the next message to the pending bytearray and scheduled the next message.""" - b = self.generate_next_message_bytes(now) - self.pending_json_bytes.extend(b) - self.next_message_time = datetime.datetime.now() + self.message_delta - - def generate_next_message_bytes(self, now): - """Generate the next message (report) from the simulated Arduino.""" - # Not worrying here about emulating the 32-bit nature of millis (wraps in 49 days) - elapsed = int((now - self.start_time).total_seconds() * 1000) - self.report_num += 1 - self.message['millis'] = elapsed - self.message['report_num'] = self.report_num - if self.command_lines: - self.message['commands'] = self.command_lines - self.command_lines = [] - - s = to_json(self.message) + '\r\n' - if 'commands' in self.message: - del self.message['commands'] - - self.logger.debug('generate_next_message -> {!r}', s) - b = s.encode(encoding='ascii') - return b - - -class FakeArduinoSerialHandler(panoptes.utils.serial.handlers.protocol_no_op.NoOpSerial): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.logger = logger - self.simulator_thread = None - self.relay_queue = queue.Queue(maxsize=1) - self.json_queue = queue.Queue(maxsize=1) - self.json_bytes = bytearray() - self.stop = threading.Event() - self.stop.set() - self.device_simulator = None - - def __del__(self): - if self.simulator_thread: # pragma: no cover - self.logger.critical('ArduinoSimulator.__del__ simulator_thread is still present') - self.stop.set() - self.simulator_thread.join(timeout=3.0) - - def open(self): - """Open port. - - Raises: - SerialException if the port cannot be opened. - """ - if not self.is_open: - self.is_open = True - self._reconfigure_port() - - def close(self): - """Close port immediately.""" - self.is_open = False - self._reconfigure_port() - - @property - def in_waiting(self): - """The number of input bytes available to read immediately.""" - if not self.is_open: - raise PortNotOpenError - # Not an accurate count because the elements of self.json_queue are arrays, not individual - # bytes. - return len(self.json_bytes) + self.json_queue.qsize() - - def reset_input_buffer(self): - """Flush input buffer, discarding all it’s contents.""" - self.json_bytes.clear() - _drain_queue(self.json_queue) - - def read(self, size=1): - """Read size bytes. - - If a timeout is set it may return fewer characters than requested. - With no timeout it will block until the requested number of bytes - is read. - - Args: - size: Number of bytes to read. - - Returns: - Bytes read from the port, of type 'bytes'. - """ - if not self.is_open: - raise PortNotOpenError - - # Not checking if the config is OK, so will try to read from a possibly - # empty queue if using the wrong baudrate, etc. This is deliberate. - - response = bytearray() - timeout_obj = serialutil.Timeout(self.timeout) - while True: - b = self._read1(timeout_obj) - if b: - response.extend(b) - if size is not None and len(response) >= size: - break - else: # pragma: no cover - # The timeout expired while in _read1. - break - if timeout_obj.expired(): # pragma: no cover - break - response = bytes(response) - return response - - def readline(self): - """Read and return one line from the simulator. - - This override exists just to support logging of the line. - """ - line = super().readline() - self.logger.debug(f'FakeArduinoSerialHandler.readline -> {line!r}') - return line - - @property - def out_waiting(self): - """The number of bytes in the output buffer.""" - if not self.is_open: - raise PortNotOpenError - # Not an accurate count because the elements of self.relay_queue are arrays, not individual - # bytes. - return self.relay_queue.qsize() - - def reset_output_buffer(self): - """Clear output buffer. - - Aborts the current output, discarding all that is in the output buffer. - """ - if not self.is_open: - raise PortNotOpenError - _drain_queue(self.relay_queue) - - def flush(self): - """Write the buffered data to the output device. - - We interpret that here as waiting until the simulator has taken all of the - entries from the queue. - """ - if not self.is_open: - raise PortNotOpenError - while not self.relay_queue.empty(): - time.sleep(0.01) - - def write(self, data): - """Write the bytes data to the port. - - Args: - data: The data to write (bytes or bytearray instance). - - Returns: - Number of bytes written. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - if not isinstance(data, (bytes, bytearray)): - raise ValueError('write takes bytes') # pragma: no cover - data = bytes(data) # Make sure it can't change. - self.logger.info('FakeArduinoSerialHandler.write({!r})', data) - try: - for n in range(len(data)): - one_byte = data[n:n + 1] - self.relay_queue.put(one_byte, block=True, timeout=self.write_timeout) - return len(data) - except queue.Full: # pragma: no cover - # This exception is "lossy" in that the caller can't tell how much was written. - raise serialutil.Timeout - - # -------------------------------------------------------------------------- - - @property - def is_config_ok(self): - """Does the caller ask for the correct serial device config?""" - # The default Arduino data, parity and stop bits are: 8 data bits, no parity, one stop bit. - v = (self.baudrate == 9600 and self.bytesize == serialutil.EIGHTBITS and - self.parity == serialutil.PARITY_NONE and not self.rtscts and not self.dsrdtr) - - # All existing tests ensure the config is OK, so we never log here. - if not v: # pragma: no cover - self.logger.critical(f'Serial config is not OK: {self.get_settings()!r}') - - return v - - def _read1(self, timeout_obj): - """Read 1 byte of input, of type bytes.""" - - # _read1 is currently called only from read(), which checks that the - # serial device is open, so is_open is always true. - if not self.is_open: # pragma: no cover - raise PortNotOpenError - - if not self.json_bytes: - try: - entry = self.json_queue.get(block=True, timeout=timeout_obj.time_left()) - assert isinstance(entry, bytes) - self.json_bytes.extend(entry) - except queue.Empty: - return None - - # Unless something has gone wrong, json_bytes is always non-empty here. - if not self.json_bytes: # pragma: no cover - return None - - c = bytes(self.json_bytes[0:1]) - del self.json_bytes[0:1] - return c - - # -------------------------------------------------------------------------- - # There are a number of methods called by SerialBase that need to be - # implemented by sub-classes, assuming their calls haven't been blocked - # by replacing the calling methods/properties. These are no-op - # implementations. - - def _reconfigure_port(self): - """Reconfigure the open port after a property has been changed. - - If you need to know which property has been changed, override the - setter for the appropriate properties. - """ - need_thread = self.is_open and self.is_config_ok - if need_thread and not self.simulator_thread: - _drain_queue(self.relay_queue) - _drain_queue(self.json_queue) - self.json_bytes.clear() - self.stop.clear() - params = self._params_from_url(self.portstr) - self._create_simulator(params) - self.simulator_thread = threading.Thread( - name='Device Simulator', target=lambda: self.device_simulator.run(), daemon=True) - self.simulator_thread.start() - elif self.simulator_thread and not need_thread: - self.stop.set() - self.simulator_thread.join(timeout=30.0) - if self.simulator_thread.is_alive(): - # Not a SerialException, but a test infrastructure error. - raise Exception(f'{self.simulator_thread.name} did not stop!') # pragma: no cover - self.simulator_thread = None - self.device_simulator = None - _drain_queue(self.relay_queue) - _drain_queue(self.json_queue) - self.json_bytes.clear() - - def _update_rts_state(self): - """Handle rts being set to some value. - - "self.rts = value" has been executed, for some value. This may not - have changed the value. - """ - # We never set rts in our tests, so this doesn't get executed. - pass # pragma: no cover - - def _update_dtr_state(self): - """Handle dtr being set to some value. - - "self.dtr = value" has been executed, for some value. This may not - have changed the value. - """ - # We never set dtr in our tests, so this doesn't get executed. - pass # pragma: no cover - - def _update_break_state(self): - """Handle break_condition being set to some value. - - "self.break_condition = value" has been executed, for some value. - This may not have changed the value. - Note that break_condition is set and then cleared by send_break(). - """ - # We never set break_condition in our tests, so this doesn't get executed. - pass # pragma: no cover - - # -------------------------------------------------------------------------- - # Internal (non-standard) methods. - - def _params_from_url(self, url): - """Extract various params from the URL.""" - expected = 'expected a string in the form "arduinosimulator://[?board=]"' - parts = urllib.parse.urlparse(url) - - # Unless we force things (break the normal protocol), scheme will always - # be 'arduinosimulator'. - if parts.scheme != 'arduinosimulator': - raise Exception(f'{expected}: got scheme {parts.scheme!r}') # pragma: no cover - int_param_names = {'chunk_size', 'read_buffer_size', 'write_buffer_size'} - params = {} - for option, values in urllib.parse.parse_qs(parts.query, True).items(): - if option == 'board' and len(values) == 1: - params[option] = values[0] - elif option == 'name' and len(values) == 1: - # This makes it easier for tests to confirm the right serial device has - # been opened. - self.name = values[0] - elif option in int_param_names and len(values) == 1: - params[option] = int(values[0]) - else: - raise Exception(f'{expected}: unknown param {option!r}') # pragma: no cover - return params - - def _create_simulator(self, params): - board = params.get('board', 'telemetry') - if board == 'telemetry': - message = from_json(""" - { - "name":"telemetry_board", - "ver":"2017-09-23", - "power": { - "computer":1, - "fan":1, - "mount":1, - "cameras":1, - "weather":1, - "main":1 - }, - "current": {"main":387,"fan":28,"mount":34,"cameras":27}, - "amps": {"main":1083.60,"fan":50.40,"mount":61.20,"cameras":27.00}, - "humidity":42.60, - "temperature":[13.01,12.81,19.75], - "temp_00":15.50 - } - """) - elif board == 'camera': - message = from_json(""" - { - "name":"camera_board", - "inputs":6, - "camera_00":1, - "camera_01":1, - "accelerometer": {"x":-7.02, "y":6.95, "z":1.70, "o": 6}, - "humidity":59.60, - "temperature":[13.01,12.81,19.75], - "temp_00":12.50 - } - """) - elif board == 'json_object': - # Produce an output that is json, but not what we expect - message = {} - else: - raise Exception(f'Unknown board: {board}') # pragma: no cover - - # The elements of these queues are of type bytes. This means we aren't fully controlling - # the baudrate unless the chunk_size is 1, but that should be OK. - chunk_size = params.get('chunk_size', 20) - self.json_queue = queue.Queue(maxsize=params.get('read_buffer_size', 10000)) - self.relay_queue = queue.Queue(maxsize=params.get('write_buffer_size', 100)) - - self.device_simulator = ArduinoSimulator(message, self.relay_queue, self.json_queue, - chunk_size, self.stop) - - -Serial = FakeArduinoSerialHandler diff --git a/src/panoptes/utils/serial/handlers/protocol_buffers.py b/src/panoptes/utils/serial/handlers/protocol_buffers.py deleted file mode 100644 index 6acbb4fa1..000000000 --- a/src/panoptes/utils/serial/handlers/protocol_buffers.py +++ /dev/null @@ -1,103 +0,0 @@ -# This module implements a handler for serial_for_url("buffers://"). - -import io -import threading -from typing import Optional - -from panoptes.utils.serial.handlers.protocol_no_op import NoOpSerial -from serial.serialutil import PortNotOpenError - -# r_buffer and w_buffer are binary I/O buffers. read(size=N) on an instance -# of Serial reads the next N bytes from r_buffer, and write(data) appends the -# bytes of data to w_buffer. -# NOTE: The caller (a test) is responsible for resetting buffers before tests. See -# the util functions below. -SERIAL_READ_BUFFER: Optional[io.BytesIO] = None -SERIAL_WRITE_BUFFER: Optional[io.BytesIO] = None - -# The above I/O buffers are not thread safe, so we need to lock them during access. -SERIAL_READ_LOCK = threading.Lock() -SERIAL_WRITE_LOCK = threading.Lock() - - -def reset_serial_buffers(read_data=None): - set_serial_read_buffer(read_data) - with SERIAL_WRITE_LOCK: - global SERIAL_WRITE_BUFFER - SERIAL_WRITE_BUFFER = io.BytesIO() - - -def set_serial_read_buffer(data): - """Sets the r buffer to data (a bytes object).""" - if data and not isinstance(data, (bytes, bytearray)): - raise TypeError('data must by a bytes or bytearray object.') - with SERIAL_READ_LOCK: - global SERIAL_READ_BUFFER - SERIAL_READ_BUFFER = io.BytesIO(data) - - -def get_serial_write_buffer(): - """Returns an immutable bytes object with the value of the w buffer.""" - with SERIAL_WRITE_LOCK: - if SERIAL_WRITE_BUFFER: - return SERIAL_WRITE_BUFFER.getvalue() - - -class BuffersSerial(NoOpSerial): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - @property - def in_waiting(self): - if not self.is_open: - raise PortNotOpenError - with SERIAL_READ_LOCK: - return len(SERIAL_READ_BUFFER.getbuffer()) - SERIAL_READ_BUFFER.tell() - - def read(self, size=1): - """Read size bytes. - - If a timeout is set it may return fewer characters than requested. - With no timeout it will block until the requested number of bytes - is read. - - Args: - size: Number of bytes to read. - - Returns: - Bytes read from the port, of type 'bytes'. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - if not self.is_open: - raise PortNotOpenError - with SERIAL_READ_LOCK: - # TODO(jamessynge): Figure out whether and how to handle timeout. - # We might choose to generate a timeout if the caller asks for data - # beyond the end of the buffer; or simply return what is left, - # including nothing (i.e. bytes()) if there is nothing left. - return SERIAL_READ_BUFFER.read(size) - - def write(self, data): - """ - Args: - data: The data to write. - - Returns: - Number of bytes written. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - if not isinstance(data, (bytes, bytearray)): - raise TypeError("data must by a bytes or bytearray object.") - if not self.is_open: - raise PortNotOpenError - with SERIAL_WRITE_LOCK: - return SERIAL_WRITE_BUFFER.write(data) - - -Serial = BuffersSerial diff --git a/src/panoptes/utils/serial/handlers/protocol_hooked.py b/src/panoptes/utils/serial/handlers/protocol_hooked.py deleted file mode 100644 index 891895dbb..000000000 --- a/src/panoptes/utils/serial/handlers/protocol_hooked.py +++ /dev/null @@ -1,31 +0,0 @@ -# This module enables a test to provide a handler for "hooked://..." urls -# passed into serial.serial_for_url. To do so, set the value of -# serial_class_for_url from your test to a function with the same API as -# ExampleSerialClassForUrl. Or assign your class to Serial. - -from panoptes.utils.serial.handlers.protocol_no_op import NoOpSerial - - -def ExampleSerialClassForUrl(url): - """Implementation of serial_class_for_url called by serial.serial_for_url. - - Returns the url, possibly modified, and a factory function to be called to - create an instance of a SerialBase sub-class (or at least behaves like it). - You can return a class as that factory function, as calling a class creates - an instance of that class. - - serial.serial_for_url will call that factory function with None as the - port parameter (the first), and after creating the instance will assign - the url to the port property of the instance. - - Returns: - A tuple (url, factory). - """ - return url, Serial - - -# Assign to this global variable from a test to override this default behavior. -serial_class_for_url = ExampleSerialClassForUrl - -# Or assign your own class to this global variable. -Serial = NoOpSerial diff --git a/src/panoptes/utils/serial/handlers/protocol_no_op.py b/src/panoptes/utils/serial/handlers/protocol_no_op.py deleted file mode 100644 index acc909102..000000000 --- a/src/panoptes/utils/serial/handlers/protocol_no_op.py +++ /dev/null @@ -1,120 +0,0 @@ -# This module implements a handler for serial_for_url("no_op://"). - -# Export it as Serial so that it will be picked up by PySerial's serial_for_url. -from serial import serialutil - - -class NoOpSerial(serialutil.SerialBase): - """No-op implementation of PySerial's SerialBase. - - Provides no-op implementation of various methods that SerialBase expects - to have implemented by the sub-class. Can be used as is for a /dev/null - type of behavior. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - @property - def in_waiting(self): - """The number of input bytes available to read immediately.""" - return 0 - - def open(self): - """Open port. - - Raises: - SerialException if the port cannot be opened. - """ - self.is_open = True - - def close(self): - """Close port immediately.""" - self.is_open = False - - def read(self, size=1): - """Read size bytes. - - If a timeout is set it may return fewer characters than requested. - With no timeout it will block until the requested number of bytes - is read. - - Args: - size: Number of bytes to read. - - Returns: - Bytes read from the port, of type 'bytes'. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - if not self.is_open: - raise serialutil.PortNotOpenError - return bytes() - - def write(self, data): - """ - Args: - data: The data to write. - - Returns: - Number of bytes written. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - if not self.is_open: - raise serialutil.PortNotOpenError - return 0 - - def reset_input_buffer(self): - """Remove any accumulated bytes from the device.""" - pass - - def reset_output_buffer(self): - """Remove any accumulated bytes not yet sent to the device.""" - pass - - # -------------------------------------------------------------------------- - # There are a number of methods called by SerialBase that need to be - # implemented by sub-classes, assuming their calls haven't been blocked - # by replacing the calling methods/properties. These are no-op - # implementations. - - def _reconfigure_port(self): - """Reconfigure the open port after a property has been changed. - - If you need to know which property has been changed, override the - setter for the appropriate properties. - """ - pass - - def _update_rts_state(self): - """Handle rts being set to some value. - - "self.rts = value" has been executed, for some value. This may not - have changed the value. - """ - pass - - def _update_dtr_state(self): - """Handle dtr being set to some value. - - "self.dtr = value" has been executed, for some value. This may not - have changed the value. - """ - pass - - def _update_break_state(self): - """Handle break_condition being set to some value. - - "self.break_condition = value" has been executed, for some value. - This may not have changed the value. - Note that break_condition is set and then cleared by send_break(). - """ - pass - - -Serial = NoOpSerial diff --git a/src/panoptes/utils/time.py b/src/panoptes/utils/time.py index b4eb15b4d..ade10429c 100644 --- a/src/panoptes/utils/time.py +++ b/src/panoptes/utils/time.py @@ -7,6 +7,7 @@ from astropy import units as u from astropy.time import Time from loguru import logger + from panoptes.utils import error @@ -101,25 +102,41 @@ def flatten_time(t): return t.isot.replace('-', '').replace(':', '').split('.')[0] -# This is a streamlined variant of PySerial's serialutil.Timeout. class CountdownTimer(object): - """Simple timer object for tracking whether a time duration has elapsed. - - - Args: - duration (int or float or astropy.units.Quantity): Amount of time to before time expires. - May be numeric seconds or an Astropy time duration (e.g. 1 * u.minute). - """ def __init__(self, duration: Union[int, float], name: str = ''): + """Simple timer object for tracking whether a time duration has elapsed. + + Examples: + + >>> timer = CountdownTimer(1) + >>> timer.time_left() > 0 + True + >>> timer.expired() + False + >>> # Sleep less than the duration returns True. + >>> timer.sleep(max_sleep=0.1) + True + >>> # Sleep more than the duration returns False. + >>> timer.sleep() + False + >>> timer.time_left() == 0 + True + >>> timer.expired() + True + >>> print(timer) + EXPIRED Timer 0.00/1.00 + + Args: + duration (int or float or astropy.units.Quantity): Amount of time to before time expires. + May be numeric seconds or an Astropy time duration (e.g. 1 * u.minute). + """ if isinstance(duration, u.Quantity): duration = duration.to(u.second).value elif not isinstance(duration, (int, float)): raise ValueError(f'duration ({duration}) is not a supported type: {type(duration)}') - #: bool: True IFF the duration is zero. assert duration >= 0, "Duration must be non-negative." - self.is_non_blocking = (duration == 0) self.name = f'{name}Timer' self.target_time = None @@ -127,13 +144,10 @@ def __init__(self, duration: Union[int, float], name: str = ''): self.restart() def __str__(self): - is_blocking = '' - if self.is_non_blocking is False: - is_blocking = '(blocking)' is_expired = '' if self.expired(): is_expired = 'EXPIRED' - return f'{is_expired} {self.name} {is_blocking} {self.time_left():.02f}/{self.duration:.02f}' + return f'{is_expired} {self.name} {self.time_left():.02f}/{self.duration:.02f}' def expired(self): """Return a boolean, telling if the timeout has expired. @@ -149,23 +163,20 @@ def time_left(self): Returns: int: Number of seconds remaining in timer, zero if ``is_non_blocking=True``. """ - if self.is_non_blocking: - return 0 + delta = self.target_time - time.monotonic() + if delta > self.duration: # pragma: no cover + # clock jumped, recalculate + self.restart() + return self.duration else: - delta = self.target_time - time.monotonic() - if delta > self.duration: - # clock jumped, recalculate - self.restart() - return self.duration - else: - return max(0.0, delta) + return max(0.0, delta) def restart(self): """Restart the timed duration.""" self.target_time = time.monotonic() + self.duration logger.debug(f'Restarting {self.name}') - def sleep(self, max_sleep: Union[int, float, None] = None, log_level:str='DEBUG'): + def sleep(self, max_sleep: Union[int, float, None] = None, log_level: str = 'DEBUG'): """Sleep until the timer expires, or for max_sleep, whichever is sooner. Args: @@ -197,7 +208,7 @@ def wait_for_events(events, ): """Wait for event(s) to be set. - This method will wait for a maximum of `timeout` seconds for all of the `events` + This method will wait for a maximum of `timeout` seconds for all the `events` to complete. Checks every `sleep_delay` seconds for the events to be set. diff --git a/src/panoptes/utils/utils.py b/src/panoptes/utils/utils.py index 974f2dada..51a4c91cc 100644 --- a/src/panoptes/utils/utils.py +++ b/src/panoptes/utils/utils.py @@ -129,7 +129,7 @@ def altaz_to_radec(alt=None, az=None, location=None, obstime=None, **kwargs): az = get_quantity_value(az, 'degree') * u.degree altaz = AltAz(obstime=obstime, location=location, alt=alt, az=az) - return SkyCoord(altaz.transform_to(ICRS)) + return SkyCoord(altaz.transform_to(ICRS())) def get_quantity_value(quantity, unit=None): diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 81a724bb3..56869a6d9 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -1,7 +1,8 @@ -import os +from pathlib import Path import pytest from astropy import units as u + from panoptes.utils.config.helpers import load_config from panoptes.utils.config.helpers import save_config from panoptes.utils.serializers import to_yaml @@ -26,21 +27,21 @@ def test_load_config_custom_file(tmp_path): temp_conf_local_file.write_text(to_yaml(temp_conf_text)) # Ignore the local name - temp_config = load_config(str(temp_conf_file.absolute()), load_local=False) + temp_config = load_config(temp_conf_file.absolute(), load_local=False) assert len(temp_config) == 2 assert temp_config['name'] == 'Temporary Name' assert temp_config['location']['elevation'] == 1234.56 * u.m assert isinstance(temp_config['location'], dict) # Load the local - temp_config = load_config(str(temp_conf_local_file.absolute()), load_local=True) + temp_config = load_config(temp_conf_local_file.absolute(), load_local=True) assert len(temp_config) == 2 assert temp_config['name'] == 'Local Name' assert temp_config['location']['elevation'] == 1234.56 * u.m assert isinstance(temp_config['location'], dict) # Reload the local but don't parse, load_local is default. - temp_config = load_config(str(temp_conf_file.absolute()), parse=False) + temp_config = load_config(temp_conf_file.absolute(), parse=False) assert len(temp_config) == 2 assert temp_config['name'] == 'Local Name' assert temp_config['location']['elevation'] == '1234.56 m' @@ -51,22 +52,21 @@ def test_save_config_custom_file(tmp_path): """Test saving with a custom file""" temp_conf_file = tmp_path / 'temp_conf.yaml' - save_config(str(temp_conf_file), dict(foo=1, bar=2), overwrite=False) + save_config(temp_conf_file, dict(foo=1, bar=2), overwrite=False) - temp_local = str(temp_conf_file).replace('.yaml', '_local.yaml') - assert os.path.exists(temp_local) + assert Path(tmp_path / 'temp_conf_local.yaml').exists() - temp_config = load_config(str(temp_conf_file), load_local=True) + temp_config = load_config(temp_conf_file, load_local=True) assert temp_config['foo'] == 1 - temp_config = load_config(str(temp_conf_file), load_local=False) + temp_config = load_config(temp_conf_file, load_local=False) assert temp_config == dict() with pytest.raises(FileExistsError): - save_config(str(temp_conf_file), dict(foo=2, bar=2), overwrite=False) + save_config(temp_conf_file, dict(foo=2, bar=2), overwrite=False) - save_config(str(temp_conf_file), dict(foo=2, bar=2), overwrite=True) - temp_config = load_config(str(temp_conf_file)) + save_config(temp_conf_file, dict(foo=2, bar=2), overwrite=True) + temp_config = load_config(temp_conf_file) assert temp_config['foo'] == 2 @@ -76,11 +76,12 @@ def test_save_config_custom_local_file(tmp_path): temp_conf_local_file = tmp_path / 'temp_conf_local.yaml' # Save the local directly. - save_config(str(temp_conf_local_file), dict(foo=1, bar=2), overwrite=False) - assert os.path.exists(temp_conf_local_file) + assert temp_conf_local_file.exists() is False + save_config(temp_conf_local_file, dict(foo=1, bar=2), overwrite=False) + assert temp_conf_local_file.exists() - temp_config = load_config(str(temp_conf_file), load_local=False) + temp_config = load_config(temp_conf_file, load_local=False) assert temp_config == dict() - temp_config = load_config(str(temp_conf_file)) + temp_config = load_config(temp_conf_file) assert temp_config['foo'] == 1 diff --git a/tests/images/test_focus_utils.py b/tests/images/test_focus_utils.py index 8ea4f43b7..2633c7fff 100644 --- a/tests/images/test_focus_utils.py +++ b/tests/images/test_focus_utils.py @@ -1,10 +1,10 @@ import os -import pytest +import pytest from astropy.io import fits -from panoptes.utils.images import mask_saturated from panoptes.utils.images import focus as focus_utils +from panoptes.utils.images.misc import mask_saturated def test_vollath_f4(data_dir): diff --git a/tests/images/test_image_utils.py b/tests/images/test_image_utils.py index 51cbb58d8..c83609db7 100644 --- a/tests/images/test_image_utils.py +++ b/tests/images/test_image_utils.py @@ -3,29 +3,53 @@ import numpy as np import pytest +from astropy import units as u from astropy.nddata import Cutout2D -from panoptes.utils import images as img_utils + from panoptes.utils import error +from panoptes.utils.images import make_pretty_image +from panoptes.utils.images.misc import crop_data, mask_saturated + + +def test_mask_saturated(): + ones = np.ones((10, 10)) + ones[0, 0] = 256 + # Bit-depth. + assert mask_saturated(ones, bit_depth=8).sum() == 99.0 + # Bit-depth with unit. + assert mask_saturated(ones, bit_depth=8 * u.bit).sum() == 99.0 + # Array has int dtype so bit_depth is inferred. + assert mask_saturated(ones.astype('int8')).sum() == 99.0 + + +def test_mask_saturated_bad(): + ones = np.ones((10, 10)) + ones[0, 0] = 256 + with pytest.raises(error.IllegalValue): + mask_saturated(ones, bit_depth=8 * u.meter) + + with pytest.raises(error.IllegalValue): + mask_saturated(ones) def test_crop_data(): ones = np.ones((201, 201)) assert ones.sum() == 40401. - cropped01 = img_utils.crop_data(ones) # False to exercise coverage. + cropped01 = crop_data(ones) # False to exercise coverage. assert cropped01.sum() == 40000. - cropped02 = img_utils.crop_data(ones, box_width=10) + cropped02 = crop_data(ones, box_width=10) assert cropped02.sum() == 100. - cropped03 = img_utils.crop_data(ones, box_width=6, center=(50, 50)) + cropped03 = crop_data(ones, box_width=6, center=(50, 50)) assert cropped03.sum() == 36. # Test the Cutout2D object - cropped04 = img_utils.crop_data(ones, - box_width=20, - center=(50, 50), - data_only=False) + cropped04 = crop_data(ones, + box_width=20, + center=(50, 50), + data_only=False) assert isinstance(cropped04, Cutout2D) assert cropped04.position_original == (50, 50) @@ -40,7 +64,7 @@ def test_make_pretty_image(solved_fits_file, tiny_fits_file, save_environ): # Can't operate on a non-existent files. with pytest.warns(UserWarning, match="File doesn't exist"): - assert not img_utils.make_pretty_image('Foobar') + assert not make_pretty_image('Foobar') # Can handle the fits file, and creating the images dir for linking # the latest image. @@ -49,42 +73,43 @@ def test_make_pretty_image(solved_fits_file, tiny_fits_file, save_environ): os.makedirs(imgdir, exist_ok=True) link_path = os.path.join(tmpdir, 'latest.jpg') - pretty = img_utils.make_pretty_image(solved_fits_file, link_path=link_path) - assert pretty - assert os.path.isfile(pretty) + pretty = make_pretty_image(solved_fits_file, link_path=link_path) + assert pretty.exists() + assert pretty.is_file() assert os.path.isdir(imgdir) - assert link_path == pretty + assert link_path == pretty.as_posix() os.remove(link_path) os.rmdir(imgdir) # Try again, but without link_path. - pretty = img_utils.make_pretty_image(tiny_fits_file, title='some text') - assert pretty - assert os.path.isfile(pretty) + pretty = make_pretty_image(tiny_fits_file, title='some text') + assert pretty.exists() + assert pretty.is_file() assert not os.path.isdir(imgdir) -@pytest.mark.skipif( - "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", - reason="Skipping this test on Travis CI.") def test_make_pretty_image_cr2_fail(): with tempfile.TemporaryDirectory() as tmpdir: tmpfile = os.path.join(tmpdir, 'bad.cr2') with open(tmpfile, 'w') as f: f.write('not an image file') - with pytest.raises(error.InvalidCommand): - img_utils.make_pretty_image(tmpfile, - title='some text') - with pytest.raises(error.InvalidCommand): - img_utils.make_pretty_image(tmpfile) + with pytest.raises(error.InvalidSystemCommand): + make_pretty_image(tmpfile, title='some text') + with pytest.raises(error.AlreadyExists): + make_pretty_image(tmpfile) + + no_image = make_pretty_image('not-a-file') + assert no_image is None def test_make_pretty_image_cr2(cr2_file, tmpdir): link_path = str(tmpdir.mkdir('images').join('latest.jpg')) - pretty_path = img_utils.make_pretty_image(cr2_file, - title='CR2 Test', - image_type='cr2', - link_path=link_path) - - assert os.path.exists(pretty_path) - assert pretty_path == link_path + print(f'link_path: {link_path} cr2_file: {cr2_file}') + pretty_path = make_pretty_image(cr2_file, + title='CR2 Test', + link_path=link_path, + remove_cr2=True) + + assert pretty_path.exists() + assert pretty_path.as_posix() == link_path + assert os.path.exists(cr2_file) is False diff --git a/tests/test_rs232.py b/tests/test_rs232.py index 78e15d2ea..cfdc3a2be 100644 --- a/tests/test_rs232.py +++ b/tests/test_rs232.py @@ -1,12 +1,8 @@ -import io - import pytest -import serial + from panoptes.utils import error from panoptes.utils import rs232 -from panoptes.utils.serial.handlers import protocol_buffers, protocol_hooked -from panoptes.utils.serial.handlers.protocol_no_op import NoOpSerial -from serial.serialutil import PortNotOpenError +from panoptes.utils.serializers import to_json def test_port_discovery(): @@ -22,181 +18,32 @@ def test_missing_port(): def test_non_existent_device(): """Doesn't complain if it can't find the device.""" port = '/dev/tty12345698765' - ser = rs232.SerialData(port=port) - assert not ser.is_connected - assert port == ser.name - # Can't connect to that device. with pytest.raises(error.BadSerialConnection): - ser.connect() - assert not ser.is_connected - - -@pytest.fixture(scope='function') -def handler(): - # Install our package that contain the test handlers. - serial.protocol_handler_packages.append('panoptes.utils.serial.handlers') - yield True - # Remove that package. - serial.protocol_handler_packages.remove('panoptes.utils.serial.handlers') - - -def test_detect_bogus_scheme(handler): - """When our handlers are installed, will still detect unknown scheme.""" - with pytest.raises(ValueError) as excinfo: - # The scheme (the part before the ://) must be a Python module name, so use - # a string that can't be a module name. - rs232.SerialData(port='# bogus #://') - assert '# bogus #' in repr(excinfo.value) - - -def test_basic_no_op(handler): - # Confirm we can create the SerialData object. - ser = rs232.SerialData(port='no_op://', name='a name', open_delay=0) - assert ser.name == 'a name' + ser = rs232.SerialData(port=port) - # Peek inside, it should have a NoOpSerial instance as member ser. - assert ser.ser - assert isinstance(ser.ser, NoOpSerial) - # Open is automatically called by SerialData. +def test_usage(): + port = 'loop://' + ser = rs232.SerialData(port=port, open_delay=0.1) assert ser.is_connected - - # connect() is idempotent. ser.connect() - assert ser.is_connected - - # Several passes of reading, writing, disconnecting and connecting. - for _ in range(3): - # no_op handler doesn't do any reading, analogous to /dev/null, which - # never produces any output. - assert '' == ser.read(retry_delay=0.01, retry_limit=2) - assert b'' == ser.read_bytes(size=1) - assert 0 == ser.write('abcdef') - ser.reset_input_buffer() - - # Disconnect from the serial port. - assert ser.is_connected - ser.disconnect() - assert not ser.is_connected - - # Should no longer be able to read or write. - with pytest.raises(AssertionError): - ser.read(retry_delay=0.01, retry_limit=1) - with pytest.raises(AssertionError): - ser.read_bytes(size=1) - with pytest.raises(AssertionError): - ser.write('a') - ser.reset_input_buffer() - - # And we should be able to reconnect. - assert not ser.is_connected - ser.connect() - assert ser.is_connected - - -def test_basic_io(handler): - protocol_buffers.reset_serial_buffers(b'abc\r\ndef\n') - ser = rs232.SerialData(port='buffers://', open_delay=0.01, retry_delay=0.01, retry_limit=2) - - # Peek inside, it should have a BuffersSerial instance as member ser. - assert isinstance(ser.ser, protocol_buffers.BuffersSerial) - # Can read two lines. Read the first as a sensor reading: - (ts, line) = ser.get_reading() - assert 'abc\r\n' == line + write_bytes = ser.write('Hello world\n') + assert write_bytes == 12 + read_line = ser.read(write_bytes) + assert read_line == 'Hello world\n' - # Read the second line from the read buffer. - assert 'def\n' == ser.read(retry_delay=0.1, retry_limit=10) + ser.write('A new line') + ts, reading = ser.get_reading() + assert reading == 'A new line' - # Another read will fail, having exhausted the contents of the read buffer. - assert '' == ser.read() - - # Can write to the "device", the handler will accumulate the results. - assert 5 == ser.write('def\r\n') - assert 6 == ser.write('done\r\n') - - assert b'def\r\ndone\r\n' == protocol_buffers.get_serial_write_buffer() - - # If we add more to the read buffer, we can read again. - protocol_buffers.set_serial_read_buffer(b'line1\r\nline2\r\ndangle') - assert 'line1\r\n' == ser.read(retry_delay=10, retry_limit=20) - assert 'line2\r\n' == ser.read(retry_delay=10, retry_limit=20) - assert 'dangle' == ser.read(retry_delay=10, retry_limit=20) - - ser.disconnect() - assert not ser.is_connected - - -class HookedSerialHandler(NoOpSerial): - """Sources a line of text repeatedly, and sinks an infinite amount of input.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.r_buffer = io.BytesIO(b"{'a': 12, 'b': [1, 2, 3, 4], 'c': {'d': 'message'}}\r\n") - - @property - def in_waiting(self): - """The number of input bytes available to read immediately.""" - if not self.is_open: - raise PortNotOpenError - total = len(self.r_buffer.getbuffer()) - avail = total - self.r_buffer.tell() - # If at end of the stream, reset the stream. - if avail <= 0: - self.r_buffer.seek(0) - avail = total - return avail - - def open(self): - """Open port.""" - self.is_open = True - - def close(self): - """Close port immediately.""" - self.is_open = False - - def read(self, size=1): - """Read until the end of self.r_buffer, then seek to beginning of self.r_buffer.""" - if not self.is_open: - raise PortNotOpenError - # If at end of the stream, reset the stream. - return self.r_buffer.read(min(size, self.in_waiting)) - - def write(self, data): - """Write noop.""" - if not self.is_open: - raise PortNotOpenError - return len(data) - - -def test_hooked_io(handler): - protocol_hooked.Serial = HookedSerialHandler - ser = rs232.SerialData(port='hooked://', open_delay=0) - - # Peek inside, it should have a PySerial instance as member ser. - assert ser.ser - assert ser.ser.__class__.__name__ == 'HookedSerialHandler' - print(str(ser.ser)) - - # Open is automatically called by SerialData. - assert ser.is_connected + ser.write(to_json(dict(message='Hello world'))) + reading = ser.get_and_parse_reading() - # Can read many identical lines from ser. - first_line = None - for n in range(20): - line = ser.read(retry_delay=10, retry_limit=20) - if first_line: - assert line == first_line - else: - first_line = line - assert 'message' in line - reading = ser.get_reading() - assert reading[1] == line + ser.reset_input_buffer() - # Can write to the "device" many times. - line = f'{"foobar" * 30}\r\n' - for n in range(20): - assert len(line) == ser.write(line) + bytes = ser.write('a') + ser.read_bytes(bytes) ser.disconnect() assert not ser.is_connected diff --git a/tests/test_serial_device.py b/tests/test_serial_device.py index be126ca4c..ba365f762 100644 --- a/tests/test_serial_device.py +++ b/tests/test_serial_device.py @@ -19,7 +19,6 @@ def test_device(): s1 = SerialDevice(port='loop://') assert str(s1) == 'SerialDevice loop:// [9600/8-N-1]' - s1.serial = False s1.disconnect() diff --git a/tests/test_time.py b/tests/test_time.py index 7db55af34..0c66528e4 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -43,7 +43,6 @@ def test_countdown_timer_bad_input(): def test_countdown_timer_non_blocking(): timer = CountdownTimer(0) - assert timer.is_non_blocking assert timer.time_left() == 0 for arg, expected_duration in [(2, 2.0), (0.5, 0.5), (1 * u.second, 1.0)]: @@ -56,7 +55,6 @@ def test_countdown_timer(): timer = CountdownTimer(count_time) assert timer.time_left() > 0 assert timer.expired() is False - assert timer.is_non_blocking is False counter = 0. while timer.time_left() > 0: @@ -66,6 +64,7 @@ def test_countdown_timer(): assert counter == pytest.approx(1) assert timer.time_left() == 0 assert timer.expired() is True + assert str(timer) == 'EXPIRED Timer 0.00/1.00' def test_countdown_timer_sleep(): @@ -73,7 +72,6 @@ def test_countdown_timer_sleep(): timer = CountdownTimer(count_time) assert timer.time_left() > 0 assert timer.expired() is False - assert timer.is_non_blocking is False counter = 0. while timer.time_left() > 0.5: