Rework LMU API

Update API to support multi dimensional memories,
arbitrary hidden components, and various quality
of life improvements.

.gitignore

@@ -3,3 +3,5 @@ lmu.egg-info
 *.swp
 *.ipynb_checkpoints
 __pycache__
+/.idea
+/docs/_build

.nengobones.yml

@@ -14,31 +14,31 @@ contributors_rst: {}
 manifest_in: {}
 
 setup_py:
-  url: https://appliedbrainresearch.com/lmu
+  python_requires: ">=3.6"
   install_req:
-    - nengolib>=0.5.1
-    - tensorflow>=2.0.0
+    - scipy
+    - tensorflow>=2.1.0
+  tests_req:
+    - pytest>=6.1.0
+    - pytest-rng>=1.0.0
   docs_req:
     - matplotlib>=3.0.2
-    - IPython>=7.2.0
-    - notebook>=5.7.4
+    - jupyter>=1.0.0
     - seaborn>=0.9.0
     - sphinx>=1.8
-    - nbsphinx
+    - nbsphinx>=0.3.5
     - nengo_sphinx_theme>=1.2.0
     - numpydoc>=0.6
-  optional_req:
-    - scipy
   classifiers:
     - "Development Status :: 3 - Alpha"
     - "Framework :: Nengo"
     - "Intended Audience :: Science/Research"
     - "License :: Free for non-commercial use"
     - "Operating System :: OS Independent"
     - "Programming Language :: Python "
-    - "Programming Language :: Python :: 3.5"
     - "Programming Language :: Python :: 3.6"
     - "Programming Language :: Python :: 3.7"
+    - "Programming Language :: Python :: 3.8"
     - "Topic :: Scientific/Engineering "
     - "Topic :: Scientific/Engineering :: Artificial Intelligence"
 
@@ -47,24 +47,59 @@ setup_cfg: {}
 docs_conf_py:
   intersphinx_mapping:
     scipy: "https://docs.scipy.org/doc/scipy/reference"
-  analytics_id: UA-41658423-2
+  tagmanager_id: GTM-KWCR2HN
   html_redirects:
     getting_started.html: getting-started.html
+  autoautosummary_change_modules:
+    lmu:
+      - lmu.layers.LMUCell
+      - lmu.layers.LMU
+      - lmu.layers.LMUFFT
+  extensions:
+    - nengo_sphinx_theme.ext.autoautosummary
+  doctest_setup:
+    - import numpy as np
+    - import tensorflow as tf
 
 travis_yml:
-  python: 3.6
+  python: 3.8
+  global_vars:
+    TF_VERSION: tensorflow
   jobs:
     - script: static
+    - script: test
+    - script: test
+      env:
+        TF_VERSION: tensorflow==2.1.0
+      python: 3.6
     - script: docs
+    - script: examples
   pypi_user: arvoelke
-  slack_notifications: "vv0lGEj/xNMFSZDbFsdoMJyEcr9BO8p43KUefmgHfjIQtodZqXdesLl+XJcXW0jxCJlNyy3H8LHYP/mEGJpIsK+tQ7dxlWscLjSGWfcoNiZGy4a1Jp4fF+MZyYvIGlTRJqxrazIrj73tGFptVr2XDE74eO0Z9YaVSJVQw4twEDrWFEAq4foWxV30SkcXfdCkhBwX+43CJyuGE3YFDD/+03me/mdccjNRqCfJ0lURRk7H5tcztryrZy2gpwHV+W73raGTybxlP1xEa1hyLYJO40eH/JfeqBqIDxa5m61Aw+BH/HJ5ZLNlTEUyUB6p7kcIYO9lyko5TY3QSqlX9pK+tK+2DojDlzI97QwgQVbx4WvTJ1JEidfgRqNcTlJOG16RvlyxQjW1u3/QV67bmINus470qQqzIBbdLfM70v+E5Ga/bk+Gk1Z29btB7DxXt4z9dH9z3NXTOLhDpH5WZzpcatrbfSrgMzKtxC+z6oLfDzzio9Fx20RiuHv3P8GtXyyR9WkelMH9GVi7xUBHVCveRVVhNKL555u7NbP5TI6Jc9NZqf7OtrNsRKIY4MfGc9KKjYa+Ks+3PT+yQZ8u/ZMMddMTv73nzLH0pU715/CBl1hQGkKkopukGtKbCpdc666PnRrFy9l21hBqSNqLo/FGPF/Yqr+yTXhuhBhvNZnvFQU="
+  slack_notifications: "wZ7l/X7cVeetmwfup7vCeN74pqFGMC5eaJfy/aqRwVGCbY3aHQKoqJaBBrVef\
+    c+DsJwPPM9HIOGs7jkPY+Y1pFbklAhWCDCvmc+f3fL4/yPWK1u7r8IIHhM3O0YvYrEHfFfZn+V1nAomx1\
+    /wttFOr07LLffaNOn+sFzXgKP1Ebv5MrlCNGY0q3T2A38pwXus60GXaGalI/I/a/vjRdkEmq38eIMqLl0\
+    v94oCy67Gqb/5mlB8eT6SffLGoFCctNpC9G9jyh/eRQp+B+YwksjZz+SOQTIy/mOlk8fU1ZL4f9wPguXh\
+    eKznckuvX+J0o+o5kspyrQveWW2lo1VjmnFfdifCVjTj4AIcmc60Zb2dMzzBcfcA+zeVlhWkWbB65LuuX\
+    kfxxsN97EwP5KVH+Lw7c2ycUyHV8XvKHNzVRLIUuVeOTKLHgGpkVgz/WasT5yBOEbnAyZxxAHQpMbXQr9\
+    POwaq92Kwb7du0sLcsP4agdd2L0YACb4JdwQeeD1c8h6RrXua2ejBJ41e+9ZkZcZGad9P6YVmxvI+hddd\
+    jdzzvBWq6+B/Ubft9AqQXmn4pg0fKkZLjrzsSYGRhXzINxhVBjZD8b5BQIb/Aeix3IFhLmDdSQMhjdETA\
+    gL3jSd9D5t1G5QNp4AdbitqcH+yNoXzZCXye76hrwYr6yrHi3JN9yrM="
   deploy_dists:
     - sdist
     - bdist_wheel
 
 ci_scripts:
   - template: static
   - template: docs
+  - template: examples
+  - template: test
+    coverage: true
+    pip_install:
+      - $TF_VERSION
   - template: deploy
 
 codecov_yml: {}
+
+pre_commit_config_yaml: {}
+
+pyproject_toml: {}

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+# Automatically generated by nengo-bones, do not edit this file directly
+
+repos:
+-   repo: https://github.com/psf/black
+    rev: 20.8b0
+    hooks:
+    - id: black

.travis.yml

@@ -1,7 +1,7 @@
 # Automatically generated by nengo-bones, do not edit this file directly
 
 language: python
-python: 3.6
+python: 3.8
 notifications:
   email:
     on_success: change
@@ -12,7 +12,7 @@ notifications:
     on_success: change
     on_failure: always
     rooms:
-      - secure: "vv0lGEj/xNMFSZDbFsdoMJyEcr9BO8p43KUefmgHfjIQtodZqXdesLl+XJcXW0jxCJlNyy3H8LHYP/mEGJpIsK+tQ7dxlWscLjSGWfcoNiZGy4a1Jp4fF+MZyYvIGlTRJqxrazIrj73tGFptVr2XDE74eO0Z9YaVSJVQw4twEDrWFEAq4foWxV30SkcXfdCkhBwX+43CJyuGE3YFDD/+03me/mdccjNRqCfJ0lURRk7H5tcztryrZy2gpwHV+W73raGTybxlP1xEa1hyLYJO40eH/JfeqBqIDxa5m61Aw+BH/HJ5ZLNlTEUyUB6p7kcIYO9lyko5TY3QSqlX9pK+tK+2DojDlzI97QwgQVbx4WvTJ1JEidfgRqNcTlJOG16RvlyxQjW1u3/QV67bmINus470qQqzIBbdLfM70v+E5Ga/bk+Gk1Z29btB7DxXt4z9dH9z3NXTOLhDpH5WZzpcatrbfSrgMzKtxC+z6oLfDzzio9Fx20RiuHv3P8GtXyyR9WkelMH9GVi7xUBHVCveRVVhNKL555u7NbP5TI6Jc9NZqf7OtrNsRKIY4MfGc9KKjYa+Ks+3PT+yQZ8u/ZMMddMTv73nzLH0pU715/CBl1hQGkKkopukGtKbCpdc666PnRrFy9l21hBqSNqLo/FGPF/Yqr+yTXhuhBhvNZnvFQU="
+      - secure: "wZ7l/X7cVeetmwfup7vCeN74pqFGMC5eaJfy/aqRwVGCbY3aHQKoqJaBBrVefc+DsJwPPM9HIOGs7jkPY+Y1pFbklAhWCDCvmc+f3fL4/yPWK1u7r8IIHhM3O0YvYrEHfFfZn+V1nAomx1/wttFOr07LLffaNOn+sFzXgKP1Ebv5MrlCNGY0q3T2A38pwXus60GXaGalI/I/a/vjRdkEmq38eIMqLl0v94oCy67Gqb/5mlB8eT6SffLGoFCctNpC9G9jyh/eRQp+B+YwksjZz+SOQTIy/mOlk8fU1ZL4f9wPguXheKznckuvX+J0o+o5kspyrQveWW2lo1VjmnFfdifCVjTj4AIcmc60Zb2dMzzBcfcA+zeVlhWkWbB65LuuXkfxxsN97EwP5KVH+Lw7c2ycUyHV8XvKHNzVRLIUuVeOTKLHgGpkVgz/WasT5yBOEbnAyZxxAHQpMbXQr9POwaq92Kwb7du0sLcsP4agdd2L0YACb4JdwQeeD1c8h6RrXua2ejBJ41e+9ZkZcZGad9P6YVmxvI+hdddjdzzvBWq6+B/Ubft9AqQXmn4pg0fKkZLjrzsSYGRhXzINxhVBjZD8b5BQIb/Aeix3IFhLmDdSQMhjdETAgL3jSd9D5t1G5QNp4AdbitqcH+yNoXzZCXye76hrwYr6yrHi3JN9yrM="
 cache: pip
 
 dist: xenial
@@ -23,19 +23,32 @@ env:
     - TEST_ARGS=""
     - BRANCH_NAME="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}"
     - PIP_USE_FEATURE="2020-resolver"
+    - TF_VERSION="tensorflow"
 
 jobs:
   include:
   -
     env:
       SCRIPT="static"
+  -
+    env:
+      SCRIPT="test"
+  -
+    env:
+      TF_VERSION="tensorflow==2.1.0"
+      SCRIPT="test"
+    python: 3.6
   -
     env:
       SCRIPT="docs"
     addons:
       apt:
         packages:
           - pandoc
+  -
+    env:
+      SCRIPT="examples"
+    services: ['xvfb']
   - stage: deploy
     if: branch =~ ^release-candidate-* OR tag =~ ^v[0-9]*
     env: SCRIPT="deploy"

CHANGES.rst

@@ -19,15 +19,59 @@ Release history
    - Removed
    - Fixed
 
-0.1.1 (unreleased)
+0.2.0 (unreleased)
 ==================
 
 **Added**
 
 - Added documentation for package description, installation, usage, API, examples,
-  and project information. (`#20 <https://github.com/abr/lmu/pull/20>`__)
-- Added LMU FFT cell variant and auto-switching LMU class
-  (`#21 <https://github.com/abr/lmu/pull/21>`__)
+  and project information. (`#20`_)
+- Added LMU FFT cell variant and auto-switching LMU class. (`#21`_)
+- LMUs can now be used with any Keras RNN cell (e.g. LSTMs or GRUs) through the
+  ``hidden_cell`` parameter. This can take an RNN cell (like
+  ``tf.keras.layers.SimpleRNNCell`` or ``tf.keras.layers.LSTMCell``) or a feedforward
+  layer (like ``tf.keras.layers.Dense``) or ``None`` (to create a memory-only LMU).
+  The output of the LMU memory component will be fed to the ``hidden_cell``.
+  (`#22`_)
+- Added ``hidden_to_memory``, ``memory_to_memory``, and ``input_to_hidden`` parameters
+  to ``LMUCell``, which can be used to enable/disable connections between components
+  of the LMU. They default to disabled. (`#22`_)
+- LMUs can now be used with multi-dimensional memory components. This is controlled
+  through a new ``memory_d`` parameter of ``LMUCell``. (`#22`_)
+- Added ``dropout`` parameter to ``LMUCell`` (which applies dropout to the input)
+  and ``recurrent_dropout`` (which applies dropout to the ``memory_to_memory``
+  connection, if it is enabled). Note that dropout can be added in the hidden
+  component through the ``hidden_cell`` object. (`#22`_)
+
+**Changed**
+
+- Renamed ``lmu.lmu`` module to ``lmu.layers``. (`#22`_)
+- Combined the ``*_encoders_initializer``parameters of ``LMUCell`` into a single
+  ``kernel_initializer`` parameter. (`#22`_)
+- Combined the ``*_kernel_initializer`` parameters of ``LMUCell`` into a single
+  ``recurrent_kernel_initializer`` parameter. (`#22`_)
+
+**Removed**
+
+- Removed ``Legendre``, ``InputScaled``, ``LMUCellODE``, and ``LMUCellGating``
+  classes. (`#22`_)
+- Removed the ``method``, ``realizer``, and ``factory`` arguments from ``LMUCell``
+  (they will take on the same default values as before, they just cannot be changed).
+  (`#22`_)
+- Removed the ``trainable_*`` arguments from ``LMUCell``. This functionality is
+  largely redundant with the new functionality added for enabling/disabling internal
+  LMU connections. These were primarily used previously for e.g. setting a connection to
+  zero and then disabling learning, which can now be done more efficiently by
+  disabling the connection entirely. (`#22`_)
+- Removed the ``units`` and ``hidden_activation`` parameters of ``LMUCell`` (these are
+  now specified directly in the ``hidden_cell``. (`#22`_)
+- Removed the dependency on ``nengolib``. (`#22`_)
+- Dropped support for Python 3.5, which reached its end of life in September 2020.
+  (`#22`_)
+
+.. _#20: https://github.com/abr/lmu/pull/20
+.. _#21: https://github.com/abr/lmu/pull/21
+.. _#22: https://github.com/abr/lmu/pull/22
 
 0.1.0 (June 22, 2020)
 =====================

docs/api-reference.rst

@@ -9,35 +9,7 @@ API reference
 LMU Layers
 ==========
 
-.. autosummary::
-   :nosignatures:
+.. automodule:: lmu.layers
 
-   lmu.LMU
-
-.. autoclass:: lmu.LMU
-
-
-LMU Cells
-=========
-
-.. autosummary::
-   :nosignatures:
-
-   lmu.LMUCell
-   lmu.LMUCellFFT
-
-.. autoclass:: lmu.LMUCell
-
-.. autoclass:: lmu.LMUCellFFT
-
-.. _api-reference-li:
-
-Legendre Initializer
-====================
-
-.. autosummary::
-   :nosignatures
-
-   lmu.Legendre
-
-.. autoclass:: lmu.Legendre
+    .. autoautosummary:: lmu.layers
+        :nosignatures: