Merge pull request #63 from HBS-HBX/#30_multiprocessing_tests

closes #30 multiprocessing tests

.travis.yml

@@ -9,6 +9,14 @@ branches:
   only:
     - master
 
+services:
+  - postgresql
+
+# by default travis uses postgres 9.2, and django 2.1 dropped support for it
+# this enables 9.4
+addons:
+  postgresql: "9.4"
+
 matrix:
   include:
     - env: TOX_ENV=py35-django110-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
@@ -19,6 +27,14 @@ matrix:
       python: 3.5
     - env: TOX_ENV=py35-django21-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
       python: 3.5
+    - env: TOX_ENV=py36-django110-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+      python: 3.6
+    - env: TOX_ENV=py36-django111-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+      python: 3.6
+    - env: TOX_ENV=py36-django20-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+      python: 3.6
+    - env: TOX_ENV=py36-django21-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+      python: 3.6
 
 before_install:
   - pip install --upgrade pip
@@ -30,8 +46,11 @@ before_install:
 
 install: pip install -r requirements/travis.txt
 
-# sleep for elasticsearch
 before_script:
+  - psql -c "CREATE DATABASE pgdb;" -U postgres
+  - psql -c "CREATE USER pguser WITH PASSWORD 'pgpass';" -U postgres
+  - psql -c "ALTER USER pguser CREATEDB;" -U postgres
+  # sleep for elasticsearch
   - sleep 10
 
 # command to run tests

CHANGELOG.rst

@@ -1,6 +1,16 @@
 Changelog
 ---------
 
+0.9.0 (2018-11-27)
+^^^^^^^^^^^^^^^^^^
+* added postgres to docker-compose and travis, and started using postgres for all tests instead of sqlite
+* fix `#30 multiprocessing tests <https://github.com/HBS-HBX/django-elastic-migrations/issues/30>`_
+* fix ``tests.search.MovieSearchDoc.get_model_full_text()`` not indexing non-title fields
+* fix ``./manage.py es_update -v=3`` not respecting the verbosity flag
+* fix ``DEMIndexManager.initialize()`` not updating ``DEMDocType`` to point to new active version after ``es_activate``
+* document ``DEMTestCaseMixin`` as the recommended way to support ``django-elastic-migrations`` in tests
+* deprecate ``DEMIndexManager.test_pre_setup``; will be removed in future version
+
 0.8.2 (2018-11-20)
 ^^^^^^^^^^^^^^^^^^
 * fix `#59 twine check error in 0.8.1 <https://github.com/HBS-HBX/django-elastic-migrations/issues/59>`_

README.rst

@@ -249,29 +249,16 @@ Django Testing
 ^^^^^^^^^^^^^^
 
 
-#. (optional) update ``DJANGO_ELASTIC_MIGRATIONS_ENVIRONMENT_PREFIX`` in
-   your Django settings. The default test prefix is ``test_``.  Every
-   test will create its own indexes.
+#. Override ``TestCase`` to provide test isolation when search indexes are involved
    ::
 
-       if 'test' in sys.argv:
-           DJANGO_ELASTIC_MIGRATIONS_ENVIRONMENT_PREFIX = 'test_'
+       from django_elastic_migrations.utils.test_utils import DEMTestCaseMixin
 
-#. Override TestCase - ``test_utilities.py``
+       class MyTestCase(DEMTestCaseMixin, TestCase):
+           """
+           Set up and tear down temporary elasticsearch test indexes for each test
+           """
 
-   .. code-block::
-
-       from django_elastic_migrations import DEMIndexManager
-
-       class MyTestCase(TestCase):
-
-           def _pre_setup(self):
-               DEMIndexManager.test_pre_setup()
-               super(MyTestCase, self)._pre_setup()
-
-           def _post_teardown(self):
-               DEMIndexManager.test_post_teardown()
-               super(MyTestCase, self)._post_teardown()
 
 Excluding from Django's ``dumpdata`` command
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -321,7 +308,7 @@ to see the available ``make`` targets.
 Elasticsearch Docker Compose
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-``docker-compose -f local.yml up``
+``docker-compose up``
 
 `See docs/docker_setup for more info <./docs/docker_setup.rst>`_
 

django_elastic_migrations/__init__.py

@@ -8,7 +8,7 @@
 from django_elastic_migrations.utils import loading
 from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger
 
-__version__ = '0.8.2'
+__version__ = '0.9.0'
 
 default_app_config = 'django_elastic_migrations.apps.DjangoElasticMigrationsConfig'  # pylint: disable=invalid-name
 

django_elastic_migrations/indexes.py

@@ -103,7 +103,8 @@ def create_and_activate_version_for_each_index_if_none_is_active(cls, create_ver
     @classmethod
     def initialize(cls, create_versions=False, activate_versions=False):
         """
-        Configure DEMIndexManager, loading settings from the database.
+        Configure DEMIndexManager.index_models and DEMIndexManager.instances,
+        index_models from the database, and loading instances from the django settings
         :param create_versions: if True, create a version if settings
                have been changed.
         :param activate_versions: if True, activate the latest version
@@ -282,11 +283,24 @@ def reinitialize_esindex_instances(cls):
 
     @classmethod
     def test_pre_setup(cls):
+        """
+        Called *AFTER* setup in DEMTestCase; this method will be removed in favor of test_pre_teardown()
+        """
+        logger.warning("DEMIndexManager.test_pre_setup is deprecated in 0.9.0 and will be removed in a future version. "
+                       "Please update your code to use DEMIndexManager.test_post_setup() instead.")
+        cls.test_post_setup()
+
+    @classmethod
+    def test_post_setup(cls):
         cls.test_post_teardown()
         DEMIndexManager.initialize(create_versions=True, activate_versions=True)
 
     @classmethod
     def test_post_teardown(cls):
+        cls.test_pre_teardown()
+
+    @classmethod
+    def test_pre_teardown(cls):
         try:
             DEMIndexManager.drop_index(
                 'all', force=True, just_prefix=es_test_prefix, hard_delete=True)
@@ -849,21 +863,28 @@ def delete(self, **kwargs):
         if index_version:
             index_version.delete()
 
-    def doc_type(self, doc_type=None):
+    def doc_type(self, doc_type=None) -> DEMDocType:
         """
-        Overrides elasticsearch_dsl.Index.doc_type().
-        Associates a DEMDocType with this DEMIndex, which is a bidirectional
-        association.
+        Overrides elasticsearch_dsl.Index.doc_type() and called during DEMIndexManager.initialize().
+
+        Associates a DEMDocType with this DEMIndex, so that commands
+        directed to this DEMIndex always go to the active index version by
+        querying the DEMIndexManager.
 
-        In the case that this DEMIndex has been instantiated
-        as DEMIndex(name, version_id) and attempting to retrieve a doc_type:
-        IF the index version requires a different version of the codebase,
-        this method will raise DEMIndexVersionCodebaseMismatchError.
+        Sets the active index version name into the DEMDocType, so that
+        DEMDocType.search() queries the active index version.
 
         :returns DEMDocType associated with this DEMIndex (if any)
         """
         if doc_type:
             self.__doc_type = doc_type
+            active_version_name = self.get_active_version_index_name()
+
+            # set the active index version name into the DEMDocType,
+            # so DEMDocType.search() is directed to the active index in elasticsearch
+            if active_version_name and self.__doc_type._doc_type:
+                self.__doc_type._doc_type.index = active_version_name
+
             return super(DEMIndex, self).doc_type(doc_type)
         else:
             if self.get_version_id() and not self.__doc_type:

django_elastic_migrations/models.py

@@ -248,7 +248,7 @@ class IndexAction(models.Model):
     def __init__(self, *args, **kwargs):
         action = self._meta.get_field('action')
         action.default = self.DEFAULT_ACTION
-        self.verbosity = kwargs.pop('verbosity', 1)
+        self.verbosity = getattr(self, 'verbosity', None) or kwargs.pop('verbosity', 1)
         super(IndexAction, self).__init__(*args, **kwargs)
 
     def __str__(self):
@@ -848,6 +848,7 @@ def __init__(self, *args, **kwargs):
             'workers': 0,
             'batch_size': 0,
             'start_date': None,
+            'verbosity': 1
         }
         self.self_kwargs = {}
         # loop through each expected kwarg, and fill in self.resume_mode, etc
@@ -1214,7 +1215,7 @@ def perform_action(self, dem_index, *args, **kwargs):
                 " # parent total docs expected: {_total_docs_expected}\n"
                 " # parent total docs remaining: {_total_docs_remaining} ({_total_docs_remaining_pct}%)\n"
                 " # parent estimated runtime remaining: {_expected_parent_runtime}\n"
-                " # num workers {_workers}\n"
+                " # num workers: {_workers}\n"
                 " # pid: {_pid}\n"
                 " # IndexAction id: {id}\n"
             ),

django_elastic_migrations/utils/test_utils.py

@@ -1,14 +1,21 @@
-from django.test import TestCase
-
 from django_elastic_migrations import DEMIndexManager
 
 
-class DEMTestCase(TestCase):
+class DEMTestCaseMixin(object):
+    """
+    Mix into TestCase or TransactionTestCase to set up and then later clear out
+    temporary elasticsearch indexes for each test
+    """
 
     def setUp(self):
-        DEMIndexManager.test_pre_setup()
-        super(DEMTestCase, self)._pre_setup()
+        super(DEMTestCaseMixin, self).setUp()
+        DEMIndexManager.test_post_setup()
 
     def tearDown(self):
+        DEMIndexManager.test_pre_teardown()
+        super(DEMTestCaseMixin, self).tearDown()
+
+    @classmethod
+    def tearDownClass(cls):
         DEMIndexManager.test_post_teardown()
-        super(DEMTestCase, self).tearDown()
+        super().tearDownClass()

local.yml → docker-compose.yml

@@ -23,6 +23,15 @@ services:
       interval: 10s
       timeout: 10s
       retries: 10
+  db:
+    image: postgres
+    restart: always
+    ports:
+    - "5432:5432"
+    environment:
+      POSTGRES_USER: "pguser"
+      POSTGRES_PASSWORD: "pgpass"
+      POSTGRES_DB: "pgdb"
 
 # uncomment to get kibana, which can be useful for analyzing index contents
 # this is currently not a required part of testing infrastructure, just left here for convenience

docs/docker_setup.rst

@@ -23,7 +23,7 @@ Build the Stack (currently unnecessary)
 Currently, the dev stack consists of prebuilt images. In the future,
 if custom docker images are added to the compose file, this may be necessary::
 
-    $ docker-compose -f local.yml build
+    $ docker-compose build --no-cache
 
 
 Run the Stack
@@ -33,19 +33,15 @@ This brings up Elasticsearch. The first time it is run it might take a while to
 
 Open a terminal at the project root and run the following for local development::
 
-    $ docker-compose -f local.yml up
-
-You can also set the environment variable ``COMPOSE_FILE`` pointing to ``local.yml`` like this::
-
-    $ export COMPOSE_FILE=local.yml
+    $ docker-compose up
 
-And then run::
+To rebuild the stack after ``docker-compose build --no-cache``, run this first::
 
-    $ docker-compose up
+    $ docker-compose down
 
-To run in a detached (background) mode, just::
+To run the psql shell for postgres::
 
-    $ docker-compose up -d
+    $ docker-compose exec db psql "postgresql://pguser:pgpass@localhost/pgdb"
 
 
 Tips & Tricks

requirements/dev.in

@@ -1,9 +1,11 @@
 # Additional requirements for development of this application
 
 edx-lint                  # includes pylint, pylint-django, etc; configured use pylintrc
+psycopg2                  # for postgres
 diff-cover                # Changeset diff test coverage
 pip-tools                 # Requirements file management
 tox                       # virtualenv management for tests
 tox-battery               # Makes tox aware of requirements file changes
+tox-pyenv                 # be able to run multiple pyenv versions with tox
 twine                     # uploading new versions to pypi
 wheel                     # For generation of wheels for PyPI

requirements/dev.txt

@@ -35,6 +35,7 @@ packaging==18.0           # via caniusepython3
 pip-tools==3.1.0
 pkginfo==1.4.2            # via twine
 pluggy==0.8.0             # via tox
+psycopg2==2.7.6.1
 py==1.7.0                 # via tox
 pycodestyle==2.4.0
 pydocstyle==3.0.0
@@ -54,11 +55,12 @@ snowballstemmer==1.2.1    # via pydocstyle
 texttable==1.5.0
 toml==0.10.0              # via tox
 tox-battery==0.5.1
+tox-pyenv==1.1.0
 tox==3.5.3
 tqdm==4.28.1              # via twine
 twine==1.12.1
 urllib3==1.24.1           # via elasticsearch, requests
 virtualenv==16.1.0        # via tox
 webencodings==0.5.1       # via bleach
-wheel==0.32.2
+wheel==0.32.3
 wrapt==1.10.11            # via astroid

requirements/test.in

@@ -2,5 +2,6 @@
 
 codecov        # for https://codecov.io/ github integration
 coverage       # for quick, one-off coverage tests
+psycopg2       # for postgres
 tox            # virtualenv management for tests
 tox-battery    # Makes tox aware of requirements file changes

requirements/test.txt

@@ -13,6 +13,7 @@ idna==2.7                 # via requests
 ipaddress==1.0.22         # via elasticsearch-dsl
 multiprocessing-logging==0.2.6
 pluggy==0.8.0             # via tox
+psycopg2==2.7.6.1
 py==1.7.0                 # via tox
 python-dateutil==2.7.5    # via elasticsearch-dsl
 pytz==2018.7              # via django

requirements/travis.in

@@ -1,5 +1,6 @@
 # Requirements for running tests in Travis
 
 codecov                   # Code coverage reporting
+psycopg2                  # for postgres
 tox                       # Virtualenv management for tests
 tox-battery               # Makes tox aware of requirements file changes

requirements/travis.txt

@@ -11,6 +11,7 @@ coverage==4.5.2           # via codecov
 filelock==3.0.10          # via tox
 idna==2.7                 # via requests
 pluggy==0.8.0             # via tox
+psycopg2==2.7.6.1
 py==1.7.0                 # via tox
 requests==2.20.1          # via codecov
 six==1.11.0               # via tox

setup.py

@@ -50,18 +50,17 @@ def get_version(*file_paths):
     include_package_data=True,
     install_requires=[
         # TBD: GH issue #3 includes support for elasticsearch-dsl>=6.2.0
-        "Django>=1.8,<2.1", "elasticsearch-dsl>=6.0.0,<6.2.0", "texttable>=1.2.1",
+        "Django>=1.8,<=2.1", "elasticsearch-dsl>=6.0.0,<6.2.0", "texttable>=1.2.1",
         "multiprocessing-logging>=0.2.6"
     ],
     zip_safe=False,
     keywords='Django Elasticsearch',
     classifiers=[
         'Development Status :: 4 - Beta',
         'Framework :: Django',
-        'Framework :: Django :: 1.9',
-        'Framework :: Django :: 1.10',
         'Framework :: Django :: 1.11',
         'Framework :: Django :: 2.0',
+        'Framework :: Django :: 2.1',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Natural Language :: English',