Compatibility with newest versions (#12)

1. Added HllExtension migration
2. Fixed python 3.10 compatibility
3. Fixed django 3.2 compatibility in tests
4. Optimized dependencies
5. Added docker testing infrastructure
6. Replaced custom hll installing with DockerHub image in GitHub testing
7. Replaced python 3.5 with python 3.10, django 2.0 with django 3.2, PostgreSQL 9.5 with PostgreSQL 13 in testing

.docker/db-init.sql

@@ -0,0 +1 @@
+CREATE DATABASE test;

.docker/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

.dockerignore

@@ -0,0 +1,15 @@
+# Docs
+docs/
+
+# Python cache files
+**/__pycache__/
+
+# Private and public keys
+*.key
+*.ppk
+*.pub
+
+# Hidden apps directories
+.github/
+.idea/
+.gitignore

.github/workflows/python-tests.yml

@@ -11,27 +11,26 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
-        postgres-version: [9.5, 9.6, 10, 11, 12]
-        django-version: [2.0, 2.1, 2.2, 3.0, 3.1]
-        exclude:
-        # Django 3.0+ doesn't support python 3.5
-        - python-version: 3.5
-          django-version: 3.0
-        - python-version: 3.5
-          django-version: 3.1
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        postgres-version: ["9.6.17", "10.12", "11.7", "12.2", "13.2"]
+        django-version: ["2.1", "2.2", "3.0", "3.1", "3.2"]
+
+    services:
+      postgres:
+        image: jasei/postgres-hll:${{ matrix.postgres-version }}
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
 
     steps:
     - uses: actions/checkout@v2
 
-    # Install postgres with hll extension. Container is better, but can't be installed with hll
-    - name: Install PostgreSQL
-      run: |
-        sudo apt-get install postgresql-${{ matrix.postgres-version }} postgresql-${{ matrix.postgres-version }}-hll
-        sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/${{ matrix.postgres-version }}/main/postgresql.conf
-        sudo sed -i -E 's/(local\s+all\s+postgres\s+)(peer)/\1trust/g' /etc/postgresql/${{ matrix.postgres-version }}/main/pg_hba.conf
-        sudo systemctl stop postgresql && sudo systemctl start postgresql@${{ matrix.postgres-version }}-main
-
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
@@ -65,13 +64,15 @@ jobs:
 
     - name: Set up test databases
       run: |
-        psql -tc 'SHOW server_version' -U postgres
-        psql -c 'CREATE ROLE test;' -U postgres
-        psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres
-        psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres
-        psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres
-        psql -c 'CREATE DATABASE test OWNER test;' -U postgres
-
+        psql -tc 'SHOW server_version' -U postgres -h localhost
+        psql -c 'CREATE ROLE test;' -U postgres -h localhost
+        psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres -h localhost
+        psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres -h localhost
+        psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres -h localhost
+        psql -c 'CREATE DATABASE test OWNER test;' -U postgres -h localhost
+      env:
+        PGPASSWORD: postgres
+
     - name: Test with unittest
       run: |
         python runtests.py

Dockerfile

@@ -0,0 +1,47 @@
+ARG PYTHON_IMAGE_TAG=latest
+
+FROM python:${PYTHON_IMAGE_TAG}  AS image_stage
+
+ARG APP_TAG="2.1.0"
+
+LABEL \
+  org.label-schema.build-date=Now \
+  org.label-schema.maintainer="[email protected]" \
+  org.label-schema.schema-version="1.0.0-rc1" \
+  org.label-schema.vcs-ref="v${APP_TAG}" \
+  org.label-schema.vcs-url="https://github.com/M1ha-Shvn/django-pg-hll" \
+  org.label-schema.vendor="M1ha-Shvn" \
+  org.label-schema.version="${APP_TAG}"
+
+ENV APP_UID ${APP_UID:-1000}
+ENV APP_GID ${APP_GID:-1000}
+ENV APP_NAME ${APP_NAME:-"app"}
+
+# Configure utf-8 locales to make sure Python correctly handles unicode filenames
+# Configure pip local path to copy data from pip_stage
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DJANGO_SETTINGS_MODULE=tests.settings PYTHONUSERBASE=/pip PATH=/pip/bin:$PATH
+
+RUN set -eu && \
+  groupadd --gid "${APP_GID}" "app" && \
+  useradd --uid ${APP_UID} --gid ${APP_GID} --create-home --shell /bin/bash -d /app app && \
+  mkdir -p /pip && \
+  chmod  755 /app /pip && \
+  chown -R ${APP_UID}:${APP_GID} /app /pip
+
+WORKDIR /app/src
+
+# Install dependencies
+# set -eu "breaks" pipeline on first error
+COPY ./requirements-test.txt /app/requirements-test.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+  set -eu && \
+  python3 -m pip install --upgrade pip setuptools wheel  && \
+  python3 -m pip install --upgrade --requirement /app/requirements-test.txt
+
+COPY . /app/src
+
+RUN python3 setup.py -q install --user
+
+USER ${APP_UID}
+
+CMD ["python3", "runtests.py"]

README.md

@@ -5,13 +5,10 @@
 Provides a django wrapper for [postgresql-hll library by CitusData](https://github.com/citusdata/postgresql-hll#the-importance-of-hashing)
 
 ## Requirements
-* Python Python 3.5+  
-* django >= 1.9 (tested 2.0+)  
-* pytz
-* six
-* typing
-* psycopg2
-* PostgreSQL 9.4+ (tested 9.5+)  
+* Python 3.5+  
+* django >= 1.9 (tested 2.1+)  
+* psycopg2-binary
+* PostgreSQL 9.4+ (tested 9.6+)  
 
 ## Installation
 Install via pip:  
@@ -23,6 +20,14 @@ or via setup.py:
 ### Prerequisites
 Install [postgresql-hll extension](https://github.com/citusdata/postgresql-hll#install)
 
+#### Creating hll extension
+If your user has super-admin privileges you can create Hll extension using migrations.
+If you use django 1.10+ you can use `django_pg_hll.migrations.HllExtension` in your migration file.
+If you have older version you can use the following:
+```python
+migrations.RunSQL('CREATE EXTENSION IF NOT EXISTS hll;', reverse_sql='DROP EXTENSION hll;')
+```
+
 ### Creating table with hll field
 * Add HllField to your model:
   ```python
@@ -226,3 +231,27 @@ MyModel.objects.bulk_update_or_create([
     ], set_functions={'hll_field': 'hll_concat'}
 )
 ```
+
+
+## Running tests
+### Running in docker
+1. Install [docker and docker-compose](https://www.docker.com/)
+2. Run `docker build . --tag django-pg-hll` in project directory
+3. Run `docker-compose run run_tests` in project directory  
+
+### Running in virtual environment
+1. Install all requirements listed above  
+2. [Create virtual environment](https://docs.python.org/3/tutorial/venv.html)  
+3. Create a superuser named 'test' on your local Postgres instance:
+  ```sql
+  CREATE ROLE test;
+  ALTER ROLE test WITH SUPERUSER;
+  ALTER ROLE test WITH LOGIN;
+  ALTER ROLE test PASSWORD 'test';
+  CREATE DATABASE test OWNER test;
+  ```   
+3. Install requirements   
+  `pip3 install -U -r requirements-test.txt`  
+4. Start tests  
+  `python3 runtests.py`  
+