From d2ac5b90b8945a39bac8dbefe8d3307640ec1d56 Mon Sep 17 00:00:00 2001 From: Dimas Date: Sun, 27 Mar 2022 10:02:27 +0700 Subject: [PATCH 1/3] Store user visit data --- .dockerignore | 25 +++++++ Dockerfile | 42 ++++++++++++ Dockerfile.testing | 23 +++++-- REQUIREMENTS.txt | 3 + docker-compose.dev.yml | 24 +++++++ qgisfeedproject/__init__.py | 0 qgisfeedproject/qgisfeed/admin.py | 56 +++++++++++++++- qgisfeedproject/qgisfeed/middleware.py | 25 +++++++ .../migrations/0006_auto_20220325_0652.py | 30 +++++++++ .../qgisfeed/migrations/0007_qgisuservisit.py | 24 +++++++ qgisfeedproject/qgisfeed/models.py | 66 ++++++++++++++++++- qgisfeedproject/qgisfeed/views.py | 2 + qgisfeedproject/qgisfeedproject/settings.py | 12 +++- .../qgisfeedproject/settings_dev.py | 16 +++++ 14 files changed, 341 insertions(+), 7 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.dev.yml create mode 100644 qgisfeedproject/__init__.py create mode 100644 qgisfeedproject/qgisfeed/middleware.py create mode 100644 qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py create mode 100644 qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py create mode 100644 qgisfeedproject/qgisfeedproject/settings_dev.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e12d0af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/__pycache__ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +README.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b770003 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# For more information, please refer to https://aka.ms/vscode-docker-python +FROM python:3.8-slim + +EXPOSE 8000 + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 + +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update +RUN echo "Installing GDAL dependencies" && \ + apt-get install -y libgdal-dev libcurl4-gnutls-dev librtmp-dev && \ + echo "Installing other depdencies" && \ + apt-get install -y wait-for-it curl sudo && \ + echo "Install C library for geoip2" && \ + apt install libmaxminddb0 libmaxminddb-dev mmdb-bin && \ + echo "Removing build dependencies and cleaning up" && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf ~/.cache/pip + +# Install pip requirements +COPY REQUIREMENTS.txt . +RUN python -m pip install -r REQUIREMENTS.txt + +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + +WORKDIR /code +COPY . /code + +# Creates a non-root user with an explicit UID and adds permission to access the /app folder +# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers +RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /code +USER appuser + +# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug +CMD ["gunicorn", "--bind", "0.0.0.0:8000", "qgisfeedproject.wsgi"] diff --git a/Dockerfile.testing b/Dockerfile.testing index 8518b4c..c9b1f9e 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,10 +1,25 @@ -FROM python:3 +FROM python:3.7 ENV PYTHONUNBUFFERED 1 -RUN apt-get update && apt install -y libgdal20 wait-for-it +RUN apt-get update +RUN echo "Installing GDAL dependencies" && \ + apt-get install -y libgdal-dev libcurl4-gnutls-dev librtmp-dev && \ + echo "Installing other depdencies" && \ + apt-get install -y wait-for-it && \ + echo "Install C library for geoip2" && \ + apt install libmaxminddb0 libmaxminddb-dev mmdb-bin && \ + echo "Removing build dependencies and cleaning up" && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf ~/.cache/pip + RUN mkdir /code WORKDIR /code + +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + COPY REQUIREMENTS.txt /code/ RUN pip install -r REQUIREMENTS.txt -COPY . /code/ -COPY ./settings_docker_testing.py /code/qgisfeedproject/qgisfeedproject/settings_local.py COPY ./entrypoint_testing.sh /code/ diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 48a39b3..fdb0978 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -6,3 +6,6 @@ django-imagekit # DEBUG=True only django-extensions + +django-user-visit==0.5.1 +geoip2==4.5.0 \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..487dd33 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,24 @@ +version: '3' +services: + postgis: + image: kartoza/postgis:14-3.1 + platform: linux/arm64, linux/amd64 + environment: + POSTGRES_USER: docker + POSTGRES_PASS: docker + POSTGRES_DBNAME: qgisfeed + + qgisfeed: + build: + context: . + dockerfile: ./Dockerfile + platform: linux/arm64, linux/amd64 + command: /code/entrypoint_testing.sh + environment: + DJANGO_SETTINGS_MODULE: qgisfeedproject.settings_dev + ports: + - "8000:8000" + depends_on: + - postgis + volumes: + - ../qgis-feed:/code \ No newline at end of file diff --git a/qgisfeedproject/__init__.py b/qgisfeedproject/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qgisfeedproject/qgisfeed/admin.py b/qgisfeedproject/qgisfeed/admin.py index 9578f7b..8808b5a 100644 --- a/qgisfeedproject/qgisfeed/admin.py +++ b/qgisfeedproject/qgisfeed/admin.py @@ -24,7 +24,10 @@ from django.urls import reverse from django.utils import timezone -from .models import QgisFeedEntry +from user_visit.admin import UserVisitAdmin +from user_visit.models import UserVisit + +from .models import QgisFeedEntry, QgisUserVisit # Get an instance of a logger logger = logging.getLogger('qgisfeed.admin') @@ -84,4 +87,55 @@ def get_form(self, request, obj=None, **kwargs): return form +class QgisUserVisitAdmin(admin.StackedInline): + readonly_fields = ('qgis_version', 'location', 'platform') + can_delete = False + model = QgisUserVisit + + +class UpdatedUserVisitAdmin(UserVisitAdmin): + inlines = [ + QgisUserVisitAdmin + ] + list_display = ("timestamp", "qgis_version", "country", "platform") + search_fields = ( + "qgisuservisit__qgis_version", + "qgisuservisit__location" + ) + readonly_fields = ( + "timestamp", + "hash", + "session_key", + "user_agent", + "ua_string", + "created_at", + ) + exclude = ['user', 'remote_addr'] + + def qgis_version(self, obj): + qgis_version = '' + if obj.qgisuservisit: + qgis_version = obj.qgisuservisit.qgis_version + if not qgis_version: + qgis_version = '-' + return qgis_version + + def country(self, obj): + country = '-' + if obj.qgisuservisit: + if obj.qgisuservisit.location and 'country_name' in obj.qgisuservisit.location: + country = obj.qgisuservisit.location['country_name'] + return country + + def platform(sel, obj): + platfrom_string = 'Unknown' + if obj.qgisuservisit: + platfrom_string = obj.qgisuservisit.platform + return platfrom_string + + qgis_version.short_description = 'QGIS Version' + + admin.site.register(QgisFeedEntry, QgisFeedEntryAdmin) +admin.site.unregister(UserVisit) +admin.site.register(UserVisit, UpdatedUserVisitAdmin) diff --git a/qgisfeedproject/qgisfeed/middleware.py b/qgisfeedproject/qgisfeed/middleware.py new file mode 100644 index 0000000..5a29581 --- /dev/null +++ b/qgisfeedproject/qgisfeed/middleware.py @@ -0,0 +1,25 @@ +import typing + +from django.http import HttpRequest, HttpResponse +from django.utils import timezone +from django.contrib.auth.models import User + +from user_visit.models import UserVisit, parse_remote_addr, parse_ua_string +from user_visit.middleware import UserVisitMiddleware, save_user_visit + + +class QgisFeedUserVisitMiddleware(UserVisitMiddleware): + """Middleware to record user visits.""" + + def __call__(self, request: HttpRequest) -> typing.Optional[HttpResponse]: + if request.user.is_anonymous: + user, _ = User.objects.get_or_create(username='qgis_user') + request.user = user + if not request.session or not request.session.session_key: + request.session.save() + + uv = UserVisit.objects.build(request, timezone.now()) + if not UserVisit.objects.filter(hash=uv.hash).exists(): + save_user_visit(uv) + + return self.get_response(request) diff --git a/qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py b/qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py new file mode 100644 index 0000000..7db1793 --- /dev/null +++ b/qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.12 on 2022-03-25 06:52 + +from django.db import migrations, models +import imagekit.models.fields +import qgisfeed.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('qgisfeed', '0005_auto_20190709_0814'), + ] + + operations = [ + migrations.AlterField( + model_name='qgisfeedentry', + name='content', + field=models.TextField(), + ), + migrations.AlterField( + model_name='qgisfeedentry', + name='image', + field=imagekit.models.fields.ProcessedImageField(blank=True, height_field='image_height', help_text='Landscape orientation, image will be cropped and scaled automatically to 500x354 px', null=True, upload_to='feedimages/%Y/%m/%d/', verbose_name='Image', width_field='image_width'), + ), + migrations.AlterField( + model_name='qgisfeedentry', + name='language_filter', + field=qgisfeed.models.QgisLanguageField(blank=True, choices=[('aa', 'Afar'), ('ab', 'Abkhazian'), ('ae', 'Avestan'), ('af', 'Afrikaans'), ('ak', 'Akan'), ('am', 'Amharic'), ('an', 'Aragonese'), ('ar', 'Arabic'), ('as', 'Assamese'), ('av', 'Avaric'), ('ay', 'Aymara'), ('az', 'Azerbaijani'), ('ba', 'Bashkir'), ('be', 'Belarusian'), ('bg', 'Bulgarian'), ('bh', 'Bihari languages'), ('bi', 'Bislama'), ('bm', 'Bambara'), ('bn', 'Bengali'), ('bo', 'Tibetan'), ('br', 'Breton'), ('bs', 'Bosnian'), ('ca', 'Catalan; Valencian'), ('ce', 'Chechen'), ('ch', 'Chamorro'), ('co', 'Corsican'), ('cr', 'Cree'), ('cs', 'Czech'), ('cu', 'Church Slavic'), ('cv', 'Chuvash'), ('cy', 'Welsh'), ('da', 'Danish'), ('de', 'German'), ('dv', 'Divehi; Dhivehi; Maldivian'), ('dz', 'Dzongkha'), ('ee', 'Ewe'), ('el', 'Greek, Modern'), ('en', 'English'), ('eo', 'Esperanto'), ('es', 'Spanish; Castilian'), ('et', 'Estonian'), ('eu', 'Basque'), ('fa', 'Persian'), ('ff', 'Fulah'), ('fi', 'Finnish'), ('fj', 'Fijian'), ('fo', 'Faroese'), ('fr', 'French'), ('fy', 'Western Frisian'), ('ga', 'Irish'), ('gd', 'Gaelic; Scottish Gaelic'), ('gl', 'Galician'), ('gn', 'Guarani'), ('gu', 'Gujarati'), ('gv', 'Manx'), ('ha', 'Hausa'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('ho', 'Hiri Motu'), ('hr', 'Croatian'), ('ht', 'Haitian; Haitian Creole'), ('hu', 'Hungarian'), ('hy', 'Armenian'), ('hz', 'Herero'), ('ia', 'Interlingua'), ('id', 'Indonesian'), ('ie', 'Interlingue; Occidental'), ('ig', 'Igbo'), ('ii', 'Sichuan Yi; Nuosu'), ('ik', 'Inupiaq'), ('io', 'Ido'), ('is', 'Icelandic'), ('it', 'Italian'), ('iu', 'Inuktitut'), ('ja', 'Japanese'), ('jv', 'Javanese'), ('ka', 'Georgian'), ('kg', 'Kongo'), ('ki', 'Kikuyu; Gikuyu'), ('kj', 'Kuanyama; Kwanyama'), ('kk', 'Kazakh'), ('kl', 'Kalaallisut; Greenlandic'), ('km', 'Central Khmer'), ('kn', 'Kannada'), ('ko', 'Korean'), ('kr', 'Kanuri'), ('ks', 'Kashmiri'), ('ku', 'Kurdish'), ('kv', 'Komi'), ('kw', 'Cornish'), ('ky', 'Kirghiz; Kyrgyz'), ('la', 'Latin'), ('lb', 'Luxembourgish; Letzeburgesch'), ('lg', 'Ganda'), ('li', 'Limburgan; Limburger; Limburgish'), ('ln', 'Lingala'), ('lo', 'Lao'), ('lt', 'Lithuanian'), ('lu', 'Luba-Katanga'), ('lv', 'Latvian'), ('mh', 'Marshallese'), ('ml', 'Malayalam'), ('mr', 'Marathi'), ('mk', 'Macedonian'), ('mg', 'Malagasy'), ('mt', 'Maltese'), ('mn', 'Mongolian'), ('mi', 'Maori'), ('ms', 'Malay (macrolanguage)'), ('my', 'Burmese'), ('na', 'Nauru'), ('nv', 'Navajo'), ('nr', 'South Ndebele'), ('nd', 'North Ndebele'), ('ng', 'Ndonga'), ('ne', 'Nepali (macrolanguage)'), ('nl', 'Dutch'), ('nn', 'Norwegian Nynorsk'), ('nb', 'Norwegian Bokmål'), ('no', 'Norwegian'), ('ny', 'Nyanja'), ('oc', 'Occitan (post 1500)'), ('oj', 'Ojibwa'), ('or', 'Oriya (macrolanguage)'), ('om', 'Oromo'), ('os', 'Ossetian'), ('pa', 'Panjabi'), ('pi', 'Pali'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('ps', 'Pushto'), ('qu', 'Quechua'), ('rm', 'Romansh'), ('ro', 'Romanian'), ('rn', 'Rundi'), ('ru', 'Russian'), ('sg', 'Sango'), ('sa', 'Sanskrit'), ('si', 'Sinhala'), ('sk', 'Slovak'), ('sl', 'Slovenian'), ('se', 'Northern Sami'), ('sm', 'Samoan'), ('sn', 'Shona'), ('sd', 'Sindhi'), ('so', 'Somali'), ('st', 'Southern Sotho'), ('es', 'Spanish'), ('sq', 'Albanian'), ('sc', 'Sardinian'), ('sr', 'Serbian'), ('ss', 'Swati'), ('su', 'Sundanese'), ('sw', 'Swahili (macrolanguage)'), ('sv', 'Swedish'), ('ty', 'Tahitian'), ('ta', 'Tamil'), ('tt', 'Tatar'), ('te', 'Telugu'), ('tg', 'Tajik'), ('tl', 'Tagalog'), ('th', 'Thai'), ('ti', 'Tigrinya'), ('to', 'Tonga (Tonga Islands)'), ('tn', 'Tswana'), ('ts', 'Tsonga'), ('tk', 'Turkmen'), ('tr', 'Turkish'), ('tw', 'Twi'), ('ug', 'Uighur'), ('uk', 'Ukrainian'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('ve', 'Venda'), ('vi', 'Vietnamese'), ('vo', 'Volapük'), ('wa', 'Walloon'), ('wo', 'Wolof'), ('xh', 'Xhosa'), ('yi', 'Yiddish'), ('yo', 'Yoruba'), ('za', 'Zhuang'), ('zh', 'Chinese'), ('zu', 'Zulu')], db_index=True, help_text='The entry will be hidden to users who have not set a matching language filter', max_length=3, null=True, verbose_name='Language filter'), + ), + ] diff --git a/qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py b/qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py new file mode 100644 index 0000000..8a9080f --- /dev/null +++ b/qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.3 on 2022-03-26 06:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_visit', '0002_add_created_at'), + ('qgisfeed', '0006_auto_20220325_0652'), + ] + + operations = [ + migrations.CreateModel( + name='QgisUserVisit', + fields=[ + ('user_visit', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='user_visit.uservisit')), + ('location', models.JSONField()), + ('qgis_version', models.CharField(blank=True, default='', max_length=255)), + ('platform', models.CharField(blank=True, default='', max_length=255)), + ], + ), + ] diff --git a/qgisfeedproject/qgisfeed/models.py b/qgisfeedproject/qgisfeed/models.py index bbd4760..2336b35 100644 --- a/qgisfeedproject/qgisfeed/models.py +++ b/qgisfeedproject/qgisfeed/models.py @@ -12,15 +12,21 @@ __date__ = '2019-05-07' __copyright__ = 'Copyright 2019, ItOpen' +from platform import platform +import re from django.contrib.auth import get_user_model from django.contrib.gis.db import models -from django.db.models import Q +from django.db.models import Q, signals from django.utils import timezone from django.utils.translation import gettext as _ +import geoip2 + from tinymce import models as tinymce_models from imagekit.models import ProcessedImageField from imagekit.processors import ResizeToFill +from user_visit.models import UserVisit + class QgisLanguageField(models.CharField): """ @@ -92,3 +98,61 @@ class Meta: verbose_name = _('QGIS Feed Entry') verbose_name_plural = _('QGIS Feed Entries') ordering = ('-sticky', '-sorting', '-publish_from') + + +class QgisUserVisit(models.Model): + + user_visit = models.OneToOneField( + UserVisit, + on_delete=models.CASCADE, + primary_key=True + ) + + location = models.JSONField() + + # Mozilla/5.0 QGIS/32400/Fedora Linux (Workstation Edition) + qgis_version = models.CharField( + max_length=255, + default='', + blank=True + ) + + platform = models.CharField( + max_length=255, + default='', + blank=True + ) + +# Post save user visit signals +def post_save_user_visit(sender, instance, **kwargs): + from django.contrib.gis.geoip2 import GeoIP2 + g = GeoIP2() + country_data = {} + qgis_version = '' + platform = '' + + if instance.remote_addr: + try: + country_data = g.city(instance.remote_addr) + except: # AddressNotFoundErrors: + country_data = {} + + version_match = re.search('QGIS(.*)\/', instance.ua_string) + + if version_match: + qgis_version = version_match.group().replace('QGIS', '').strip('/') + platform = instance.ua_string[version_match.end():] + + if not platform: + if instance.user_agent: + platform = instance.user_agent.get_os() + + QgisUserVisit.objects.create( + user_visit=instance, + location=country_data, + qgis_version=qgis_version, + platform=platform + ) + + +signals.post_save.connect(post_save_user_visit, sender=UserVisit) diff --git a/qgisfeedproject/qgisfeed/views.py b/qgisfeedproject/qgisfeed/views.py index 278bea5..dc6b401 100644 --- a/qgisfeedproject/qgisfeed/views.py +++ b/qgisfeedproject/qgisfeed/views.py @@ -24,6 +24,8 @@ from .languages import LANGUAGE_KEYS import json +from user_visit.models import UserVisit + QGISFEED_MAX_RECORDS=getattr(settings, 'QGISFEED_MAX_RECORDS', 20) diff --git a/qgisfeedproject/qgisfeedproject/settings.py b/qgisfeedproject/qgisfeedproject/settings.py index bc442af..c3f583d 100644 --- a/qgisfeedproject/qgisfeedproject/settings.py +++ b/qgisfeedproject/qgisfeedproject/settings.py @@ -11,6 +11,9 @@ """ import os +import logging + +logger = logging.getLogger(__name__) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -45,6 +48,9 @@ # Dependencies 'tinymce', # HTML field 'imagekit', # Image crop and resize + + # User visit + 'user_visit', ] # Useful debugging extensions @@ -61,6 +67,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'qgisfeed.middleware.QgisFeedUserVisitMiddleware', ] ROOT_URLCONF = 'qgisfeedproject.urls' @@ -140,7 +147,10 @@ MEDIA_ROOT=os.path.join(BASE_DIR, 'media') MEDIA_URL = '/media/' +GEOIP_PATH='/var/opt/maxmind/' + + try: from .settings_local import * except ImportError as ex: - raise ex \ No newline at end of file + pass diff --git a/qgisfeedproject/qgisfeedproject/settings_dev.py b/qgisfeedproject/qgisfeedproject/settings_dev.py new file mode 100644 index 0000000..f6f3aa4 --- /dev/null +++ b/qgisfeedproject/qgisfeedproject/settings_dev.py @@ -0,0 +1,16 @@ +from .settings import * # noqa + +# Settings local for docker compose + +ALLOWED_HOSTS=['*'] + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': 'qgisfeed', + 'USER': 'docker', + 'PASSWORD': 'docker', + 'HOST': 'postgis', + 'PORT': '5432' + } +} From d3084cf4e07cc910c516ab587d1d15a121c643a6 Mon Sep 17 00:00:00 2001 From: dimasciput Date: Fri, 8 Apr 2022 14:35:35 +0700 Subject: [PATCH 2/3] Remove ip address on save --- Dockerfile | 2 +- qgisfeedproject/qgisfeed/models.py | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index b770003..c93d02e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN echo "Installing GDAL dependencies" && \ rm -rf ~/.cache/pip # Install pip requirements -COPY REQUIREMENTS.txt . +ADD REQUIREMENTS.txt . RUN python -m pip install -r REQUIREMENTS.txt RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ diff --git a/qgisfeedproject/qgisfeed/models.py b/qgisfeedproject/qgisfeed/models.py index 2336b35..77ae155 100644 --- a/qgisfeedproject/qgisfeed/models.py +++ b/qgisfeedproject/qgisfeed/models.py @@ -129,7 +129,10 @@ def post_save_user_visit(sender, instance, **kwargs): g = GeoIP2() country_data = {} qgis_version = '' - platform = '' + platform_name = '' + + if hasattr(instance, '_dirty'): + return if instance.remote_addr: try: @@ -141,18 +144,25 @@ def post_save_user_visit(sender, instance, **kwargs): if version_match: qgis_version = version_match.group().replace('QGIS', '').strip('/') - platform = instance.ua_string[version_match.end():] + platform_name = instance.ua_string[version_match.end():] - if not platform: + if not platform_name: if instance.user_agent: - platform = instance.user_agent.get_os() + platform_name = instance.user_agent.get_os() - QgisUserVisit.objects.create( + QgisUserVisit.objects.get_or_create( user_visit=instance, location=country_data, qgis_version=qgis_version, - platform=platform + platform=platform_name ) + instance.remote_addr = '' + try: + instance._dirty = True + instance.save() + finally: + del instance._dirty + signals.post_save.connect(post_save_user_visit, sender=UserVisit) From 453c9d44fc0132da295b272ca4c95587e5211e6c Mon Sep 17 00:00:00 2001 From: dimasciput Date: Sun, 10 Apr 2022 11:31:15 +0700 Subject: [PATCH 3/3] Add function to aggregate daily visit, remove remote addr --- Dockerfile | 8 +- docker-compose.dev.yml | 8 +- qgisfeedproject/qgisfeed/admin.py | 14 +- qgisfeedproject/qgisfeed/apps.py | 4 +- .../qgisfeed/management/__init__.py | 0 .../qgisfeed/management/commands/__init__.py | 0 .../commands/aggregate_user_visit_data.py | 11 ++ .../migrations/0008_dailyqgisuservisit.py | 23 +++ qgisfeedproject/qgisfeed/models.py | 150 ++++++++++++------ qgisfeedproject/qgisfeed/signals.py | 37 +++++ qgisfeedproject/qgisfeed/tests.py | 41 ++++- 11 files changed, 238 insertions(+), 58 deletions(-) create mode 100644 qgisfeedproject/qgisfeed/management/__init__.py create mode 100644 qgisfeedproject/qgisfeed/management/commands/__init__.py create mode 100644 qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py create mode 100644 qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py diff --git a/Dockerfile b/Dockerfile index c93d02e..0b77be9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,12 +31,8 @@ RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3 ENV GEOIP_PATH=/var/opt/maxmind/ WORKDIR /code -COPY . /code - -# Creates a non-root user with an explicit UID and adds permission to access the /app folder -# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers -RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /code -USER appuser +ADD . /code +# Creates a non-root user # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug CMD ["gunicorn", "--bind", "0.0.0.0:8000", "qgisfeedproject.wsgi"] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 487dd33..7bc30b1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,8 +1,8 @@ -version: '3' +version: '3.7' services: postgis: image: kartoza/postgis:14-3.1 - platform: linux/arm64, linux/amd64 + platform: linux/amd64 environment: POSTGRES_USER: docker POSTGRES_PASS: docker @@ -12,12 +12,14 @@ services: build: context: . dockerfile: ./Dockerfile - platform: linux/arm64, linux/amd64 + platform: linux/amd64 command: /code/entrypoint_testing.sh environment: DJANGO_SETTINGS_MODULE: qgisfeedproject.settings_dev ports: - "8000:8000" + links: + - postgis depends_on: - postgis volumes: diff --git a/qgisfeedproject/qgisfeed/admin.py b/qgisfeedproject/qgisfeed/admin.py index 8808b5a..a2fa423 100644 --- a/qgisfeedproject/qgisfeed/admin.py +++ b/qgisfeedproject/qgisfeed/admin.py @@ -27,7 +27,7 @@ from user_visit.admin import UserVisitAdmin from user_visit.models import UserVisit -from .models import QgisFeedEntry, QgisUserVisit +from .models import QgisFeedEntry, QgisUserVisit, DailyQgisUserVisit # Get an instance of a logger logger = logging.getLogger('qgisfeed.admin') @@ -91,7 +91,16 @@ class QgisUserVisitAdmin(admin.StackedInline): readonly_fields = ('qgis_version', 'location', 'platform') can_delete = False model = QgisUserVisit - + + +class DailyQgisUserVisitAdmin(admin.ModelAdmin): + list_display = ( + 'date', + ) + + def has_add_permission(self, request): + return False + class UpdatedUserVisitAdmin(UserVisitAdmin): inlines = [ @@ -139,3 +148,4 @@ def platform(sel, obj): admin.site.register(QgisFeedEntry, QgisFeedEntryAdmin) admin.site.unregister(UserVisit) admin.site.register(UserVisit, UpdatedUserVisitAdmin) +admin.site.register(DailyQgisUserVisit, DailyQgisUserVisitAdmin) diff --git a/qgisfeedproject/qgisfeed/apps.py b/qgisfeedproject/qgisfeed/apps.py index 0b5f85e..c7347db 100644 --- a/qgisfeedproject/qgisfeed/apps.py +++ b/qgisfeedproject/qgisfeed/apps.py @@ -21,7 +21,9 @@ class QgisFeedConfig(AppConfig): name = 'qgisfeed' def ready(self): - from .signals import setup_group + from .signals import setup_group, post_save_user_visit from django.contrib.auth.models import User + from user_visit.models import UserVisit post_save.connect(setup_group, sender=User) + post_save.connect(post_save_user_visit, sender=UserVisit) diff --git a/qgisfeedproject/qgisfeed/management/__init__.py b/qgisfeedproject/qgisfeed/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qgisfeedproject/qgisfeed/management/commands/__init__.py b/qgisfeedproject/qgisfeed/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py b/qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py new file mode 100644 index 0000000..5e7f199 --- /dev/null +++ b/qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py @@ -0,0 +1,11 @@ +# coding=utf-8 +from django.core.management.base import BaseCommand +from qgisfeed.models import aggregate_user_visit_data + + +class Command(BaseCommand): + """Add site codes + """ + + def handle(self, *args, **options): + aggregate_user_visit_data() diff --git a/qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py b/qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py new file mode 100644 index 0000000..9f04242 --- /dev/null +++ b/qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py @@ -0,0 +1,23 @@ +# Generated by Django 4.0.3 on 2022-04-08 09:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('qgisfeed', '0007_qgisuservisit'), + ] + + operations = [ + migrations.CreateModel( + name='DailyQgisUserVisit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateField(auto_now_add=True)), + ('qgis_version', models.JSONField()), + ('platform', models.JSONField()), + ('country', models.JSONField()), + ], + ), + ] diff --git a/qgisfeedproject/qgisfeed/models.py b/qgisfeedproject/qgisfeed/models.py index 77ae155..193e79d 100644 --- a/qgisfeedproject/qgisfeed/models.py +++ b/qgisfeedproject/qgisfeed/models.py @@ -12,17 +12,12 @@ __date__ = '2019-05-07' __copyright__ = 'Copyright 2019, ItOpen' -from platform import platform -import re - from django.contrib.auth import get_user_model from django.contrib.gis.db import models -from django.db.models import Q, signals +from django.db.models import Q, F, Count from django.utils import timezone from django.utils.translation import gettext as _ -import geoip2 -from tinymce import models as tinymce_models from imagekit.models import ProcessedImageField from imagekit.processors import ResizeToFill from user_visit.models import UserVisit @@ -123,46 +118,111 @@ class QgisUserVisit(models.Model): blank=True ) -# Post save user visit signals -def post_save_user_visit(sender, instance, **kwargs): - from django.contrib.gis.geoip2 import GeoIP2 - g = GeoIP2() - country_data = {} - qgis_version = '' - platform_name = '' - - if hasattr(instance, '_dirty'): - return - - if instance.remote_addr: - try: - country_data = g.city(instance.remote_addr) - except: # AddressNotFoundErrors: - country_data = {} - - version_match = re.search('QGIS(.*)\/', instance.ua_string) - - if version_match: - qgis_version = version_match.group().replace('QGIS', '').strip('/') - platform_name = instance.ua_string[version_match.end():] - - if not platform_name: - if instance.user_agent: - platform_name = instance.user_agent.get_os() - - QgisUserVisit.objects.get_or_create( - user_visit=instance, - location=country_data, - qgis_version=qgis_version, - platform=platform_name + +class DailyQgisUserVisit(models.Model): + + date = models.DateField( + auto_now_add=True, + blank=True ) - instance.remote_addr = '' - try: - instance._dirty = True - instance.save() - finally: - del instance._dirty + qgis_version = models.JSONField() + platform = models.JSONField() + + country = models.JSONField() + + +def aggregate_user_visit_data(): + user_visits = QgisUserVisit.objects.all() + + # Group by date + user_visit_dates = ( + user_visits.annotate( + date=F('user_visit__timestamp__date') + ).values_list('date', flat=True).distinct() + ) -signals.post_save.connect(post_save_user_visit, sender=UserVisit) + for user_visit_date in user_visit_dates: + + daily_visit, _ = DailyQgisUserVisit.objects.get_or_create( + date=user_visit_date, + defaults={ + 'qgis_version': {}, + 'platform': {}, + 'country': {} + } + ) + + qgis_user_visit = user_visits.filter( + user_visit__timestamp__date=user_visit_date + ) + + total_platform_data = dict( + qgis_user_visit.values( + 'platform' + ).annotate(total_platform=Count('platform')).values_list( + 'platform', 'total_platform' + ) + ) + + total_country = dict( + qgis_user_visit.filter( + location__country_name__isnull=False + ).values( + 'location__country_name' + ).annotate( + total_country=Count('location__country_name') + ).values_list( + 'location__country_name', 'total_country' + ) + ) + + total_qgis_version = dict( + qgis_user_visit.exclude( + qgis_version='' + ).values( + 'qgis_version' + ).annotate( + total_qgis_version=Count('qgis_version') + ).values_list( + 'qgis_version', 'total_qgis_version' + ) + ) + + if total_platform_data: + daily_platform_data = daily_visit.platform + for platform, value in total_platform_data.items(): + if platform not in daily_platform_data: + daily_platform_data[platform] = ( + value + ) + else: + daily_platform_data[platform] += ( + value + ) + daily_visit.platform = daily_platform_data + + if total_country: + daily_country = daily_visit.country + for country, value in total_country.items(): + if country not in daily_country: + daily_country[country] = value + else: + daily_country[country] += value + daily_visit.country = daily_country + + if total_qgis_version: + daily_qgis_version = daily_visit.qgis_version + for qgis_version, value in total_qgis_version.items(): + if qgis_version not in daily_qgis_version: + daily_qgis_version[qgis_version] = value + else: + daily_qgis_version[qgis_version] += value + daily_visit.qgis_version = daily_qgis_version + + daily_visit.save() + + UserVisit.objects.filter( + timestamp__date=user_visit_date + ).delete() diff --git a/qgisfeedproject/qgisfeed/signals.py b/qgisfeedproject/qgisfeed/signals.py index 727e9f9..75f621a 100644 --- a/qgisfeedproject/qgisfeed/signals.py +++ b/qgisfeedproject/qgisfeed/signals.py @@ -24,3 +24,40 @@ def setup_group(sender, **kwargs): for staff_user in User.objects.filter(is_staff=True, is_superuser=False): group.user_set.add(staff_user) + +# Post save user visit signals +def post_save_user_visit(sender, instance, **kwargs): + import re + from django.contrib.gis.geoip2 import GeoIP2 + from qgisfeed.models import QgisUserVisit + from user_visit.models import UserVisit + + g = GeoIP2() + country_data = {} + qgis_version = '' + platform_name = '' + + if instance.remote_addr: + try: + country_data = g.city(instance.remote_addr) + except: # AddressNotFoundErrors: + country_data = {} + + version_match = re.search('QGIS(.*)\/', instance.ua_string) + + if version_match: + qgis_version = version_match.group().replace('QGIS', '').strip('/') + platform_name = instance.ua_string[version_match.end():] + + if not platform_name: + if instance.user_agent: + platform_name = instance.user_agent.get_os() + + QgisUserVisit.objects.get_or_create( + user_visit=instance, + location=country_data, + qgis_version=qgis_version, + platform=platform_name + ) + + UserVisit.objects.filter(pk=instance.pk).update(remote_addr='') diff --git a/qgisfeedproject/qgisfeed/tests.py b/qgisfeedproject/qgisfeed/tests.py index 6dbb2a5..1c88a01 100644 --- a/qgisfeedproject/qgisfeed/tests.py +++ b/qgisfeedproject/qgisfeed/tests.py @@ -20,7 +20,9 @@ from django.contrib.admin.sites import AdminSite from django.utils import timezone -from .models import QgisFeedEntry +from .models import ( + QgisFeedEntry, QgisUserVisit, DailyQgisUserVisit, aggregate_user_visit_data +) from .admin import QgisFeedEntryAdmin @@ -188,3 +190,40 @@ def test_admin_author_is_set(self): ma.save_model(request, obj, form, False) self.assertEqual(obj.author, request.user) + +class QgisUserVisitTestCase(TestCase): + + def test_user_visit(self): + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Fedora ' + 'Linux (Workstation Edition)') + c.get('/') + user_visit = QgisUserVisit.objects.filter( + platform__icontains='Fedora Linux (Workstation Edition)') + self.assertEqual(user_visit.count(), 1) + self.assertEqual(user_visit.first().qgis_version, '32400') + + def test_ip_address_removed(self): + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Fedora ' + 'Linux (Workstation Edition)', + REMOTE_ADDR='180.247.213.170') + c.get('/') + qgis_visit = QgisUserVisit.objects.first() + self.assertTrue(qgis_visit.user_visit.remote_addr == '') + self.assertTrue(qgis_visit.location['country_name'] == 'Indonesia') + + def test_aggregate_visit(self): + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/31400/Fedora ' + 'Linux (Workstation Edition)', + REMOTE_ADDR='180.247.213.170') + c.get('/') + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Windows 10', + REMOTE_ADDR='180.247.213.160') + c.get('/') + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Windows XP', + REMOTE_ADDR='180.247.213.160') + c.get('/') + aggregate_user_visit_data() + daily_visit = DailyQgisUserVisit.objects.first() + self.assertTrue(daily_visit.platform['Windows 10'] == 1) + self.assertTrue(daily_visit.qgis_version['32400'] == 2) + self.assertTrue(daily_visit.country['Indonesia'] == 3)