diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e12d0af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/__pycache__ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +README.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0b77be9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# For more information, please refer to https://aka.ms/vscode-docker-python +FROM python:3.8-slim + +EXPOSE 8000 + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 + +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update +RUN echo "Installing GDAL dependencies" && \ + apt-get install -y libgdal-dev libcurl4-gnutls-dev librtmp-dev && \ + echo "Installing other depdencies" && \ + apt-get install -y wait-for-it curl sudo && \ + echo "Install C library for geoip2" && \ + apt install libmaxminddb0 libmaxminddb-dev mmdb-bin && \ + echo "Removing build dependencies and cleaning up" && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf ~/.cache/pip + +# Install pip requirements +ADD REQUIREMENTS.txt . +RUN python -m pip install -r REQUIREMENTS.txt + +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + +WORKDIR /code +ADD . /code + +# Creates a non-root user +# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug +CMD ["gunicorn", "--bind", "0.0.0.0:8000", "qgisfeedproject.wsgi"] diff --git a/Dockerfile.testing b/Dockerfile.testing index 8518b4c..c9b1f9e 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,10 +1,25 @@ -FROM python:3 +FROM python:3.7 ENV PYTHONUNBUFFERED 1 -RUN apt-get update && apt install -y libgdal20 wait-for-it +RUN apt-get update +RUN echo "Installing GDAL dependencies" && \ + apt-get install -y libgdal-dev libcurl4-gnutls-dev librtmp-dev && \ + echo "Installing other depdencies" && \ + apt-get install -y wait-for-it && \ + echo "Install C library for geoip2" && \ + apt install libmaxminddb0 libmaxminddb-dev mmdb-bin && \ + echo "Removing build dependencies and cleaning up" && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf ~/.cache/pip + RUN mkdir /code WORKDIR /code + +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + COPY REQUIREMENTS.txt /code/ RUN pip install -r REQUIREMENTS.txt -COPY . /code/ -COPY ./settings_docker_testing.py /code/qgisfeedproject/qgisfeedproject/settings_local.py COPY ./entrypoint_testing.sh /code/ diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 48a39b3..fdb0978 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -6,3 +6,6 @@ django-imagekit # DEBUG=True only django-extensions + +django-user-visit==0.5.1 +geoip2==4.5.0 \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..7bc30b1 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,26 @@ +version: '3.7' +services: + postgis: + image: kartoza/postgis:14-3.1 + platform: linux/amd64 + environment: + POSTGRES_USER: docker + POSTGRES_PASS: docker + POSTGRES_DBNAME: qgisfeed + + qgisfeed: + build: + context: . + dockerfile: ./Dockerfile + platform: linux/amd64 + command: /code/entrypoint_testing.sh + environment: + DJANGO_SETTINGS_MODULE: qgisfeedproject.settings_dev + ports: + - "8000:8000" + links: + - postgis + depends_on: + - postgis + volumes: + - ../qgis-feed:/code \ No newline at end of file diff --git a/qgisfeedproject/__init__.py b/qgisfeedproject/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qgisfeedproject/qgisfeed/admin.py b/qgisfeedproject/qgisfeed/admin.py index 9578f7b..a2fa423 100644 --- a/qgisfeedproject/qgisfeed/admin.py +++ b/qgisfeedproject/qgisfeed/admin.py @@ -24,7 +24,10 @@ from django.urls import reverse from django.utils import timezone -from .models import QgisFeedEntry +from user_visit.admin import UserVisitAdmin +from user_visit.models import UserVisit + +from .models import QgisFeedEntry, QgisUserVisit, DailyQgisUserVisit # Get an instance of a logger logger = logging.getLogger('qgisfeed.admin') @@ -84,4 +87,65 @@ def get_form(self, request, obj=None, **kwargs): return form +class QgisUserVisitAdmin(admin.StackedInline): + readonly_fields = ('qgis_version', 'location', 'platform') + can_delete = False + model = QgisUserVisit + + +class DailyQgisUserVisitAdmin(admin.ModelAdmin): + list_display = ( + 'date', + ) + + def has_add_permission(self, request): + return False + + +class UpdatedUserVisitAdmin(UserVisitAdmin): + inlines = [ + QgisUserVisitAdmin + ] + list_display = ("timestamp", "qgis_version", "country", "platform") + search_fields = ( + "qgisuservisit__qgis_version", + "qgisuservisit__location" + ) + readonly_fields = ( + "timestamp", + "hash", + "session_key", + "user_agent", + "ua_string", + "created_at", + ) + exclude = ['user', 'remote_addr'] + + def qgis_version(self, obj): + qgis_version = '' + if obj.qgisuservisit: + qgis_version = obj.qgisuservisit.qgis_version + if not qgis_version: + qgis_version = '-' + return qgis_version + + def country(self, obj): + country = '-' + if obj.qgisuservisit: + if obj.qgisuservisit.location and 'country_name' in obj.qgisuservisit.location: + country = obj.qgisuservisit.location['country_name'] + return country + + def platform(sel, obj): + platfrom_string = 'Unknown' + if obj.qgisuservisit: + platfrom_string = obj.qgisuservisit.platform + return platfrom_string + + qgis_version.short_description = 'QGIS Version' + + admin.site.register(QgisFeedEntry, QgisFeedEntryAdmin) +admin.site.unregister(UserVisit) +admin.site.register(UserVisit, UpdatedUserVisitAdmin) +admin.site.register(DailyQgisUserVisit, DailyQgisUserVisitAdmin) diff --git a/qgisfeedproject/qgisfeed/apps.py b/qgisfeedproject/qgisfeed/apps.py index 0b5f85e..c7347db 100644 --- a/qgisfeedproject/qgisfeed/apps.py +++ b/qgisfeedproject/qgisfeed/apps.py @@ -21,7 +21,9 @@ class QgisFeedConfig(AppConfig): name = 'qgisfeed' def ready(self): - from .signals import setup_group + from .signals import setup_group, post_save_user_visit from django.contrib.auth.models import User + from user_visit.models import UserVisit post_save.connect(setup_group, sender=User) + post_save.connect(post_save_user_visit, sender=UserVisit) diff --git a/qgisfeedproject/qgisfeed/management/__init__.py b/qgisfeedproject/qgisfeed/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qgisfeedproject/qgisfeed/management/commands/__init__.py b/qgisfeedproject/qgisfeed/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py b/qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py new file mode 100644 index 0000000..5e7f199 --- /dev/null +++ b/qgisfeedproject/qgisfeed/management/commands/aggregate_user_visit_data.py @@ -0,0 +1,11 @@ +# coding=utf-8 +from django.core.management.base import BaseCommand +from qgisfeed.models import aggregate_user_visit_data + + +class Command(BaseCommand): + """Add site codes + """ + + def handle(self, *args, **options): + aggregate_user_visit_data() diff --git a/qgisfeedproject/qgisfeed/middleware.py b/qgisfeedproject/qgisfeed/middleware.py new file mode 100644 index 0000000..5a29581 --- /dev/null +++ b/qgisfeedproject/qgisfeed/middleware.py @@ -0,0 +1,25 @@ +import typing + +from django.http import HttpRequest, HttpResponse +from django.utils import timezone +from django.contrib.auth.models import User + +from user_visit.models import UserVisit, parse_remote_addr, parse_ua_string +from user_visit.middleware import UserVisitMiddleware, save_user_visit + + +class QgisFeedUserVisitMiddleware(UserVisitMiddleware): + """Middleware to record user visits.""" + + def __call__(self, request: HttpRequest) -> typing.Optional[HttpResponse]: + if request.user.is_anonymous: + user, _ = User.objects.get_or_create(username='qgis_user') + request.user = user + if not request.session or not request.session.session_key: + request.session.save() + + uv = UserVisit.objects.build(request, timezone.now()) + if not UserVisit.objects.filter(hash=uv.hash).exists(): + save_user_visit(uv) + + return self.get_response(request) diff --git a/qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py b/qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py new file mode 100644 index 0000000..7db1793 --- /dev/null +++ b/qgisfeedproject/qgisfeed/migrations/0006_auto_20220325_0652.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.12 on 2022-03-25 06:52 + +from django.db import migrations, models +import imagekit.models.fields +import qgisfeed.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('qgisfeed', '0005_auto_20190709_0814'), + ] + + operations = [ + migrations.AlterField( + model_name='qgisfeedentry', + name='content', + field=models.TextField(), + ), + migrations.AlterField( + model_name='qgisfeedentry', + name='image', + field=imagekit.models.fields.ProcessedImageField(blank=True, height_field='image_height', help_text='Landscape orientation, image will be cropped and scaled automatically to 500x354 px', null=True, upload_to='feedimages/%Y/%m/%d/', verbose_name='Image', width_field='image_width'), + ), + migrations.AlterField( + model_name='qgisfeedentry', + name='language_filter', + field=qgisfeed.models.QgisLanguageField(blank=True, choices=[('aa', 'Afar'), ('ab', 'Abkhazian'), ('ae', 'Avestan'), ('af', 'Afrikaans'), ('ak', 'Akan'), ('am', 'Amharic'), ('an', 'Aragonese'), ('ar', 'Arabic'), ('as', 'Assamese'), ('av', 'Avaric'), ('ay', 'Aymara'), ('az', 'Azerbaijani'), ('ba', 'Bashkir'), ('be', 'Belarusian'), ('bg', 'Bulgarian'), ('bh', 'Bihari languages'), ('bi', 'Bislama'), ('bm', 'Bambara'), ('bn', 'Bengali'), ('bo', 'Tibetan'), ('br', 'Breton'), ('bs', 'Bosnian'), ('ca', 'Catalan; Valencian'), ('ce', 'Chechen'), ('ch', 'Chamorro'), ('co', 'Corsican'), ('cr', 'Cree'), ('cs', 'Czech'), ('cu', 'Church Slavic'), ('cv', 'Chuvash'), ('cy', 'Welsh'), ('da', 'Danish'), ('de', 'German'), ('dv', 'Divehi; Dhivehi; Maldivian'), ('dz', 'Dzongkha'), ('ee', 'Ewe'), ('el', 'Greek, Modern'), ('en', 'English'), ('eo', 'Esperanto'), ('es', 'Spanish; Castilian'), ('et', 'Estonian'), ('eu', 'Basque'), ('fa', 'Persian'), ('ff', 'Fulah'), ('fi', 'Finnish'), ('fj', 'Fijian'), ('fo', 'Faroese'), ('fr', 'French'), ('fy', 'Western Frisian'), ('ga', 'Irish'), ('gd', 'Gaelic; Scottish Gaelic'), ('gl', 'Galician'), ('gn', 'Guarani'), ('gu', 'Gujarati'), ('gv', 'Manx'), ('ha', 'Hausa'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('ho', 'Hiri Motu'), ('hr', 'Croatian'), ('ht', 'Haitian; Haitian Creole'), ('hu', 'Hungarian'), ('hy', 'Armenian'), ('hz', 'Herero'), ('ia', 'Interlingua'), ('id', 'Indonesian'), ('ie', 'Interlingue; Occidental'), ('ig', 'Igbo'), ('ii', 'Sichuan Yi; Nuosu'), ('ik', 'Inupiaq'), ('io', 'Ido'), ('is', 'Icelandic'), ('it', 'Italian'), ('iu', 'Inuktitut'), ('ja', 'Japanese'), ('jv', 'Javanese'), ('ka', 'Georgian'), ('kg', 'Kongo'), ('ki', 'Kikuyu; Gikuyu'), ('kj', 'Kuanyama; Kwanyama'), ('kk', 'Kazakh'), ('kl', 'Kalaallisut; Greenlandic'), ('km', 'Central Khmer'), ('kn', 'Kannada'), ('ko', 'Korean'), ('kr', 'Kanuri'), ('ks', 'Kashmiri'), ('ku', 'Kurdish'), ('kv', 'Komi'), ('kw', 'Cornish'), ('ky', 'Kirghiz; Kyrgyz'), ('la', 'Latin'), ('lb', 'Luxembourgish; Letzeburgesch'), ('lg', 'Ganda'), ('li', 'Limburgan; Limburger; Limburgish'), ('ln', 'Lingala'), ('lo', 'Lao'), ('lt', 'Lithuanian'), ('lu', 'Luba-Katanga'), ('lv', 'Latvian'), ('mh', 'Marshallese'), ('ml', 'Malayalam'), ('mr', 'Marathi'), ('mk', 'Macedonian'), ('mg', 'Malagasy'), ('mt', 'Maltese'), ('mn', 'Mongolian'), ('mi', 'Maori'), ('ms', 'Malay (macrolanguage)'), ('my', 'Burmese'), ('na', 'Nauru'), ('nv', 'Navajo'), ('nr', 'South Ndebele'), ('nd', 'North Ndebele'), ('ng', 'Ndonga'), ('ne', 'Nepali (macrolanguage)'), ('nl', 'Dutch'), ('nn', 'Norwegian Nynorsk'), ('nb', 'Norwegian Bokmål'), ('no', 'Norwegian'), ('ny', 'Nyanja'), ('oc', 'Occitan (post 1500)'), ('oj', 'Ojibwa'), ('or', 'Oriya (macrolanguage)'), ('om', 'Oromo'), ('os', 'Ossetian'), ('pa', 'Panjabi'), ('pi', 'Pali'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('ps', 'Pushto'), ('qu', 'Quechua'), ('rm', 'Romansh'), ('ro', 'Romanian'), ('rn', 'Rundi'), ('ru', 'Russian'), ('sg', 'Sango'), ('sa', 'Sanskrit'), ('si', 'Sinhala'), ('sk', 'Slovak'), ('sl', 'Slovenian'), ('se', 'Northern Sami'), ('sm', 'Samoan'), ('sn', 'Shona'), ('sd', 'Sindhi'), ('so', 'Somali'), ('st', 'Southern Sotho'), ('es', 'Spanish'), ('sq', 'Albanian'), ('sc', 'Sardinian'), ('sr', 'Serbian'), ('ss', 'Swati'), ('su', 'Sundanese'), ('sw', 'Swahili (macrolanguage)'), ('sv', 'Swedish'), ('ty', 'Tahitian'), ('ta', 'Tamil'), ('tt', 'Tatar'), ('te', 'Telugu'), ('tg', 'Tajik'), ('tl', 'Tagalog'), ('th', 'Thai'), ('ti', 'Tigrinya'), ('to', 'Tonga (Tonga Islands)'), ('tn', 'Tswana'), ('ts', 'Tsonga'), ('tk', 'Turkmen'), ('tr', 'Turkish'), ('tw', 'Twi'), ('ug', 'Uighur'), ('uk', 'Ukrainian'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('ve', 'Venda'), ('vi', 'Vietnamese'), ('vo', 'Volapük'), ('wa', 'Walloon'), ('wo', 'Wolof'), ('xh', 'Xhosa'), ('yi', 'Yiddish'), ('yo', 'Yoruba'), ('za', 'Zhuang'), ('zh', 'Chinese'), ('zu', 'Zulu')], db_index=True, help_text='The entry will be hidden to users who have not set a matching language filter', max_length=3, null=True, verbose_name='Language filter'), + ), + ] diff --git a/qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py b/qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py new file mode 100644 index 0000000..8a9080f --- /dev/null +++ b/qgisfeedproject/qgisfeed/migrations/0007_qgisuservisit.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.3 on 2022-03-26 06:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_visit', '0002_add_created_at'), + ('qgisfeed', '0006_auto_20220325_0652'), + ] + + operations = [ + migrations.CreateModel( + name='QgisUserVisit', + fields=[ + ('user_visit', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='user_visit.uservisit')), + ('location', models.JSONField()), + ('qgis_version', models.CharField(blank=True, default='', max_length=255)), + ('platform', models.CharField(blank=True, default='', max_length=255)), + ], + ), + ] diff --git a/qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py b/qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py new file mode 100644 index 0000000..9f04242 --- /dev/null +++ b/qgisfeedproject/qgisfeed/migrations/0008_dailyqgisuservisit.py @@ -0,0 +1,23 @@ +# Generated by Django 4.0.3 on 2022-04-08 09:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('qgisfeed', '0007_qgisuservisit'), + ] + + operations = [ + migrations.CreateModel( + name='DailyQgisUserVisit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateField(auto_now_add=True)), + ('qgis_version', models.JSONField()), + ('platform', models.JSONField()), + ('country', models.JSONField()), + ], + ), + ] diff --git a/qgisfeedproject/qgisfeed/models.py b/qgisfeedproject/qgisfeed/models.py index bbd4760..193e79d 100644 --- a/qgisfeedproject/qgisfeed/models.py +++ b/qgisfeedproject/qgisfeed/models.py @@ -12,15 +12,16 @@ __date__ = '2019-05-07' __copyright__ = 'Copyright 2019, ItOpen' - from django.contrib.auth import get_user_model from django.contrib.gis.db import models -from django.db.models import Q +from django.db.models import Q, F, Count from django.utils import timezone from django.utils.translation import gettext as _ -from tinymce import models as tinymce_models + from imagekit.models import ProcessedImageField from imagekit.processors import ResizeToFill +from user_visit.models import UserVisit + class QgisLanguageField(models.CharField): """ @@ -92,3 +93,136 @@ class Meta: verbose_name = _('QGIS Feed Entry') verbose_name_plural = _('QGIS Feed Entries') ordering = ('-sticky', '-sorting', '-publish_from') + + +class QgisUserVisit(models.Model): + + user_visit = models.OneToOneField( + UserVisit, + on_delete=models.CASCADE, + primary_key=True + ) + + location = models.JSONField() + + # Mozilla/5.0 QGIS/32400/Fedora Linux (Workstation Edition) + qgis_version = models.CharField( + max_length=255, + default='', + blank=True + ) + + platform = models.CharField( + max_length=255, + default='', + blank=True + ) + + +class DailyQgisUserVisit(models.Model): + + date = models.DateField( + auto_now_add=True, + blank=True + ) + + qgis_version = models.JSONField() + + platform = models.JSONField() + + country = models.JSONField() + + +def aggregate_user_visit_data(): + user_visits = QgisUserVisit.objects.all() + + # Group by date + user_visit_dates = ( + user_visits.annotate( + date=F('user_visit__timestamp__date') + ).values_list('date', flat=True).distinct() + ) + + for user_visit_date in user_visit_dates: + + daily_visit, _ = DailyQgisUserVisit.objects.get_or_create( + date=user_visit_date, + defaults={ + 'qgis_version': {}, + 'platform': {}, + 'country': {} + } + ) + + qgis_user_visit = user_visits.filter( + user_visit__timestamp__date=user_visit_date + ) + + total_platform_data = dict( + qgis_user_visit.values( + 'platform' + ).annotate(total_platform=Count('platform')).values_list( + 'platform', 'total_platform' + ) + ) + + total_country = dict( + qgis_user_visit.filter( + location__country_name__isnull=False + ).values( + 'location__country_name' + ).annotate( + total_country=Count('location__country_name') + ).values_list( + 'location__country_name', 'total_country' + ) + ) + + total_qgis_version = dict( + qgis_user_visit.exclude( + qgis_version='' + ).values( + 'qgis_version' + ).annotate( + total_qgis_version=Count('qgis_version') + ).values_list( + 'qgis_version', 'total_qgis_version' + ) + ) + + if total_platform_data: + daily_platform_data = daily_visit.platform + for platform, value in total_platform_data.items(): + if platform not in daily_platform_data: + daily_platform_data[platform] = ( + value + ) + else: + daily_platform_data[platform] += ( + value + ) + daily_visit.platform = daily_platform_data + + if total_country: + daily_country = daily_visit.country + for country, value in total_country.items(): + if country not in daily_country: + daily_country[country] = value + else: + daily_country[country] += value + daily_visit.country = daily_country + + if total_qgis_version: + daily_qgis_version = daily_visit.qgis_version + for qgis_version, value in total_qgis_version.items(): + if qgis_version not in daily_qgis_version: + daily_qgis_version[qgis_version] = value + else: + daily_qgis_version[qgis_version] += value + daily_visit.qgis_version = daily_qgis_version + + daily_visit.save() + + UserVisit.objects.filter( + timestamp__date=user_visit_date + ).delete() diff --git a/qgisfeedproject/qgisfeed/signals.py b/qgisfeedproject/qgisfeed/signals.py index 727e9f9..75f621a 100644 --- a/qgisfeedproject/qgisfeed/signals.py +++ b/qgisfeedproject/qgisfeed/signals.py @@ -24,3 +24,40 @@ def setup_group(sender, **kwargs): for staff_user in User.objects.filter(is_staff=True, is_superuser=False): group.user_set.add(staff_user) + +# Post save user visit signals +def post_save_user_visit(sender, instance, **kwargs): + import re + from django.contrib.gis.geoip2 import GeoIP2 + from qgisfeed.models import QgisUserVisit + from user_visit.models import UserVisit + + g = GeoIP2() + country_data = {} + qgis_version = '' + platform_name = '' + + if instance.remote_addr: + try: + country_data = g.city(instance.remote_addr) + except: # AddressNotFoundErrors: + country_data = {} + + version_match = re.search('QGIS(.*)\/', instance.ua_string) + + if version_match: + qgis_version = version_match.group().replace('QGIS', '').strip('/') + platform_name = instance.ua_string[version_match.end():] + + if not platform_name: + if instance.user_agent: + platform_name = instance.user_agent.get_os() + + QgisUserVisit.objects.get_or_create( + user_visit=instance, + location=country_data, + qgis_version=qgis_version, + platform=platform_name + ) + + UserVisit.objects.filter(pk=instance.pk).update(remote_addr='') diff --git a/qgisfeedproject/qgisfeed/tests.py b/qgisfeedproject/qgisfeed/tests.py index 6dbb2a5..1c88a01 100644 --- a/qgisfeedproject/qgisfeed/tests.py +++ b/qgisfeedproject/qgisfeed/tests.py @@ -20,7 +20,9 @@ from django.contrib.admin.sites import AdminSite from django.utils import timezone -from .models import QgisFeedEntry +from .models import ( + QgisFeedEntry, QgisUserVisit, DailyQgisUserVisit, aggregate_user_visit_data +) from .admin import QgisFeedEntryAdmin @@ -188,3 +190,40 @@ def test_admin_author_is_set(self): ma.save_model(request, obj, form, False) self.assertEqual(obj.author, request.user) + +class QgisUserVisitTestCase(TestCase): + + def test_user_visit(self): + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Fedora ' + 'Linux (Workstation Edition)') + c.get('/') + user_visit = QgisUserVisit.objects.filter( + platform__icontains='Fedora Linux (Workstation Edition)') + self.assertEqual(user_visit.count(), 1) + self.assertEqual(user_visit.first().qgis_version, '32400') + + def test_ip_address_removed(self): + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Fedora ' + 'Linux (Workstation Edition)', + REMOTE_ADDR='180.247.213.170') + c.get('/') + qgis_visit = QgisUserVisit.objects.first() + self.assertTrue(qgis_visit.user_visit.remote_addr == '') + self.assertTrue(qgis_visit.location['country_name'] == 'Indonesia') + + def test_aggregate_visit(self): + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/31400/Fedora ' + 'Linux (Workstation Edition)', + REMOTE_ADDR='180.247.213.170') + c.get('/') + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Windows 10', + REMOTE_ADDR='180.247.213.160') + c.get('/') + c = Client(HTTP_USER_AGENT='Mozilla/5.0 QGIS/32400/Windows XP', + REMOTE_ADDR='180.247.213.160') + c.get('/') + aggregate_user_visit_data() + daily_visit = DailyQgisUserVisit.objects.first() + self.assertTrue(daily_visit.platform['Windows 10'] == 1) + self.assertTrue(daily_visit.qgis_version['32400'] == 2) + self.assertTrue(daily_visit.country['Indonesia'] == 3) diff --git a/qgisfeedproject/qgisfeed/views.py b/qgisfeedproject/qgisfeed/views.py index 278bea5..dc6b401 100644 --- a/qgisfeedproject/qgisfeed/views.py +++ b/qgisfeedproject/qgisfeed/views.py @@ -24,6 +24,8 @@ from .languages import LANGUAGE_KEYS import json +from user_visit.models import UserVisit + QGISFEED_MAX_RECORDS=getattr(settings, 'QGISFEED_MAX_RECORDS', 20) diff --git a/qgisfeedproject/qgisfeedproject/settings.py b/qgisfeedproject/qgisfeedproject/settings.py index bc442af..c3f583d 100644 --- a/qgisfeedproject/qgisfeedproject/settings.py +++ b/qgisfeedproject/qgisfeedproject/settings.py @@ -11,6 +11,9 @@ """ import os +import logging + +logger = logging.getLogger(__name__) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -45,6 +48,9 @@ # Dependencies 'tinymce', # HTML field 'imagekit', # Image crop and resize + + # User visit + 'user_visit', ] # Useful debugging extensions @@ -61,6 +67,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'qgisfeed.middleware.QgisFeedUserVisitMiddleware', ] ROOT_URLCONF = 'qgisfeedproject.urls' @@ -140,7 +147,10 @@ MEDIA_ROOT=os.path.join(BASE_DIR, 'media') MEDIA_URL = '/media/' +GEOIP_PATH='/var/opt/maxmind/' + + try: from .settings_local import * except ImportError as ex: - raise ex \ No newline at end of file + pass diff --git a/qgisfeedproject/qgisfeedproject/settings_dev.py b/qgisfeedproject/qgisfeedproject/settings_dev.py new file mode 100644 index 0000000..f6f3aa4 --- /dev/null +++ b/qgisfeedproject/qgisfeedproject/settings_dev.py @@ -0,0 +1,16 @@ +from .settings import * # noqa + +# Settings local for docker compose + +ALLOWED_HOSTS=['*'] + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': 'qgisfeed', + 'USER': 'docker', + 'PASSWORD': 'docker', + 'HOST': 'postgis', + 'PORT': '5432' + } +}