diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 6159a3d9b..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/python:3.6.8-stretch - steps: - - checkout - - run: - name: Run tests - command: | - set -x - sudo apt-get install libgdal-dev osmctools gdal-bin libspatialindex-dev - virtualenv ./venv - source venv/bin/activate - export CPLUS_INCLUDE_PATH=/usr/include/gdal - export C_INCLUDE_PATH=/usr/include/gdal - export PIP_PROGRESS_BAR=off - pip install -r requirements-dev.txt - make test diff --git a/.github/workflows/test.yaml b/.github/workflows/ci.yaml similarity index 93% rename from .github/workflows/test.yaml rename to .github/workflows/ci.yaml index e5e332f9a..b30e98c9b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/ci.yaml @@ -9,8 +9,7 @@ on: jobs: deploy: - runs-on: - ubuntu-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python 3 @@ -31,4 +30,4 @@ jobs: pip install -r requirements-dev.txt - name: Run Tests run: | - make test \ No newline at end of file + make test diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 936a0b093..1dae9e527 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -12,4 +12,4 @@ on: jobs: publish_docs: - uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.0.1 + uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@main diff --git a/Makefile b/Makefile index 7f909f07e..3992982ed 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ django_test: python manage.py test api.tests jobs.tests tasks.tests test: - nosetests hdx_exports/tests/test_hdx_export_set.py + pytest hdx_exports/tests/test_hdx_export_set.py worker: PYTHONPATH=../osm-export-tool-python/ DJANGO_SETTINGS_MODULE=core.settings.project dramatiq tasks.task_runners --processes 1 --threads 1 \ No newline at end of file diff --git a/api/migrations/0001_create_application.py b/api/migrations/0001_create_application.py deleted file mode 100644 index 62066a2a3..000000000 --- a/api/migrations/0001_create_application.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2017-07-26 20:29 -from __future__ import unicode_literals - -from django.db import migrations -from oauth2_provider.models import Application - - -class Migration(migrations.Migration): - def add_default_application(apps, schema_editor): - Application.objects.create( - name="OSM Export Tool UI", - redirect_uris= - "http://localhost/authorized http://localhost:8080/authorized http://localhost:8000/authorized", - client_type=Application.CLIENT_PUBLIC, - authorization_grant_type=Application.GRANT_IMPLICIT, - skip_authorization=True) - - dependencies = [ - ("oauth2_provider", "0005_auto_20170514_1141"), - ] - - operations = [ - migrations.RunPython(add_default_application), - ] diff --git a/api/serializers.py b/api/serializers.py index a773bb54f..3352fef9f 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -11,7 +11,14 @@ import django.core.exceptions from django.contrib.auth.models import User from django.db import transaction -from jobs.models import HDXExportRegion, Job, SavedFeatureSelection, validate_aoi, validate_mbtiles, PartnerExportRegion +from jobs.models import ( + HDXExportRegion, + Job, + SavedFeatureSelection, + validate_aoi, + validate_mbtiles, + PartnerExportRegion, +) from rest_framework import serializers from rest_framework_gis import serializers as geo_serializers from tasks.models import ExportRun, ExportTask @@ -23,83 +30,117 @@ class UserSerializer(serializers.ModelSerializer): class Meta: model = User - fields = ('username', ) + fields = ("username",) class ExportTaskSerializer(serializers.ModelSerializer): class Meta: model = ExportTask - fields = ('uid', 'name', 'status', 'started_at', 'finished_at', - 'duration', 'filesize_bytes', 'download_urls') + fields = ( + "uid", + "name", + "status", + "started_at", + "finished_at", + "duration", + "filesize_bytes", + "download_urls", + ) class ExportRunSerializer(serializers.ModelSerializer): tasks = ExportTaskSerializer(many=True, read_only=True) - user = UserSerializer( - read_only=True, default=serializers.CurrentUserDefault()) + user = UserSerializer(read_only=True, default=serializers.CurrentUserDefault()) class Meta: model = ExportRun - lookup_field = 'uid' - fields = ('uid','created_at', 'started_at', 'finished_at', 'duration', - 'elapsed_time', 'user', 'size','hdx_sync_status', 'status', 'tasks') + lookup_field = "uid" + fields = ( + "uid", + "created_at", + "started_at", + "finished_at", + "duration", + "elapsed_time", + "user", + "size", + "hdx_sync_status", + "status", + "tasks", + ) class ConfigurationSerializer(serializers.ModelSerializer): - user = UserSerializer( - read_only=True, default=serializers.CurrentUserDefault()) + user = UserSerializer(read_only=True, default=serializers.CurrentUserDefault()) class Meta: model = SavedFeatureSelection - fields = ('uid', 'name', 'description', 'yaml', 'public', 'user','pinned') + fields = ("uid", "name", "description", "yaml", "public", "user", "pinned") class JobGeomSerializer(serializers.ModelSerializer): - """ Since Job Geoms can be large, these are serialized separately, + """Since Job Geoms can be large, these are serialized separately, instead of nested within Jobs.""" class Meta: model = Job - fields = ('the_geom', ) + fields = ("the_geom",) + class JobSerializer(serializers.ModelSerializer): - user = UserSerializer( - read_only=True, default=serializers.CurrentUserDefault()) + user = UserSerializer(default=serializers.CurrentUserDefault()) class Meta: model = Job - fields = ('id', 'uid', 'user', 'name', 'description', 'event', - 'export_formats', 'published', 'feature_selection', - 'buffer_aoi', 'osma_link', 'created_at', 'area', 'the_geom', - 'simplified_geom', 'mbtiles_source', 'mbtiles_minzoom', 'mbtiles_maxzoom','pinned','unfiltered','preserve_geom') + fields = ( + "id", + "uid", + "user", + "name", + "description", + "event", + "export_formats", + "published", + "feature_selection", + "buffer_aoi", + "osma_link", + "created_at", + "area", + "the_geom", + "simplified_geom", + "mbtiles_source", + "mbtiles_minzoom", + "mbtiles_maxzoom", + "pinned", + "unfiltered", + "preserve_geom", + ) extra_kwargs = { - 'the_geom': { - 'write_only': True - }, - 'simplified_geom': { - 'read_only': True - } + "the_geom": {"write_only": True}, + "simplified_geom": {"read_only": True}, } - def validate(self,data): + def validate(self, data): try: - validate_aoi(data['the_geom']) + validate_aoi(data["the_geom"]) except django.core.exceptions.ValidationError as e: - raise serializers.ValidationError({'the_geom':e.messages[0]}) + raise serializers.ValidationError({"the_geom": e.messages[0]}) try: validate_mbtiles(data) except django.core.exceptions.ValidationError as e: - raise serializers.ValidationError({'mbtiles_source': e.messages[0]}) + raise serializers.ValidationError({"mbtiles_source": e.messages[0]}) return data + def validate_model(model): try: model.full_clean() except django.core.exceptions.ValidationError as e: raise serializers.ValidationError(e.message_dict) + class PartnerExportRegionListSerializer(serializers.ModelSerializer): export_formats = serializers.ListField() feature_selection = serializers.CharField() @@ -108,10 +149,21 @@ class PartnerExportRegionListSerializer(serializers.ModelSerializer): class Meta: # noqa model = PartnerExportRegion - fields = ('id', 'feature_selection', - 'schedule_period', 'schedule_hour', 'export_formats', - 'name', 'last_run', 'next_run', - 'simplified_geom', 'job_uid', 'last_size','group_name') + fields = ( + "id", + "feature_selection", + "schedule_period", + "schedule_hour", + "export_formats", + "name", + "last_run", + "next_run", + "simplified_geom", + "job_uid", + "last_size", + "group_name", + ) + class PartnerExportRegionSerializer(serializers.ModelSerializer): # noqa export_formats = serializers.ListField() @@ -124,47 +176,74 @@ class PartnerExportRegionSerializer(serializers.ModelSerializer): # noqa class Meta: # noqa model = PartnerExportRegion - fields = ('id', 'feature_selection', - 'schedule_period', 'schedule_hour', 'export_formats', - 'name', 'event', 'description', 'last_run', 'next_run', - 'simplified_geom', 'job_uid', - 'the_geom','group','planet_file', 'polygon_centroid') + fields = ( + "id", + "feature_selection", + "schedule_period", + "schedule_hour", + "export_formats", + "name", + "event", + "description", + "last_run", + "next_run", + "simplified_geom", + "job_uid", + "the_geom", + "group", + "planet_file", + "polygon_centroid", + ) extra_kwargs = { - 'simplified_geom': { - 'read_only': True - }, - 'the_geom': { - 'write_only': True - } + "simplified_geom": {"read_only": True}, + "the_geom": {"write_only": True}, } def create(self, validated_data): # noqa def slice_dict(in_dict, wanted_keys): return dict((k, in_dict[k]) for k in wanted_keys if k in in_dict) - job_dict = slice_dict(validated_data, [ - 'the_geom', 'export_formats', 'feature_selection', - ]) - job_dict['user'] = self.context['request'].user - job_dict['name'] = validated_data.get('name') - job_dict['event'] = validated_data.get('event') or "" - job_dict['description'] = validated_data.get('description') or "" - - region_dict = slice_dict(validated_data, [ - 'schedule_period', 'schedule_hour','group','planet_file', 'polygon_centroid' - ]) + job_dict = slice_dict( + validated_data, + [ + "the_geom", + "export_formats", + "feature_selection", + ], + ) + job_dict["user"] = self.context["request"].user + job_dict["name"] = validated_data.get("name") + job_dict["event"] = validated_data.get("event") or "" + job_dict["description"] = validated_data.get("description") or "" + + region_dict = slice_dict( + validated_data, + [ + "schedule_period", + "schedule_hour", + "group", + "planet_file", + "polygon_centroid", + ], + ) job = Job(**job_dict) job.hidden = True job.unlimited_extent = True validate_model(job) # check on creation that i'm a member of the group - if not self.context['request'].user.groups.filter(name=region_dict['group'].name).exists(): - raise serializers.ValidationError({'group':'You are not a member of this group.'}) + if ( + not self.context["request"] + .user.groups.filter(name=region_dict["group"].name) + .exists() + ): + raise serializers.ValidationError( + {"group": "You are not a member of this group."} + ) with transaction.atomic(): job.save() - region_dict['job'] = job + region_dict["job"] = job region = PartnerExportRegion(**region_dict) validate_model(region) region.save() @@ -177,29 +256,44 @@ def update_attrs(model, v_data, keys): setattr(model, key, v_data[key]) # if re-assigning, check group membership - if not self.context['request'].user.groups.filter(name= validated_data['group'].name).exists(): - raise serializers.ValidationError({'group':'You are not a member of this group.'}) + if ( + not self.context["request"] + .user.groups.filter(name=validated_data["group"].name) + .exists() + ): + raise serializers.ValidationError( + {"group": "You are not a member of this group."} + ) job = instance.job - update_attrs(job, validated_data, [ - 'the_geom', 'export_formats', 'feature_selection' - ]) - job.name = validated_data.get('name') - job.event = validated_data.get('event') or "" - job.description = validated_data.get('description') or "" + update_attrs( + job, validated_data, ["the_geom", "export_formats", "feature_selection"] + ) + job.name = validated_data.get("name") + job.event = validated_data.get("event") or "" + job.description = validated_data.get("description") or "" validate_model(job) - update_attrs(instance, validated_data, [ - 'schedule_period', 'schedule_hour', 'group','planet_file', 'polygon_centroid' - ]) + update_attrs( + instance, + validated_data, + [ + "schedule_period", + "schedule_hour", + "group", + "planet_file", + "polygon_centroid", + ], + ) validate_model(instance) with transaction.atomic(): instance.save() job.save() return instance + class HDXExportRegionListSerializer(serializers.ModelSerializer): # noqa - """ The list serializer does not expose the Geom, as it can be large.""" + """The list serializer does not expose the Geom, as it can be large.""" export_formats = serializers.ListField() dataset_prefix = serializers.CharField() @@ -210,16 +304,33 @@ class HDXExportRegionListSerializer(serializers.ModelSerializer): # noqa class Meta: # noqa model = HDXExportRegion - fields = ('id', 'dataset_prefix', 'datasets', 'feature_selection', - 'schedule_period', 'schedule_hour', 'export_formats', - 'locations', 'name', 'last_run', 'next_run', - 'simplified_geom', 'dataset_prefix', 'job_uid', 'license', - 'subnational', 'extra_notes', 'is_private', 'buffer_aoi', 'last_size') + fields = ( + "id", + "dataset_prefix", + "datasets", + "feature_selection", + "schedule_period", + "schedule_hour", + "export_formats", + "locations", + "name", + "last_run", + "next_run", + "simplified_geom", + "dataset_prefix", + "job_uid", + "license", + "subnational", + "extra_notes", + "is_private", + "buffer_aoi", + "last_size", + ) class HDXExportRegionSerializer(serializers.ModelSerializer): # noqa - """ Internally, an export region is a job model + an export region model - but to API users, it appears as a single entity. """ + """Internally, an export region is a job model + an export region model + but to API users, it appears as a single entity.""" export_formats = serializers.ListField() dataset_prefix = serializers.CharField() @@ -228,53 +339,79 @@ class HDXExportRegionSerializer(serializers.ModelSerializer): # noqa the_geom = geo_serializers.GeometryField() name = serializers.CharField() buffer_aoi = serializers.BooleanField() + def validate(self, data): """ Check for export formats for country exports. """ - if HDXExportRegion.objects.filter(schedule_period='daily').count() > 6: - raise serializers.ValidationError("Maximum daily run limit of 6 for hdx job exceeded. Please change the frequency") + if HDXExportRegion.objects.filter(schedule_period="daily").count() > 6: + raise serializers.ValidationError( + "Maximum daily run limit of 6 for hdx job exceeded. Please change the frequency" + ) return data class Meta: # noqa model = HDXExportRegion - fields = ('id', 'dataset_prefix', 'datasets', 'feature_selection', - 'schedule_period', 'schedule_hour', 'export_formats', - 'locations', 'name', 'last_run', 'next_run', - 'simplified_geom', 'dataset_prefix', 'job_uid', 'license', - 'subnational', 'extra_notes', 'is_private', 'buffer_aoi', - 'the_geom','planet_file') + fields = ( + "id", + "dataset_prefix", + "datasets", + "feature_selection", + "schedule_period", + "schedule_hour", + "export_formats", + "locations", + "name", + "last_run", + "next_run", + "simplified_geom", + "dataset_prefix", + "job_uid", + "license", + "subnational", + "extra_notes", + "is_private", + "buffer_aoi", + "the_geom", + "planet_file", + ) extra_kwargs = { - 'simplified_geom': { - 'read_only': True - }, - 'the_geom': { - 'write_only': True - } + "simplified_geom": {"read_only": True}, + "the_geom": {"write_only": True}, } def create(self, validated_data): # noqa def slice_dict(in_dict, wanted_keys): return dict((k, in_dict[k]) for k in wanted_keys if k in in_dict) - job_dict = slice_dict(validated_data, [ - 'the_geom', 'export_formats', 'feature_selection', 'buffer_aoi' - ]) - job_dict['user'] = self.context['request'].user - job_dict['name'] = validated_data.get('dataset_prefix') - job_dict['description'] = validated_data.get('name') - - region_dict = slice_dict(validated_data, [ - 'extra_notes', 'is_private', 'locations', 'license', - 'schedule_period', 'schedule_hour', 'subnational','planet_file' - ]) + job_dict = slice_dict( + validated_data, + ["the_geom", "export_formats", "feature_selection", "buffer_aoi"], + ) + job_dict["user"] = self.context["request"].user + job_dict["name"] = validated_data.get("dataset_prefix") + job_dict["description"] = validated_data.get("name") + + region_dict = slice_dict( + validated_data, + [ + "extra_notes", + "is_private", + "locations", + "license", + "schedule_period", + "schedule_hour", + "subnational", + "planet_file", + ], + ) job = Job(**job_dict) job.hidden = True job.unlimited_extent = True validate_model(job) with transaction.atomic(): job.save() - region_dict['job'] = job + region_dict["job"] = job region = HDXExportRegion(**region_dict) validate_model(region) region.save() @@ -287,17 +424,29 @@ def update_attrs(model, v_data, keys): setattr(model, key, v_data[key]) job = instance.job - update_attrs(job, validated_data, [ - 'the_geom', 'export_formats', 'feature_selection', 'buffer_aoi' - ]) - job.name = validated_data.get('dataset_prefix') - job.description = validated_data.get('name') + update_attrs( + job, + validated_data, + ["the_geom", "export_formats", "feature_selection", "buffer_aoi"], + ) + job.name = validated_data.get("dataset_prefix") + job.description = validated_data.get("name") validate_model(job) - update_attrs(instance, validated_data, [ - 'extra_notes', 'is_private', 'locations', 'license', - 'schedule_period', 'schedule_hour', 'subnational', 'planet_file' - ]) + update_attrs( + instance, + validated_data, + [ + "extra_notes", + "is_private", + "locations", + "license", + "schedule_period", + "schedule_hour", + "subnational", + "planet_file", + ], + ) validate_model(instance) with transaction.atomic(): instance.save() diff --git a/api/templates/rest_framework/api.html b/api/templates/rest_framework/api.html index 1622c25b0..c5ed8cd7e 100644 --- a/api/templates/rest_framework/api.html +++ b/api/templates/rest_framework/api.html @@ -1,5 +1,5 @@ {% extends "rest_framework/base.html" %} -{% load staticfiles %} +{% load static %} {% block head %} {% block meta %} diff --git a/api/templates/rest_framework/login_base.html b/api/templates/rest_framework/login_base.html index 48ecd1573..c89271a8d 100644 --- a/api/templates/rest_framework/login_base.html +++ b/api/templates/rest_framework/login_base.html @@ -1,5 +1,5 @@ {% extends "rest_framework/base.html" %} -{% load staticfiles %} +{% load static %} {% load rest_framework %} {% block body %} diff --git a/api/urls.py b/api/urls.py index 02fe4eecc..3d0f0c5fb 100644 --- a/api/urls.py +++ b/api/urls.py @@ -1,41 +1,48 @@ -# -*- coding: utf-8 -*- -"""API url configuration.""" - -from django.conf.urls import url +from django.urls import re_path, path from rest_framework.routers import DefaultRouter - -from .views import (ConfigurationViewSet, ExportRunViewSet, - HDXExportRegionViewSet, PartnerExportRegionViewSet, JobViewSet, permalink, get_overpass_timestamp, - cancel_run, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status) +from .views import ( + ConfigurationViewSet, + ExportRunViewSet, + HDXExportRegionViewSet, + PartnerExportRegionViewSet, + JobViewSet, + permalink, + get_overpass_timestamp, + cancel_run, + get_user_permissions, + request_geonames, + get_overpass_status, + get_groups, + stats, + run_stats, + request_nominatim, + machine_status, +) router = DefaultRouter(trailing_slash=False) -router.register(r'jobs', JobViewSet, base_name='jobs') -router.register(r'runs', ExportRunViewSet, base_name='runs') +router.register(r"jobs", JobViewSet, basename="jobs") +router.register(r"runs", ExportRunViewSet, basename="runs") +router.register(r"configurations", ConfigurationViewSet, basename="configurations") router.register( - r'configurations', ConfigurationViewSet, base_name='configurations') + r"hdx_export_regions", HDXExportRegionViewSet, basename="hdx_export_regions" +) router.register( - r'hdx_export_regions', - HDXExportRegionViewSet, - base_name='hdx_export_regions') -router.register( - r'partner_export_regions', + r"partner_export_regions", PartnerExportRegionViewSet, - base_name='partner_export_regions') - - -urlpatterns = router.urls - -urlpatterns += [ - url(r'^permalink/(?P[a-z0-9\-]+)$', permalink), - url(r'^request_nominatim$', request_nominatim), - url(r'^request_geonames$', request_geonames), - url(r'^overpass_timestamp$', get_overpass_timestamp), - url(r'^overpass_status$', get_overpass_status), - url(r'^permissions$', get_user_permissions), - url(r'^groups$',get_groups), - url(r'^stats$', stats), - url(r'^run_stats$', run_stats), - url(r'^status$', machine_status), - url(r'^cancel_run$', cancel_run), + basename="partner_export_regions", +) +app_name = "api" +urlpatterns = router.urls + [ + re_path(r"^permalink/(?P[a-z0-9\-]+)$", permalink), + path("request_nominatim", request_nominatim), + path("request_geonames", request_geonames), + path("overpass_timestamp", get_overpass_timestamp), + path("overpass_status", get_overpass_status), + path("permissions", get_user_permissions), + path("groups", get_groups), + path("stats", stats), + path("run_stats", run_stats), + path("status", machine_status), + path("cancel_run", cancel_run), ] diff --git a/api/views.py b/api/views.py index c91b2f82f..47a7b4d69 100644 --- a/api/views.py +++ b/api/views.py @@ -31,7 +31,7 @@ from django.core.exceptions import ValidationError as DjangoValidationError from jobs.models import HDXExportRegion, PartnerExportRegion, Job, SavedFeatureSelection from rest_framework import filters, permissions, status, viewsets -from rest_framework.decorators import detail_route +from rest_framework.decorators import action from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework.serializers import ValidationError @@ -156,7 +156,7 @@ def perform_create(self, serializer): task_runner = ExportTaskRunner() task_runner.run_task(job_uid=str(job.uid)) - @detail_route() + @action(detail=True) def geom(self, request, uid=None): job = Job.objects.get(uid=uid) geom_serializer = JobGeomSerializer(job) @@ -618,15 +618,14 @@ def request_geonames(request): keyword = request.GET.get("q") response = {"totalResultsCount": 0, "geonames": []} if not ( - str(keyword).lower().startswith("boundary") + str(keyword).lower().startswith("hdx") or str(keyword).lower().startswith("osm") or str(keyword).lower().startswith("tm") ): response = requests.get(geonames_url, params=payload).json() - print(response) assert isinstance(response, dict) if RAW_DATA_API_URL: - if str(keyword).lower().startswith("boundary"): + if str(keyword).lower().startswith("hdx"): lst = keyword.split(" ") if len(lst) > 1: keyword = lst[1] @@ -646,7 +645,7 @@ def request_geonames(request): } add_resp = { "bbox": geojson, - "adminName2": feature["properties"]["iso_3"], + "adminName2": f'ISO3 : {feature["properties"]["iso_3"]}', "name": f'{request.GET.get("q")} -> {feature["properties"]["description"]}', "countryName": feature["properties"][ "dataset_name" @@ -656,6 +655,7 @@ def request_geonames(request): if "geonames" in response: response["geonames"].append(add_resp) + response["totalResultsCount"] += 1 if str(keyword).lower().startswith("osm"): lst = keyword.split(" ") @@ -805,13 +805,15 @@ def cancel_run(request): if message_id: LOG.debug("Canceling task_message_id:{0} ".format(message_id)) if run.status == "SUBMITTED": + run.status = "FAILED" + run.save() abort(message_id, mode="cancel") elif run.status == "RUNNING": - abort(message_id) # cancel if its in queue or in progress - - run.status = "FAILED" - run.worker_message_id = None # set back message id - run.save() + run.status = "FAILED" + run.save() + abort(message_id) + run.worker_message_id = None + run.save() except (Job.DoesNotExist, ExportRun.DoesNotExist, ExportTask.DoesNotExist): LOG.warn("ExportRun doesnot exist . Exiting") return JsonResponse( diff --git a/core/settings/base.py b/core/settings/base.py index ef61ee878..af3bf90ff 100644 --- a/core/settings/base.py +++ b/core/settings/base.py @@ -9,35 +9,33 @@ # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. # In a Windows environment this must be set to your system time zone. -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" # default DEBUG setting # Set debug to true for development -DEBUG = bool(os.getenv('DEBUG')) +DEBUG = bool(os.getenv("DEBUG")) # from django.utils.crypto import get_random_string # secret_key = get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') -if 'SECRET_KEY' not in os.environ: +if "SECRET_KEY" not in os.environ: print("WARNING: secret key not set - setting a default for development.") -SECRET_KEY = os.getenv('SECRET_KEY', 'default_secret_key') +SECRET_KEY = os.getenv("SECRET_KEY", "default_secret_key") # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en' +LANGUAGE_CODE = "en" LANGUAGES = ( - ('en', _('English')), - ('id', _('Bahasa Indonesia')), - ('de', _('German')), - ('es', _('Spanish')), - ('ja', _('Japanese')), - ('fr', _('French')), + ("en", _("English")), + ("id", _("Bahasa Indonesia")), + ("de", _("German")), + ("es", _("Spanish")), + ("ja", _("Japanese")), + ("fr", _("French")), ) -LOCALE_PATHS = ( - ABS_PATH('locales'), -) +LOCALE_PATHS = (ABS_PATH("locales"),) # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. @@ -52,22 +50,22 @@ # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/var/www/example.com/media/" -MEDIA_ROOT = ABS_PATH('media') +MEDIA_ROOT = ABS_PATH("media") # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. # Examples: "http://example.com/media/", "http://media.example.com/" -MEDIA_URL = '/media/' +MEDIA_URL = "/media/" # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = ABS_PATH('../static') +STATIC_ROOT = ABS_PATH("../static") # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" -STATIC_URL = '/static/' +STATIC_URL = "/static/" # Additional locations of static files STATICFILES_DIRS = ( @@ -80,57 +78,54 @@ # default middleware classes MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'oauth2_provider.middleware.OAuth2TokenMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - 'social_django.middleware.SocialAuthExceptionMiddleware', - 'corsheaders.middleware.CorsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.locale.LocaleMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "oauth2_provider.middleware.OAuth2TokenMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + "social_django.middleware.SocialAuthExceptionMiddleware", + "corsheaders.middleware.CorsMiddleware", ] CORS_ORIGIN_ALLOW_ALL = True -ROOT_URLCONF = 'core.urls' +ROOT_URLCONF = "core.urls" # Python dotted path to the WSGI application used by Django's runserver. -WSGI_APPLICATION = 'core.wsgi.application' +WSGI_APPLICATION = "core.wsgi.application" INSTALLED_APPS = ( - 'django.contrib.admin.apps.SimpleAdminConfig', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.gis', - 'django.contrib.postgres', - 'raven.contrib.django.raven_compat', - 'oauth2_provider', - 'corsheaders', + "django.contrib.admin.apps.SimpleAdminConfig", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.gis", + "django.contrib.postgres", + "raven.contrib.django.raven_compat", + "oauth2_provider", + "corsheaders", ) # enable cached storage -STATICFILES_STORAGE = ( - 'django.contrib.staticfiles.storage.CachedStaticFilesStorage' -) +STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder' + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", ) AUTHENTICATION_BACKENDS = ( - 'social_core.backends.openstreetmap.OpenStreetMapOAuth', - 'oauth2_provider.backends.OAuth2Backend', - 'social_core.backends.email.EmailAuth', - 'social_core.backends.username.UsernameAuth', - 'django.contrib.auth.backends.ModelBackend', + "social_core.backends.openstreetmap.OpenStreetMapOAuth", + "oauth2_provider.backends.OAuth2Backend", + "social_core.backends.email.EmailAuth", + "social_core.backends.username.UsernameAuth", + "django.contrib.auth.backends.ModelBackend", ) diff --git a/core/settings/project.py b/core/settings/project.py index cccd6816f..5a2176e64 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -92,7 +92,7 @@ """ JOB_MAX_EXTENT = 2500000 # default export max extent in sq km -HOSTNAME = os.getenv("HOSTNAME", "export.hotosm.org") +HOSTNAME = os.getenv("HOSTNAME", "localhost") # Comment if you are not running behind proxy USE_X_FORWARDED_HOST = bool(os.getenv("USE_X_FORWARDED_HOST")) @@ -100,7 +100,7 @@ # Hosts/domain names that are valid for this site # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts -ALLOWED_HOSTS = ["exports-prod.hotosm.org", HOSTNAME] +ALLOWED_HOSTS = ["exports-prod.hotosm.org", "export.hotosm.org", HOSTNAME, "127.0.0.1"] """ Overpass Element limit diff --git a/core/urls.py b/core/urls.py index ea6907400..d2a7502a9 100644 --- a/core/urls.py +++ b/core/urls.py @@ -1,69 +1,47 @@ -# -*- coding: utf-8 -*- -""" -HOT Export Tool URL Configuration -""" -from api.urls import router +from django.urls import path, include, re_path from django.conf import settings -from django.conf.urls import include, url -from django.conf.urls.i18n import i18n_patterns from django.conf.urls.static import static from django.contrib import admin from django.views.generic import TemplateView, RedirectView from django.views.i18n import JavaScriptCatalog -from ui.views import (authorized, login, logout, - redirect_to_v3, v3, worker_dashboard) -urlpatterns = [] - -urlpatterns += i18n_patterns( - url(r'^$', redirect_to_v3, name='index'), - url(r'^v3/', v3, name="v3"), - url(r'^worker-dashboard/$', worker_dashboard), - url(r'^login/$', login, name="login"), - url(r'^logout$', logout, name='logout'), - url(r'^email/$', TemplateView.as_view(template_name='osm/email.html'), - name='require_email'), -) - -urlpatterns += [ - url(r'^authorized', authorized, name="authorized"), -] - -urlpatterns += i18n_patterns( - url(r'^admin/login/', RedirectView.as_view(pattern_name='login', permanent=False)), - url(r'^admin/', include(admin.site.urls)), -) - -# OAuth client urls -urlpatterns += i18n_patterns( - url('^osm/', include('social_django.urls', namespace='osm')), - url('^osm/email_verify_sent/$', TemplateView.as_view( - template_name='osm/email_verify_sent.html'), name='email_verify_sent'), - url('^osm/error$', TemplateView.as_view(template_name='osm/error.html'), - name='login_error') -) - -# OAuth provider urls -urlpatterns += [ - url(r'^o/', include('oauth2_provider.urls', namespace='oauth2_provider')), -] - -# don't apply i18n patterns here.. api uses Accept-Language header -urlpatterns += [ - url(r'^api/', include(router.urls, namespace='api')), - url(r'^api/', include('rest_framework.urls', namespace='rest_framework')), -] - -# i18n for js -js_info_dict = { - 'packages': ('hot_osm',), -} - -urlpatterns += [ - url(r'^jsi18n/$', JavaScriptCatalog.as_view(packages=['hot_osm']), name='javascript-catalog'), - url(r'^i18n/', include('django.conf.urls.i18n')), +from ui.views import authorized, login, logout, redirect_to_v3, v3, worker_dashboard + +urlpatterns = [ + path("", redirect_to_v3, name="index"), + path("v3/", v3, name="v3"), + path("worker-dashboard/", worker_dashboard, name="worker_dashboard"), + path("login/", login, name="login"), + path("logout/", logout, name="logout"), + path( + "email/", + TemplateView.as_view(template_name="osm/email.html"), + name="require_email", + ), + re_path(r"^authorized", authorized, name="authorized"), + path("admin/login/", RedirectView.as_view(pattern_name="login", permanent=False)), + path("admin/", admin.site.urls), + path("osm/", include("social_django.urls", namespace="osm")), + path( + "osm/email_verify_sent/", + TemplateView.as_view(template_name="osm/email_verify_sent.html"), + name="email_verify_sent", + ), + path( + "osm/error", + TemplateView.as_view(template_name="osm/error.html"), + name="login_error", + ), + path("o/", include("oauth2_provider.urls", namespace="oauth2_provider")), + path("api/", include("api.urls", namespace="api")), + path("api-auth/", include("rest_framework.urls", namespace="rest_framework")), + path( + "jsi18n/", + JavaScriptCatalog.as_view(packages=["hot_osm"]), + name="javascript-catalog", + ), + path("i18n/", include("django.conf.urls.i18n")), ] if settings.DEBUG: - urlpatterns += static(settings.STATIC_URL, - document_root=settings.STATIC_ROOT) + urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/hdx_exports/hdx_export_set.py b/hdx_exports/hdx_export_set.py index cd485330d..b8bbd941f 100644 --- a/hdx_exports/hdx_export_set.py +++ b/hdx_exports/hdx_export_set.py @@ -23,13 +23,15 @@ information. """ + def slugify(str): s = django.utils.text.slugify(str) - return s.replace('-','_') + return s.replace("-", "_") + -def sync_datasets(datasets,update_dataset_date=False): +def sync_datasets(datasets, update_dataset_date=False): for dataset in datasets: - exists = Dataset.read_from_hdx(dataset['name']) + exists = Dataset.read_from_hdx(dataset["name"]) if exists: if update_dataset_date: dataset.set_date_of_dataset(datetime.now()) @@ -38,12 +40,13 @@ def sync_datasets(datasets,update_dataset_date=False): dataset.set_date_of_dataset(datetime.now()) dataset.create_in_hdx(allow_no_resources=True) -def sync_region(region,files=[],public_dir=''): + +def sync_region(region, files=[], public_dir=""): export_set = HDXExportSet( Mapping(region.feature_selection), region.dataset_prefix, region.name, - region.extra_notes + region.extra_notes, ) datasets = export_set.datasets( region.is_private, @@ -51,87 +54,108 @@ def sync_region(region,files=[],public_dir=''): region.update_frequency, region.locations, files, - public_dir + public_dir, ) - sync_datasets(datasets,len(files) > 0) + sync_datasets(datasets, len(files) > 0) + class HDXExportSet(object): - def __init__(self,mapping,dataset_prefix,name,extra_notes=''): + def __init__(self, mapping, dataset_prefix, name, extra_notes=""): self._dataset_prefix = dataset_prefix - self._extra_notes = extra_notes + '\n' if extra_notes else '' + self._extra_notes = extra_notes + "\n" if extra_notes else "" self._mapping = mapping self._name = name # used in the serializer - def dataset_links(self,hdx_prefix_url): - return [{ - 'name': '{}_{}'.format(self._dataset_prefix, slugify(theme.name)), - 'url': '{}/dataset/{}_{}'.format( - hdx_prefix_url, self._dataset_prefix, slugify(theme.name)), - } for theme in self._mapping.themes] + def dataset_links(self, hdx_prefix_url): + return [ + { + "name": "{}_{}".format(self._dataset_prefix, slugify(theme.name)), + "url": "{}/dataset/{}_{}".format( + hdx_prefix_url, self._dataset_prefix, slugify(theme.name) + ), + } + for theme in self._mapping.themes + ] def hdx_note(self, theme): columns = [] for key in theme.keys: - columns.append('- [{0}](http://wiki.openstreetmap.org/wiki/Key:{0})'.format(key)) - columns = '\n'.join(columns) + columns.append( + "- [{0}](http://wiki.openstreetmap.org/wiki/Key:{0})".format(key) + ) + columns = "\n".join(columns) criteria = theme.matcher.to_sql() filter_str = FILTER_CRITERIA.format(criteria=criteria) return self._extra_notes + MARKDOWN.format( - region=self._name, - columns=columns, - filter_str=filter_str - ) - - def datasets(self,is_private,subnational,data_update_frequency,locations,files,public_dir): + region=self._name, columns=columns, filter_str=filter_str + ) + + def datasets( + self, + is_private, + subnational, + data_update_frequency, + locations, + files, + public_dir, + ): HDX_FORMATS = { - 'geojson':'geojson', - 'shp': 'SHP', - 'geopackage': 'Geopackage', - 'gpkg': 'Geopackage', - 'garmin_img': 'Garmin IMG', - 'kml': 'KML', - 'csv': 'CSV' + "geojson": "geojson", + "shp": "SHP", + "geopackage": "Geopackage", + "gpkg": "Geopackage", + "garmin_img": "Garmin IMG", + "kml": "KML", + "csv": "CSV", } HDX_DESCRIPTIONS = { - 'geojson': 'geojson', - 'shp':'ESRI Shapefile', - 'geopackage':'Geopackage, SQLite compatible', - 'gpkg':'Geopackage, SQLite compatible', - 'garmin_img':'.IMG for Garmin GPS Devices (All OSM layers for area)', - 'kml':'Google Earth .KML', - 'csv' : 'CSV Output along with centroid of feature' + "geojson": "GeoJSON", + "shp": "ESRI Shapefile", + "geopackage": "Geopackage, SQLite compatible", + "gpkg": "Geopackage, SQLite compatible", + "garmin_img": ".IMG for Garmin GPS Devices (All OSM layers for area)", + "kml": "Google Earth .KML", + "csv": "CSV Output along with centroid of feature", } d = [] - updated_by_script = f'HOT Export Tool ({datetime.now().strftime("%Y-%m-%dT%H:%M:%S")})' + updated_by_script = ( + f'HOT Export Tool ({datetime.now().strftime("%Y-%m-%dT%H:%M:%S")})' + ) for theme in self._mapping.themes: dataset = Dataset() - dataset['owner_org'] = '225b9f7d-e7cb-4156-96a6-44c9c58d31e3' - dataset['maintainer'] = '6a0688ce-8521-46e2-8edd-8e26c0851ebd' - dataset['dataset_source'] = 'OpenStreetMap contributors' - dataset['methodology'] = 'Other' - dataset['methodology_other'] = 'Volunteered geographic information' - dataset['license_id'] = 'hdx-odc-odbl' - dataset['updated_by_script'] = updated_by_script - dataset['groups'] = [] - - dataset['name'] = '{0}_{1}'.format(self._dataset_prefix, slugify(theme.name)) - dataset['title'] = '{0} {1} (OpenStreetMap Export)'.format(self._name, theme.name) - dataset['notes'] = self.hdx_note(theme) - - dataset['private'] = is_private - dataset['subnational'] = str(int(subnational)) - dataset['data_update_frequency'] = str(data_update_frequency) - - if 'hdx' in theme.extra: - if 'caveats' in theme.extra['hdx']: - dataset['caveats'] = theme.extra['hdx']['caveats'] - if 'tags' in theme.extra['hdx']: - tags = [tag.strip() for tag in theme.extra['hdx']['tags'].split(',')] + dataset["owner_org"] = "225b9f7d-e7cb-4156-96a6-44c9c58d31e3" + dataset["maintainer"] = "6a0688ce-8521-46e2-8edd-8e26c0851ebd" + dataset["dataset_source"] = "OpenStreetMap contributors" + dataset["methodology"] = "Other" + dataset["methodology_other"] = "Volunteered geographic information" + dataset["license_id"] = "hdx-odc-odbl" + dataset["updated_by_script"] = updated_by_script + dataset["groups"] = [] + + dataset["name"] = "{0}_{1}".format( + self._dataset_prefix, slugify(theme.name) + ) + dataset["title"] = "{0} {1} (OpenStreetMap Export)".format( + self._name, theme.name + ) + dataset["notes"] = self.hdx_note(theme) + + dataset["private"] = is_private + dataset["subnational"] = str(int(subnational)) + dataset["data_update_frequency"] = str(data_update_frequency) + + if "hdx" in theme.extra: + if "caveats" in theme.extra["hdx"]: + dataset["caveats"] = theme.extra["hdx"]["caveats"] + if "tags" in theme.extra["hdx"]: + tags = [ + tag.strip() for tag in theme.extra["hdx"]["tags"].split(",") + ] dataset.add_tags(tags) for location in locations: @@ -140,31 +164,34 @@ def datasets(self,is_private,subnational,data_update_frequency,locations,files,p resources = [] for f in files: - if isinstance(f, dict): # it is coming from galaxy - if f['theme'] == theme.name: - file_name = f['file_name'] # only one part: the zip file - resources.append({ - 'name': f"{file_name}.zip", - 'format': HDX_FORMATS[f['output_name']], - 'description': HDX_DESCRIPTIONS[f['output_name']], - 'url': f['download_url'], - 'last_modified':datetime.now().isoformat() - }) + if isinstance(f, dict): # it is coming from galaxy + if f["theme"] == theme.name: + file_name = f"{f['file_name']}_{f['output_name']}" # only one part: the zip file + resources.append( + { + "name": f"{file_name}.zip", + "format": HDX_FORMATS[f["output_name"]], + "description": HDX_DESCRIPTIONS[f["output_name"]], + "url": f["download_url"], + "last_modified": datetime.now().isoformat(), + } + ) else: - if 'theme' not in f.extra or f.extra['theme'] == theme.name: - file_name = os.path.basename(f.parts[0]) # only one part: the zip file - resources.append({ - 'name': file_name, - 'format': HDX_FORMATS[f.output_name], - 'description': HDX_DESCRIPTIONS[f.output_name], - 'url': os.path.join(public_dir,file_name), - 'last_modified':datetime.now().isoformat() - }) + if "theme" not in f.extra or f.extra["theme"] == theme.name: + file_name = os.path.basename( + f.parts[0] + ) # only one part: the zip file + resources.append( + { + "name": file_name, + "format": HDX_FORMATS[f.output_name], + "description": HDX_DESCRIPTIONS[f.output_name], + "url": os.path.join(public_dir, file_name), + "last_modified": datetime.now().isoformat(), + } + ) # stable sort, but put shapefiles first for Geopreview to pick up correctly - resources.sort(key=lambda x: 0 if x['format'] == 'zipped shapefile' else 1) + resources.sort(key=lambda x: 0 if x["format"] == "zipped shapefile" else 1) dataset.add_update_resources(resources) d.append(dataset) return d - - - diff --git a/api/migrations/__init__.py b/hdx_exports/tests/__init__.py similarity index 100% rename from api/migrations/__init__.py rename to hdx_exports/tests/__init__.py diff --git a/hdx_exports/tests/test_hdx_export_set.py b/hdx_exports/tests/test_hdx_export_set.py index b640e2bf4..8d4f8eab1 100644 --- a/hdx_exports/tests/test_hdx_export_set.py +++ b/hdx_exports/tests/test_hdx_export_set.py @@ -2,39 +2,45 @@ import json import unittest -from hdx_exports.hdx_export_set import HDXExportSet +from ..hdx_export_set import HDXExportSet from hdx.api.configuration import Configuration from django.contrib.gis.geos import GEOSGeometry from hdx.data.hdxobject import HDXError -DAKAR_GEOJSON_POLYGON = json.dumps({ +DAKAR_GEOJSON_POLYGON = json.dumps( + { "type": "Polygon", "coordinates": [ - [ - [-17.465,14.719], - [-17.442,14.719], - [-17.442,14.741], - [-17.465,14.741], - [-17.465,14.719] - ] - ] - }) - -DAKAR_GEOJSON_MULTIPOLYGON = json.dumps({ + [ + [-17.465, 14.719], + [-17.442, 14.719], + [-17.442, 14.741], + [-17.465, 14.741], + [-17.465, 14.719], + ] + ], + } +) + +DAKAR_GEOJSON_MULTIPOLYGON = json.dumps( + { "type": "MultiPolygon", - "coordinates": [[ - [ - [-17.465,14.719], - [-17.442,14.719], - [-17.442,14.741], - [-17.465,14.741], - [-17.465,14.719] - ]] - ] - }) - -yaml = ''' + "coordinates": [ + [ + [ + [-17.465, 14.719], + [-17.442, 14.719], + [-17.442, 14.741], + [-17.465, 14.741], + [-17.465, 14.719], + ] + ] + ], + } +) + +yaml = """ Some Buildings: types: - polygons @@ -49,8 +55,8 @@ select: - natural where: natural in ('waterway') -''' -#BASIC_FEATURE_SELECTION = FeatureSelection(yaml) +""" +# BASIC_FEATURE_SELECTION = FeatureSelection(yaml) SINGLE_THEME_NOTE = """ OpenStreetMap exports for use in GIS applications. @@ -86,58 +92,59 @@ information. """ + @unittest.skip("HDX Configuration requires network access") class TestHDXExportSet(unittest.TestCase): maxDiff = None def test_minimal_export_set(self): - h = HDXExportSet( dataset_prefix="hot_dakar", name="Dakar Urban Area", extent=DAKAR_GEOJSON_POLYGON, - feature_selection=BASIC_FEATURE_SELECTION + feature_selection=BASIC_FEATURE_SELECTION, ) datasets = h.datasets - self.assertEquals(len(datasets),2) - self.assertEquals(datasets['Some Buildings']['name'],'hot_dakar_some_buildings') - self.assertEquals(datasets['waterways']['name'],'hot_dakar_waterways') + self.assertEquals(len(datasets), 2) + self.assertEquals( + datasets["Some Buildings"]["name"], "hot_dakar_some_buildings" + ) + self.assertEquals(datasets["waterways"]["name"], "hot_dakar_waterways") def test_extent_not_polygon_or_multipolygon(self): with self.assertRaises(AssertionError): h = HDXExportSet( - dataset_prefix="hot_dakar", - name="Dakar Urban Area", - extent=GEOSGeometry("{'type':'LineString','coordinates':[]}"), - feature_selection=BASIC_FEATURE_SELECTION + dataset_prefix="hot_dakar", + name="Dakar Urban Area", + extent=GEOSGeometry("{'type':'LineString','coordinates':[]}"), + feature_selection=BASIC_FEATURE_SELECTION, ) - def test_single_theme_note(self): - yaml = ''' + yaml = """ all: select: - name - ''' + """ h = HDXExportSet( dataset_prefix="hot_dakar", name="Dakar Urban Area", extent=DAKAR_GEOJSON_POLYGON, - feature_selection=FeatureSelection(yaml) + feature_selection=FeatureSelection(yaml), ) - self.assertMultiLineEqual(h.hdx_note('all'),SINGLE_THEME_NOTE) + self.assertMultiLineEqual(h.hdx_note("all"), SINGLE_THEME_NOTE) def test_filtered_note(self): - yaml = ''' + yaml = """ some: select: - name where: highway IS NOT NULL - ''' + """ h = HDXExportSet( dataset_prefix="hot_dakar", name="Dakar Urban Area", extent=DAKAR_GEOJSON_POLYGON, - feature_selection=FeatureSelection(yaml) + feature_selection=FeatureSelection(yaml), ) - self.assertMultiLineEqual(h.hdx_note('some'),SINGLE_FILTER_NOTE) + self.assertMultiLineEqual(h.hdx_note("some"), SINGLE_FILTER_NOTE) diff --git a/jobs/migrations/0001_auto_20150813_0654.py b/jobs/migrations/0001_auto_20150813_0654.py deleted file mode 100644 index c75cffc15..000000000 --- a/jobs/migrations/0001_auto_20150813_0654.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', 'create_tag_gin_index'), - ] - - operations = [ - migrations.RenameField( - model_name='exportconfig', - old_name='visible', - new_name='published', - ), - ] diff --git a/jobs/migrations/0001_auto_20151003_1441.py b/jobs/migrations/0001_auto_20151003_1441.py deleted file mode 100644 index 79e798da4..000000000 --- a/jobs/migrations/0001_auto_20151003_1441.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', 'install_default_group'), - ] - - operations = [ - migrations.AlterField( - model_name='tag', - name='key', - field=models.CharField(default='', max_length=50, db_index=True), - ), - migrations.AlterField( - model_name='tag', - name='value', - field=models.CharField(default='', max_length=50, db_index=True), - ), - ] diff --git a/jobs/migrations/0001_initial.py b/jobs/migrations/0001_initial.py index e3651ae3a..f1b07dd3a 100644 --- a/jobs/migrations/0001_initial.py +++ b/jobs/migrations/0001_initial.py @@ -1,106 +1,311 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +# Generated by Django 3.2.22 on 2023-10-31 04:36 -from django.db import models, migrations +from django.conf import settings import django.contrib.gis.db.models.fields +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion import django.utils.timezone import jobs.models -from django.db.models.fields import CharField -from django.conf import settings import uuid -class LowerCaseCharField(CharField): - """ - Defines a charfield which automatically converts all inputs to - lowercase and saves. - """ - - def pre_save(self, model_instance, add): - """ - Converts the string to lowercase before saving. - """ - current_value = getattr(model_instance, self.attname) - setattr(model_instance, self.attname, current_value.lower()) - return getattr(model_instance, self.attname) class Migration(migrations.Migration): + initial = True dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("auth", "__first__"), ] operations = [ migrations.CreateModel( - name='ExportFormat', + name="Job", fields=[ - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('updated_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(default=uuid.uuid4, unique=True, editable=False, db_index=True)), - ('name', models.CharField(max_length=100)), - ('slug', LowerCaseCharField(default='', unique=True, max_length=10)), - ('description', models.CharField(max_length=255)), - ('cmd', models.TextField(max_length=1000)), + ( + "id", + models.AutoField(editable=False, primary_key=True, serialize=False), + ), + ( + "uid", + models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + ("name", models.CharField(db_index=True, max_length=100)), + ( + "description", + models.CharField( + blank=True, db_index=True, default="", max_length=1000 + ), + ), + ( + "event", + models.CharField( + blank=True, db_index=True, default="", max_length=100 + ), + ), + ( + "export_formats", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=10), + size=None, + validators=[jobs.models.validate_export_formats], + ), + ), + ("published", models.BooleanField(db_index=True, default=False)), + ( + "the_geom", + django.contrib.gis.db.models.fields.GeometryField( + srid=4326, verbose_name="Uploaded geometry" + ), + ), + ( + "simplified_geom", + django.contrib.gis.db.models.fields.GeometryField( + blank=True, + null=True, + srid=4326, + verbose_name="Simplified geometry", + ), + ), + ( + "feature_selection", + models.TextField( + validators=[jobs.models.validate_feature_selection] + ), + ), + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, editable=False + ), + ), + ( + "updated_at", + models.DateTimeField( + default=django.utils.timezone.now, editable=False + ), + ), + ("mbtiles_maxzoom", models.IntegerField(blank=True, null=True)), + ("mbtiles_minzoom", models.IntegerField(blank=True, null=True)), + ("mbtiles_source", models.TextField(blank=True, null=True)), + ("buffer_aoi", models.BooleanField(default=False)), + ("unlimited_extent", models.BooleanField(default=False)), + ("hidden", models.BooleanField(default=False)), + ("expire_old_runs", models.BooleanField(default=True)), + ("pinned", models.BooleanField(default=False)), + ("unfiltered", models.BooleanField(default=False)), + ("preserve_geom", models.BooleanField(default=False)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="owner", + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'db_table': 'export_formats', - 'managed': True, + "db_table": "jobs", + "managed": True, }, ), migrations.CreateModel( - name='ExportTask', + name="SavedFeatureSelection", fields=[ - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('updated_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(blank=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + db_index=True, default=django.utils.timezone.now, editable=False + ), + ), + ( + "updated_at", + models.DateTimeField( + default=django.utils.timezone.now, editable=False + ), + ), + ("name", models.CharField(db_index=True, max_length=100)), + ( + "description", + models.CharField( + blank=True, db_index=True, default="", max_length=1000 + ), + ), + ( + "yaml", + models.TextField( + validators=[jobs.models.validate_feature_selection] + ), + ), + ("public", models.BooleanField(default=False)), + ("deleted", models.BooleanField(default=False)), + ( + "uid", + models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + ("pinned", models.BooleanField(default=False)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], - options={ - 'abstract': False, - }, ), migrations.CreateModel( - name='Job', + name="PartnerExportRegion", fields=[ - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('updated_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(default=uuid.uuid4, unique=True, editable=False, db_index=True)), - ('name', models.CharField(max_length=100)), - ('description', models.CharField(max_length=1000)), - ('status', models.CharField(max_length=30)), - ('the_geom', django.contrib.gis.db.models.fields.PolygonField(default='', srid=4326, verbose_name='Extent for export')), - ('the_geom_webmercator', django.contrib.gis.db.models.fields.PolygonField(default='', srid=3857, verbose_name='Mercator extent for export')), - ('the_geog', django.contrib.gis.db.models.fields.PolygonField(default='', srid=4326, verbose_name='Geographic extent for export', geography=True)), - ('formats', models.ManyToManyField(related_name='formats', to='jobs.ExportFormat')), - ('user', models.ForeignKey(related_name='user', to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "schedule_period", + models.CharField( + choices=[ + ("6hrs", "Every 6 hours"), + ("daily", "Every day"), + ("weekly", "Every Sunday"), + ("2wks", "Every two weeks"), + ("3wks", "Every three weeks"), + ("monthly", "The 1st of every month"), + ("quarterly", "Every quarter (90 days)"), + ("semiyearly", "Every 6 months"), + ("yearly", "Every year"), + ("disabled", "Disabled"), + ], + default="disabled", + max_length=10, + ), + ), + ( + "schedule_hour", + models.IntegerField( + choices=[ + (0, 0), + (1, 1), + (2, 2), + (3, 3), + (4, 4), + (5, 5), + (6, 6), + (7, 7), + (8, 8), + (9, 9), + (10, 10), + (11, 11), + (12, 12), + (13, 13), + (14, 14), + (15, 15), + (16, 16), + (17, 17), + (18, 18), + (19, 19), + (20, 20), + (21, 21), + (22, 22), + (23, 23), + ], + default=0, + ), + ), + ("deleted", models.BooleanField(default=False)), + ("planet_file", models.BooleanField(default=False)), + ("polygon_centroid", models.BooleanField(default=False)), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="auth.group" + ), + ), + ( + "job", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="jobs.job", + ), + ), ], - options={ - 'db_table': 'jobs', - 'managed': True, - }, + bases=(models.Model, jobs.models.RegionMixin), ), migrations.CreateModel( - name='Region', + name="HDXExportRegion", fields=[ - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('updated_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(default=uuid.uuid4, unique=True, editable=False)), - ('name', models.CharField(max_length=100, db_index=True)), - ('description', models.CharField(max_length=1000, blank=True)), - ('the_geom', django.contrib.gis.db.models.fields.PolygonField(default='', srid=4326, verbose_name='HOT Export Region')), - ('the_geom_webmercator', django.contrib.gis.db.models.fields.PolygonField(default='', srid=3857, verbose_name='Mercator extent for export region')), - ('the_geog', django.contrib.gis.db.models.fields.PolygonField(default='', srid=4326, verbose_name='Geographic extent for export region', geography=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "schedule_period", + models.CharField( + choices=[ + ("6hrs", "Every 6 hours"), + ("daily", "Every day"), + ("weekly", "Every Sunday"), + ("2wks", "Every two weeks"), + ("3wks", "Every three weeks"), + ("monthly", "The 1st of every month"), + ("quarterly", "Every quarter (90 days)"), + ("semiyearly", "Every 6 months"), + ("yearly", "Every year"), + ("disabled", "Disabled"), + ], + default="disabled", + max_length=10, + ), + ), + ("schedule_hour", models.IntegerField(choices=[], default=0)), + ("deleted", models.BooleanField(default=False)), + ("is_private", models.BooleanField(default=False)), + ( + "locations", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=32), null=True, size=None + ), + ), + ("license", models.CharField(blank=True, max_length=32, null=True)), + ("subnational", models.BooleanField(default=True)), + ("extra_notes", models.TextField(blank=True, null=True)), + ("planet_file", models.BooleanField(default=False)), + ( + "job", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="hdx_export_region_set", + to="jobs.job", + ), + ), ], options={ - 'db_table': 'regions', - 'managed': True, + "db_table": "hdx_export_regions", }, - ), - migrations.AddField( - model_name='exporttask', - name='job', - field=models.ForeignKey(related_name='job', to='jobs.Job'), + bases=(models.Model, jobs.models.RegionMixin), ), ] diff --git a/jobs/migrations/0002_auto_20150813_0655.py b/jobs/migrations/0002_auto_20150813_0655.py deleted file mode 100644 index 714379439..000000000 --- a/jobs/migrations/0002_auto_20150813_0655.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0001_auto_20150813_0654'), - ] - - operations = [ - migrations.AlterField( - model_name='exportconfig', - name='published', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0002_merge.py b/jobs/migrations/0002_merge.py deleted file mode 100644 index 822504554..000000000 --- a/jobs/migrations/0002_merge.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', 'insert_export_formats'), - ] - - operations = [ - ] diff --git a/jobs/migrations/0003_job_published.py b/jobs/migrations/0003_job_published.py deleted file mode 100644 index 18f34a8bb..000000000 --- a/jobs/migrations/0003_job_published.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0002_auto_20150813_0655'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='published', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0003_job_region.py b/jobs/migrations/0003_job_region.py deleted file mode 100644 index 63c8b4f79..000000000 --- a/jobs/migrations/0003_job_region.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0002_merge'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='region', - field=models.OneToOneField(null=True, to='jobs.Region'), - ), - ] diff --git a/jobs/migrations/0004_auto_20150526_1523.py b/jobs/migrations/0004_auto_20150526_1523.py deleted file mode 100644 index 671e42dc9..000000000 --- a/jobs/migrations/0004_auto_20150526_1523.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0003_job_region'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='region', - field=models.ForeignKey(to='jobs.Region', null=True), - ), - ] diff --git a/jobs/migrations/0004_auto_20150825_1141.py b/jobs/migrations/0004_auto_20150825_1141.py deleted file mode 100644 index 445036f29..000000000 --- a/jobs/migrations/0004_auto_20150825_1141.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.db.models.fields import CharField -import jobs.models - -class LowerCaseCharField(CharField): - """ - Defines a charfield which automatically converts all inputs to - lowercase and saves. - """ - - def pre_save(self, model_instance, add): - """ - Converts the string to lowercase before saving. - """ - current_value = getattr(model_instance, self.attname) - setattr(model_instance, self.attname, current_value.lower()) - return getattr(model_instance, self.attname) - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0003_job_published'), - ] - - operations = [ - migrations.AlterField( - model_name='exportformat', - name='slug', - field=LowerCaseCharField(default='', unique=True, max_length=10), - ), - ] diff --git a/jobs/migrations/0005_auto_20150601_1027.py b/jobs/migrations/0005_auto_20150601_1027.py deleted file mode 100644 index 164c37e08..000000000 --- a/jobs/migrations/0005_auto_20150601_1027.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.contrib.gis.db.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0004_auto_20150526_1523'), - ] - - operations = [ - migrations.CreateModel( - name='RegionMask', - fields=[ - ('id', models.IntegerField(serialize=False, primary_key=True)), - ('the_geom', django.contrib.gis.db.models.fields.MultiPolygonField(srid=4326, verbose_name='Mask for export regions')), - ], - options={ - 'db_table': 'region_mask', - 'managed': False, - }, - ), - migrations.RemoveField( - model_name='exporttask', - name='job', - ), - migrations.DeleteModel( - name='ExportTask', - ), - ] diff --git a/jobs/migrations/0005_tag_groups.py b/jobs/migrations/0005_tag_groups.py deleted file mode 100644 index 75ae05581..000000000 --- a/jobs/migrations/0005_tag_groups.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.contrib.postgres.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0004_auto_20150825_1141'), - ] - - operations = [ - migrations.AddField( - model_name='tag', - name='groups', - field=django.contrib.postgres.fields.ArrayField(default=[], base_field=models.CharField(default='', max_length=100, blank=True), size=None), - ), - ] diff --git a/jobs/migrations/0006_remove_job_status.py b/jobs/migrations/0006_remove_job_status.py deleted file mode 100644 index 26508ab48..000000000 --- a/jobs/migrations/0006_remove_job_status.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0005_auto_20150601_1027'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='status', - ), - ] diff --git a/jobs/migrations/0006_tag_name.py b/jobs/migrations/0006_tag_name.py deleted file mode 100644 index 39f35d048..000000000 --- a/jobs/migrations/0006_tag_name.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0005_tag_groups'), - ] - - operations = [ - migrations.AddField( - model_name='tag', - name='name', - field=models.CharField(default='', max_length=100, db_index=True), - ), - ] diff --git a/jobs/migrations/0007_auto_20150828_1317.py b/jobs/migrations/0007_auto_20150828_1317.py deleted file mode 100644 index 03f082b46..000000000 --- a/jobs/migrations/0007_auto_20150828_1317.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0006_tag_name'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='feature_pub', - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name='job', - name='feature_save', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0007_exportconfig.py b/jobs/migrations/0007_exportconfig.py deleted file mode 100644 index b8034d5e5..000000000 --- a/jobs/migrations/0007_exportconfig.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone -import jobs.models -import uuid - -def get_upload_path(instance, filename): - """ - Construct the path to where the uploaded config file is to be stored. - """ - configtype = instance.config_type.lower() - # sanitize the filename here.. - path = 'export/config/{0}/{1}'.format(configtype, instance.filename) - return path - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0006_remove_job_status'), - ] - - operations = [ - migrations.CreateModel( - name='ExportConfig', - fields=[ - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('updated_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(default=uuid.uuid4, unique=True, editable=False)), - ('name', models.CharField(max_length=100, db_index=True)), - ('description', models.CharField(max_length=255, blank=True)), - ('type', models.CharField(default='PRESET', max_length=10, choices=[('PRESET', 'Preset'), ('TRANSLATION', 'Translation'), ('TRANSFORM', 'Transform')])), - ('filename', models.CharField(max_length=255)), - ('upload', models.FileField(max_length=255, upload_to=get_upload_path)), - ], - options={ - 'abstract': False, - }, - ), - ] diff --git a/jobs/migrations/0008_auto_20150603_0938.py b/jobs/migrations/0008_auto_20150603_0938.py deleted file mode 100644 index 15f59bdc4..000000000 --- a/jobs/migrations/0008_auto_20150603_0938.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0007_exportconfig'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exportconfig', - options={'managed': True}, - ), - migrations.AlterModelTable( - name='exportconfig', - table='export_configurations', - ), - ] diff --git a/jobs/migrations/0008_auto_20150905_0952.py b/jobs/migrations/0008_auto_20150905_0952.py deleted file mode 100644 index ccfc496b2..000000000 --- a/jobs/migrations/0008_auto_20150905_0952.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0007_auto_20150828_1317'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='event', - field=models.CharField(default='', max_length=100, db_index=True, blank=True), - ), - ] diff --git a/jobs/migrations/0009_auto_20150907_1140.py b/jobs/migrations/0009_auto_20150907_1140.py deleted file mode 100644 index b64c187d7..000000000 --- a/jobs/migrations/0009_auto_20150907_1140.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0008_auto_20150905_0952'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='description', - field=models.CharField(max_length=1000, db_index=True), - ), - migrations.AlterField( - model_name='job', - name='feature_pub', - field=models.BooleanField(default=False, db_index=True), - ), - migrations.AlterField( - model_name='job', - name='feature_save', - field=models.BooleanField(default=False, db_index=True), - ), - migrations.AlterField( - model_name='job', - name='name', - field=models.CharField(max_length=100, db_index=True), - ), - migrations.AlterField( - model_name='job', - name='published', - field=models.BooleanField(default=False, db_index=True), - ), - ] diff --git a/jobs/migrations/0009_job_configs.py b/jobs/migrations/0009_job_configs.py deleted file mode 100644 index b7065f30f..000000000 --- a/jobs/migrations/0009_job_configs.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0008_auto_20150603_0938'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='configs', - field=models.ManyToManyField(related_name='configs', to='jobs.ExportConfig'), - ), - ] diff --git a/jobs/migrations/0010_auto_20150603_0945.py b/jobs/migrations/0010_auto_20150603_0945.py deleted file mode 100644 index 5b9346c43..000000000 --- a/jobs/migrations/0010_auto_20150603_0945.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.conf import settings - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('jobs', '0009_job_configs'), - ] - - operations = [ - migrations.AddField( - model_name='exportconfig', - name='user', - field=models.ForeignKey(related_name='user', default='', to=settings.AUTH_USER_MODEL), - preserve_default=False, - ), - migrations.AlterField( - model_name='job', - name='user', - field=models.ForeignKey(related_name='owner', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/jobs/migrations/0010_exportmaxextent.py b/jobs/migrations/0010_exportmaxextent.py deleted file mode 100644 index c0bfe7ab3..000000000 --- a/jobs/migrations/0010_exportmaxextent.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0009_auto_20150907_1140'), - ] - - operations = [ - migrations.CreateModel( - name='ExportMaxExtent', - fields=[ - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('name', models.CharField(default='', max_length=100)), - ('max_extent', models.IntegerField()), - ], - ), - ] diff --git a/jobs/migrations/0011_auto_20150603_1030.py b/jobs/migrations/0011_auto_20150603_1030.py deleted file mode 100644 index 506f3712f..000000000 --- a/jobs/migrations/0011_auto_20150603_1030.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0010_auto_20150603_0945'), - ] - - operations = [ - migrations.RenameField( - model_name='exportconfig', - old_name='type', - new_name='config_type', - ), - ] diff --git a/jobs/migrations/0011_auto_20150908_1225.py b/jobs/migrations/0011_auto_20150908_1225.py deleted file mode 100644 index 39eac2b0c..000000000 --- a/jobs/migrations/0011_auto_20150908_1225.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0006_require_contenttypes_0002'), - ('jobs', '0010_exportmaxextent'), - ] - - operations = [ - migrations.CreateModel( - name='GroupProfile', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('name', models.CharField(default='', max_length=100)), - ('max_extent', models.IntegerField()), - ('group', models.OneToOneField(related_name='group_profile', to='auth.Group')), - ], - options={ - 'db_table': 'group_max_extents', - 'managed': True, - }, - ), - migrations.DeleteModel( - name='ExportMaxExtent', - ), - ] diff --git a/jobs/migrations/0012_auto_20150603_1212.py b/jobs/migrations/0012_auto_20150603_1212.py deleted file mode 100644 index b89b5e3c1..000000000 --- a/jobs/migrations/0012_auto_20150603_1212.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0011_auto_20150603_1030'), - ] - - operations = [ - migrations.AlterField( - model_name='exportconfig', - name='config_type', - field=models.CharField(default='PRESET', max_length=11, choices=[('PRESET', 'Preset'), ('TRANSLATION', 'Translation'), ('TRANSFORM', 'Transform')]), - ), - ] diff --git a/jobs/migrations/0012_auto_20150908_1226.py b/jobs/migrations/0012_auto_20150908_1226.py deleted file mode 100644 index c17a8bbf2..000000000 --- a/jobs/migrations/0012_auto_20150908_1226.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0006_require_contenttypes_0002'), - ('jobs', '0011_auto_20150908_1225'), - ] - - operations = [ - migrations.CreateModel( - name='ExportProfile', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('name', models.CharField(default='', max_length=100)), - ('max_extent', models.IntegerField()), - ('group', models.OneToOneField(related_name='export_profile', to='auth.Group')), - ], - options={ - 'db_table': 'export_profiles', - 'managed': True, - }, - ), - migrations.RemoveField( - model_name='groupprofile', - name='group', - ), - migrations.DeleteModel( - name='GroupProfile', - ), - ] diff --git a/jobs/migrations/0013_auto_20150603_1223.py b/jobs/migrations/0013_auto_20150603_1223.py deleted file mode 100644 index 6740f543b..000000000 --- a/jobs/migrations/0013_auto_20150603_1223.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0012_auto_20150603_1212'), - ] - - operations = [ - migrations.RemoveField( - model_name='exportconfig', - name='description', - ), - migrations.RemoveField( - model_name='exportconfig', - name='name', - ), - ] diff --git a/jobs/migrations/0014_exportconfig_content_type.py b/jobs/migrations/0014_exportconfig_content_type.py deleted file mode 100644 index 11a964f64..000000000 --- a/jobs/migrations/0014_exportconfig_content_type.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0013_auto_20150603_1223'), - ] - - operations = [ - migrations.AddField( - model_name='exportconfig', - name='content_type', - field=models.CharField(default='text/plain', max_length=30, editable=False), - preserve_default=False, - ), - ] diff --git a/jobs/migrations/0015_auto_20150605_1037.py b/jobs/migrations/0015_auto_20150605_1037.py deleted file mode 100644 index 65a4bc51a..000000000 --- a/jobs/migrations/0015_auto_20150605_1037.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0014_exportconfig_content_type'), - ] - - operations = [ - migrations.AddField( - model_name='exportconfig', - name='name', - field=models.CharField(default='', max_length=255, db_index=True), - ), - migrations.AddField( - model_name='exportconfig', - name='visible', - field=models.BooleanField(default=True), - ), - ] diff --git a/jobs/migrations/0016_merge.py b/jobs/migrations/0016_merge.py deleted file mode 100644 index 1a8eaa0af..000000000 --- a/jobs/migrations/0016_merge.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0015_auto_20150605_1037'), - ('jobs', 'create_hstore_extension'), - ] - - operations = [ - ] diff --git a/jobs/migrations/0017_auto_20150616_1544.py b/jobs/migrations/0017_auto_20150616_1544.py deleted file mode 100644 index 931e39232..000000000 --- a/jobs/migrations/0017_auto_20150616_1544.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.contrib.postgres.fields.hstore - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0016_merge'), - ] - - operations = [ - migrations.CreateModel( - name='Tag', - fields=[ - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('name', models.CharField(max_length=50)), - ('geom_types', django.contrib.postgres.fields.hstore.HStoreField()), - ], - ), - migrations.AddField( - model_name='job', - name='tags', - field=models.ManyToManyField(related_name='tags', to='jobs.Tag'), - ), - ] diff --git a/jobs/migrations/0018_auto_20150616_1547.py b/jobs/migrations/0018_auto_20150616_1547.py deleted file mode 100644 index e6c0e524c..000000000 --- a/jobs/migrations/0018_auto_20150616_1547.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0017_auto_20150616_1544'), - ] - - operations = [ - migrations.AlterModelOptions( - name='tag', - options={'managed': True}, - ), - migrations.AlterModelTable( - name='tag', - table='tags', - ), - ] diff --git a/jobs/migrations/0019_job_event.py b/jobs/migrations/0019_job_event.py deleted file mode 100644 index c9e656471..000000000 --- a/jobs/migrations/0019_job_event.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0018_auto_20150616_1547'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='event', - field=models.CharField(default='', max_length=100, db_index=True), - ), - ] diff --git a/jobs/migrations/0020_job_filters.py b/jobs/migrations/0020_job_filters.py deleted file mode 100644 index c66f55f14..000000000 --- a/jobs/migrations/0020_job_filters.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.contrib.postgres.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0019_job_event'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='filters', - field=django.contrib.postgres.fields.ArrayField(default=[], base_field=models.CharField(default='', max_length=50, blank=True), size=None), - ), - ] diff --git a/jobs/migrations/0021_remove_job_filters.py b/jobs/migrations/0021_remove_job_filters.py deleted file mode 100644 index 51dbec5a8..000000000 --- a/jobs/migrations/0021_remove_job_filters.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0020_job_filters'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='filters', - ), - ] diff --git a/jobs/migrations/0022_remove_job_tags.py b/jobs/migrations/0022_remove_job_tags.py deleted file mode 100644 index 1ae086569..000000000 --- a/jobs/migrations/0022_remove_job_tags.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0021_remove_job_filters'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='tags', - ), - ] diff --git a/jobs/migrations/0023_delete_tag.py b/jobs/migrations/0023_delete_tag.py deleted file mode 100644 index 271452aac..000000000 --- a/jobs/migrations/0023_delete_tag.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0022_remove_job_tags'), - ] - - operations = [ - migrations.DeleteModel( - name='Tag', - ), - ] diff --git a/jobs/migrations/0024_tag.py b/jobs/migrations/0024_tag.py deleted file mode 100644 index 06354ced0..000000000 --- a/jobs/migrations/0024_tag.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0023_delete_tag'), - ] - - operations = [ - migrations.CreateModel( - name='Tag', - fields=[ - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('key', models.CharField(default='', max_length=30, db_index=True)), - ('value', models.CharField(default='', max_length=30, db_index=True)), - ('job', models.ForeignKey(related_name='tags', to='jobs.Job')), - ], - options={ - 'db_table': 'tags', - 'managed': True, - }, - ), - ] diff --git a/jobs/migrations/0025_auto_20150731_1033.py b/jobs/migrations/0025_auto_20150731_1033.py deleted file mode 100644 index 51153fc93..000000000 --- a/jobs/migrations/0025_auto_20150731_1033.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.contrib.postgres.fields.hstore - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0024_tag'), - ] - - operations = [ - migrations.AddField( - model_name='tag', - name='data_model', - field=models.CharField(default='', max_length=10, db_index=True), - ), - migrations.AddField( - model_name='tag', - name='geom_types', - field=django.contrib.postgres.fields.hstore.HStoreField(default={}), - ), - ] diff --git a/jobs/migrations/0026_remove_tag_geom_types.py b/jobs/migrations/0026_remove_tag_geom_types.py deleted file mode 100644 index e81ec6092..000000000 --- a/jobs/migrations/0026_remove_tag_geom_types.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0025_auto_20150731_1033'), - ] - - operations = [ - migrations.RemoveField( - model_name='tag', - name='geom_types', - ), - ] diff --git a/jobs/migrations/0027_tag_geom_types.py b/jobs/migrations/0027_tag_geom_types.py deleted file mode 100644 index 7ee077d3b..000000000 --- a/jobs/migrations/0027_tag_geom_types.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.contrib.postgres.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0026_remove_tag_geom_types'), - ] - - operations = [ - migrations.AddField( - model_name='tag', - name='geom_types', - field=django.contrib.postgres.fields.ArrayField(default=[], base_field=models.CharField(default='', max_length=10, blank=True), size=None), - ), - ] diff --git a/jobs/migrations/0028_promote_pbf_export_format.py b/jobs/migrations/0028_promote_pbf_export_format.py deleted file mode 100644 index b1e623695..000000000 --- a/jobs/migrations/0028_promote_pbf_export_format.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - def promote_pbf_export_format(apps, schema_editor): - ExportFormat = apps.get_model('jobs', 'ExportFormat') - ExportFormat.objects.create(name='PBF Format', description='OSM PBF', - slug='PBF') - - dependencies = [ - ('jobs', '0001_auto_20151003_1441'), - ] - - operations = [ - migrations.RunPython(promote_pbf_export_format), - ] diff --git a/jobs/migrations/0029_auto_20170323_2111.py b/jobs/migrations/0029_auto_20170323_2111.py deleted file mode 100644 index f887270d2..000000000 --- a/jobs/migrations/0029_auto_20170323_2111.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-23 21:11 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0028_promote_pbf_export_format'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='region', - ), - migrations.DeleteModel( - name='Region', - ), - ] diff --git a/jobs/migrations/0030_add_gpkg_export_format.py b/jobs/migrations/0030_add_gpkg_export_format.py deleted file mode 100644 index fef716858..000000000 --- a/jobs/migrations/0030_add_gpkg_export_format.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-24 20:45 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - def add_gpkg_export_format(apps, schema_editor): - ExportFormat = apps.get_model('jobs', 'ExportFormat') - ExportFormat.objects.create(name='GeoPackage Format (OSM)', description='GeoPackage (OSM Schema)', - slug='GPKG') - ExportFormat.objects.create(name='GeoPackage Format (Thematic)', description='GeoPackage (Thematic Schema)', - slug='THEME_GPKG') - - dependencies = [ - ('jobs', '0029_auto_20170323_2111'), - ] - - operations = [ - migrations.RunPython(add_gpkg_export_format), - ] diff --git a/jobs/migrations/0031_auto_20170328_2342.py b/jobs/migrations/0031_auto_20170328_2342.py deleted file mode 100644 index 5d9700701..000000000 --- a/jobs/migrations/0031_auto_20170328_2342.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-28 23:42 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0030_add_gpkg_export_format'), - ] - - operations = [ - migrations.AlterField( - model_name='exportconfig', - name='config_type', - field=models.CharField(choices=[('PRESET', 'Preset')], default='PRESET', max_length=11), - ), - ] diff --git a/jobs/migrations/0031_convert_export_format_to_slug.py b/jobs/migrations/0031_convert_export_format_to_slug.py deleted file mode 100644 index 7b3d3218f..000000000 --- a/jobs/migrations/0031_convert_export_format_to_slug.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-28 23:01 -from __future__ import unicode_literals - -from django.contrib.postgres.fields import ArrayField -from django.db import migrations -from django.db.models.fields import CharField - - -class Migration(migrations.Migration): - def convert_export_format_to_slug(apps, schema_editor): - Job = apps.get_model('jobs', 'Job') - for job in Job.objects.all(): - job.export_formats = map(lambda f: f.slug, job.formats.all()) - job.save() - - - dependencies = [ - ('jobs', '0030_add_gpkg_export_format'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='export_formats', - field=ArrayField(base_field=CharField(max_length=10), default=list, size=None), - ), - migrations.RunPython(convert_export_format_to_slug), - migrations.RemoveField( - model_name='job', - name='formats', - ), - migrations.DeleteModel( - name='ExportFormat', - ), - ] diff --git a/jobs/migrations/0032_merge.py b/jobs/migrations/0032_merge.py deleted file mode 100644 index 682fb2234..000000000 --- a/jobs/migrations/0032_merge.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-29 04:30 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0031_auto_20170328_2342'), - ('jobs', '0031_convert_export_format_to_slug'), - ] - - operations = [ - ] diff --git a/jobs/migrations/0033_job_config.py b/jobs/migrations/0033_job_config.py deleted file mode 100644 index dd478a0d4..000000000 --- a/jobs/migrations/0033_job_config.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-29 16:49 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - def populate_config_column(apps, schema_editor): - Job = apps.get_model('jobs', 'Job') - for job in Job.objects.all(): - job.config = job.configs.first() - job.save() - - dependencies = [ - ('jobs', '0032_merge'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='config', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='config', to='jobs.ExportConfig'), - ), - migrations.RunPython(populate_config_column) - ] - - - - diff --git a/jobs/migrations/0034_remove_job_configs.py b/jobs/migrations/0034_remove_job_configs.py deleted file mode 100644 index 30ce0d170..000000000 --- a/jobs/migrations/0034_remove_job_configs.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-29 16:56 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0033_job_config'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='configs', - ), - ] diff --git a/jobs/migrations/0035_hdxexportregion.py b/jobs/migrations/0035_hdxexportregion.py deleted file mode 100644 index 5133aa764..000000000 --- a/jobs/migrations/0035_hdxexportregion.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-30 14:35 -from __future__ import unicode_literals - -import django.contrib.gis.db.models.fields -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0034_remove_job_configs'), - ] - - operations = [ - migrations.CreateModel( - name='HDXExportRegion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('dataset_prefix', models.CharField(max_length=100)), - ('feature_selection', models.TextField()), - ('schedule_period', models.CharField(choices=[('6hrs', 'Every 7 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10)), - ('schedule_hour', models.IntegerField(choices=[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17), (18, 18), (19, 19), (20, 20), (21, 21), (22, 22), (23, 23)])), - ('export_formats', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(choices=[('kml', 'Google Earth KMZ'), ('pbf', 'OSM PBF'), ('sqlite', 'SQLite Database'), ('geopackage', 'GeoPackage Format (OSM)'), ('theme_gpkg', 'GeoPackage (Thematic'), ('obf', 'OSMAnd OBF'), ('shp', 'ESRI Shapefile format (OSM)'), ('thematic', 'ESRI Shapefile format (Thematic)'), ('garmin', 'Garmin Map')], max_length=10), size=None)), - ('the_geom', django.contrib.gis.db.models.fields.PolygonField(srid=4326, verbose_name='Extent for export')), - ], - ), - ] diff --git a/jobs/migrations/0036_auto_20170330_2134.py b/jobs/migrations/0036_auto_20170330_2134.py deleted file mode 100644 index 2bbf8ba01..000000000 --- a/jobs/migrations/0036_auto_20170330_2134.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-30 21:34 -from __future__ import unicode_literals - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - def backfill_names(apps, schema_editor): - HDXExportRegion = apps.get_model('jobs', 'HDXExportRegion') - for region in HDXExportRegion.objects.all(): - region.name = region.dataset_prefix - region.save() - - dependencies = [ - ('jobs', '0035_hdxexportregion'), - ] - - operations = [ - migrations.AddField( - model_name='hdxexportregion', - name='country_codes', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=3), null=True, size=None), - ), - migrations.AddField( - model_name='hdxexportregion', - name='deleted', - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name='hdxexportregion', - name='name', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_hour', - field=models.IntegerField(choices=[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17), (18, 18), (19, 19), (20, 20), (21, 21), (22, 22), (23, 23)], default=0), - ), - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - migrations.RunPython(backfill_names) - ] diff --git a/jobs/migrations/0037_hdxexportregion_last_run.py b/jobs/migrations/0037_hdxexportregion_last_run.py deleted file mode 100644 index 571fc53d5..000000000 --- a/jobs/migrations/0037_hdxexportregion_last_run.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-04 22:00 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0036_auto_20170330_2134'), - ] - - operations = [ - migrations.AddField( - model_name='hdxexportregion', - name='last_run', - field=models.DateTimeField(default=None, null=True), - ), - ] diff --git a/jobs/migrations/0038_auto_20170406_1557.py b/jobs/migrations/0038_auto_20170406_1557.py deleted file mode 100644 index 45e840fe4..000000000 --- a/jobs/migrations/0038_auto_20170406_1557.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-06 15:57 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0037_hdxexportregion_last_run'), - ] - - operations = [ - migrations.RemoveField( - model_name='exportprofile', - name='group', - ), - migrations.DeleteModel( - name='ExportProfile', - ), - ] diff --git a/jobs/migrations/0039_auto_20170406_1631.py b/jobs/migrations/0039_auto_20170406_1631.py deleted file mode 100644 index 009bf3f1d..000000000 --- a/jobs/migrations/0039_auto_20170406_1631.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-06 16:31 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0038_auto_20170406_1557'), - ] - - operations = [ - migrations.RemoveField( - model_name='hdxexportregion', - name='export_formats', - ), - migrations.RemoveField( - model_name='hdxexportregion', - name='feature_selection', - ), - migrations.RemoveField( - model_name='hdxexportregion', - name='last_run', - ), - migrations.RemoveField( - model_name='hdxexportregion', - name='name', - ), - migrations.RemoveField( - model_name='hdxexportregion', - name='the_geom', - ), - migrations.AddField( - model_name='hdxexportregion', - name='job', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='jobs.Job'), - ), - migrations.AddField( - model_name='job', - name='feature_selection', - field=models.TextField(blank=True), - ), - ] diff --git a/jobs/migrations/0040_remove_hdxexportregion_dataset_prefix.py b/jobs/migrations/0040_remove_hdxexportregion_dataset_prefix.py deleted file mode 100644 index b41d3eb45..000000000 --- a/jobs/migrations/0040_remove_hdxexportregion_dataset_prefix.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-06 18:40 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0039_auto_20170406_1631'), - ] - - operations = [ - migrations.RemoveField( - model_name='hdxexportregion', - name='dataset_prefix', - ), - ] diff --git a/jobs/migrations/0041_hdxexportregion_is_private.py b/jobs/migrations/0041_hdxexportregion_is_private.py deleted file mode 100644 index ffe030817..000000000 --- a/jobs/migrations/0041_hdxexportregion_is_private.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-07 16:23 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0040_remove_hdxexportregion_dataset_prefix'), - ] - - operations = [ - migrations.AddField( - model_name='hdxexportregion', - name='is_private', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0042_auto_20170411_2155.py b/jobs/migrations/0042_auto_20170411_2155.py deleted file mode 100644 index abfa0d6b8..000000000 --- a/jobs/migrations/0042_auto_20170411_2155.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-11 21:55 -from __future__ import unicode_literals - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0041_hdxexportregion_is_private'), - ] - - operations = [ - migrations.RemoveField( - model_name='hdxexportregion', - name='country_codes', - ), - migrations.AddField( - model_name='hdxexportregion', - name='license', - field=models.CharField(max_length=32, null=True), - ), - migrations.AddField( - model_name='hdxexportregion', - name='locations', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=32), null=True, size=None), - ), - migrations.AddField( - model_name='hdxexportregion', - name='subnational', - field=models.BooleanField(default=True), - ), - migrations.AddField( - model_name='hdxexportregion', - name='extra_notes', - field=models.TextField(null=True), - ), - ] diff --git a/jobs/migrations/0043_auto_20170419_2127.py b/jobs/migrations/0043_auto_20170419_2127.py deleted file mode 100644 index ed0888757..000000000 --- a/jobs/migrations/0043_auto_20170419_2127.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-19 21:27 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0042_auto_20170411_2155'), - ] - - operations = [ - migrations.AlterModelTable( - name='hdxexportregion', - table='hdx_export_regions', - ), - ] diff --git a/jobs/migrations/0044_auto_20170424_1815.py b/jobs/migrations/0044_auto_20170424_1815.py deleted file mode 100644 index c228a5614..000000000 --- a/jobs/migrations/0044_auto_20170424_1815.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-24 18:15 -from __future__ import unicode_literals - -import django.contrib.gis.db.models.fields -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0043_auto_20170419_2127'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='the_geog', - ), - migrations.RemoveField( - model_name='job', - name='the_geom_webmercator', - ), - migrations.AlterField( - model_name='job', - name='the_geom', - field=django.contrib.gis.db.models.fields.GeometryField(default='', srid=4326, verbose_name='Extent for export'), - ), - ] diff --git a/jobs/migrations/0045_auto_20170424_1929.py b/jobs/migrations/0045_auto_20170424_1929.py deleted file mode 100644 index 7a0a7417c..000000000 --- a/jobs/migrations/0045_auto_20170424_1929.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-24 19:29 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0044_auto_20170424_1815'), - ] - - operations = [ - migrations.AlterField( - model_name='hdxexportregion', - name='job', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='hdx_export_region_set', to='jobs.Job'), - ), - ] diff --git a/jobs/migrations/0046_job_buffer_aoi.py b/jobs/migrations/0046_job_buffer_aoi.py deleted file mode 100644 index 5d80081c4..000000000 --- a/jobs/migrations/0046_job_buffer_aoi.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-24 22:37 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0045_auto_20170424_1929'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='buffer_aoi', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0047_auto_20170522_2220.py b/jobs/migrations/0047_auto_20170522_2220.py deleted file mode 100644 index 27222e592..000000000 --- a/jobs/migrations/0047_auto_20170522_2220.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-05-22 22:20 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0046_job_buffer_aoi'), - ] - - operations = [ - migrations.RemoveField( - model_name='exportconfig', - name='user', - ), - migrations.RemoveField( - model_name='tag', - name='job', - ), - migrations.RemoveField( - model_name='job', - name='config', - ), - migrations.DeleteModel( - name='ExportConfig', - ), - migrations.DeleteModel( - name='Tag', - ), - ] diff --git a/jobs/migrations/0048_auto_20170610_0634.py b/jobs/migrations/0048_auto_20170610_0634.py deleted file mode 100644 index b38ee3bcf..000000000 --- a/jobs/migrations/0048_auto_20170610_0634.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-10 06:34 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0047_auto_20170522_2220'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='feature_pub', - ), - migrations.RemoveField( - model_name='job', - name='feature_save', - ), - ] diff --git a/jobs/migrations/0049_auto_20170610_0736.py b/jobs/migrations/0049_auto_20170610_0736.py deleted file mode 100644 index 08d541fbf..000000000 --- a/jobs/migrations/0049_auto_20170610_0736.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-10 07:36 -from __future__ import unicode_literals - -import django.contrib.gis.db.models.fields -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0048_auto_20170610_0634'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='the_geom', - field=django.contrib.gis.db.models.fields.GeometryField(srid=4326, verbose_name='Extent for export'), - ), - ] diff --git a/jobs/migrations/0050_auto_20170610_0842.py b/jobs/migrations/0050_auto_20170610_0842.py deleted file mode 100644 index e627db1f8..000000000 --- a/jobs/migrations/0050_auto_20170610_0842.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-10 08:42 -from __future__ import unicode_literals - -import django.contrib.postgres.fields -from django.db import migrations, models -import jobs.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0049_auto_20170610_0736'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='export_formats', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=10), default=list, size=None, validators=[jobs.models.validate_export_formats]), - ), - ] diff --git a/jobs/migrations/0051_auto_20170610_2251.py b/jobs/migrations/0051_auto_20170610_2251.py deleted file mode 100644 index bd1b98e53..000000000 --- a/jobs/migrations/0051_auto_20170610_2251.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-10 22:51 -from __future__ import unicode_literals - -import django.contrib.gis.db.models.fields -import django.contrib.postgres.fields -from django.db import migrations, models -import jobs.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0050_auto_20170610_0842'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='description', - field=models.CharField(blank=True, db_index=True, default='', max_length=1000), - ), - migrations.AlterField( - model_name='job', - name='export_formats', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=10), size=None, validators=[jobs.models.validate_export_formats]), - ), - migrations.AlterField( - model_name='job', - name='the_geom', - field=django.contrib.gis.db.models.fields.GeometryField(srid=4326, validators=[], verbose_name='Extent for export'), - ), - ] diff --git a/jobs/migrations/0052_auto_20170611_0956.py b/jobs/migrations/0052_auto_20170611_0956.py deleted file mode 100644 index c381d4906..000000000 --- a/jobs/migrations/0052_auto_20170611_0956.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-11 09:56 -from __future__ import unicode_literals - -from django.db import migrations, models -import jobs.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0051_auto_20170610_2251'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='feature_selection', - field=models.TextField(validators=[jobs.models.validate_feature_selection]), - ), - ] diff --git a/jobs/migrations/0053_auto_20170612_0313.py b/jobs/migrations/0053_auto_20170612_0313.py deleted file mode 100644 index 5be34e0ee..000000000 --- a/jobs/migrations/0053_auto_20170612_0313.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-12 03:13 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0052_auto_20170611_0956'), - ] - - operations = [ - migrations.AlterField( - model_name='hdxexportregion', - name='extra_notes', - field=models.TextField(blank=True, null=True), - ), - migrations.AlterField( - model_name='hdxexportregion', - name='license', - field=models.CharField(blank=True, max_length=32, null=True), - ), - ] diff --git a/jobs/migrations/0054_savedfeatureselection.py b/jobs/migrations/0054_savedfeatureselection.py deleted file mode 100644 index 115f70a21..000000000 --- a/jobs/migrations/0054_savedfeatureselection.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-26 05:00 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone -import jobs.models - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('jobs', '0053_auto_20170612_0313'), - ] - - operations = [ - migrations.CreateModel( - name='SavedFeatureSelection', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('updated_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('name', models.CharField(db_index=True, max_length=100)), - ('description', models.CharField(blank=True, db_index=True, default='', max_length=1000)), - ('yaml', models.TextField(validators=[jobs.models.validate_feature_selection])), - ('public', models.BooleanField(default=False)), - ('deleted', models.BooleanField(default=False)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - ), - ] diff --git a/jobs/migrations/0055_savedfeatureselection_uid.py b/jobs/migrations/0055_savedfeatureselection_uid.py deleted file mode 100644 index 5bb69ffab..000000000 --- a/jobs/migrations/0055_savedfeatureselection_uid.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-26 12:06 -from __future__ import unicode_literals - -from django.db import migrations, models -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0054_savedfeatureselection'), - ] - - operations = [ - migrations.AddField( - model_name='savedfeatureselection', - name='uid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True), - ), - ] diff --git a/jobs/migrations/0056_auto_20170628_1141.py b/jobs/migrations/0056_auto_20170628_1141.py deleted file mode 100644 index ef7903bed..000000000 --- a/jobs/migrations/0056_auto_20170628_1141.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-28 11:41 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0055_savedfeatureselection_uid'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='expire_old_runs', - field=models.BooleanField(default=True), - ), - migrations.AddField( - model_name='job', - name='hidden', - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name='job', - name='unlimited_extent', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0057_job_per_theme.py b/jobs/migrations/0057_job_per_theme.py deleted file mode 100644 index d67c3fca9..000000000 --- a/jobs/migrations/0057_job_per_theme.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-06-28 19:38 -from __future__ import unicode_literals - -from django.db import migrations, models - -def hdx_pertheme(apps, schema_editor): - # We can't import the Person model directly as it may be a newer - # version than this migration expects. We use the historical version. - HDXExportRegion = apps.get_model('jobs', 'HDXExportRegion') - for region in HDXExportRegion.objects.all(): - job = region.job - job.per_theme = True - job.hidden = True - job.unlimited_extent = True - job.expire_old_runs = False - job.save() - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0056_auto_20170628_1141'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='per_theme', - field=models.BooleanField(default=False), - ), - migrations.RunPython(hdx_pertheme) - ] diff --git a/jobs/migrations/0058_auto_20170703_2355.py b/jobs/migrations/0058_auto_20170703_2355.py deleted file mode 100644 index 6970687d0..000000000 --- a/jobs/migrations/0058_auto_20170703_2355.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-07-03 23:55 -from __future__ import unicode_literals - -import django.contrib.gis.db.models.fields -from django.db import migrations -import jobs.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0057_job_per_theme'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='simplified_geom', - field=django.contrib.gis.db.models.fields.GeometryField(null=True,blank=True, srid=4326, verbose_name='Simplified geometry'), - ), - migrations.AlterField( - model_name='job', - name='the_geom', - field=django.contrib.gis.db.models.fields.GeometryField(srid=4326, validators=[], verbose_name='Uploaded geometry'), - ), - migrations.RunSQL( - "UPDATE jobs SET simplified_geom=ST_SimplifyPreserveTopology(the_geom, 0.01)" - ) - ] diff --git a/jobs/migrations/0059_auto_20170711_0108.py b/jobs/migrations/0059_auto_20170711_0108.py deleted file mode 100644 index 07f407212..000000000 --- a/jobs/migrations/0059_auto_20170711_0108.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-07-11 01:08 -from __future__ import unicode_literals - -from django.db import migrations -from utils.aoi_utils import simplify_geom - -def resimplify(apps, schema_editor): - Job = apps.get_model('jobs', 'Job') - for job in Job.objects.all(): - job.simplified_geom = simplify_geom(job.the_geom,force_buffer=job.buffer_aoi) - job.save() - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0058_auto_20170703_2355'), - ] - - operations = [ - migrations.RunPython(resimplify,reverse_code=lambda x,y:None) - ] diff --git a/jobs/migrations/0060_auto_20170825_2343.py b/jobs/migrations/0060_auto_20170825_2343.py deleted file mode 100644 index c1b12668c..000000000 --- a/jobs/migrations/0060_auto_20170825_2343.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.4 on 2017-08-25 23:43 -from __future__ import unicode_literals - -import django.contrib.gis.db.models.fields -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0059_auto_20170711_0108'), - ] - - operations = [ - migrations.AlterField( - model_name='job', - name='the_geom', - field=django.contrib.gis.db.models.fields.GeometryField(srid=4326, verbose_name='Uploaded geometry'), - ), - ] diff --git a/jobs/migrations/0061_auto_20170825_2351.py b/jobs/migrations/0061_auto_20170825_2351.py deleted file mode 100644 index a0c7a105a..000000000 --- a/jobs/migrations/0061_auto_20170825_2351.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.4 on 2017-08-25 23:51 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0060_auto_20170825_2343'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='mbtiles_maxzoom', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='job', - name='mbtiles_minzoom', - field=models.IntegerField(null=True), - ), - migrations.AddField( - model_name='job', - name='mbtiles_source', - field=models.TextField(null=True), - ), - ] diff --git a/jobs/migrations/0062_auto_20170925_1457.py b/jobs/migrations/0062_auto_20170925_1457.py deleted file mode 100644 index 80663cd7c..000000000 --- a/jobs/migrations/0062_auto_20170925_1457.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2017-09-25 14:57 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0061_auto_20170825_2351'), - ] - - operations = [ - migrations.AddField( - model_name='savedfeatureselection', - name='pinned', - field=models.BooleanField(default=False), - ), - migrations.AlterField( - model_name='job', - name='mbtiles_maxzoom', - field=models.IntegerField(blank=True, null=True), - ), - migrations.AlterField( - model_name='job', - name='mbtiles_minzoom', - field=models.IntegerField(blank=True, null=True), - ), - migrations.AlterField( - model_name='job', - name='mbtiles_source', - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/jobs/migrations/0063_auto_20190814_1737.py b/jobs/migrations/0063_auto_20190814_1737.py deleted file mode 100644 index 217dd57f6..000000000 --- a/jobs/migrations/0063_auto_20190814_1737.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.22 on 2019-08-14 17:37 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0008_alter_user_username_max_length'), - ('jobs', '0062_auto_20170925_1457'), - ] - - operations = [ - migrations.CreateModel( - name='PartnerExportRegion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('schedule_period', models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10)), - ('schedule_hour', models.IntegerField(choices=[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17), (18, 18), (19, 19), (20, 20), (21, 21), (22, 22), (23, 23)], default=0)), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.Group')), - ('job', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='jobs.Job')), - ], - ), - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_hour', - field=models.IntegerField(default=0), - ), - ] diff --git a/jobs/migrations/0064_partnerexportregion_deleted.py b/jobs/migrations/0064_partnerexportregion_deleted.py deleted file mode 100644 index 95b94ec06..000000000 --- a/jobs/migrations/0064_partnerexportregion_deleted.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.22 on 2019-08-14 18:33 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0063_auto_20190814_1737'), - ] - - operations = [ - migrations.AddField( - model_name='partnerexportregion', - name='deleted', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0065_auto_20191020_1210.py b/jobs/migrations/0065_auto_20191020_1210.py deleted file mode 100644 index 078b63af1..000000000 --- a/jobs/migrations/0065_auto_20191020_1210.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.24 on 2019-10-20 12:10 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0064_partnerexportregion_deleted'), - ] - - operations = [ - migrations.AddField( - model_name='hdxexportregion', - name='planet_file', - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name='partnerexportregion', - name='planet_file', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0066_job_pinned.py b/jobs/migrations/0066_job_pinned.py deleted file mode 100644 index 9f5621a60..000000000 --- a/jobs/migrations/0066_job_pinned.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.25 on 2019-11-12 02:13 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0065_auto_20191020_1210'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='pinned', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0067_remove_job_per_theme.py b/jobs/migrations/0067_remove_job_per_theme.py deleted file mode 100644 index 33ed478a2..000000000 --- a/jobs/migrations/0067_remove_job_per_theme.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.25 on 2019-11-19 20:51 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0066_job_pinned'), - ] - - operations = [ - migrations.RemoveField( - model_name='job', - name='per_theme', - ), - ] diff --git a/jobs/migrations/0068_job_unfiltered.py b/jobs/migrations/0068_job_unfiltered.py deleted file mode 100644 index 2c08db36c..000000000 --- a/jobs/migrations/0068_job_unfiltered.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.25 on 2019-11-19 20:52 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0067_remove_job_per_theme'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='unfiltered', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0069_partnerexportregion_polygon_centroid.py b/jobs/migrations/0069_partnerexportregion_polygon_centroid.py deleted file mode 100644 index 07698174f..000000000 --- a/jobs/migrations/0069_partnerexportregion_polygon_centroid.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-09-22 15:10 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0068_job_unfiltered'), - ] - - operations = [ - migrations.AddField( - model_name='partnerexportregion', - name='polygon_centroid', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0070_auto_20210129_2028.py b/jobs/migrations/0070_auto_20210129_2028.py deleted file mode 100644 index 284518054..000000000 --- a/jobs/migrations/0070_auto_20210129_2028.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-01-29 20:28 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0069_partnerexportregion_polygon_centroid'), - ] - - operations = [ - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('two_weeks', 'Every two weeks'), ('three_weeks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - migrations.AlterField( - model_name='partnerexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('two_weeks', 'Every two weeks'), ('three_weeks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - ] diff --git a/jobs/migrations/0071_auto_20210209_1235.py b/jobs/migrations/0071_auto_20210209_1235.py deleted file mode 100644 index 31cfb9050..000000000 --- a/jobs/migrations/0071_auto_20210209_1235.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-02-09 12:35 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0070_auto_20210129_2028'), - ] - - operations = [ - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('2wks', 'Every two weeks'), ('3wks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - migrations.AlterField( - model_name='partnerexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('2wks', 'Every two weeks'), ('3wks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - ] diff --git a/jobs/migrations/0072_job_preserve_geom.py b/jobs/migrations/0072_job_preserve_geom.py deleted file mode 100644 index 6b8405858..000000000 --- a/jobs/migrations/0072_job_preserve_geom.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2022-11-07 09:28 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0071_auto_20210209_1235'), - ] - - operations = [ - migrations.AddField( - model_name='job', - name='preserve_geom', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0073_auto_20221115_0541.py b/jobs/migrations/0073_auto_20221115_0541.py deleted file mode 100644 index a1e85552d..000000000 --- a/jobs/migrations/0073_auto_20221115_0541.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-11-15 05:41 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0072_job_preserve_geom'), - ] - - operations = [ - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours check'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('2wks', 'Every two weeks'), ('3wks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('quarterly', 'Every quarter (90 days)'), ('semiyearly', 'Every 6 months'), ('yearly', 'Every year'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - migrations.AlterField( - model_name='partnerexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours check'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('2wks', 'Every two weeks'), ('3wks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('quarterly', 'Every quarter (90 days)'), ('semiyearly', 'Every 6 months'), ('yearly', 'Every year'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - ] diff --git a/jobs/migrations/0074_hdxexportregion_country_export.py b/jobs/migrations/0074_hdxexportregion_country_export.py deleted file mode 100644 index 862f8e8ee..000000000 --- a/jobs/migrations/0074_hdxexportregion_country_export.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-11-19 10:44 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0073_auto_20221115_0541'), - ] - - operations = [ - migrations.AddField( - model_name='hdxexportregion', - name='country_export', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0075_hdxexportregion_sync_status.py b/jobs/migrations/0075_hdxexportregion_sync_status.py deleted file mode 100644 index 4b4f1c27c..000000000 --- a/jobs/migrations/0075_hdxexportregion_sync_status.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-12-01 11:53 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0074_hdxexportregion_country_export'), - ] - - operations = [ - migrations.AddField( - model_name='hdxexportregion', - name='sync_status', - field=models.BooleanField(default=False), - ), - ] diff --git a/jobs/migrations/0076_remove_hdxexportregion_sync_status.py b/jobs/migrations/0076_remove_hdxexportregion_sync_status.py deleted file mode 100644 index 881220a27..000000000 --- a/jobs/migrations/0076_remove_hdxexportregion_sync_status.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-12-01 12:20 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0075_hdxexportregion_sync_status'), - ] - - operations = [ - migrations.RemoveField( - model_name='hdxexportregion', - name='sync_status', - ), - ] diff --git a/jobs/migrations/0077_auto_20221203_0607.py b/jobs/migrations/0077_auto_20221203_0607.py deleted file mode 100644 index e2c33b5c6..000000000 --- a/jobs/migrations/0077_auto_20221203_0607.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-12-03 06:07 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0076_remove_hdxexportregion_sync_status'), - ] - - operations = [ - migrations.AlterField( - model_name='hdxexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('2wks', 'Every two weeks'), ('3wks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('quarterly', 'Every quarter (90 days)'), ('semiyearly', 'Every 6 months'), ('yearly', 'Every year'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - migrations.AlterField( - model_name='partnerexportregion', - name='schedule_period', - field=models.CharField(choices=[('6hrs', 'Every 6 hours'), ('daily', 'Every day'), ('weekly', 'Every Sunday'), ('2wks', 'Every two weeks'), ('3wks', 'Every three weeks'), ('monthly', 'The 1st of every month'), ('quarterly', 'Every quarter (90 days)'), ('semiyearly', 'Every 6 months'), ('yearly', 'Every year'), ('disabled', 'Disabled')], default='disabled', max_length=10), - ), - ] diff --git a/jobs/migrations/0078_remove_hdxexportregion_country_export.py b/jobs/migrations/0078_remove_hdxexportregion_country_export.py deleted file mode 100644 index 73d6eff99..000000000 --- a/jobs/migrations/0078_remove_hdxexportregion_country_export.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2023-03-23 09:36 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0077_auto_20221203_0607'), - ] - - operations = [ - migrations.RemoveField( - model_name='hdxexportregion', - name='country_export', - ), - ] diff --git a/jobs/migrations/create_hstore_extension.py b/jobs/migrations/create_hstore_extension.py deleted file mode 100644 index 25c109080..000000000 --- a/jobs/migrations/create_hstore_extension.py +++ /dev/null @@ -1,13 +0,0 @@ -from django.contrib.postgres.operations import HStoreExtension -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0001_initial'), - ] - - operations = [ - HStoreExtension(), - ] diff --git a/jobs/migrations/create_tag_gin_index.py b/jobs/migrations/create_tag_gin_index.py deleted file mode 100644 index 14ec35db6..000000000 --- a/jobs/migrations/create_tag_gin_index.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0027_tag_geom_types'), - ] - - operations = [ - migrations.RunSQL('CREATE INDEX geom_types_gin_idx ON tags USING gin(geom_types);') - ] diff --git a/jobs/migrations/insert_export_formats.py b/jobs/migrations/insert_export_formats.py deleted file mode 100644 index 0a16a4d4b..000000000 --- a/jobs/migrations/insert_export_formats.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.contrib.auth.models import User -from rest_framework.authtoken.models import Token - - -class Migration(migrations.Migration): - - def insert_export_formats(apps, schema_editor): - ExportFormat = apps.get_model('jobs', 'ExportFormat') - ExportFormat.objects.create(name='OBF Format', description='OSMAnd OBF', - slug='OBF') - ExportFormat.objects.create(name='ESRI Shapefile Format', description='Esri SHP (OSM Schema)', - slug='SHP') - ExportFormat.objects.create(name='KML Format', description='Google Earth KMZ', - slug='KML') - ExportFormat.objects.create(name='SQLITE Format', description='SQlite SQL', - slug='SQLITE') - ExportFormat.objects.create(name='Garmin Map Format', description='Garmin Map', - slug='GARMIN') - ExportFormat.objects.create(name='ESRI Shapefile Format (Thematic)', description='Esri SHP (Thematic Schema)', - slug='THEMATIC') - - dependencies = [ - ('jobs', '0001_initial'), - ] - - operations = [ - migrations.RunPython(insert_export_formats), - ] diff --git a/jobs/migrations/install_default_group.py b/jobs/migrations/install_default_group.py deleted file mode 100644 index dfb64ed5f..000000000 --- a/jobs/migrations/install_default_group.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -import os -from django.db import models, migrations -from django.contrib.auth.models import Group - - -class Migration(migrations.Migration): - - def insert_default_group(apps, schema_editor): - """ - Set up the default group and group profile. - """ - Group = apps.get_model('auth', 'Group') - ExportProfile = apps.get_model('jobs', 'ExportProfile') - group = Group.objects.create(name='DefaultExportExtentGroup') - profile = ExportProfile.objects.create( - name='DefaultExportProfile', - max_extent=2500000, - group=group - ) - - dependencies = [ - ('jobs', 'install_region_mask'), - ] - - operations = [ - migrations.RunPython(insert_default_group), - ] diff --git a/jobs/migrations/install_region_mask.py b/jobs/migrations/install_region_mask.py deleted file mode 100644 index a913319cf..000000000 --- a/jobs/migrations/install_region_mask.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -import os -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('jobs', '0012_auto_20150908_1226'), - ] - - operations = [ - migrations.RunSQL('DROP VIEW IF EXISTS region_mask;'), - migrations.RunSQL("""CREATE OR REPLACE VIEW region_mask AS - select 1 as id, - st_multi(st_symdifference(st_polyfromtext('POLYGON((-180 90, -180 -90, 180 -90, 180 90, -180 90))', 4326), st_union(the_geom))) - AS the_geom - FROM regions;""") - ] diff --git a/jobs/models.py b/jobs/models.py index 65c746773..023d1db6a 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -34,9 +34,10 @@ MAX_TILE_COUNT = 10000 DIR = os.path.dirname(os.path.abspath(__file__)) -RASTER = rasterio.open(os.path.join(DIR,'osm_nodes.tif')) +RASTER = rasterio.open(os.path.join(DIR, "osm_nodes.tif")) + +Group.add_to_class("is_partner", models.BooleanField(null=False, default=False)) -Group.add_to_class('is_partner', models.BooleanField(null=False, default=False)) def get_geodesic_area(geom): bbox = geom.envelope @@ -56,59 +57,88 @@ def get_geodesic_area(geom): if length > 2: for x in range(length - 1): p1 = coords[x] - p2 = coords[x+1] - area += math.radians(p2[0] - p1[0]) * (2 + math.sin(math.radians(p1[1])) - + math.sin(math.radians(p2[1]))) + p2 = coords[x + 1] + area += math.radians(p2[0] - p1[0]) * ( + 2 + math.sin(math.radians(p1[1])) + math.sin(math.radians(p2[1])) + ) area = abs(int(area * 6378137 * 6378137 / 2.0 / 1000 / 1000)) return area + MAX_NODES = 10000000 -ValidateResult = namedtuple('ValidateResult',['valid','message','params']) +ValidateResult = namedtuple("ValidateResult", ["valid", "message", "params"]) -def check_extent(aoi,url): + +def check_extent(aoi, url): if not aoi.valid: - return ValidateResult(False,aoi.valid_reason,None) + return ValidateResult(False, aoi.valid_reason, None) aoi.srid = 4326 - transformed = aoi.transform(3857,clone=True) - masked = mask.mask(RASTER,[json.loads(transformed.json)],all_touched=False) + transformed = aoi.transform(3857, clone=True) + masked = mask.mask(RASTER, [json.loads(transformed.json)], all_touched=False) nodes = masked[0].sum() * 1000 if nodes > MAX_NODES: - return ValidateResult(False, "The selected area's bounding box contains about %(nodes)s nodes.\ + return ValidateResult( + False, + "The selected area's bounding box contains about %(nodes)s nodes.\ The maximum is %(maxnodes)s. Please choose a smaller area.", - {'nodes':nodes,'maxnodes':MAX_NODES}) - return ValidateResult(True,None,None) + {"nodes": nodes, "maxnodes": MAX_NODES}, + ) + return ValidateResult(True, None, None) + def validate_export_formats(value): if not value: - raise ValidationError( - "Must choose at least one export format." - ) + raise ValidationError("Must choose at least one export format.") for format_name in value: - if format_name not in ['shp','geojson','fgb','csv','sql','geopackage','garmin_img','kml','mwm','osmand_obf','osm_pbf','osm_xml','bundle','mbtiles','full_pbf']: + if format_name not in [ + "shp", + "geojson", + "fgb", + "csv", + "sql", + "geopackage", + "garmin_img", + "kml", + "mwm", + "osmand_obf", + "osm_pbf", + "osm_xml", + "bundle", + "mbtiles", + "full_pbf", + ]: raise ValidationError( "Bad format name: %(format_name)s", - params={'format_name': format_name}, + params={"format_name": format_name}, ) + def validate_feature_selection(value): from osm_export_tool.mapping import Mapping + m, errors = Mapping.validate(value) if not m: raise ValidationError(errors) + def validate_aoi(aoi): - result = check_extent(aoi,settings.OVERPASS_API_URL) + result = check_extent(aoi, settings.OVERPASS_API_URL) if not result.valid: - raise ValidationError(result.message,params=result.params) + raise ValidationError(result.message, params=result.params) + def validate_mbtiles(job): if "mbtiles" in job["export_formats"]: if job.get("mbtiles_source") is None: - raise ValidationError("A source is required when generating an MBTiles archive.") + raise ValidationError( + "A source is required when generating an MBTiles archive." + ) if job.get("mbtiles_maxzoom") is None or job.get("mbtiles_minzoom") is None: - raise ValidationError("A zoom range must be provided when generating an MBTiles archive.") + raise ValidationError( + "A zoom range must be provided when generating an MBTiles archive." + ) bounds = job["the_geom"].extent tile_count = 0 @@ -125,36 +155,51 @@ def validate_mbtiles(job): if tile_count > MAX_TILE_COUNT: raise ValidationError( "%(tile_count)s tiles would be rendered; please reduce the zoom range, the size of your AOI, or split the export into pieces covering specific areas in order to render fewer than %(max_tile_count)s in each.", - params={'tile_count': tile_count, 'max_tile_count': MAX_TILE_COUNT}, + params={"tile_count": tile_count, "max_tile_count": MAX_TILE_COUNT}, ) + class Job(models.Model): """ Database model for an 'Export'. Immutable, except in the case of HDX Export Regions. """ + id = models.AutoField(primary_key=True, editable=False) - uid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False, db_index=True) - user = models.ForeignKey(User, related_name='owner') + uid = models.UUIDField( + unique=True, default=uuid.uuid4, editable=False, db_index=True + ) + user = models.ForeignKey(User, related_name="owner", on_delete=models.CASCADE) name = models.CharField(max_length=100, db_index=True, blank=False) - description = models.CharField(max_length=1000, db_index=True, default='', blank=True) - event = models.CharField(max_length=100, db_index=True, default='', blank=True) - export_formats = ArrayField(models.CharField(max_length=10),validators=[validate_export_formats],blank=False) + description = models.CharField( + max_length=1000, db_index=True, default="", blank=True + ) + event = models.CharField(max_length=100, db_index=True, default="", blank=True) + export_formats = ArrayField( + models.CharField(max_length=10), + validators=[validate_export_formats], + blank=False, + ) published = models.BooleanField(default=False, db_index=True) - the_geom = models.GeometryField(verbose_name='Uploaded geometry', srid=4326, blank=False) - simplified_geom = models.GeometryField(verbose_name='Simplified geometry', srid=4326, blank=True,null=True) - objects = models.GeoManager() - feature_selection = models.TextField(blank=False,validators=[validate_feature_selection]) + the_geom = models.GeometryField( + verbose_name="Uploaded geometry", srid=4326, blank=False + ) + simplified_geom = models.GeometryField( + verbose_name="Simplified geometry", srid=4326, blank=True, null=True + ) + feature_selection = models.TextField( + blank=False, validators=[validate_feature_selection] + ) created_at = models.DateTimeField(default=timezone.now, editable=False) updated_at = models.DateTimeField(default=timezone.now, editable=False) - mbtiles_maxzoom = models.IntegerField(null=True,blank=True) - mbtiles_minzoom = models.IntegerField(null=True,blank=True) - mbtiles_source = models.TextField(null=True,blank=True) + mbtiles_maxzoom = models.IntegerField(null=True, blank=True) + mbtiles_minzoom = models.IntegerField(null=True, blank=True) + mbtiles_source = models.TextField(null=True, blank=True) # flags buffer_aoi = models.BooleanField(default=False) unlimited_extent = models.BooleanField(default=False) - hidden = models.BooleanField(default=False) # hidden from the list page + hidden = models.BooleanField(default=False) # hidden from the list page expire_old_runs = models.BooleanField(default=True) pinned = models.BooleanField(default=False) unfiltered = models.BooleanField(default=False) @@ -162,7 +207,7 @@ class Job(models.Model): class Meta: # pragma: no cover managed = True - db_table = 'jobs' + db_table = "jobs" @property def last_run_status(self): @@ -174,7 +219,6 @@ def last_run_date(self): if self.runs.count() > 0: return self.runs.all()[self.runs.count() - 1].started_at - @property def is_hdx(self): if HDXExportRegion.objects.filter(job_id=self.id).exists(): @@ -184,7 +228,9 @@ def is_hdx(self): @property def osma_link(self): bounds = self.the_geom.extent - return "http://osm-analytics.org/#/show/bbox:{0},{1},{2},{3}/buildings/recency".format(*bounds) + return "http://osm-analytics.org/#/show/bbox:{0},{1},{2},{3}/buildings/recency".format( + *bounds + ) @property def area(self): @@ -192,7 +238,9 @@ def area(self): def save(self, *args, **kwargs): self.the_geom = force2d(self.the_geom) - self.simplified_geom = simplify_geom(self.the_geom,force_buffer=self.buffer_aoi) + self.simplified_geom = simplify_geom( + self.the_geom, force_buffer=self.buffer_aoi + ) super(Job, self).save(*args, **kwargs) def __str__(self): @@ -200,41 +248,53 @@ def __str__(self): class SavedFeatureSelection(models.Model): - """ Mutable database record for a saved YAML configuration.""" - created_at = models.DateTimeField(default=timezone.now,db_index=True, editable=False) + """Mutable database record for a saved YAML configuration.""" + + created_at = models.DateTimeField( + default=timezone.now, db_index=True, editable=False + ) updated_at = models.DateTimeField(default=timezone.now, editable=False) - user = models.ForeignKey(User) - name = models.CharField(max_length=100,db_index=True,blank=False) - description = models.CharField(max_length=1000, db_index=True, default='', blank=True) - yaml = models.TextField(blank=False,validators=[validate_feature_selection]) + user = models.ForeignKey(User, on_delete=models.CASCADE) + name = models.CharField(max_length=100, db_index=True, blank=False) + description = models.CharField( + max_length=1000, db_index=True, default="", blank=True + ) + yaml = models.TextField(blank=False, validators=[validate_feature_selection]) public = models.BooleanField(default=False) deleted = models.BooleanField(default=False) - uid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False, db_index=True) + uid = models.UUIDField( + unique=True, default=uuid.uuid4, editable=False, db_index=True + ) pinned = models.BooleanField(default=False) def __str__(self): return str(self.name) + PERIOD_CHOICES = ( - ('6hrs', 'Every 6 hours'), - ('daily', 'Every day'), - ('weekly', 'Every Sunday'), - ('2wks', 'Every two weeks'), - ('3wks', 'Every three weeks'), - ('monthly', 'The 1st of every month'), - ('quarterly', 'Every quarter (90 days)'), - ('semiyearly', 'Every 6 months'), - ('yearly', 'Every year'), - ('disabled', 'Disabled'), + ("6hrs", "Every 6 hours"), + ("daily", "Every day"), + ("weekly", "Every Sunday"), + ("2wks", "Every two weeks"), + ("3wks", "Every three weeks"), + ("monthly", "The 1st of every month"), + ("quarterly", "Every quarter (90 days)"), + ("semiyearly", "Every 6 months"), + ("yearly", "Every year"), + ("disabled", "Disabled"), ) -HOUR_CHOICES = zip(range(0, 24), range(0, 24)) +HOUR_CHOICES = list(zip(range(0, 24), range(0, 24))) + class RegionMixin: @property - def last_run(self): # noqa + def last_run(self): # noqa if self.job.runs.count() > 0: - - last_run_time = self.job.runs.all()[self.job.runs.count() - 1].finished_at or self.job.runs.all()[self.job.runs.count() - 1].started_at or self.job.runs.all()[self.job.runs.count() - 1].created_at + last_run_time = ( + self.job.runs.all()[self.job.runs.count() - 1].finished_at + or self.job.runs.all()[self.job.runs.count() - 1].started_at + or self.job.runs.all()[self.job.runs.count() - 1].created_at + ) return last_run_time @property @@ -245,32 +305,38 @@ def last_run_status(self): @property def last_run_duration(self): if self.job.runs.count() > 0: - i=1 + i = 1 last_run_duration = None - while (last_run_duration is None): # get previous run size if current is running/submitted - if i >= self.job.runs.count() and i != 1: + while ( + last_run_duration is None + ): # get previous run size if current is running/submitted + if i >= self.job.runs.count() and i != 1: break - last_run_duration = self.job.runs.all()[self.job.runs.count() - i].duration - i+=1 - return time.strftime('%H:%M:%S', time.gmtime(last_run_duration)) - + last_run_duration = self.job.runs.all()[ + self.job.runs.count() - i + ].duration + i += 1 + return time.strftime("%H:%M:%S", time.gmtime(last_run_duration)) @property def last_size(self): if self.job.runs.count() > 0: - i=1 + i = 1 last_run_size = 0 - while (last_run_size == 0): # get previous run size if current is running/submitted - if i >= self.job.runs.count() and i != 1: + while ( + last_run_size == 0 + ): # get previous run size if current is running/submitted + if i >= self.job.runs.count() and i != 1: break last_run_size = self.job.runs.all()[self.job.runs.count() - i].size - i+=1 + i += 1 return last_run_size @property def last_export_size(self): if self.last_size: return size(self.last_size) + @property def last_run_hdx_sync(self): if self.job.runs.count() > 0: @@ -284,19 +350,18 @@ def next_run_hum(self): def last_run_hum(self): return humanize.naturaltime(self.last_run) - @property - def next_run(self): # noqa + def next_run(self): # noqa now = timezone.now().replace(minute=0, second=0, microsecond=0) - if self.schedule_period == '6hrs': + if self.schedule_period == "6hrs": delta = 6 - (self.schedule_hour + now.hour % 6) return now + timedelta(hours=delta) now = now.replace(hour=self.schedule_hour) - if self.schedule_period == 'daily': + if self.schedule_period == "daily": anchor = now if timezone.now() < anchor: @@ -304,7 +369,7 @@ def next_run(self): # noqa return anchor + timedelta(days=1) - if self.schedule_period == 'weekly': + if self.schedule_period == "weekly": # adjust so the week starts on Sunday anchor = now - timedelta((now.weekday() + 1) % 7) @@ -313,7 +378,7 @@ def next_run(self): # noqa return anchor + timedelta(days=7) - if self.schedule_period == '2wks': + if self.schedule_period == "2wks": # adjust so the week starts on Sunday anchor = now - timedelta((now.weekday() + 1) % 7) @@ -322,7 +387,7 @@ def next_run(self): # noqa return anchor + timedelta(days=14) - if self.schedule_period == '3wks': + if self.schedule_period == "3wks": # adjust so the week starts on Sunday anchor = now - timedelta((now.weekday() + 1) % 7) @@ -331,7 +396,7 @@ def next_run(self): # noqa return anchor + timedelta(days=21) - if self.schedule_period == 'monthly': + if self.schedule_period == "monthly": (_, num_days) = calendar.monthrange(now.year, now.month) anchor = now.replace(day=1) @@ -340,7 +405,7 @@ def next_run(self): # noqa return anchor + timedelta(days=num_days) - if self.schedule_period == 'quarterly': + if self.schedule_period == "quarterly": # adjust so the week starts on Sunday anchor = now - timedelta((now.weekday() + 1) % 7) @@ -349,7 +414,7 @@ def next_run(self): # noqa return anchor + timedelta(days=90) - if self.schedule_period == 'semiyearly': + if self.schedule_period == "semiyearly": # adjust so the week starts on Sunday anchor = now - timedelta((now.weekday() + 1) % 7) @@ -358,7 +423,7 @@ def next_run(self): # noqa return anchor + timedelta(days=180) - if self.schedule_period == 'yearly': + if self.schedule_period == "yearly": # adjust so the week starts on Sunday anchor = now - timedelta((now.weekday() + 1) % 7) @@ -368,36 +433,36 @@ def next_run(self): # noqa return anchor + timedelta(days=365) @property - def delta(self): # noqa - if self.schedule_period == '6hrs': + def delta(self): # noqa + if self.schedule_period == "6hrs": return timedelta(hours=6) - if self.schedule_period == 'daily': + if self.schedule_period == "daily": return timedelta(days=1) - if self.schedule_period == 'weekly': + if self.schedule_period == "weekly": return timedelta(days=7) - if self.schedule_period == '2wks': + if self.schedule_period == "2wks": return timedelta(days=14) - if self.schedule_period == '3wks': + if self.schedule_period == "3wks": return timedelta(days=21) - if self.schedule_period == 'monthly': + if self.schedule_period == "monthly": return timedelta(days=31) - if self.schedule_period == 'quarterly': + if self.schedule_period == "quarterly": return timedelta(days=90) - if self.schedule_period == 'semiyearly': + if self.schedule_period == "semiyearly": return timedelta(days=189) - if self.schedule_period == 'yearly': + if self.schedule_period == "yearly": return timedelta(days=365) @property - def feature_selection(self): # noqa + def feature_selection(self): # noqa return self.job.feature_selection @property @@ -405,7 +470,7 @@ def the_geom(self): return self.job.the_geom @property - def simplified_geom(self): # noqa + def simplified_geom(self): # noqa return self.job.simplified_geom @property @@ -417,27 +482,28 @@ def created_by(self): return self.job.user @property - def export_formats(self): # noqa + def export_formats(self): # noqa return self.job.export_formats + class PartnerExportRegion(models.Model, RegionMixin): schedule_period = models.CharField( - blank=False, max_length=10, default="disabled", choices=PERIOD_CHOICES) - schedule_hour = models.IntegerField( - blank=False, choices=HOUR_CHOICES, default=0) - job = models.ForeignKey(Job, null=True) + blank=False, max_length=10, default="disabled", choices=PERIOD_CHOICES + ) + schedule_hour = models.IntegerField(blank=False, choices=HOUR_CHOICES, default=0) + job = models.ForeignKey(Job, null=True, on_delete=models.SET_NULL) # the owning group, which determines access control. - group = models.ForeignKey(Group) + group = models.ForeignKey(Group, on_delete=models.CASCADE) deleted = models.BooleanField(default=False) planet_file = models.BooleanField(default=False) polygon_centroid = models.BooleanField(default=False) @property - def export_formats(self): # noqa + def export_formats(self): # noqa return self.job.export_formats @property - def name(self): # noqa + def name(self): # noqa return self.job.name @property @@ -452,54 +518,57 @@ def event(self): def group_name(self): return self.group.name -class HDXExportRegion(models.Model, RegionMixin): # noqa - """ Mutable database table for hdx - additional attributes on a Job.""" + +class HDXExportRegion(models.Model, RegionMixin): # noqa + """Mutable database table for hdx - additional attributes on a Job.""" + schedule_period = models.CharField( - blank=False, max_length=10, default="disabled", choices=PERIOD_CHOICES) - schedule_hour = models.IntegerField( - blank=False, choices=HOUR_CHOICES, default=0) + blank=False, max_length=10, default="disabled", choices=PERIOD_CHOICES + ) + schedule_hour = models.IntegerField(blank=False, choices=HOUR_CHOICES, default=0) deleted = models.BooleanField(default=False) # a job should really be required, but that interferes with DRF validation lifecycle. - job = models.ForeignKey(Job, null=True, related_name='hdx_export_region_set') + job = models.ForeignKey( + Job, null=True, related_name="hdx_export_region_set", on_delete=models.SET_NULL + ) is_private = models.BooleanField(default=False) - locations = ArrayField( - models.CharField(blank=False, max_length=32), null=True) - license = models.CharField(max_length=32, null=True,blank=True) + locations = ArrayField(models.CharField(blank=False, max_length=32), null=True) + license = models.CharField(max_length=32, null=True, blank=True) subnational = models.BooleanField(default=True) - extra_notes = models.TextField(null=True,blank=True) + extra_notes = models.TextField(null=True, blank=True) planet_file = models.BooleanField(default=False) - class Meta: # noqa - db_table = 'hdx_export_regions' - + class Meta: # noqa + db_table = "hdx_export_regions" def __str__(self): return self.name + " (prefix: " + self.dataset_prefix + ")" - def clean(self): - if self.job and not re.match(r'^[a-z0-9-_]+$',self.job.name): - raise ValidationError({'dataset_prefix':"Invalid dataset_prefix: {0}".format(self.job.name)}) + if self.job and not re.match(r"^[a-z0-9-_]+$", self.job.name): + raise ValidationError( + {"dataset_prefix": "Invalid dataset_prefix: {0}".format(self.job.name)} + ) @property - def buffer_aoi(self): # noqa + def buffer_aoi(self): # noqa return self.job.buffer_aoi @property - def name(self): # noqa + def name(self): # noqa return self.job.description @property - def dataset_prefix(self): # noqa + def dataset_prefix(self): # noqa return self.job.name @property - def datasets(self): # noqa + def datasets(self): # noqa export_set = HDXExportSet( Mapping(self.feature_selection), self.dataset_prefix, self.name, - self.extra_notes + self.extra_notes, ) return export_set.dataset_links(settings.HDX_URL_PREFIX) @@ -539,31 +608,31 @@ def update_frequency(self): # "semiyearly": "180", # "annually": "365", # "yearly": "365", - if self.schedule_period == '6hrs': + if self.schedule_period == "6hrs": return 1 - if self.schedule_period == 'daily': + if self.schedule_period == "daily": return 1 - if self.schedule_period == 'weekly': + if self.schedule_period == "weekly": return 7 - if self.schedule_period == '2wks': + if self.schedule_period == "2wks": return 14 - if self.schedule_period == '3wks': + if self.schedule_period == "3wks": return 30 - if self.schedule_period == 'monthly': + if self.schedule_period == "monthly": return 30 - if self.schedule_period == 'quarterly': + if self.schedule_period == "quarterly": return 90 - if self.schedule_period == 'semiyearly': + if self.schedule_period == "semiyearly": return 180 - if self.schedule_period == 'yearly': + if self.schedule_period == "yearly": return 365 return -2 # returning as needed as default instead of live diff --git a/ops/README.md b/ops/README.md index dc9fa88ea..5df2f6082 100644 --- a/ops/README.md +++ b/ops/README.md @@ -106,7 +106,7 @@ sudo systemctl start database_backup.timer TODO `docker.postgresql-backup.{service,timer}` define a unit that runs daily to back the database up to S3. To check its schedule, run `systemctl list-timers`. -### Pre-compiled libraries + diff --git a/requirements.txt b/requirements.txt index 6d0264435..87aaa0904 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,39 +1,42 @@ -Django~=1.11.3 +Django~=3.2.22 cachetools dj_database_url -django-oauth-toolkit==1.1.2 +django-oauth-toolkit==1.3.2 django-cors-middleware -djangorestframework-gis~=0.11.2 -djangorestframework~=3.6.3 +djangorestframework-gis~=0.15 +djangorestframework~=3.11.0 hdx-python-api==5.9.2 +hdx-python-utilities==3.6.2 mercantile~=0.10.0 -gitdb==4.0.9 -gevent==21.12.0 -psycopg2==2.8.3 -hdx-python-utilities==3.4.7 -python3-openid -social-auth-app-django==2.1.0 -social-auth-core==1.7.0 +psycopg2 +python3-openid==3.2.0 +social-auth-app-django==5.4.0 +social-auth-core==4.4.2 pytz -pyyaml==5.4 -raven~=6.1.0 -requests==2.26.0 -urllib3==1.26.12 +pyyaml>=5.3 +raven +requests~=2.26 requests_oauthlib==0.8.0 -pyparsing>=2.2.0 -oauthlib==2.0.7 +pyparsing~=2.4 +oauthlib==3.1.0 + gunicorn==19.9.0 osmium~=3.5.0 dramatiq[redis,watch]~=1.13.0 boto3~=1.23.10 # needed for cloudwatch -rasterio~=1.0.25 +rasterio # osm-export-tool==0.0.25 #manual setup is needed for now osm-export-tool-python == 2.0.10 -rtree==0.9.1 +rtree==1.1.0 validators==0.20.0 geonamescache==1.5.0 psutil==5.9.1 dramatiq-dashboard==0.4.0 deepdiff==5.7.0 hurry.filesize==0.9 -dramatiq-abort==1.0 \ No newline at end of file +dramatiq-abort==1.0 + +python3-openid + +##testing +pytest==7.4.3 \ No newline at end of file diff --git a/tasks/migrations/0001_initial.py b/tasks/migrations/0001_initial.py index e5de7a1e4..9b7fe04dd 100644 --- a/tasks/migrations/0001_initial.py +++ b/tasks/migrations/0001_initial.py @@ -1,14 +1,19 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +# Generated by Django 3.2.22 on 2023-10-31 04:36 -from django.db import models, migrations +from django.conf import settings +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion import django.utils.timezone import uuid class Migration(migrations.Migration): + initial = True + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('jobs', '__first__'), ] @@ -16,44 +21,39 @@ class Migration(migrations.Migration): migrations.CreateModel( name='ExportRun', fields=[ - ('started_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('finished_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(default=uuid.uuid4, unique=True, editable=False)), - ('type', models.CharField(max_length=20, db_index=True)), - ('job', models.ForeignKey(related_name='job', to='jobs.Job')), + ('id', models.AutoField(editable=False, primary_key=True, serialize=False)), + ('uid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), + ('worker_message_id', models.CharField(blank=True, editable=False, max_length=50, null=True)), + ('hdx_sync_status', models.BooleanField(default=False)), + ('status', models.CharField(blank=True, db_index=True, default='', max_length=20)), + ('started_at', models.DateTimeField(editable=False, null=True)), + ('finished_at', models.DateTimeField(editable=False, null=True)), + ('job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='jobs.job')), + ('user', models.ForeignKey(default=0, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='runs', to=settings.AUTH_USER_MODEL)), ], options={ 'db_table': 'export_runs', - 'managed': True, + 'ordering': ['created_at'], }, ), migrations.CreateModel( name='ExportTask', fields=[ - ('started_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('finished_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('uid', models.UUIDField(blank=True)), + ('id', models.AutoField(editable=False, primary_key=True, serialize=False)), + ('uid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('name', models.CharField(max_length=50)), + ('status', models.CharField(blank=True, db_index=True, max_length=20)), + ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), + ('started_at', models.DateTimeField(editable=False, null=True)), + ('finished_at', models.DateTimeField(editable=False, null=True)), + ('filesize_bytes', models.BigIntegerField(null=True)), + ('filenames', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(null=True), default=list, size=None)), + ('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='tasks.exportrun')), ], options={ 'db_table': 'export_tasks', - 'managed': True, - }, - ), - migrations.CreateModel( - name='ExportTaskResult', - fields=[ - ('task', models.OneToOneField(primary_key=True, serialize=False, to='tasks.ExportTask')), - ], - options={ - 'db_table': 'export_task_results', - 'managed': True, + 'ordering': ['created_at'], }, ), - migrations.AddField( - model_name='exporttask', - name='run', - field=models.ForeignKey(related_name='run', to='tasks.ExportRun'), - ), ] diff --git a/tasks/migrations/0002_exporttaskresult_output_url.py b/tasks/migrations/0002_exporttaskresult_output_url.py deleted file mode 100644 index c6e6eafcb..000000000 --- a/tasks/migrations/0002_exporttaskresult_output_url.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='exporttaskresult', - name='output_url', - field=models.URLField(default='', verbose_name='Url to export task result.'), - preserve_default=False, - ), - ] diff --git a/tasks/migrations/0003_exportrun_run_type.py b/tasks/migrations/0003_exportrun_run_type.py deleted file mode 100644 index 1f09c978d..000000000 --- a/tasks/migrations/0003_exportrun_run_type.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0002_exporttaskresult_output_url'), - ] - - operations = [ - migrations.AddField( - model_name='exportrun', - name='run_type', - field=models.CharField(default='EXPORT', max_length=6, editable=False, db_index=True), - ), - ] diff --git a/tasks/migrations/0004_auto_20150601_1122.py b/tasks/migrations/0004_auto_20150601_1122.py deleted file mode 100644 index b7b8003ad..000000000 --- a/tasks/migrations/0004_auto_20150601_1122.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0003_exportrun_run_type'), - ] - - operations = [ - migrations.RemoveField( - model_name='exportrun', - name='run_type', - ), - migrations.AlterField( - model_name='exporttask', - name='run', - field=models.ForeignKey(related_name='tasks', to='tasks.ExportRun'), - ), - ] diff --git a/tasks/migrations/0005_auto_20150602_1309.py b/tasks/migrations/0005_auto_20150602_1309.py deleted file mode 100644 index d89bb8737..000000000 --- a/tasks/migrations/0005_auto_20150602_1309.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0004_auto_20150601_1122'), - ] - - operations = [ - migrations.AddField( - model_name='exporttask', - name='status', - field=models.CharField(db_index=True, max_length=20, blank=True), - ), - migrations.AlterField( - model_name='exportrun', - name='finished_at', - field=models.DateTimeField(editable=False), - ), - migrations.AlterField( - model_name='exportrun', - name='job', - field=models.ForeignKey(related_name='runs', to='jobs.Job'), - ), - migrations.AlterField( - model_name='exporttask', - name='finished_at', - field=models.DateTimeField(editable=False), - ), - migrations.AlterField( - model_name='exporttaskresult', - name='task', - field=models.OneToOneField(related_name='result', primary_key=True, serialize=False, to='tasks.ExportTask'), - ), - ] diff --git a/tasks/migrations/0006_auto_20150602_1312.py b/tasks/migrations/0006_auto_20150602_1312.py deleted file mode 100644 index 062bf3e0f..000000000 --- a/tasks/migrations/0006_auto_20150602_1312.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0005_auto_20150602_1309'), - ] - - operations = [ - migrations.AlterField( - model_name='exportrun', - name='finished_at', - field=models.DateTimeField(editable=False, blank=True), - ), - migrations.AlterField( - model_name='exporttask', - name='finished_at', - field=models.DateTimeField(editable=False, blank=True), - ), - ] diff --git a/tasks/migrations/0007_remove_exportrun_type.py b/tasks/migrations/0007_remove_exportrun_type.py deleted file mode 100644 index dc06533f1..000000000 --- a/tasks/migrations/0007_remove_exportrun_type.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0006_auto_20150602_1312'), - ] - - operations = [ - migrations.RemoveField( - model_name='exportrun', - name='type', - ), - ] diff --git a/tasks/migrations/0008_auto_20150602_1319.py b/tasks/migrations/0008_auto_20150602_1319.py deleted file mode 100644 index b1b2ba87e..000000000 --- a/tasks/migrations/0008_auto_20150602_1319.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0007_remove_exportrun_type'), - ] - - operations = [ - migrations.AlterField( - model_name='exportrun', - name='finished_at', - field=models.DateTimeField(null=True, editable=False), - ), - migrations.AlterField( - model_name='exporttask', - name='finished_at', - field=models.DateTimeField(null=True, editable=False), - ), - ] diff --git a/tasks/migrations/0009_exporttask_name.py b/tasks/migrations/0009_exporttask_name.py deleted file mode 100644 index 98a3ddf3e..000000000 --- a/tasks/migrations/0009_exporttask_name.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0008_auto_20150602_1319'), - ] - - operations = [ - migrations.AddField( - model_name='exporttask', - name='name', - field=models.CharField(default='', max_length=50), - preserve_default=False, - ), - ] diff --git a/tasks/migrations/0010_exporttaskexception.py b/tasks/migrations/0010_exporttaskexception.py deleted file mode 100644 index 052b9743f..000000000 --- a/tasks/migrations/0010_exporttaskexception.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0009_exporttask_name'), - ] - - operations = [ - migrations.CreateModel( - name='ExportTaskException', - fields=[ - ('id', models.AutoField(serialize=False, editable=False, primary_key=True)), - ('timestamp', models.DateTimeField(default=django.utils.timezone.now, editable=False)), - ('exception', models.CharField(max_length=1000, editable=False)), - ('task', models.ForeignKey(related_name='exceptions', to='tasks.ExportTask')), - ], - options={ - 'db_table': 'export_task_exceptions', - 'managed': True, - }, - ), - ] diff --git a/tasks/migrations/0011_auto_20150602_1914.py b/tasks/migrations/0011_auto_20150602_1914.py deleted file mode 100644 index 370ffb7ed..000000000 --- a/tasks/migrations/0011_auto_20150602_1914.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0010_exporttaskexception'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttaskexception', - name='exception', - field=models.CharField(max_length=5000, editable=False), - ), - ] diff --git a/tasks/migrations/0012_auto_20150619_1650.py b/tasks/migrations/0012_auto_20150619_1650.py deleted file mode 100644 index 35576a972..000000000 --- a/tasks/migrations/0012_auto_20150619_1650.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0011_auto_20150602_1914'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttaskexception', - name='exception', - field=models.TextField(editable=False), - ), - ] diff --git a/tasks/migrations/0013_auto_20150622_1837.py b/tasks/migrations/0013_auto_20150622_1837.py deleted file mode 100644 index f7e481061..000000000 --- a/tasks/migrations/0013_auto_20150622_1837.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0012_auto_20150619_1650'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttask', - name='uid', - field=models.UUIDField(null=True), - ), - ] diff --git a/tasks/migrations/0014_auto_20150622_1848.py b/tasks/migrations/0014_auto_20150622_1848.py deleted file mode 100644 index 0bf48d62f..000000000 --- a/tasks/migrations/0014_auto_20150622_1848.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0013_auto_20150622_1837'), - ] - - operations = [ - migrations.AddField( - model_name='exporttask', - name='celery_uid', - field=models.UUIDField(null=True), - ), - migrations.AlterField( - model_name='exporttask', - name='uid', - field=models.UUIDField(default=uuid.uuid4, unique=True, editable=False), - ), - ] diff --git a/tasks/migrations/0015_auto_20150624_1503.py b/tasks/migrations/0015_auto_20150624_1503.py deleted file mode 100644 index 7c0f4d35a..000000000 --- a/tasks/migrations/0015_auto_20150624_1503.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0014_auto_20150622_1848'), - ] - - operations = [ - migrations.RemoveField( - model_name='exporttaskresult', - name='output_url', - ), - migrations.AddField( - model_name='exporttaskresult', - name='download_url', - field=models.URLField(default='', verbose_name='Url to export task result output.'), - preserve_default=False, - ), - ] diff --git a/tasks/migrations/0016_auto_20150624_1506.py b/tasks/migrations/0016_auto_20150624_1506.py deleted file mode 100644 index 426385777..000000000 --- a/tasks/migrations/0016_auto_20150624_1506.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0015_auto_20150624_1503'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttaskresult', - name='download_url', - field=models.URLField(max_length=254, verbose_name='Url to export task result output.'), - ), - ] diff --git a/tasks/migrations/0017_exportrun_status.py b/tasks/migrations/0017_exportrun_status.py deleted file mode 100644 index b85126342..000000000 --- a/tasks/migrations/0017_exportrun_status.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0016_auto_20150624_1506'), - ] - - operations = [ - migrations.AddField( - model_name='exportrun', - name='status', - field=models.CharField(default='', max_length=20, db_index=True, blank=True), - ), - ] diff --git a/tasks/migrations/0018_auto_20150624_1736.py b/tasks/migrations/0018_auto_20150624_1736.py deleted file mode 100644 index 870590920..000000000 --- a/tasks/migrations/0018_auto_20150624_1736.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0017_exportrun_status'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttask', - name='started_at', - field=models.DateTimeField(null=True, editable=False), - ), - ] diff --git a/tasks/migrations/0019_auto_20150624_1808.py b/tasks/migrations/0019_auto_20150624_1808.py deleted file mode 100644 index 1d6a28f3a..000000000 --- a/tasks/migrations/0019_auto_20150624_1808.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0018_auto_20150624_1736'), - ] - - operations = [ - migrations.AddField( - model_name='exporttaskresult', - name='filename', - field=models.CharField(max_length=100, editable=False, blank=True), - ), - migrations.AddField( - model_name='exporttaskresult', - name='size', - field=models.FloatField(null=True, editable=False), - ), - ] diff --git a/tasks/migrations/0020_auto_20150724_1413.py b/tasks/migrations/0020_auto_20150724_1413.py deleted file mode 100644 index 19b97af4e..000000000 --- a/tasks/migrations/0020_auto_20150724_1413.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0019_auto_20150624_1808'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exporttask', - options={'ordering': ['started_at'], 'managed': True}, - ), - migrations.AddField( - model_name='exportrun', - name='created_at', - field=models.DateTimeField(default=django.utils.timezone.now, editable=False), - ), - ] diff --git a/tasks/migrations/0021_remove_exportrun_created_at.py b/tasks/migrations/0021_remove_exportrun_created_at.py deleted file mode 100644 index 490cf1771..000000000 --- a/tasks/migrations/0021_remove_exportrun_created_at.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0020_auto_20150724_1413'), - ] - - operations = [ - migrations.RemoveField( - model_name='exportrun', - name='created_at', - ), - ] diff --git a/tasks/migrations/0022_exportrun_created_at.py b/tasks/migrations/0022_exportrun_created_at.py deleted file mode 100644 index 4d5d1f7d8..000000000 --- a/tasks/migrations/0022_exportrun_created_at.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0021_remove_exportrun_created_at'), - ] - - operations = [ - migrations.AddField( - model_name='exportrun', - name='created_at', - field=models.DateTimeField(default=django.utils.timezone.now, editable=False), - ), - ] diff --git a/tasks/migrations/0023_auto_20150724_1436.py b/tasks/migrations/0023_auto_20150724_1436.py deleted file mode 100644 index e33a60468..000000000 --- a/tasks/migrations/0023_auto_20150724_1436.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0022_exportrun_created_at'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exporttask', - options={'ordering': ['-started_at'], 'managed': True}, - ), - ] diff --git a/tasks/migrations/0024_auto_20150724_1436.py b/tasks/migrations/0024_auto_20150724_1436.py deleted file mode 100644 index 3c456896a..000000000 --- a/tasks/migrations/0024_auto_20150724_1436.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0023_auto_20150724_1436'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exporttask', - options={'ordering': ['started_at'], 'managed': True}, - ), - ] diff --git a/tasks/migrations/0025_exporttask_created_at.py b/tasks/migrations/0025_exporttask_created_at.py deleted file mode 100644 index 32ad9c311..000000000 --- a/tasks/migrations/0025_exporttask_created_at.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0024_auto_20150724_1436'), - ] - - operations = [ - migrations.AddField( - model_name='exporttask', - name='created_at', - field=models.DateTimeField(default=django.utils.timezone.now, editable=False), - ), - ] diff --git a/tasks/migrations/0026_auto_20150724_1437.py b/tasks/migrations/0026_auto_20150724_1437.py deleted file mode 100644 index 66a0cf5e6..000000000 --- a/tasks/migrations/0026_auto_20150724_1437.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0025_exporttask_created_at'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exporttask', - options={'ordering': ['created_at'], 'managed': True}, - ), - ] diff --git a/tasks/migrations/0027_exportrun_user.py b/tasks/migrations/0027_exportrun_user.py deleted file mode 100644 index 4e8241fb4..000000000 --- a/tasks/migrations/0027_exportrun_user.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -from django.conf import settings - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('tasks', '0026_auto_20150724_1437'), - ] - - operations = [ - migrations.AddField( - model_name='exportrun', - name='user', - field=models.ForeignKey(related_name='runs', default=0, to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/tasks/migrations/0028_auto_20170329_0105.py b/tasks/migrations/0028_auto_20170329_0105.py deleted file mode 100644 index 8044dd754..000000000 --- a/tasks/migrations/0028_auto_20170329_0105.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-03-29 01:05 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0027_exportrun_user'), - ] - - operations = [ - migrations.RemoveField( - model_name='exporttaskexception', - name='task', - ), - migrations.DeleteModel( - name='ExportTaskException', - ), - ] diff --git a/tasks/migrations/0029_remove_exporttask_celery_uid.py b/tasks/migrations/0029_remove_exporttask_celery_uid.py deleted file mode 100644 index d35f14db0..000000000 --- a/tasks/migrations/0029_remove_exporttask_celery_uid.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-06 00:11 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0028_auto_20170329_0105'), - ] - - operations = [ - migrations.RemoveField( - model_name='exporttask', - name='celery_uid', - ), - ] diff --git a/tasks/migrations/0030_auto_20170406_0045.py b/tasks/migrations/0030_auto_20170406_0045.py deleted file mode 100644 index 073f9500a..000000000 --- a/tasks/migrations/0030_auto_20170406_0045.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-06 00:45 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0029_remove_exporttask_celery_uid'), - ] - - operations = [ - migrations.RemoveField( - model_name='exporttaskresult', - name='task', - ), - migrations.AddField( - model_name='exporttask', - name='filesize_bytes', - field=models.IntegerField(null=True), - ), - migrations.DeleteModel( - name='ExportTaskResult', - ), - ] diff --git a/tasks/migrations/0031_exporttask_filename.py b/tasks/migrations/0031_exporttask_filename.py deleted file mode 100644 index 2c151232f..000000000 --- a/tasks/migrations/0031_exporttask_filename.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-06 04:57 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0030_auto_20170406_0045'), - ] - - operations = [ - migrations.AddField( - model_name='exporttask', - name='filename', - field=models.CharField(max_length=50, null=True), - ), - ] diff --git a/tasks/migrations/0032_auto_20170410_1909.py b/tasks/migrations/0032_auto_20170410_1909.py deleted file mode 100644 index 323d6e270..000000000 --- a/tasks/migrations/0032_auto_20170410_1909.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-10 19:09 -from __future__ import unicode_literals - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - def filename_to_filenames(apps, schema_editor): - ExportTask = apps.get_model('tasks', 'ExportTask') - for task in ExportTask.objects.all(): - task.filenames = [task.filename] - task.save() - - dependencies = [ - ('tasks', '0031_exporttask_filename'), - ] - - operations = [ - migrations.AddField( - model_name='exporttask', - name='filenames', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=50, null=True), default=list, size=None), - ), - migrations.RunPython(filename_to_filenames) - ] diff --git a/tasks/migrations/0033_remove_exporttask_filename.py b/tasks/migrations/0033_remove_exporttask_filename.py deleted file mode 100644 index 288eaa93b..000000000 --- a/tasks/migrations/0033_remove_exporttask_filename.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-10 19:17 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0032_auto_20170410_1909'), - ] - - operations = [ - migrations.RemoveField( - model_name='exporttask', - name='filename', - ), - ] diff --git a/tasks/migrations/0034_auto_20170424_1815.py b/tasks/migrations/0034_auto_20170424_1815.py deleted file mode 100644 index 5145ff4b2..000000000 --- a/tasks/migrations/0034_auto_20170424_1815.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-24 18:15 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0033_remove_exporttask_filename'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exportrun', - options={}, - ), - ] diff --git a/tasks/migrations/0035_auto_20170424_1929.py b/tasks/migrations/0035_auto_20170424_1929.py deleted file mode 100644 index c12a100d2..000000000 --- a/tasks/migrations/0035_auto_20170424_1929.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-04-24 19:29 -from __future__ import unicode_literals - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0034_auto_20170424_1815'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttask', - name='filenames', - field=django.contrib.postgres.fields.ArrayField(base_field=models.TextField(null=True), default=list, size=None), - ), - ] diff --git a/tasks/migrations/0036_auto_20170522_2220.py b/tasks/migrations/0036_auto_20170522_2220.py deleted file mode 100644 index 6d0aef876..000000000 --- a/tasks/migrations/0036_auto_20170522_2220.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2017-05-22 22:20 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0035_auto_20170424_1929'), - ] - - operations = [ - migrations.AlterModelOptions( - name='exportrun', - options={'ordering': ['created_at']}, - ), - migrations.AlterModelOptions( - name='exporttask', - options={'ordering': ['created_at']}, - ), - ] diff --git a/tasks/migrations/0037_auto_20221107_1313.py b/tasks/migrations/0037_auto_20221107_1313.py deleted file mode 100644 index 39b2ff075..000000000 --- a/tasks/migrations/0037_auto_20221107_1313.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2022-11-07 13:13 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0036_auto_20170522_2220'), - ] - - operations = [ - migrations.AlterField( - model_name='exporttask', - name='filesize_bytes', - field=models.BigIntegerField(null=True), - ), - ] diff --git a/tasks/migrations/0038_exportrun_worker_message_id.py b/tasks/migrations/0038_exportrun_worker_message_id.py deleted file mode 100644 index b62a074b7..000000000 --- a/tasks/migrations/0038_exportrun_worker_message_id.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-11-15 05:41 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0037_auto_20221107_1313'), - ] - - operations = [ - migrations.AddField( - model_name='exportrun', - name='worker_message_id', - field=models.UUIDField(blank=True, null=True), - ), - ] diff --git a/tasks/migrations/0039_auto_20221115_0620.py b/tasks/migrations/0039_auto_20221115_0620.py deleted file mode 100644 index ddf4b52ea..000000000 --- a/tasks/migrations/0039_auto_20221115_0620.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-11-15 06:20 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0038_exportrun_worker_message_id'), - ] - - operations = [ - migrations.AlterField( - model_name='exportrun', - name='worker_message_id', - field=models.CharField(blank=True, editable=False, max_length=50, null=True), - ), - ] diff --git a/tasks/migrations/0040_auto_20221201_0652.py b/tasks/migrations/0040_auto_20221201_0652.py deleted file mode 100644 index 4da710309..000000000 --- a/tasks/migrations/0040_auto_20221201_0652.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-12-01 06:52 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0039_auto_20221115_0620'), - ] - - operations = [ - migrations.AlterField( - model_name='exportrun', - name='started_at', - field=models.DateTimeField(editable=False, null=True), - ), - ] diff --git a/tasks/migrations/0041_exportrun_hdx_sync_status.py b/tasks/migrations/0041_exportrun_hdx_sync_status.py deleted file mode 100644 index f20a89945..000000000 --- a/tasks/migrations/0041_exportrun_hdx_sync_status.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2022-12-01 12:20 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('tasks', '0040_auto_20221201_0652'), - ] - - operations = [ - migrations.AddField( - model_name='exportrun', - name='hdx_sync_status', - field=models.BooleanField(default=False), - ), - ] diff --git a/tasks/models.py b/tasks/models.py index 353861ecb..2e64bfbe3 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -15,7 +15,8 @@ from django.contrib import admin from django.contrib.gis.admin import GeoModelAdmin from django.utils.safestring import mark_safe -from django.core.urlresolvers import reverse + +from django.urls import reverse import validators import time @@ -78,8 +79,11 @@ class ExportRun(models.Model): id = models.AutoField(primary_key=True, editable=False) uid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False) created_at = models.DateTimeField(default=timezone.now, editable=False) - job = models.ForeignKey(Job, related_name="runs") - user = models.ForeignKey(User, related_name="runs", default=0) + + job = models.ForeignKey(Job, related_name="runs", on_delete=models.CASCADE) + user = models.ForeignKey( + User, related_name="runs", default=0, on_delete=models.SET_DEFAULT + ) worker_message_id = models.CharField( max_length=50, null=True, blank=True, editable=False ) # used to store worker message id for run to abort @@ -154,7 +158,7 @@ class ExportTask(models.Model): id = models.AutoField(primary_key=True, editable=False) uid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=50) - run = models.ForeignKey(ExportRun, related_name="tasks") + run = models.ForeignKey(ExportRun, related_name="tasks", on_delete=models.CASCADE) status = models.CharField(blank=True, max_length=20, db_index=True) created_at = models.DateTimeField(default=timezone.now, editable=False) @@ -207,14 +211,9 @@ def fdownload(fname): name = value[-1] split_name = name.split("_uid_") file_name = split_name[0] - - if ( - file_name[-(2 * len(self.name) + 1) :] - == f"{self.name}_{self.name}" - ): - # filename has duplicated export formats - file_name = file_name[: -(2 * len(self.name) + 2)] - download_name = f"{file_name}.zip" # getting human redable name ignoring unique id + download_name = ( + f"{file_name}.zip" if ".zip" not in file_name else file_name + ) # getting human redable name fname = download_name except: fname = f"""{self.run.job.name}_{self.name}.zip""" diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 032a601ad..cde6ad593 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -249,6 +249,7 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) "fgb", "csv", "sql", + "mbtiles", ] if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset( set(galaxy_supported_outputs) @@ -954,6 +955,42 @@ def add_metadata(z, theme): stop_task("kml") raise ex + if "mbtiles" in export_formats: + try: + mbtiles = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + ) + start_task("mbtiles") + LOG.debug("Galaxy fetch started for mbtiles run: {0}".format(run_uid)) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = mbtiles.fetch( + "mbtiles", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + for r in response_back: + config = configparser.ConfigParser() + config["FileInfo"] = {"FileSize": str(r["zip_file_size_bytes"])} + size_path = join( + download_dir, f"{r['download_url'].split('/')[-1]}_size.ini" + ) + with open(size_path, "w") as configfile: + config.write(configfile) + + LOG.debug("Galaxy fetch ended for mbtiles run: {0}".format(run_uid)) + finish_task("mbtiles", response_back=response_back) + + except Exception as ex: + stop_task("mbtiles") + raise ex + if "garmin_img" in export_formats: start_task("garmin_img") try: @@ -1011,27 +1048,6 @@ def add_metadata(z, theme): stop_task("osmand_obf") raise ex - if "mbtiles" in export_formats: - start_task("mbtiles") - try: - mbtiles_files = nontabular.mbtiles( - geom, - join(stage_dir, valid_name + ".mbtiles"), - job.mbtiles_source, - job.mbtiles_minzoom, - job.mbtiles_maxzoom, - ) - bundle_files += mbtiles_files - zipped = create_package( - join(download_dir, valid_name + "_mbtiles.zip"), - mbtiles_files, - boundary_geom=geom, - ) - finish_task("mbtiles", [zipped]) - except Exception as ex: - stop_task("mbtiles") - raise ex - if "osm_pbf" in export_formats: bundle_files += [osm_export_tool.File("osm_pbf", [source_path], "")] diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index cf4a0c4f5..6b9e70f06 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -151,7 +151,7 @@ const Details = ({ exportInfo }) => { - + {/* { /> - + */} ); @@ -249,7 +249,8 @@ class ExportRuns extends Component { run_uid: run.uid } }); - window.location.reload(); + + // window.location.reload(); } catch (err) { console.warn(err); } diff --git a/ui/app/components/ExportForm.js b/ui/app/components/ExportForm.js index f5a7a8275..d46fe020e 100644 --- a/ui/app/components/ExportForm.js +++ b/ui/app/components/ExportForm.js @@ -284,7 +284,7 @@ export class ExportForm extends Component { diff --git a/ui/app/components/SelectFeatures.js b/ui/app/components/SelectFeatures.js index 421f1c0e2..7939c8b41 100644 --- a/ui/app/components/SelectFeatures.js +++ b/ui/app/components/SelectFeatures.js @@ -43,7 +43,7 @@ class SelectFeatures extends Component { diff --git a/ui/app/components/Summary.js b/ui/app/components/Summary.js index ee493914d..686852d37 100644 --- a/ui/app/components/Summary.js +++ b/ui/app/components/Summary.js @@ -11,10 +11,10 @@ const messages = defineMessages({ id: "export.buffer_aoi.description", defaultMessage: "Buffer AOI - expand an uploaded boundary by 0.02 degrees" }, - bundleForPOSM: { - id: "export.bundle_for_posm.description", - defaultMessage: "Bundle for POSM" - }, + // bundleForPOSM: { + // id: "export.bundle_for_posm.description", + // defaultMessage: "Bundle for POSM" + // }, preserveGeometry: { id: "export.preserve_geom.description", defaultMessage: "Preserve Geometry - Avoid simplify ( Only supports for geojson )" @@ -78,12 +78,12 @@ export default injectIntl( component={renderCheckbox} type="checkbox" /> - + /> */} {/* { var geojson; - if (result.adminName2.startsWith('HOTOSM')){ + if (result.adminName2.startsWith('ISO3')){ try { geojson = JSON.parse(JSON.stringify(result.bbox)); } catch (e) { diff --git a/ui/app/components/aoi/SearchAOIToolbar.js b/ui/app/components/aoi/SearchAOIToolbar.js index 618d364b0..26ca44b9d 100644 --- a/ui/app/components/aoi/SearchAOIToolbar.js +++ b/ui/app/components/aoi/SearchAOIToolbar.js @@ -121,7 +121,7 @@ export class SearchAOIToolbar extends Component { disabled={this.props.toolbarIcons.search === "INACTIVE"} options={this.state.suggestions} onChange={this.handleEnter} - placeholder={"Search for location or TM Project as TM id or bbox as 'minX, minY, maxX, maxY'"} + placeholder={"Search Location or TM pid or osm id or bbox as 'minX, minY, maxX, maxY'"} onInputChange={this.debouncer} labelKey={"name"} paginate={false} diff --git a/ui/app/components/help/ExportFormats.js b/ui/app/components/help/ExportFormats.js index d0b110134..2f51eb9f2 100644 --- a/ui/app/components/help/ExportFormats.js +++ b/ui/app/components/help/ExportFormats.js @@ -389,7 +389,7 @@ export default () => -
+ {/*

MAPS.ME .mwm

Maps.me is a GPS Navigation and map application for Android and @@ -432,7 +432,7 @@ export default () => -

+
*/}

MBTiles .mbtiles

diff --git a/ui/app/components/utils.js b/ui/app/components/utils.js index 86aaa9d6b..58fbcb9b0 100644 --- a/ui/app/components/utils.js +++ b/ui/app/components/utils.js @@ -73,11 +73,11 @@ export const AVAILABLE_EXPORT_FORMATS = { OSM .pbf ), - mwm: ( - - MAPS.ME .mwm - - ), + // mwm: ( + // + // MAPS.ME .mwm + // + // ), osmand_obf: ( OsmAnd .obf diff --git a/ui/pipeline.py b/ui/pipeline.py index cb0e6e5b2..69ee83a59 100644 --- a/ui/pipeline.py +++ b/ui/pipeline.py @@ -10,7 +10,7 @@ from django.core import signing from django.core.mail import EmailMultiAlternatives from django.core.signing import BadSignature -from django.core.urlresolvers import reverse +from django.urls import reverse from django.shortcuts import redirect from django.template.loader import get_template @@ -32,15 +32,18 @@ def require_email(strategy, backend, request, details, user=None, is_new=False, def email_validation(strategy, backend, code, partial_token): # pragma: no cover """ - Send an email with an embedded verification code and the necessary details to restore the required session - elements to complete the verification and sign-in, regardless of what browser the user completes the - verification from. + Send an email with an embedded verification code and the necessary details to + restore the required session elements to complete the verification and sign-in, + regardless of what browser the user completes the verification from. """ signature = signing.dumps({"session_key": strategy.session.session_key, "email": code.email}, key=settings.SECRET_KEY) - verifyURL = "{0}?verification_code={1}&signature={2}".format( + verifyURL = "{0}?verification_code={1}&signature={2}&partial_token={3}".format( reverse('osm:complete', args=(backend.name,)), - code.code, signature) + code.code, + signature, + partial_token + ) verifyURL = strategy.request.build_absolute_uri(verifyURL) ctx = { 'verifyUrl': verifyURL, @@ -48,7 +51,9 @@ def email_validation(strategy, backend, code, partial_token): # pragma: no cove subject = "Please verify your email address" text = get_template('osm/verify_osm_email.txt').render(ctx) html = get_template('osm/verify_osm_email.html').render(ctx) - msg = EmailMultiAlternatives(subject, text, to=[code.email], from_email="HOT Export Tool ") + msg = EmailMultiAlternatives( + subject, text, to=[code.email], from_email="HOT Export Tool " + ) msg.attach_alternative(html, "text/html") msg.send() @@ -91,4 +96,5 @@ def partial_pipeline_data(backend, user=None, *args, **kwargs): # pragma: no co else: backend.strategy.clean_partial_pipeline() + utils.partial_pipeline_data = partial_pipeline_data diff --git a/ui/templates/osm/email.html b/ui/templates/osm/email.html index d37eb9d4c..3239f274e 100644 --- a/ui/templates/osm/email.html +++ b/ui/templates/osm/email.html @@ -1,5 +1,5 @@ {% extends "ui/base.html" %} -{% load staticfiles %} +{% load static %} {% load i18n %} {% block container %} diff --git a/ui/templates/osm/email_verify_sent.html b/ui/templates/osm/email_verify_sent.html index 7ac983975..3480d92d1 100644 --- a/ui/templates/osm/email_verify_sent.html +++ b/ui/templates/osm/email_verify_sent.html @@ -1,5 +1,5 @@ {% extends "ui/base.html" %} -{% load staticfiles %} +{% load static %} {% load i18n %} {% block container %} diff --git a/ui/templates/osm/error.html b/ui/templates/osm/error.html index a9ba6135b..a1e11bd12 100644 --- a/ui/templates/osm/error.html +++ b/ui/templates/osm/error.html @@ -1,5 +1,5 @@ {% extends "ui/base.html" %} -{% load staticfiles %} +{% load static %} {% load i18n %} {% block container %} diff --git a/ui/templates/ui/authorized.html b/ui/templates/ui/authorized.html index 25b1f8492..26805af8f 100644 --- a/ui/templates/ui/authorized.html +++ b/ui/templates/ui/authorized.html @@ -1,5 +1,5 @@ {% extends "ui/base.html" %} -{% load staticfiles %} +{% load static %} {% block container %}

Thank you for authorizing! diff --git a/ui/templates/ui/base.html b/ui/templates/ui/base.html index e2438f798..95c157d96 100644 --- a/ui/templates/ui/base.html +++ b/ui/templates/ui/base.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} {% load i18n %} diff --git a/ui/templates/ui/v3.html b/ui/templates/ui/v3.html index 96f9e16eb..22aeee8a8 100644 --- a/ui/templates/ui/v3.html +++ b/ui/templates/ui/v3.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} diff --git a/ui/views.py b/ui/views.py index 8a6d45fa3..7c23987a1 100644 --- a/ui/views.py +++ b/ui/views.py @@ -2,14 +2,20 @@ """UI view definitions.""" from django.contrib.auth import logout as auth_logout -from django.core.urlresolvers import reverse +from django.urls import reverse from django.shortcuts import redirect, render -from django.views.decorators.http import require_http_methods from oauth2_provider.models import Application from django.contrib import admin from django.contrib.auth.admin import User, UserAdmin from django.conf import settings -from django.http import JsonResponse, HttpResponseRedirect, HttpResponse, HttpResponseNotFound, HttpResponseForbidden +from django.http import ( + JsonResponse, + HttpResponseRedirect, + HttpResponse, + HttpResponseNotFound, + HttpResponseForbidden, +) +from django.views.decorators.http import require_http_methods def authorized(request): @@ -21,44 +27,55 @@ def authorized(request): def login(request): - if not request.user.is_authenticated(): + if not request.user.is_authenticated: # preserve redirects ("next" in request.GET) return redirect( - reverse('osm:begin', args=['openstreetmap']) + '?' + - request.GET.urlencode()) + reverse("osm:begin", args=["openstreetmap"]) + "?" + request.GET.urlencode() + ) else: - return redirect('/v3/') + return redirect("/v3/") def logout(request): """Logs out user""" auth_logout(request) - return redirect('/v3/') + return redirect("/v3/") def v3(request): - ui_app = Application.objects.get(name='OSM Export Tool UI') + try: + ui_app = Application.objects.get(name="OSM Export Tool UI") + except Application.DoesNotExist: + ui_app = Application.objects.create( + name="OSM Export Tool UI", + redirect_uris="http://localhost/authorized http://localhost:8080/authorized http://localhost:8000/authorized", + client_type=Application.CLIENT_PUBLIC, + authorization_grant_type=Application.GRANT_IMPLICIT, + skip_authorization=True, + ) context = dict(client_id=ui_app.client_id) if settings.MATOMO_URL is not None and settings.MATOMO_SITEID is not None: - context.update({ - 'MATOMO_URL': settings.MATOMO_URL, - 'MATOMO_SITEID': settings.MATOMO_SITEID - }) - return render(request, 'ui/v3.html', context) + context.update( + {"MATOMO_URL": settings.MATOMO_URL, "MATOMO_SITEID": settings.MATOMO_SITEID} + ) + return render(request, "ui/v3.html", context) def redirect_to_v3(request): - return redirect('/v3/') + return redirect("/v3/") + -@require_http_methods(['GET']) +@require_http_methods(["GET"]) def worker_dashboard(request): if not request.user.is_superuser: return HttpResponseForbidden() # return HttpResponse('test') return HttpResponseRedirect(f"/{settings.WORKER_SECRET_KEY}/") + class ApplicationAdmin(admin.ModelAdmin): - raw_id_fields = ("user", ) + raw_id_fields = ("user",) + admin.site.register(Application, ApplicationAdmin)