From b500c8cce8af664d3f7ba03d6a41155853b48d2e Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Wed, 8 Apr 2020 10:30:13 -0500 Subject: [PATCH 1/9] mapserver endpoint update to use 443 protocol. added comment for endpoint documentation notes. --- src/data_hub/lore/viewsets.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/data_hub/lore/viewsets.py b/src/data_hub/lore/viewsets.py index 074f01d6..475dcd50 100644 --- a/src/data_hub/lore/viewsets.py +++ b/src/data_hub/lore/viewsets.py @@ -72,6 +72,14 @@ class MapserverViewSet(viewsets.ViewSet): """ Retrieve TNRIS mapserver instance mapfiles list from S3 content objects """ + # ******************************************************************* + # *** this endpoint is not currently used by any of our apps/pages *** + # ******************************************************************* + # This is an alternative, direct-from-s3 (what services truly exist) list of + # mapserver service URLs. Same information can be/currently is retrieved by using the + # '_service_url' fields from the /api/v1/historical/collections + # endpoint but if the LORE database and those fields are not properly populated, + # then that result will be different from this endpoint permission_classes = (AllowAny,) def list(self, request): @@ -103,7 +111,7 @@ def get_mapfiles(token): name = key.replace('mapfiles/', '').replace('.map', '') key_obj['name'] = name key_obj['label'] = name.replace("_", " ").title() - key_obj['wms'] = 'http://mapserver.tnris.org/wms/?map=/' + key + key_obj['wms'] = 'https://mapserver.tnris.org/wms/?map=/' + key if len(name.split("_")) == 4: key_obj['org'] = 'county' From fd350a4356c01b1f8f7df5a20e29c885c8b8b713 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Tue, 28 Apr 2020 13:52:56 -0500 Subject: [PATCH 2/9] hedge against alternative s3 url links by always replacing bad formats --- src/data_hub/lore/admin.py | 3 ++- src/data_hub/lore/forms.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/data_hub/lore/admin.py b/src/data_hub/lore/admin.py index 44b11350..a19e087a 100644 --- a/src/data_hub/lore/admin.py +++ b/src/data_hub/lore/admin.py @@ -5,7 +5,7 @@ from .filters import (CollectionAgencyNameFilter, CollectionCountyFilter, CountyDropdownFilter) -from .forms import CollectionForm, ProductForm +from .forms import CollectionForm, ProductForm, ScannedPhotoIndexLinkForm from .models import (Agency, Collection, County, CountyRelate, FrameSize, LineIndex, MicroficheIndex, PhotoIndex, Product, ScannedPhotoIndexLink) @@ -41,6 +41,7 @@ class ScannedPhotoIndexLinkInlineAdmin(admin.StackedInline): classes = ('grp-collapse grp-closed',) inline_classes = ('grp-collapse grp-closed',) model = ScannedPhotoIndexLink + form = ScannedPhotoIndexLinkForm extra = 0 diff --git a/src/data_hub/lore/forms.py b/src/data_hub/lore/forms.py index 94eb57de..59b0c1fa 100644 --- a/src/data_hub/lore/forms.py +++ b/src/data_hub/lore/forms.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError from django.db.utils import ProgrammingError -from .models import Collection, County, CountyRelate, Product +from .models import Collection, County, CountyRelate, Product, ScannedPhotoIndexLink class ProductForm(forms.ModelForm): @@ -89,3 +89,17 @@ def save(self, commit=True): for add in adds: CountyRelate(county_id=add, collection=self.instance).save() return super(CollectionForm, self).save(commit=commit) + + +class ScannedPhotoIndexLinkForm(forms.ModelForm): + class Meta: + model = ScannedPhotoIndexLink + fields = ('__all__') + + def clean(self): + link = self.cleaned_data['link'] + link = link.replace('http://', 'https://') + link = link.replace('https://tnris-ls4.s3.amazonaws.com/', 'https://s3.amazonaws.com/tnris-ls4/') + self.cleaned_data['link'] = link + super(ScannedPhotoIndexLinkForm, self).save(commit=False) + return From b4c3f8bbe674d5f2df0b95b4a17af2d654692987 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Tue, 28 Apr 2020 14:31:33 -0500 Subject: [PATCH 3/9] new lore service url validation for consistency and compliance to frontend protocols --- src/data_hub/lore/forms.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/data_hub/lore/forms.py b/src/data_hub/lore/forms.py index 59b0c1fa..ae44bebc 100644 --- a/src/data_hub/lore/forms.py +++ b/src/data_hub/lore/forms.py @@ -51,30 +51,54 @@ def clean(self): mosaic = self.cleaned_data['mosaic_service_url'] links = [] + mapserver_url = 'https://mapserver.tnris.org/wms/?map=/mapfiles/' if index is not None: + # validate url belongs to proper product type if '_index' not in index: raise forms.ValidationError('That is not an Index Service URL!') + # validate url is 443 protocol + if 'http://' in index: + raise forms.ValidationError('Index Service URL must be https protocol!') + # validate the url structure matches the tnris mapserver instance + if mapserver_url not in index: + raise forms.ValidationError('Index Service URL must belong to the TNRIS mapserver: "https://mapserver.tnris.org/wms/?map=/mapfiles/"') idx_link = index.split('/')[-1].replace('.map', '').replace('_index', '').replace('_', ' ').upper() if idx_link not in links: links.append(idx_link) if frames is not None: + # validate url belongs to proper product type if '_frames' not in frames: raise forms.ValidationError('That is not an Frames Service URL!') + # validate url is 443 protocol + if 'http://' in frames: + raise forms.ValidationError('Frames Service URL must be https protocol!') + # validate the url structure matches the tnris mapserver instance + if mapserver_url not in frames: + raise forms.ValidationError('Frames Service URL must belong to the TNRIS mapserver: "https://mapserver.tnris.org/wms/?map=/mapfiles/"') frm_link = frames.split('/')[-1].replace('.map', '').replace('_frames', '').replace('_', ' ').upper() if frm_link not in links: links.append(frm_link) if mosaic is not None: + # validate url belongs to proper product type if '_mosaic' not in mosaic: raise forms.ValidationError('That is not an Mosaic Service URL!') + # validate url is 443 protocol + if 'http://' in mosaic: + raise forms.ValidationError('Mosaic Service URL must be https protocol!') + # validate the url structure matches the tnris mapserver instance + if mapserver_url not in mosaic: + raise forms.ValidationError('Mosaic Service URL must belong to the TNRIS mapserver: "https://mapserver.tnris.org/wms/?map=/mapfiles/"') msc_link = mosaic.split('/')[-1].replace('.map', '').replace('_mosaic', '').replace('_', ' ').upper() if msc_link not in links: links.append(msc_link) + # cross validate consistency across all 3 service urls if len(links) > 1: raise forms.ValidationError('Index, Frames, and Mosaic Service URLs have inconsistent collections!') elif len(links) == 1: self.instance.ls4_link = links[0] else: self.instance.ls4_link = None + return def save(self, commit=True): updated_counties = self.cleaned_data['counties'] @@ -97,6 +121,7 @@ class Meta: fields = ('__all__') def clean(self): + # force 443 protocol on urls and reformat url structure for consistency link = self.cleaned_data['link'] link = link.replace('http://', 'https://') link = link.replace('https://tnris-ls4.s3.amazonaws.com/', 'https://s3.amazonaws.com/tnris-ls4/') From cebd73ddec1591a14b005547644c2d81efdd1bfc Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Wed, 29 Apr 2020 10:36:43 -0500 Subject: [PATCH 4/9] initial historical images table. admin console management of inline images --- src/data_hub/lore/admin.py | 14 +++- src/data_hub/lore/forms.py | 48 +++++++++++- src/data_hub/lore/migrations/0018_image.py | 31 ++++++++ src/data_hub/lore/models.py | 89 ++++++++++++++++++++++ 4 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 src/data_hub/lore/migrations/0018_image.py diff --git a/src/data_hub/lore/admin.py b/src/data_hub/lore/admin.py index 44b11350..b93cd527 100644 --- a/src/data_hub/lore/admin.py +++ b/src/data_hub/lore/admin.py @@ -5,9 +5,9 @@ from .filters import (CollectionAgencyNameFilter, CollectionCountyFilter, CountyDropdownFilter) -from .forms import CollectionForm, ProductForm +from .forms import CollectionForm, ProductForm, ImageForm from .models import (Agency, Collection, County, CountyRelate, FrameSize, - LineIndex, MicroficheIndex, PhotoIndex, Product, + Image, LineIndex, MicroficheIndex, PhotoIndex, Product, ScannedPhotoIndexLink) from .actions import (export_collection, export_product, export_photo_index, export_scanned_photo_index_link, export_county, export_line_index, @@ -75,6 +75,14 @@ class CountyRelateInlineAdmin(admin.StackedInline): ordering = ('county__name',) +class ImageInlineAdmin(admin.StackedInline): + classes = ('grp-collapse grp-closed',) + inline_classes = ('grp-collapse grp-open',) + model = Image + form = ImageForm + extra = 0 + + class CollectionAdmin(admin.ModelAdmin): model = Collection form = CollectionForm @@ -90,7 +98,7 @@ class CollectionAdmin(admin.ModelAdmin): ) inlines = [PhotoIndexInlineAdmin, LineIndexInlineAdmin, MicroficheIndexInlineAdmin, ProductInlineAdmin, - ScannedPhotoIndexLinkInlineAdmin] + ScannedPhotoIndexLinkInlineAdmin, ImageInlineAdmin] list_display = ( 'collection', 'id', 'agency', 'from_date', 'to_date', 'county_names', 'public' ) diff --git a/src/data_hub/lore/forms.py b/src/data_hub/lore/forms.py index 94eb57de..d048d29b 100644 --- a/src/data_hub/lore/forms.py +++ b/src/data_hub/lore/forms.py @@ -2,8 +2,10 @@ from django.core.exceptions import ValidationError from django.db.utils import ProgrammingError -from .models import Collection, County, CountyRelate, Product +from .models import Collection, County, CountyRelate, Product, Image +from lcd.forms import PictureWidget +import boto3, uuid class ProductForm(forms.ModelForm): class Meta: @@ -14,6 +16,17 @@ class Meta: clean_status = forms.BooleanField(label='Clean Status', help_text='Clean Status refers to collections that have been reviewed and are ready to been scanned, no erasing of frames needed. Default is False/Unchecked.', required=False) +class ImageForm(forms.ModelForm): + class Meta: + model = Image + fields = ('__all__') + help_texts = { + 'caption': 'Caption will not be saved until chosen image is uploaded and saved to database.', + } + + image_url = forms.FileField(required=True, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply.") + + class CollectionForm(forms.ModelForm): class Meta: model = Collection @@ -23,6 +36,9 @@ class Meta: fields = ('collection', 'agency', 'from_date', 'to_date', 'counties', 'remarks') + # boto3 s3 object + client = boto3.client('s3') + # added a try/except for initial migration without data try: county_choices = ( @@ -45,6 +61,28 @@ def __init__(self, *args, **kwargs): choices=county_choices ) + def inline_image_handler(self, file): + new_uuid = uuid.uuid4() + file_ext = str(file).split('.')[-1] + # upload image + key = "%s/assets/%s.%s" % (self.instance.id, new_uuid, file_ext) + # print(key + file_ext) + response = self.client.put_object( + Bucket='data.tnris.org', + ACL='public-read', + Key=key, + Body=file + ) + print('%s upload success!' % key) + # update link in database table + args = { + 'image_id': new_uuid, + 'image_url': "https://s3.amazonaws.com/data.tnris.org/" + key, + 'collection_id': self.instance + } + Image(**args).save() + return + def clean(self): index = self.cleaned_data['index_service_url'] frames = self.cleaned_data['frames_service_url'] @@ -88,4 +126,12 @@ def save(self, commit=True): county=remove).filter(collection=self.instance.id).delete() for add in adds: CountyRelate(county_id=add, collection=self.instance).save() + + # check for files + files = self.files + # if files and field not marked for deletion then upload to s3 + for f in files: + if 'image_collections' in f: + self.inline_image_handler(files[f]) + return super(CollectionForm, self).save(commit=commit) diff --git a/src/data_hub/lore/migrations/0018_image.py b/src/data_hub/lore/migrations/0018_image.py new file mode 100644 index 00000000..9656e909 --- /dev/null +++ b/src/data_hub/lore/migrations/0018_image.py @@ -0,0 +1,31 @@ +# Generated by Django 2.0.13 on 2020-04-29 14:39 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('lore', '0017_delete_scale'), + ] + + operations = [ + migrations.CreateModel( + name='Image', + fields=[ + ('image_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='Image ID')), + ('image_url', models.URLField(max_length=255, verbose_name='Image URL')), + ('caption', models.CharField(blank=True, max_length=255, null=True, verbose_name='Image Caption')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('last_modified', models.DateTimeField(auto_now=True, verbose_name='Last Modified')), + ('collection_id', models.ForeignKey(db_column='collection_id', on_delete=django.db.models.deletion.CASCADE, related_name='image_collections', to='lore.Collection')), + ], + options={ + 'verbose_name': 'Historical Image', + 'verbose_name_plural': 'Historical Images', + 'db_table': 'historical_image', + }, + ), + ] diff --git a/src/data_hub/lore/models.py b/src/data_hub/lore/models.py index 11737be2..ff88e5f0 100644 --- a/src/data_hub/lore/models.py +++ b/src/data_hub/lore/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import uuid +import boto3 from django.db import models @@ -234,9 +235,97 @@ class Meta: qr_code_url = models.URLField('QR Code URL', max_length=256, null=True, blank=True) number_of_boxes = models.PositiveIntegerField('Number of Boxes', null=True, blank=True) + def delete_s3_files(self): + # do that boto dance + client = boto3.client('s3') + # set aside list for compiling keys + key_list = [] + # list Objects + collection_prefix = str(self.id) + '/assets' + response = client.list_objects_v2( + Bucket='data.tnris.org', + Prefix=collection_prefix + ) + + if 'Contents' in response.keys(): + # add image keys to list + for image in response['Contents']: + key_list.append({'Key':image['Key']}) + + response = client.delete_objects( + Bucket='data.tnris.org', + Delete={'Objects': key_list} + ) + print('%s s3 files: delete success!' % self.collection) + return + + # overwrite default model delete method so that all associated + # s3 files get deleted as well + def delete(self, *args, **kwargs): + self.delete_s3_files() + super().delete(*args, **kwargs) + def __str__(self): return self.collection + +class Image(models.Model): + """ + Defines available image resources. + Related to :model:`lore.collection`. + """ + + class Meta: + db_table = 'historical_image' + verbose_name = 'Image' + verbose_name_plural = 'Images' + + image_id = models.UUIDField( + 'Image ID', + primary_key=True, + default=uuid.uuid4, + editable=False + ) + collection_id = models.ForeignKey( + 'Collection', + db_column='collection_id', + on_delete=models.CASCADE, + related_name='image_collections' + ) + image_url = models.URLField( + 'Image URL', + max_length=255 + ) + caption = models.CharField( + 'Image Caption', + max_length=255, + null=True, + blank=True + ) + created = models.DateTimeField( + 'Created', + auto_now_add=True + ) + last_modified = models.DateTimeField( + 'Last Modified', + auto_now=True + ) + # delete s3 image files + def delete(self, *args, **kwargs): + client = boto3.client('s3') + key = str(self).replace('https://s3.amazonaws.com/data.tnris.org/', '') + print(key) + response = client.delete_object( + Bucket='data.tnris.org', + Key=key + ) + print(self) + super().delete(*args, **kwargs) + + def __str__(self): + return self.image_url + + """ ********** Database Views ********** **** Used as the API endpoints **** From e5d67583943c0f48d8705b0d55f2287107075ed1 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Wed, 29 Apr 2020 14:08:32 -0500 Subject: [PATCH 5/9] lore collections added thumbnail field to model and updated admin console to respect uploaded images --- src/data_hub/lore/admin.py | 3 ++- src/data_hub/lore/forms.py | 27 +++++++++++++++++++ .../migrations/0019_auto_20200429_1117.py | 22 +++++++++++++++ src/data_hub/lore/models.py | 11 ++++++-- 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 src/data_hub/lore/migrations/0019_auto_20200429_1117.py diff --git a/src/data_hub/lore/admin.py b/src/data_hub/lore/admin.py index b93cd527..d3d5ff72 100644 --- a/src/data_hub/lore/admin.py +++ b/src/data_hub/lore/admin.py @@ -90,7 +90,8 @@ class CollectionAdmin(admin.ModelAdmin): ('Collection Information', { 'fields': ('collection', 'agency', 'from_date', 'to_date', 'index_service_url', 'frames_service_url', 'mosaic_service_url', - 'counties', 'number_of_boxes', 'photo_index_only', 'public', 'fully_scanned', 'qr_code_url'), + 'counties', 'number_of_boxes', 'photo_index_only', 'public', + 'fully_scanned', 'thumbnail_image', 'qr_code_url'), }), ('Remarks', { 'fields': ('remarks',) diff --git a/src/data_hub/lore/forms.py b/src/data_hub/lore/forms.py index d048d29b..42bc9c5b 100644 --- a/src/data_hub/lore/forms.py +++ b/src/data_hub/lore/forms.py @@ -55,11 +55,38 @@ def __init__(self, *args, **kwargs): "county_id").filter(collection=self.instance.id) ] self.fields['counties'].initial = self.initial_counties + self.fields['thumbnail_image'].choices = self.create_image_choices('image_url', 'image_id', Image, 'image_id') + self.fields['thumbnail_image'].help_text = self.selected_thumbnail() counties = forms.MultipleChoiceField( widget=forms.SelectMultiple(attrs={'title': 'Hold down ctrl to select multiple counties',}), choices=county_choices ) + thumbnail_image = forms.ChoiceField(required=False, choices=[]) + + # general function to create a form dropdown for thumbnail image + def create_image_choices(self, id_field, label_field, type_table, order_field): + # start with the model field default which resides in s3 outside of any dbase record + choices = [('https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg', 'Default')] + # get the relate type choices from the type table + try: + uploaded_images = [ + (getattr(b, id_field), getattr(b, label_field)) for b in type_table.objects.filter(collection_id=self.instance.id).order_by(order_field)] + + choices.extend(uploaded_images) + except ProgrammingError as e: + print(e) + return choices + + # retrieve selected thumbnail id for helper text to display + def selected_thumbnail(self): + text = "NO THUMBNAIL SELECTED! Please choose an image UUID from the dropdown and 'Save' the form." + if self.instance.thumbnail_image is not None and self.instance.thumbnail_image != "": + filename = self.instance.thumbnail_image.split("/")[-1] + if filename == 'historical_thumbnail.jpg': + filename = 'Default' + text = "Currently Selected: %s" % filename + return text def inline_image_handler(self, file): new_uuid = uuid.uuid4() diff --git a/src/data_hub/lore/migrations/0019_auto_20200429_1117.py b/src/data_hub/lore/migrations/0019_auto_20200429_1117.py new file mode 100644 index 00000000..69e61710 --- /dev/null +++ b/src/data_hub/lore/migrations/0019_auto_20200429_1117.py @@ -0,0 +1,22 @@ +# Generated by Django 2.0.13 on 2020-04-29 16:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lore', '0018_image'), + ] + + operations = [ + migrations.AlterModelOptions( + name='image', + options={'verbose_name': 'Image', 'verbose_name_plural': 'Images'}, + ), + migrations.AddField( + model_name='collection', + name='thumbnail_image', + field=models.TextField(blank=True, default='https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg', max_length=120, null=True, verbose_name='Thumb Image'), + ), + ] diff --git a/src/data_hub/lore/models.py b/src/data_hub/lore/models.py index ff88e5f0..8122e38f 100644 --- a/src/data_hub/lore/models.py +++ b/src/data_hub/lore/models.py @@ -226,14 +226,21 @@ class Meta: public = models.BooleanField('Public', default=True) fully_scanned = models.BooleanField('Fully Scanned', default=False) remarks = models.TextField(null=True, blank=True) - created = models.DateTimeField('Created', auto_now_add=True) - last_modified = models.DateTimeField('Last Modified', auto_now=True) index_service_url = models.URLField('Index Service URL', max_length=256, null=True, blank=True) frames_service_url = models.URLField('Frames Service URL', max_length=256, null=True, blank=True) mosaic_service_url = models.URLField('Mosaic Service URL', max_length=256, null=True, blank=True) ls4_link = models.CharField(max_length=200, null=True, blank=True) qr_code_url = models.URLField('QR Code URL', max_length=256, null=True, blank=True) number_of_boxes = models.PositiveIntegerField('Number of Boxes', null=True, blank=True) + thumbnail_image = models.TextField( + 'Thumb Image', + max_length=120, + null=True, + blank=True, + default='https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg' + ) + created = models.DateTimeField('Created', auto_now_add=True) + last_modified = models.DateTimeField('Last Modified', auto_now=True) def delete_s3_files(self): # do that boto dance From b9aa42454a7eeece48981d59998b8b5a8af60259 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Wed, 29 Apr 2020 14:22:19 -0500 Subject: [PATCH 6/9] updated Chc view database table to use new thumbnail image field for LORE records. model restriction updates. --- .../compiled_historical_collection.sql | 2 +- .../lcd/migrations/0045_auto_20200429_1417.py | 18 ++++++++++++++++++ src/data_hub/lcd/models.py | 2 +- .../lore/migrations/0020_auto_20200429_1417.py | 18 ++++++++++++++++++ src/data_hub/lore/models.py | 2 +- 5 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 src/data_hub/lcd/migrations/0045_auto_20200429_1417.py create mode 100644 src/data_hub/lore/migrations/0020_auto_20200429_1417.py diff --git a/database_management/compiled_historical_collection.sql b/database_management/compiled_historical_collection.sql index 68a17429..c3ad7770 100644 --- a/database_management/compiled_historical_collection.sql +++ b/database_management/compiled_historical_collection.sql @@ -15,6 +15,7 @@ SELECT historical_collection.id as collection_id, historical_collection.index_service_url, historical_collection.frames_service_url, historical_collection.mosaic_service_url, + historical_collection.thumbnail_image, string_agg(distinct county.name, ', ' order by county.name) as counties, agency.name as source_name, agency.abbreviation as source_abbreviation, @@ -39,7 +40,6 @@ SELECT historical_collection.id as collection_id, END AS name, 'historical-aerial' as template, 'Order_Only' as availability, - 'https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg' as thumbnail_image, 'Historic_Imagery' as category, 'Historical Use,Research' as recommended_use, 'Public Domain (Creative Commons CC0)' as license_name, diff --git a/src/data_hub/lcd/migrations/0045_auto_20200429_1417.py b/src/data_hub/lcd/migrations/0045_auto_20200429_1417.py new file mode 100644 index 00000000..9d634625 --- /dev/null +++ b/src/data_hub/lcd/migrations/0045_auto_20200429_1417.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2020-04-29 19:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lcd', '0044_auto_20191121_1208'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='thumbnail_image', + field=models.TextField(blank=True, max_length=200, null=True, verbose_name='Thumb Image'), + ), + ] diff --git a/src/data_hub/lcd/models.py b/src/data_hub/lcd/models.py index 5bd3c1c2..dad9b232 100644 --- a/src/data_hub/lcd/models.py +++ b/src/data_hub/lcd/models.py @@ -832,7 +832,7 @@ class Meta: ) thumbnail_image = models.TextField( 'Thumb Image', - max_length=120, + max_length=200, null=True, blank=True ) diff --git a/src/data_hub/lore/migrations/0020_auto_20200429_1417.py b/src/data_hub/lore/migrations/0020_auto_20200429_1417.py new file mode 100644 index 00000000..a8e8c946 --- /dev/null +++ b/src/data_hub/lore/migrations/0020_auto_20200429_1417.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2020-04-29 19:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lore', '0019_auto_20200429_1117'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='thumbnail_image', + field=models.TextField(blank=True, default='https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg', max_length=200, null=True, verbose_name='Thumb Image'), + ), + ] diff --git a/src/data_hub/lore/models.py b/src/data_hub/lore/models.py index 8122e38f..d79b8803 100644 --- a/src/data_hub/lore/models.py +++ b/src/data_hub/lore/models.py @@ -234,7 +234,7 @@ class Meta: number_of_boxes = models.PositiveIntegerField('Number of Boxes', null=True, blank=True) thumbnail_image = models.TextField( 'Thumb Image', - max_length=120, + max_length=200, null=True, blank=True, default='https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg' From 71fca88b19805efb55c74fa985a1ac0de5027bc6 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Wed, 29 Apr 2020 14:37:16 -0500 Subject: [PATCH 7/9] image upload help_text include aspect ratio --- src/data_hub/lcd/forms.py | 6 +++++- src/data_hub/lore/forms.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/data_hub/lcd/forms.py b/src/data_hub/lcd/forms.py index d9b7345d..a3807969 100644 --- a/src/data_hub/lcd/forms.py +++ b/src/data_hub/lcd/forms.py @@ -60,7 +60,11 @@ class Meta: 'caption': 'Caption will not be saved until chosen image is uploaded and saved to database.', } - image_url = forms.FileField(required=True, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply.") + image_url = forms.FileField( + required=True, + widget=PictureWidget, + help_text="DataHub Images should be 16:9 ratio (1920 x 1080 pixels).
Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply." + ) class CollectionForm(forms.ModelForm): diff --git a/src/data_hub/lore/forms.py b/src/data_hub/lore/forms.py index 42bc9c5b..122f7410 100644 --- a/src/data_hub/lore/forms.py +++ b/src/data_hub/lore/forms.py @@ -24,7 +24,11 @@ class Meta: 'caption': 'Caption will not be saved until chosen image is uploaded and saved to database.', } - image_url = forms.FileField(required=True, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply.") + image_url = forms.FileField( + required=True, + widget=PictureWidget, + help_text="DataHub Images should be 16:9 ratio (1920 x 1080 pixels).
Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply." + ) class CollectionForm(forms.ModelForm): From fa368a8c83be9e574e7bf33f762a3054dd5f29f0 Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Thu, 30 Apr 2020 11:45:36 -0500 Subject: [PATCH 8/9] updated ChC materialized view to include new image table records --- .../collection_catalog_records.sql | 4 ++-- .../compiled_historical_collection.sql | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/database_management/collection_catalog_records.sql b/database_management/collection_catalog_records.sql index a34a8611..b37739a6 100644 --- a/database_management/collection_catalog_records.sql +++ b/database_management/collection_catalog_records.sql @@ -166,12 +166,12 @@ LEFT JOIN license_type ON license_type.license_type_id=collection.license_type_i LEFT JOIN template_type ON template_type.template_type_id=collection.template_type_id -LEFT JOIN image ON image.collection_id = collection.collection_id +LEFT JOIN image ON image.collection_id=collection.collection_id LEFT JOIN collection_county_relate ON collection_county_relate.collection_id=collection.collection_id LEFT JOIN area_type ON area_type.area_type_id=collection_county_relate.area_type_id -LEFT JOIN outside_entity_services ON outside_entity_services.collection_id = collection.collection_id +LEFT JOIN outside_entity_services ON outside_entity_services.collection_id=collection.collection_id GROUP BY collection.collection_id, source_type.source_name, diff --git a/database_management/compiled_historical_collection.sql b/database_management/compiled_historical_collection.sql index c3ad7770..d9de3682 100644 --- a/database_management/compiled_historical_collection.sql +++ b/database_management/compiled_historical_collection.sql @@ -22,7 +22,18 @@ SELECT historical_collection.id as collection_id, agency.about as about, agency.general_scale as general_scale, agency.media_type as media_type, - agency.sample_image_url as images, + string_agg(distinct historical_image.image_url, ',') as bull, + agency.sample_image_url as crap, + -- use conditional logic to combine agency sample image and uploaded images for carousel if they exist + CASE + WHEN agency.sample_image_url IS NOT NULL AND string_agg(distinct historical_image.image_url, ',') IS NOT NULL + THEN CONCAT(agency.sample_image_url, ',', string_agg(distinct historical_image.image_url, ',')) + WHEN agency.sample_image_url IS NOT NULL AND string_agg(distinct historical_image.image_url, ',') IS NULL + THEN agency.sample_image_url + WHEN agency.sample_image_url IS NULL AND string_agg(distinct historical_image.image_url, ',') IS NOT NULL + THEN string_agg(distinct historical_image.image_url, ',') + ELSE agency.sample_image_url + END AS images, array_to_string(ARRAY(SELECT json_build_object('coverage', product.coverage, 'number_of_frames', product.number_of_frames, 'medium', product.medium, @@ -35,7 +46,8 @@ SELECT historical_collection.id as collection_id, WHEN ( (string_agg(distinct county.name, ',' order by county.name) ~ '.*(,).*') - ) THEN CONCAT('Multi-County ', agency.abbreviation, ' Historic Imagery') + ) + THEN CONCAT('Multi-County ', agency.abbreviation, ' Historic Imagery') ELSE CONCAT(string_agg(distinct county.name, ',' order by county.name), ' ', agency.abbreviation, ' Historic Imagery') END AS name, 'historical-aerial' as template, @@ -59,6 +71,8 @@ LEFT JOIN county ON county.id=county_relate.county_id LEFT JOIN agency ON agency.id=historical_collection.agency_id +LEFT JOIN historical_image ON historical_image.collection_id=historical_collection.id + GROUP BY historical_collection.id, agency.name, agency.abbreviation, From 62e9366166535cdcce0f8e71acf3adbc1c939f7f Mon Sep 17 00:00:00 2001 From: "Marlboro Blend No. 27" Date: Mon, 4 May 2020 13:03:49 -0500 Subject: [PATCH 9/9] added training_link field to tnris.org training model --- src/data_hub/lore/forms.py | 1 + .../0025_tnristraining_training_link.py | 18 ++++++++++++++++++ .../migrations/0026_auto_20200504_1301.py | 19 +++++++++++++++++++ src/data_hub/tnris_org/models.py | 5 +++++ src/data_hub/tnris_org/serializers.py | 1 + 5 files changed, 44 insertions(+) create mode 100644 src/data_hub/tnris_org/migrations/0025_tnristraining_training_link.py create mode 100644 src/data_hub/tnris_org/migrations/0026_auto_20200504_1301.py diff --git a/src/data_hub/lore/forms.py b/src/data_hub/lore/forms.py index 70044eaa..d5e86643 100644 --- a/src/data_hub/lore/forms.py +++ b/src/data_hub/lore/forms.py @@ -3,6 +3,7 @@ from django.db.utils import ProgrammingError from .models import Collection, County, CountyRelate, Product, Image, ScannedPhotoIndexLink +from lcd.forms import PictureWidget import boto3, uuid diff --git a/src/data_hub/tnris_org/migrations/0025_tnristraining_training_link.py b/src/data_hub/tnris_org/migrations/0025_tnristraining_training_link.py new file mode 100644 index 00000000..9f2966fd --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0025_tnristraining_training_link.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.13 on 2020-05-04 17:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0024_tnrisdocument_sgm_note'), + ] + + operations = [ + migrations.AddField( + model_name='tnristraining', + name='training_link', + field=models.URLField(blank=True, max_length=255, null=True, verbose_name='Training Course Registration Link'), + ), + ] diff --git a/src/data_hub/tnris_org/migrations/0026_auto_20200504_1301.py b/src/data_hub/tnris_org/migrations/0026_auto_20200504_1301.py new file mode 100644 index 00000000..1de3a4ad --- /dev/null +++ b/src/data_hub/tnris_org/migrations/0026_auto_20200504_1301.py @@ -0,0 +1,19 @@ +# Generated by Django 2.0.13 on 2020-05-04 18:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tnris_org', '0025_tnristraining_training_link'), + ] + + operations = [ + migrations.AlterField( + model_name='tnristraining', + name='training_link', + field=models.URLField(default='https://events.eply.com/TNRIS2020', max_length=255, verbose_name='Training Course Registration Link'), + preserve_default=False, + ), + ] diff --git a/src/data_hub/tnris_org/models.py b/src/data_hub/tnris_org/models.py index de33ce9d..8a0b506f 100644 --- a/src/data_hub/tnris_org/models.py +++ b/src/data_hub/tnris_org/models.py @@ -161,6 +161,11 @@ class Meta: description = models.TextField( 'Training Description' ) + training_link = models.URLField( + 'Training Course Registration Link', + max_length=255, + null=False + ) created = models.DateTimeField( 'Created', auto_now_add=True diff --git a/src/data_hub/tnris_org/serializers.py b/src/data_hub/tnris_org/serializers.py index 9252a0c5..0a803e4e 100644 --- a/src/data_hub/tnris_org/serializers.py +++ b/src/data_hub/tnris_org/serializers.py @@ -23,6 +23,7 @@ class Meta: 'cost', 'registration_open', 'description', + 'training_link', 'created', 'last_modified', 'public',)