Merge branch 'develop'

database_management/collection_catalog_records.sql

@@ -166,12 +166,12 @@ LEFT JOIN license_type ON license_type.license_type_id=collection.license_type_i
 
 LEFT JOIN template_type ON template_type.template_type_id=collection.template_type_id
 
-LEFT JOIN image ON image.collection_id = collection.collection_id
+LEFT JOIN image ON image.collection_id=collection.collection_id
 
 LEFT JOIN collection_county_relate ON collection_county_relate.collection_id=collection.collection_id
 LEFT JOIN area_type ON area_type.area_type_id=collection_county_relate.area_type_id
 
-LEFT JOIN outside_entity_services ON outside_entity_services.collection_id = collection.collection_id
+LEFT JOIN outside_entity_services ON outside_entity_services.collection_id=collection.collection_id
 
 GROUP BY collection.collection_id,
         source_type.source_name,

database_management/compiled_historical_collection.sql

@@ -15,13 +15,25 @@ SELECT historical_collection.id as collection_id,
   historical_collection.index_service_url,
   historical_collection.frames_service_url,
   historical_collection.mosaic_service_url,
+  historical_collection.thumbnail_image,
   string_agg(distinct county.name, ', ' order by county.name) as counties,
   agency.name as source_name,
   agency.abbreviation as source_abbreviation,
   agency.about as about,
   agency.general_scale as general_scale,
   agency.media_type as media_type,
-  agency.sample_image_url as images,
+  string_agg(distinct historical_image.image_url, ',') as bull,
+  agency.sample_image_url as crap,
+  -- use conditional logic to combine agency sample image and uploaded images for carousel if they exist
+  CASE
+    WHEN agency.sample_image_url IS NOT NULL AND string_agg(distinct historical_image.image_url, ',') IS NOT NULL
+    THEN CONCAT(agency.sample_image_url, ',', string_agg(distinct historical_image.image_url, ','))
+    WHEN agency.sample_image_url IS NOT NULL AND string_agg(distinct historical_image.image_url, ',') IS NULL
+    THEN agency.sample_image_url
+    WHEN agency.sample_image_url IS NULL AND string_agg(distinct historical_image.image_url, ',') IS NOT NULL
+    THEN string_agg(distinct historical_image.image_url, ',')
+    ELSE agency.sample_image_url
+  END AS images,
   array_to_string(ARRAY(SELECT json_build_object('coverage', product.coverage,
                                  'number_of_frames', product.number_of_frames,
                                  'medium', product.medium,
@@ -34,12 +46,12 @@ SELECT historical_collection.id as collection_id,
     WHEN (
       (string_agg(distinct county.name, ',' order by county.name)
       ~ '.*(,).*')
-      ) THEN CONCAT('Multi-County ', agency.abbreviation, ' Historic Imagery')
+      )
+    THEN CONCAT('Multi-County ', agency.abbreviation, ' Historic Imagery')
     ELSE CONCAT(string_agg(distinct county.name, ',' order by county.name), ' ', agency.abbreviation, ' Historic Imagery')
   END AS name,
   'historical-aerial' as template,
   'Order_Only' as availability,
-  'https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg' as thumbnail_image,
   'Historic_Imagery' as category,
   'Historical Use,Research' as recommended_use,
   'Public Domain (Creative Commons CC0)' as license_name,
@@ -59,6 +71,8 @@ LEFT JOIN county ON county.id=county_relate.county_id
 
 LEFT JOIN agency ON agency.id=historical_collection.agency_id
 
+LEFT JOIN historical_image ON historical_image.collection_id=historical_collection.id
+
 GROUP BY historical_collection.id,
          agency.name,
          agency.abbreviation,

src/data_hub/lcd/forms.py

@@ -60,7 +60,11 @@ class Meta:
             'caption': 'Caption will not be saved until chosen image is uploaded and saved to database.',
         }
 
-    image_url = forms.FileField(required=True, widget=PictureWidget, help_text="Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply.")
+    image_url = forms.FileField(
+        required=True,
+        widget=PictureWidget, 
+        help_text="DataHub Images should be 16:9 ratio (1920 x 1080 pixels).<br/>Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply."
+        )
 
 
 class CollectionForm(forms.ModelForm):

src/data_hub/lcd/migrations/0045_auto_20200429_1417.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.13 on 2020-04-29 19:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lcd', '0044_auto_20191121_1208'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collection',
+            name='thumbnail_image',
+            field=models.TextField(blank=True, max_length=200, null=True, verbose_name='Thumb Image'),
+        ),
+    ]

src/data_hub/lcd/models.py

@@ -832,7 +832,7 @@ class Meta:
     )
     thumbnail_image = models.TextField(
         'Thumb Image',
-        max_length=120,
+        max_length=200,
         null=True,
         blank=True
     )

src/data_hub/lore/admin.py

@@ -5,9 +5,11 @@
 
 from .filters import (CollectionAgencyNameFilter, CollectionCountyFilter,
                       CountyDropdownFilter)
-from .forms import CollectionForm, ProductForm
+
+from .forms import CollectionForm, ProductForm, ImageForm, ScannedPhotoIndexLinkForm
+
 from .models import (Agency, Collection, County, CountyRelate, FrameSize,
-                     LineIndex, MicroficheIndex, PhotoIndex, Product,
+                     Image, LineIndex, MicroficheIndex, PhotoIndex, Product,
                      ScannedPhotoIndexLink)
 from .actions import (export_collection, export_product, export_photo_index,
                       export_scanned_photo_index_link, export_county, export_line_index,
@@ -41,6 +43,7 @@ class ScannedPhotoIndexLinkInlineAdmin(admin.StackedInline):
     classes = ('grp-collapse grp-closed',)
     inline_classes = ('grp-collapse grp-closed',)
     model = ScannedPhotoIndexLink
+    form = ScannedPhotoIndexLinkForm
     extra = 0
 
 
@@ -75,22 +78,31 @@ class CountyRelateInlineAdmin(admin.StackedInline):
     ordering = ('county__name',)
 
 
+class ImageInlineAdmin(admin.StackedInline):
+    classes = ('grp-collapse grp-closed',)
+    inline_classes = ('grp-collapse grp-open',)
+    model = Image
+    form = ImageForm
+    extra = 0
+
+
 class CollectionAdmin(admin.ModelAdmin):
     model = Collection
     form = CollectionForm
     fieldsets = (
         ('Collection Information', {
              'fields': ('collection', 'agency', 'from_date', 'to_date',
                        'index_service_url', 'frames_service_url', 'mosaic_service_url',
-                       'counties', 'number_of_boxes', 'photo_index_only', 'public', 'fully_scanned', 'qr_code_url'),
+                       'counties', 'number_of_boxes', 'photo_index_only', 'public',
+                       'fully_scanned', 'thumbnail_image', 'qr_code_url'),
         }),
         ('Remarks', {
             'fields': ('remarks',)
         })
     )
     inlines = [PhotoIndexInlineAdmin, LineIndexInlineAdmin,
                MicroficheIndexInlineAdmin, ProductInlineAdmin,
-               ScannedPhotoIndexLinkInlineAdmin]
+               ScannedPhotoIndexLinkInlineAdmin, ImageInlineAdmin]
     list_display = (
         'collection', 'id', 'agency', 'from_date', 'to_date', 'county_names', 'public'
     )

src/data_hub/lore/forms.py

@@ -2,8 +2,10 @@
 from django.core.exceptions import ValidationError
 from django.db.utils import ProgrammingError
 
-from .models import Collection, County, CountyRelate, Product
+from .models import Collection, County, CountyRelate, Product, Image, ScannedPhotoIndexLink
+from lcd.forms import PictureWidget
 
+import boto3, uuid
 
 class ProductForm(forms.ModelForm):
     class Meta:
@@ -14,6 +16,21 @@ class Meta:
     clean_status = forms.BooleanField(label='Clean Status', help_text='Clean Status refers to collections that have been reviewed and are ready to been scanned, no erasing of frames needed. Default is False/Unchecked.', required=False)
 
 
+class ImageForm(forms.ModelForm):
+    class Meta:
+        model = Image
+        fields = ('__all__')
+        help_texts = {
+            'caption': 'Caption will not be saved until chosen image is uploaded and saved to database.',
+        }
+
+    image_url = forms.FileField(
+        required=True,
+        widget=PictureWidget,
+        help_text="DataHub Images should be 16:9 ratio (1920 x 1080 pixels).<br/>Choose an image file and 'Save' this form to upload & save it to the database. After saving, you can populate a Caption and re-save to apply."
+        )
+
+
 class CollectionForm(forms.ModelForm):
     class Meta:
         model = Collection
@@ -23,6 +40,9 @@ class Meta:
         fields = ('collection', 'agency', 'from_date', 'to_date', 'counties',
                   'remarks')
 
+    # boto3 s3 object
+    client = boto3.client('s3')
+
     # added a try/except for initial migration without data
     try:
         county_choices = (
@@ -39,42 +59,115 @@ def __init__(self, *args, **kwargs):
                     "county_id").filter(collection=self.instance.id)
             ]
             self.fields['counties'].initial = self.initial_counties
+            self.fields['thumbnail_image'].choices = self.create_image_choices('image_url', 'image_id', Image, 'image_id')
+            self.fields['thumbnail_image'].help_text = self.selected_thumbnail()
 
     counties = forms.MultipleChoiceField(
         widget=forms.SelectMultiple(attrs={'title': 'Hold down ctrl to select multiple counties',}),
         choices=county_choices
     )
+    thumbnail_image = forms.ChoiceField(required=False, choices=[])
+
+    # general function to create a form dropdown for thumbnail image
+    def create_image_choices(self, id_field, label_field, type_table, order_field):
+        # start with the model field default which resides in s3 outside of any dbase record
+        choices = [('https://s3.amazonaws.com/data.tnris.org/historical_images/historical_thumbnail.jpg', 'Default')]
+        # get the relate type choices from the type table
+        try:
+            uploaded_images = [
+                (getattr(b, id_field), getattr(b, label_field)) for b in type_table.objects.filter(collection_id=self.instance.id).order_by(order_field)]
+
+            choices.extend(uploaded_images)
+        except ProgrammingError as e:
+            print(e)
+        return choices
+
+    # retrieve selected thumbnail id for helper text to display
+    def selected_thumbnail(self):
+        text = "NO THUMBNAIL SELECTED! Please choose an image UUID from the dropdown and 'Save' the form."
+        if self.instance.thumbnail_image is not None and self.instance.thumbnail_image != "":
+            filename = self.instance.thumbnail_image.split("/")[-1]
+            if filename == 'historical_thumbnail.jpg':
+                filename = 'Default'
+            text = "Currently Selected: %s" % filename 
+        return text
+
+    def inline_image_handler(self, file):
+        new_uuid = uuid.uuid4()
+        file_ext = str(file).split('.')[-1]
+        # upload image
+        key = "%s/assets/%s.%s" % (self.instance.id, new_uuid, file_ext)
+        # print(key + file_ext)
+        response = self.client.put_object(
+            Bucket='data.tnris.org',
+            ACL='public-read',
+            Key=key,
+            Body=file
+        )
+        print('%s upload success!' % key)
+        # update link in database table
+        args = {
+            'image_id': new_uuid,
+            'image_url': "https://s3.amazonaws.com/data.tnris.org/" + key,
+            'collection_id': self.instance
+        }
+        Image(**args).save()
+        return
 
     def clean(self):
         index = self.cleaned_data['index_service_url']
         frames = self.cleaned_data['frames_service_url']
         mosaic = self.cleaned_data['mosaic_service_url']
 
         links = []
+        mapserver_url = 'https://mapserver.tnris.org/wms/?map=/mapfiles/'
         if index is not None:
+            # validate url belongs to proper product type
             if '_index' not in index:
                 raise forms.ValidationError('That is not an Index Service URL!')
+            # validate url is 443 protocol
+            if 'http://' in index:
+                raise forms.ValidationError('Index Service URL must be https protocol!')
+            # validate the url structure matches the tnris mapserver instance
+            if mapserver_url not in index:
+                raise forms.ValidationError('Index Service URL must belong to the TNRIS mapserver: "https://mapserver.tnris.org/wms/?map=/mapfiles/<mapfile>"')
             idx_link = index.split('/')[-1].replace('.map', '').replace('_index', '').replace('_', ' ').upper()
             if idx_link not in links:
                 links.append(idx_link)
         if frames is not None:
+            # validate url belongs to proper product type
             if '_frames' not in frames:
                 raise forms.ValidationError('That is not an Frames Service URL!')
+            # validate url is 443 protocol
+            if 'http://' in frames:
+                raise forms.ValidationError('Frames Service URL must be https protocol!')
+            # validate the url structure matches the tnris mapserver instance
+            if mapserver_url not in frames:
+                raise forms.ValidationError('Frames Service URL must belong to the TNRIS mapserver: "https://mapserver.tnris.org/wms/?map=/mapfiles/<mapfile>"')
             frm_link = frames.split('/')[-1].replace('.map', '').replace('_frames', '').replace('_', ' ').upper()
             if frm_link not in links:
                 links.append(frm_link)
         if mosaic is not None:
+            # validate url belongs to proper product type
             if '_mosaic' not in mosaic:
                 raise forms.ValidationError('That is not an Mosaic Service URL!')
+            # validate url is 443 protocol
+            if 'http://' in mosaic:
+                raise forms.ValidationError('Mosaic Service URL must be https protocol!')
+            # validate the url structure matches the tnris mapserver instance
+            if mapserver_url not in mosaic:
+                raise forms.ValidationError('Mosaic Service URL must belong to the TNRIS mapserver: "https://mapserver.tnris.org/wms/?map=/mapfiles/<mapfile>"')
             msc_link = mosaic.split('/')[-1].replace('.map', '').replace('_mosaic', '').replace('_', ' ').upper()
             if msc_link not in links:
                 links.append(msc_link)
+        # cross validate consistency across all 3 service urls
         if len(links) > 1:
             raise forms.ValidationError('Index, Frames, and Mosaic Service URLs have inconsistent collections!')
         elif len(links) == 1:
             self.instance.ls4_link = links[0]
         else:
             self.instance.ls4_link = None
+        return
 
     def save(self, commit=True):
         updated_counties = self.cleaned_data['counties']
@@ -88,4 +181,27 @@ def save(self, commit=True):
                 county=remove).filter(collection=self.instance.id).delete()
         for add in adds:
             CountyRelate(county_id=add, collection=self.instance).save()
+
+        # check for files
+        files = self.files
+        # if files and field not marked for deletion then upload to s3
+        for f in files:
+            if 'image_collections' in f:
+                self.inline_image_handler(files[f])
+
         return super(CollectionForm, self).save(commit=commit)
+
+
+class ScannedPhotoIndexLinkForm(forms.ModelForm):
+    class Meta:
+        model = ScannedPhotoIndexLink
+        fields = ('__all__')
+
+    def clean(self):
+        # force 443 protocol on urls and reformat url structure for consistency
+        link = self.cleaned_data['link']
+        link = link.replace('http://', 'https://')
+        link = link.replace('https://tnris-ls4.s3.amazonaws.com/', 'https://s3.amazonaws.com/tnris-ls4/')
+        self.cleaned_data['link'] = link
+        super(ScannedPhotoIndexLinkForm, self).save(commit=False)
+        return 

src/data_hub/lore/migrations/0018_image.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.0.13 on 2020-04-29 14:39
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lore', '0017_delete_scale'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Image',
+            fields=[
+                ('image_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='Image ID')),
+                ('image_url', models.URLField(max_length=255, verbose_name='Image URL')),
+                ('caption', models.CharField(blank=True, max_length=255, null=True, verbose_name='Image Caption')),
+                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')),
+                ('last_modified', models.DateTimeField(auto_now=True, verbose_name='Last Modified')),
+                ('collection_id', models.ForeignKey(db_column='collection_id', on_delete=django.db.models.deletion.CASCADE, related_name='image_collections', to='lore.Collection')),
+            ],
+            options={
+                'verbose_name': 'Historical Image',
+                'verbose_name_plural': 'Historical Images',
+                'db_table': 'historical_image',
+            },
+        ),
+    ]