Create functions to download specific IQA datasets

[ocampor]

Datasets:

• Live
• TID2008
• TID2013
• CSIQ

[KamalLamichhane]

Dear Sir, I would like to build up a dataset builder based on CSIQ dataset. In the first stage, I tried to download CSIQ dataset with the following codes:
import os.path

import tensorflow as tf
import tensorflow_datasets.public_api as tfds

#from . import CHECKSUMS_PATH

#tfds.download.add_checksums_dir(CHECKSUMS_PATH)
#dl_manager.manual_dir()

CITATION = r"""
@Article{larson2010most,
title={Most apparent distortion: full-reference image quality assessment and the role of strategy},
author={Larson, Eric Cooper and Chandler, Damon Michael},
journal={Journal of electronic imaging},
volume={19},
number={1},
pages={011006},
year={2010},
publisher={International Society for Optics and Photonics}
}
"""
DESCRIPTION = """
The CSIQ image database is a popular database for testing image quality assessment algorithms and other aspects of image quality. The database consists of 30 original images, each distorted using one of six types of distortions, each at four to five different levels of distortion. The CSIQ images were subjectively rated based on a linear displacement of the images across four calibrated LCD monitors placed side-by-side with equal viewing distance to the observer. The database contains 5000 subjective ratings from 35 different observers, and the ratings are reported in the form of DMOS (a larger denotes greater visual distortion compared to the corresponding reference image).
"""
URL = u"http://vision.eng.shizuoka.ac.jp/mod/page/view.php?id=23"
LICENSE = """
©2020 Damon M. Chandler
CSIQ Lab | Dept. of Electrical and Electronic Engineering | College of Engineering | Shizuoka University

〒432-8561 | 浜松市中区城北3-5-1 | 静岡大学 工学部 電気電子工学科

Portions of this site contain images © David Vignoni. LGPL 2.1.

The Essential theme for Moodle is developed by Gareth J Barnard
"""
SUPERVISED_KEYS = ("distorted_image", "dmos")

class CSIQ(tfds.core.GeneratorBasedBuilder):
VERSION = tfds.core.Version("1.0.0")

def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "dst_idx": tf.int32,#tfds.features.Text(),
            "dst_type":tfds.features.Text(),
            #"index": tf.int32,
            "Distorted_image": tfds.features.Image(),
            "Reference_image": tfds.features.Image(),
            "dst_lev": tf.int32,
            "dmos_std": tf.float32,
            "dmos": tf.float32,
        }),
        supervised_keys=SUPERVISED_KEYS,
        homepage=URL,
        citation=CITATION,
        #redistribution_info={
           # 'license': LICENSE,
        #},
    )

def _split_generators(self, manager):
    #"CSIQ_LBL"= "http://vision.eng.shizuoka.ac.jp/pluginfile.php/69/mod_page/content/25/csiq.DMOS.xlsx"
    extracted_path = manager.download_and_extract({
        "CSIQ_DEST": "http://vision.eng.shizuoka.ac.jp/csiq/dst_imgs.zip",
        "CSIQ_REF": "http://vision.eng.shizuoka.ac.jp/csiq/src_imgs.zip",
        #"force_checksums_validation": bool = False,
        #"register_checksums": bool = False
        })
    extracted_path['CSIQ_DEST']
    extracted_path['CSIQ_REF']
    #images_path = os.path.join(extracted_path, "CSIQ_DEST","CSIQ_REF")
    #images_path = os.path.join(Datadir)
    directory_url = 'http://vision.eng.shizuoka.ac.jp/pluginfile.php/69/mod_page/content/25/'
    file_names = ['csiq.DMOS.xlsx']
    file_paths = [ 
        tf.keras.utils.get_file(file_name, directory_url + file_name)
        for file_name in file_names
        ]

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            gen_kwargs={
                "images_path": images_path,
                #"labels": os.path.join(images_path,"dmos1.csv")
                "labels": tf.io.read_file(file_paths)
                #"labels":pd.read_csv(r"C:\CSIQ_all\dmos1.csv")
            },
        )
    ]

def _generate_examples(self, images_path, labels):
    with tf.io.gfile.GFile(labels) as f:
        lines = f.readlines()

    for image_id, line in enumerate(lines[1:]):
        values = line.split(",")
        yield image_id, {
            "dst_idx": values[0],
            "dst_type": values[1],
            "Distorted": os.path.join(images_path, values[2]),
            "Reference": os.path.join(images_path, values[3]),
            "DST_lev": values[4],
            "dmos_std": values[5],
            "dmos": values[6],
            #"dmos_realigned": values[0],
            #"dmos_realigned_std": values[0],
        }

WHile building a pipeline, with following commands:
builder = CSIQ()
builder.download_and_prepare()
ds = builder.as_dataset(shuffle_files=True)['train']
ds = ds.shuffle(1024).batch(1)
next(iter(ds)).keys()
for features in ds.take(1):
distorted_image = features['Distorted']
reference_image = features['Reference']
dmos = tf.round(features['dmos'][0], 2)#disable for Tid2013
distortion = features['distortion'][0] # disable for Tid2013
#print function should disable in TID2013 or need to modify
print(f'The distortion of the image is {dmos} with'
f'a distortion {distortion} and shape {distorted_image.shape}')
show_images([reference_image, distorted_image])
It results: Checksum error.

---

NonMatchingChecksumError Traceback (most recent call last)
in ()
2 #builder = imquality.datasets.CSIQ()
3 builder = CSIQ()
----> 4 builder.download_and_prepare()
5 ds = builder.as_dataset(shuffle_files=True)['train']
6 ds = ds.shuffle(1024).batch(1)

17 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_datasets/core/download/download_manager.py in _handle_download_result(self, resource, tmp_dir_path, sha256, dl_size)
214 self._record_sizes_checksums()
215 elif (dl_size, sha256) != self._sizes_checksums.get(resource.url, None):
--> 216 raise NonMatchingChecksumError(resource.url, tmp_path)
217 download_path = self._get_final_dl_path(resource.url, sha256)
218 resource_lib.write_info_file(resource, download_path, self._dataset_name,

NonMatchingChecksumError: Artifact http://vision.eng.shizuoka.ac.jp/csiq/dst_imgs.zip, downloaded to /root/tensorflow_datasets/downloads/vision.eng.shizuoka.ac.jp_csiq_dst_imgsX-ETcy6zbVXku6A1J7b7xRNgTdP0_qLW84MAch1ss5c.zip.tmp.b3e7b23d848648c69a92d6b1566a39d6/dst_imgs.zip, has wrong checksum.

I would be glad if you could really guide me on this issue.
Thank you!

[chun2925084]

Hi @KamalLamichhane, I want to download the CSIQ databaes, but the link to it can not be located anymore. Do you have any solution to deal with it?

Thanks