From bda1a3c5739fad61c7747edf0b4d1a06ac1a2d00 Mon Sep 17 00:00:00 2001 From: Programmer-RD-AI Date: Fri, 14 Oct 2022 13:43:08 +0530 Subject: [PATCH] dfsd --- .history/.gitignore_20220510084905 | 189 ------ .history/.gitignore_20220510084910 | 190 ------ .history/.gitignore_20220510084916 | 190 ------ .../Model/dataset/download_20220313204243.py | 549 ----------------- .../Model/dataset/download_20220509222907.py | 549 ----------------- .../Model/dataset/download_20220509222918.py | 549 ----------------- .../Model/dataset/download_20220509222920.py | 546 ----------------- .../Model/dataset/download_20220509222923.py | 573 ----------------- .../Model/dataset/download_20220509222938.py | 577 ------------------ .../Model/dataset/download_20220509223014.py | 576 ----------------- .../Model/dataset/download_20220509224638.py | 574 ----------------- .../Model/dataset/download_20220509231219.py | 574 ----------------- .../Model/dataset/download_20220509231223.py | 574 ----------------- .../Model/dataset/download_20220509231230.py | 572 ----------------- .../Model/dataset/download_20220509231420.py | 576 ----------------- .../Model/dataset/download_20220509231423.py | 576 ----------------- .../Model/dataset/download_20220509231425.py | 543 ---------------- .../Model/dataset/download_20220509231428.py | 543 ---------------- .../Model/dataset/download_20220510064024.py | 543 ---------------- .../Model/dataset/download_20220510091201.py | 539 ---------------- .../Model/dataset/download_20220515113355.py | 539 ---------------- .../Model/dataset/download_20220515113403.py | 539 ---------------- .../Model/dataset/download_20220515113413.py | 536 ---------------- .../Model/dataset/download_20220515113423.py | 536 ---------------- .../Model/dataset/download_20220515113424.py | 536 ---------------- .../Model/dataset/download_20220515113433.py | 538 ---------------- .../Model/dataset/download_20220515113439.py | 543 ---------------- .../Model/dataset/download_20220515113441.py | 544 ----------------- .../Model/dataset/download_20220515113451.py | 544 ----------------- .../Model/dataset/download_20220515113459.py | 545 ----------------- .../Model/dataset/download_20220515113512.py | 546 ----------------- .../Model/dataset/download_20220515113521.py | 553 ----------------- .../Model/dataset/download_20220515113747.py | 553 ----------------- .../modelling/detectron2_20220429065528.py | 533 ---------------- .../modelling/detectron2_20220509224846.py | 530 ---------------- .../modelling/detectron2_20220509224849.py | 527 ---------------- .history/ML/run_20220313204243.py | 23 - .history/ML/run_20220509224632.py | 2 - .history/ML/run_20220509224639.py | 5 - .history/README_20220510101058.md | 6 - .history/README_20220515114046.md | 5 - .history/README_20220515114052.md | 6 - 42 files changed, 18791 deletions(-) delete mode 100644 .history/.gitignore_20220510084905 delete mode 100644 .history/.gitignore_20220510084910 delete mode 100644 .history/.gitignore_20220510084916 delete mode 100644 .history/ML/Model/dataset/download_20220313204243.py delete mode 100644 .history/ML/Model/dataset/download_20220509222907.py delete mode 100644 .history/ML/Model/dataset/download_20220509222918.py delete mode 100644 .history/ML/Model/dataset/download_20220509222920.py delete mode 100644 .history/ML/Model/dataset/download_20220509222923.py delete mode 100644 .history/ML/Model/dataset/download_20220509222938.py delete mode 100644 .history/ML/Model/dataset/download_20220509223014.py delete mode 100644 .history/ML/Model/dataset/download_20220509224638.py delete mode 100644 .history/ML/Model/dataset/download_20220509231219.py delete mode 100644 .history/ML/Model/dataset/download_20220509231223.py delete mode 100644 .history/ML/Model/dataset/download_20220509231230.py delete mode 100644 .history/ML/Model/dataset/download_20220509231420.py delete mode 100644 .history/ML/Model/dataset/download_20220509231423.py delete mode 100644 .history/ML/Model/dataset/download_20220509231425.py delete mode 100644 .history/ML/Model/dataset/download_20220509231428.py delete mode 100644 .history/ML/Model/dataset/download_20220510064024.py delete mode 100644 .history/ML/Model/dataset/download_20220510091201.py delete mode 100644 .history/ML/Model/dataset/download_20220515113355.py delete mode 100644 .history/ML/Model/dataset/download_20220515113403.py delete mode 100644 .history/ML/Model/dataset/download_20220515113413.py delete mode 100644 .history/ML/Model/dataset/download_20220515113423.py delete mode 100644 .history/ML/Model/dataset/download_20220515113424.py delete mode 100644 .history/ML/Model/dataset/download_20220515113433.py delete mode 100644 .history/ML/Model/dataset/download_20220515113439.py delete mode 100644 .history/ML/Model/dataset/download_20220515113441.py delete mode 100644 .history/ML/Model/dataset/download_20220515113451.py delete mode 100644 .history/ML/Model/dataset/download_20220515113459.py delete mode 100644 .history/ML/Model/dataset/download_20220515113512.py delete mode 100644 .history/ML/Model/dataset/download_20220515113521.py delete mode 100644 .history/ML/Model/dataset/download_20220515113747.py delete mode 100644 .history/ML/Model/modelling/detectron2_20220429065528.py delete mode 100644 .history/ML/Model/modelling/detectron2_20220509224846.py delete mode 100644 .history/ML/Model/modelling/detectron2_20220509224849.py delete mode 100644 .history/ML/run_20220313204243.py delete mode 100644 .history/ML/run_20220509224632.py delete mode 100644 .history/ML/run_20220509224639.py delete mode 100644 .history/README_20220510101058.md delete mode 100644 .history/README_20220515114046.md delete mode 100644 .history/README_20220515114052.md diff --git a/.history/.gitignore_20220510084905 b/.history/.gitignore_20220510084905 deleted file mode 100644 index 8ccb1f8c..00000000 --- a/.history/.gitignore_20220510084905 +++ /dev/null @@ -1,189 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ -Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv -Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv -Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv -Model/dataset/open_image_raw_data/test-annotations-bbox.csv -Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/test-images-with-rotation.csv -Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv -Model/dataset/open_image_raw_data/validation-annotations-bbox.csv -Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/validation-images-with-rotation.csv -Model/dataset/data/Labels_and_Image_ID.csv -Model/dataset/data/BBOX.csv -Model/dataset/data/BBOX.json -Model/dataset/data/Image_Urls.csv -Model/dataset/data/Image_Urls.json -Model/dataset/data/Labels_and_Image_ID.json -Model/dataset/Img/ -.Other/V2/download/Img/ -.Other/V2/wandb/ -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09 (2).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.08.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.08 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09 (2).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.08.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.08 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.07.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -.Other/V2/detectron2/ -Model/test_imgs/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -Model/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -*.jpeg - -ML/checking/torch_object_detection/PennFudanPed/ -ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv -ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv -ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv -ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv -ML/Model/dataset/Img/ diff --git a/.history/.gitignore_20220510084910 b/.history/.gitignore_20220510084910 deleted file mode 100644 index 1df14777..00000000 --- a/.history/.gitignore_20220510084910 +++ /dev/null @@ -1,190 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ -Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv -Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv -Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv -Model/dataset/open_image_raw_data/test-annotations-bbox.csv -Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/test-images-with-rotation.csv -Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv -Model/dataset/open_image_raw_data/validation-annotations-bbox.csv -Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/validation-images-with-rotation.csv -Model/dataset/data/Labels_and_Image_ID.csv -Model/dataset/data/BBOX.csv -Model/dataset/data/BBOX.json -Model/dataset/data/Image_Urls.csv -Model/dataset/data/Image_Urls.json -Model/dataset/data/Labels_and_Image_ID.json -Model/dataset/Img/ -.Other/V2/download/Img/ -.Other/V2/wandb/ -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09 (2).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.08.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.08 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09 (2).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.08.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.08 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.07.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -.Other/V2/detectron2/ -Model/test_imgs/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -Model/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -*.jpeg - -ML/checking/torch_object_detection/PennFudanPed/ -ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv -ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv -ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv -ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv -ML/Model/dataset/Img/ -.histroy/ diff --git a/.history/.gitignore_20220510084916 b/.history/.gitignore_20220510084916 deleted file mode 100644 index 5b23ab85..00000000 --- a/.history/.gitignore_20220510084916 +++ /dev/null @@ -1,190 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ -Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv -Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv -Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv -Model/dataset/open_image_raw_data/test-annotations-bbox.csv -Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/test-images-with-rotation.csv -Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv -Model/dataset/open_image_raw_data/validation-annotations-bbox.csv -Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv -Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv -Model/dataset/open_image_raw_data/validation-images-with-rotation.csv -Model/dataset/data/Labels_and_Image_ID.csv -Model/dataset/data/BBOX.csv -Model/dataset/data/BBOX.json -Model/dataset/data/Image_Urls.csv -Model/dataset/data/Image_Urls.json -Model/dataset/data/Labels_and_Image_ID.json -Model/dataset/Img/ -.Other/V2/download/Img/ -.Other/V2/wandb/ -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09 (2).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.09 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.08.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.08 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07.jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -.Other/V2/test_imgs/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09 (2).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.09 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.08.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.08 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.07.jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -.Other/V2/preds/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -.Other/V2/detectron2/ -Model/test_imgs/WhatsApp Image 2021-10-19 at 21.27.06.jpeg -Model/test_imgs/WhatsApp Image 2021-10-19 at 21.27.07 (1).jpeg -*.jpeg - -ML/checking/torch_object_detection/PennFudanPed/ -ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv -ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv -ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv -ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv -ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv -ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv -ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv -ML/Model/dataset/Img/ -.histroy diff --git a/.history/ML/Model/dataset/download_20220313204243.py b/.history/ML/Model/dataset/download_20220313204243.py deleted file mode 100644 index 67d4e410..00000000 --- a/.history/ML/Model/dataset/download_20220313204243.py +++ /dev/null @@ -1,549 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/016vt9", - "/m/01sdgj", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_r is None: - labels_r = [ - "/m/016vt9", - "/m/01sdgj", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220509222907.py b/.history/ML/Model/dataset/download_20220509222907.py deleted file mode 100644 index 84162925..00000000 --- a/.history/ML/Model/dataset/download_20220509222907.py +++ /dev/null @@ -1,549 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/01sdgj", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_r is None: - labels_r = [ - "/m/016vt9", - "/m/01sdgj", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220509222918.py b/.history/ML/Model/dataset/download_20220509222918.py deleted file mode 100644 index a799c2f4..00000000 --- a/.history/ML/Model/dataset/download_20220509222918.py +++ /dev/null @@ -1,549 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_r is None: - labels_r = [ - "/m/016vt9", - "/m/01sdgj", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220509222920.py b/.history/ML/Model/dataset/download_20220509222920.py deleted file mode 100644 index 3b6fadb2..00000000 --- a/.history/ML/Model/dataset/download_20220509222920.py +++ /dev/null @@ -1,546 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - - ] - if labels_r is None: - labels_r = [ - "/m/016vt9", - "/m/01sdgj", - "/m/02h5d", - "/m/02wvcj0", - "/m/09vh0m", - "/m/0d7pp", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220509222923.py b/.history/ML/Model/dataset/download_20220509222923.py deleted file mode 100644 index 7bd03004..00000000 --- a/.history/ML/Model/dataset/download_20220509222923.py +++ /dev/null @@ -1,573 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) diff --git a/.history/ML/Model/dataset/download_20220509222938.py b/.history/ML/Model/dataset/download_20220509222938.py deleted file mode 100644 index 5df23560..00000000 --- a/.history/ML/Model/dataset/download_20220509222938.py +++ /dev/null @@ -1,577 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) - - -d = Download() -d.download() diff --git a/.history/ML/Model/dataset/download_20220509223014.py b/.history/ML/Model/dataset/download_20220509223014.py deleted file mode 100644 index f5dd5d80..00000000 --- a/.history/ML/Model/dataset/download_20220509223014.py +++ /dev/null @@ -1,576 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) - - -d = Download() -d.download() diff --git a/.history/ML/Model/dataset/download_20220509224638.py b/.history/ML/Model/dataset/download_20220509224638.py deleted file mode 100644 index a7ae1b07..00000000 --- a/.history/ML/Model/dataset/download_20220509224638.py +++ /dev/null @@ -1,574 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) - - diff --git a/.history/ML/Model/dataset/download_20220509231219.py b/.history/ML/Model/dataset/download_20220509231219.py deleted file mode 100644 index 9d15fab1..00000000 --- a/.history/ML/Model/dataset/download_20220509231219.py +++ /dev/null @@ -1,574 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data# [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) - - diff --git a/.history/ML/Model/dataset/download_20220509231223.py b/.history/ML/Model/dataset/download_20220509231223.py deleted file mode 100644 index 17597bab..00000000 --- a/.history/ML/Model/dataset/download_20220509231223.py +++ /dev/null @@ -1,574 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data[: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) - - diff --git a/.history/ML/Model/dataset/download_20220509231230.py b/.history/ML/Model/dataset/download_20220509231230.py deleted file mode 100644 index 923d5883..00000000 --- a/.history/ML/Model/dataset/download_20220509231230.py +++ /dev/null @@ -1,572 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data[:1250] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) diff --git a/.history/ML/Model/dataset/download_20220509231420.py b/.history/ML/Model/dataset/download_20220509231420.py deleted file mode 100644 index 036c08ff..00000000 --- a/.history/ML/Model/dataset/download_20220509231420.py +++ /dev/null @@ -1,576 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data[:1250] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) diff --git a/.history/ML/Model/dataset/download_20220509231423.py b/.history/ML/Model/dataset/download_20220509231423.py deleted file mode 100644 index cad58cb1..00000000 --- a/.history/ML/Model/dataset/download_20220509231423.py +++ /dev/null @@ -1,576 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError( - f""" - In the indexing parameters - there was a error occurred - \n - {e}""" - ) - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError( - f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""" - ) - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError( - f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """ - ) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError( - f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""" - ) - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError( - f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """ - ) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i]) - ) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError( - f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""" - ) - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""" - ) - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append(pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError( - f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), labels_and_imageid["ImageID"] - ): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""" - ) - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - is True - ): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""" - ) - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and ( - os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) - is True - ): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates() - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0] - ].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data[:1250] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError( - f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""" - ) - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError( - f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""" - ) - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError( - f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""" - ) diff --git a/.history/ML/Model/dataset/download_20220509231425.py b/.history/ML/Model/dataset/download_20220509231425.py deleted file mode 100644 index 013f534d..00000000 --- a/.history/ML/Model/dataset/download_20220509231425.py +++ /dev/null @@ -1,543 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data[:1250] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220509231428.py b/.history/ML/Model/dataset/download_20220509231428.py deleted file mode 100644 index 19344e1c..00000000 --- a/.history/ML/Model/dataset/download_20220509231428.py +++ /dev/null @@ -1,543 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data[:1250] - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220510064024.py b/.history/ML/Model/dataset/download_20220510064024.py deleted file mode 100644 index c6703a79..00000000 --- a/.history/ML/Model/dataset/download_20220510064024.py +++ /dev/null @@ -1,543 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - urlretrieve( - ourl, - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{self.idx}.png", - ) - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220510091201.py b/.history/ML/Model/dataset/download_20220510091201.py deleted file mode 100644 index a95db03a..00000000 --- a/.history/ML/Model/dataset/download_20220510091201.py +++ /dev/null @@ -1,539 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113355.py b/.history/ML/Model/dataset/download_20220515113355.py deleted file mode 100644 index a2bd3d33..00000000 --- a/.history/ML/Model/dataset/download_20220515113355.py +++ /dev/null @@ -1,539 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/025rxrz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113403.py b/.history/ML/Model/dataset/download_20220515113403.py deleted file mode 100644 index a7f2277c..00000000 --- a/.history/ML/Model/dataset/download_20220515113403.py +++ /dev/null @@ -1,539 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113413.py b/.history/ML/Model/dataset/download_20220515113413.py deleted file mode 100644 index 5d79e56e..00000000 --- a/.history/ML/Model/dataset/download_20220515113413.py +++ /dev/null @@ -1,536 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = ["/m/01h44", "/m/01m0p1", "/m/02hhhb"] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113423.py b/.history/ML/Model/dataset/download_20220515113423.py deleted file mode 100644 index 6366d4f0..00000000 --- a/.history/ML/Model/dataset/download_20220515113423.py +++ /dev/null @@ -1,536 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = ["/m/01h44", "/m/01m0p1", "/m/02hhhb", "/m/0449p"] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113424.py b/.history/ML/Model/dataset/download_20220515113424.py deleted file mode 100644 index 6366d4f0..00000000 --- a/.history/ML/Model/dataset/download_20220515113424.py +++ /dev/null @@ -1,536 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = ["/m/01h44", "/m/01m0p1", "/m/02hhhb", "/m/0449p"] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113433.py b/.history/ML/Model/dataset/download_20220515113433.py deleted file mode 100644 index 6b0ea2fe..00000000 --- a/.history/ML/Model/dataset/download_20220515113433.py +++ /dev/null @@ -1,538 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", "/m/01m0p1", "/m/02hhhb", "/m/0449p", "/m/0464z4" - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113439.py b/.history/ML/Model/dataset/download_20220515113439.py deleted file mode 100644 index 8d828170..00000000 --- a/.history/ML/Model/dataset/download_20220515113439.py +++ /dev/null @@ -1,543 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113441.py b/.history/ML/Model/dataset/download_20220515113441.py deleted file mode 100644 index 588a3018..00000000 --- a/.history/ML/Model/dataset/download_20220515113441.py +++ /dev/null @@ -1,544 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113451.py b/.history/ML/Model/dataset/download_20220515113451.py deleted file mode 100644 index 80541e30..00000000 --- a/.history/ML/Model/dataset/download_20220515113451.py +++ /dev/null @@ -1,544 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113459.py b/.history/ML/Model/dataset/download_20220515113459.py deleted file mode 100644 index 067517a3..00000000 --- a/.history/ML/Model/dataset/download_20220515113459.py +++ /dev/null @@ -1,545 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - "/m/0cqdf", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113512.py b/.history/ML/Model/dataset/download_20220515113512.py deleted file mode 100644 index e38815a3..00000000 --- a/.history/ML/Model/dataset/download_20220515113512.py +++ /dev/null @@ -1,546 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - "/m/0cqdf", - "/m/0jbk", - ] - if labels_r is None: - labels_r = [ - "/m/0r3vbqy", - "/m/025rxrz", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113521.py b/.history/ML/Model/dataset/download_20220515113521.py deleted file mode 100644 index 05840561..00000000 --- a/.history/ML/Model/dataset/download_20220515113521.py +++ /dev/null @@ -1,553 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - "/m/0cqdf", - "/m/0jbk", - ] - if labels_r is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - "/m/0cqdf", - "/m/0jbk", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/dataset/download_20220515113747.py b/.history/ML/Model/dataset/download_20220515113747.py deleted file mode 100644 index 269d98fb..00000000 --- a/.history/ML/Model/dataset/download_20220515113747.py +++ /dev/null @@ -1,553 +0,0 @@ -from Model import * -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True - - -class Download: - """ - This class is used to download all of classes, - for object detection - """ - - def __init__( - self, - idx: int = 0, - idx_1: int = 0, - idx_2: int = 0, - idx_3: int = 0, - data: dict = None, - labels: list = None, - labels_r: list = None, - labels_and_imageids: list = None, - bboxs: list = None, - image_urls: list = None, - init_imageids: list = None, - images_and_bbox_and_imgid_: list = None, - imgids: list = None, - download_url_data: pd.DataFrame = None, - imageids_labels: dict = None, - recover: bool = False, - number_of_split: int = 25000, - ) -> None: - """summary_line - Keyword arguments: - argument - idx,idx_1,idx_2,idx_3 = the init starting index - data = The data that is going to be download, - if want to add new info can be done. - labels = labels from open images (Name of label) - labels_r = labels from open images (Label Code of label) - labels_and_imageids = loading files for getting the labels_and_imageids - bboxs = loading files for getting the bboxs - image_urls = loading files for getting the image_urls - init_imageids = init imageids - images_and_bbox_and_imgid_ = init images_and_bbox_and_imgid_ - init imgids = init imgids - Return: None - """ - self.number_of_split = number_of_split - if imageids_labels is None: - imageids_labels = {} - self.imageids_labels = imageids_labels - if data is None: - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - } - if download_url_data is None: - self.download_url_data = [] - if labels is None: - labels = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - "/m/0cqdf", - "/m/0jbk", - ] - if labels_r is None: - labels_r = [ - "/m/01h44", - "/m/01m0p1", - "/m/02hhhb", - "/m/0449p", - "/m/0464z4", - "/m/04lmyz", - "/m/09141t", - "/m/0cqdf", - "/m/0jbk", - ] - if labels_and_imageids is None: - labels_and_imageids = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-human-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-annotations-human-imagelabels-boxable.csv", - # "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/Model/dataset/open_image_raw_data/train-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-machine-imagelabels.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-human-imagelabels-boxable.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-machine-imagelabels.csv", - ] - if bboxs is None: - bboxs = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-annotations-bbox.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-annotations-bbox.csv", - ] - if image_urls is None: - image_urls = [ - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/oidv6-train-images-with-labels-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/test-images-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/train-images-boxable-with-rotation.csv", - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/open_image_raw_data/validation-images-with-rotation.csv", - ] - if init_imageids is None: - init_imageids = [] - if images_and_bbox_and_imgid_ is None: - images_and_bbox_and_imgid_ = [] - if imgids is None: - imgids = [] - try: - self.recover = recover - except Exception as e: - raise ValueError(f"Recovering Error: {e}") - try: - # Indexing - self.idx = idx - self.idx_1 = idx_1 - self.idx_2 = idx_2 - self.idx_3 = idx_3 - except Exception as e: - raise ValueError(f""" - In the indexing parameters - there was a error occurred - \n - {e}""") - try: - # Initinalizing Data Storage - self.data = data - except Exception as e: - raise ValueError(f""" - In the Initinalizing Data Storage - parameters there was a error occurred. - \n - {e}""") - try: - # Labels of data - self.labels = labels - self.labels_r = labels_r - except Exception as e: - raise ValueError(f""" - In the Labels of data - parameters there was a error occurred - \n - {e} - """) - try: - # Data Loading file paths - self.labels_and_imageids = labels_and_imageids - self.bboxs = bboxs - self.image_urls = image_urls - except Exception as e: - raise ValueError(f""" - In the Data Loading - file paths parameters there - was a error occurred. - \n - {e}""") - try: - # Collection of data - self.imageids = init_imageids - self.images_and_bbox_and_imgid_ = images_and_bbox_and_imgid_ - self.imgids = imgids - except Exception as e: - raise ValueError(f""" - In the Collection of data - parameters there was a error occurred - \n - {e} - """) - - # Loading data section - - def load_labels_and_imageid(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_labels_and_imageid - Return: pd.DataFrame - """ - try: - print("load_labels_and_imageid") - labels_and_imageid = pd.read_csv(self.labels_and_imageids[0]) - loader_iter = tqdm(range(1, len(self.labels_and_imageids))) - for i in loader_iter: - labels_and_imageid = labels_and_imageid.append( - pd.read_csv(self.labels_and_imageids[i])) - loader_iter.set_description(str(len(labels_and_imageid))) - labels_and_imageid.sample(frac=1) - return labels_and_imageid - except Exception as e: - raise ValueError(f""" - The function - self.load_labels_and_imageid() - or - Download().load_labels_and_imageid() - is not working correctly. - \n - {e}""") - - def load_bbox(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_bbox - Return: pd.DataFrame - """ - try: - - print("Loading bbox") - bboxs_df = pd.read_csv(self.bboxs[0]) - loader_iter = tqdm(range(1, len(self.bboxs))) - for i in loader_iter: - loader_iter.set_description(str(len(bboxs_df))) - bboxs_df = bboxs_df.append(pd.read_csv(self.bboxs[i])) - bboxs_df.sample(frac=1) - - return bboxs_df - except Exception as e: - raise ValueError(f""" - The function - self.load_bbox() - or - Download().load_bbox() - is not working correctly. - \n - {e}""") - - def load_image_urls(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument load_image_urls - Return: pd.DataFrame - """ - try: - - print("Loading Image Urls") - image_urls_df = pd.read_csv(self.image_urls[0]) - loader_iter = tqdm(range(1, len(self.image_urls))) - for i in loader_iter: - loader_iter.set_description(str(len(image_urls_df))) - image_urls_df = image_urls_df.append( - pd.read_csv(self.image_urls[i])) - image_urls_df.sample(frac=1) - - print("Loaded Image Urls") - return image_urls_df - except Exception as e: - raise ValueError(f""" - The function - self.load_image_urls() - or - Download().load_image_urls() - is not working correctly. - \n - {e}""") - - # Creating data section - - def create_imageids(self) -> bool: - """summary_line - Keyword arguments: - argument create_imageids - Return: bool - """ - try: - - print("Creating imageids") - labels_and_imageid = self.load_labels_and_imageid() - for labelname, imageid in zip( - tqdm(labels_and_imageid["LabelName"]), - labels_and_imageid["ImageID"]): - if labelname in self.labels_r: - self.idx_1 += 1 - threading.Thread( - target=self.imageids.append, - args=[imageid], - ).start() - self.imageids_labels[imageid] = labelname - del labels_and_imageid - - print("Created imageids") - print(f"Number of Images : {self.idx_1}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_imageids() - or - Download().create_imageids() - is not working correctly. - \n - {e}""") - - def create_bbox(self) -> bool: - """summary_line - Keyword arguments: - argument create_bbox - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) is True): - self.images_and_bbox_and_imgid_ = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv" - ) - return True - print("Creating Box") - bboxs = self.load_bbox() - for imgid in zip( - tqdm(bboxs["ImageID"]), - bboxs["XMin"], - bboxs["YMin"], - bboxs["XMax"], - bboxs["YMax"], - ): - imgid = list(imgid) - if str(imgid[0]) in self.imageids: - threading.Thread( - target=imgid.append, - args=[self.imageids_labels[imgid[0]]], - ).start() - self.idx_2 += 1 - threading.Thread( - target=self.images_and_bbox_and_imgid_.append, - args=[imgid], - ).start() - threading.Thread( - target=self.imgids.append, - args=[imgid[0]], - ).start() - np.save( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/imageids.npy", - self.imgids, - ) - del bboxs - - self.images_and_bbox_and_imgid_ = pd.DataFrame( - self.images_and_bbox_and_imgid_, - columns=["ImageID", "XMin", "YMin", "XMax", "YMax", "Labels"], - ) - self.images_and_bbox_and_imgid_.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_bbox.csv", - index=False, - ) - print("Created Box") - print(f"Number of Images : {self.idx_2}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_bbox() - or - Download().create_bbox() - is not working correctly. - \n - {e}""") - - def create_image_urls(self) -> bool: - """summary_line - Keyword arguments: - argument create_image_urls - Return: bool - """ - try: - - if self.recover is True and (os.path.exists( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) is True): - self.download_url_data = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv" - ) # [: self.number_of_split] - self.download_url_data = self.download_url_data.drop_duplicates( - ) - return True - print("Creating Image Urls") - data = { - "ImageID": [], - "OriginalURL": [], - "OriginalLandingURL": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "Labels": [], - } - image_urls = self.load_image_urls() - for imgid in zip( - tqdm((image_urls["ImageID"])), - image_urls["OriginalURL"], - image_urls["OriginalLandingURL"], - ): - if imgid[0] in self.imgids: - imgid_of_iabaid = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == imgid[0]] - for idx_3 in range(len(imgid_of_iabaid)): - imgid_of_iabaid_iter = self.images_and_bbox_and_imgid_[ - self.images_and_bbox_and_imgid_["ImageID"] == - imgid[0]].iloc[idx_3] - threading.Thread( - target=data["ImageID"].append, - args=[imgid[0]], - ).start() - threading.Thread( - target=data["OriginalURL"].append, - args=[imgid[1]], - ).start() - threading.Thread( - target=data["OriginalLandingURL"].append, - args=[imgid[2]], - ).start() - threading.Thread( - target=data["XMin"].append, - args=[imgid_of_iabaid_iter["XMin"]], - ).start() - threading.Thread( - target=data["YMin"].append, - args=[imgid_of_iabaid_iter["YMin"]], - ).start() - threading.Thread( - target=data["XMax"].append, - args=[imgid_of_iabaid_iter["XMax"]], - ).start() - threading.Thread( - target=data["YMax"].append, - args=[imgid_of_iabaid_iter["YMax"]], - ).start() - threading.Thread( - target=data["Labels"].append, - args=[imgid_of_iabaid_iter["Labels"]], - ).start() - del self.images_and_bbox_and_imgid_ - - data = pd.DataFrame(data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/create_image_urls.csv", - index=False, - ) - self.download_url_data = data - self.download_url_data = self.download_url_data.drop_duplicates() - print("Created Image Urls") - print(f"Number of Images : {len(data)}") - - return True - except Exception as e: - raise ValueError(f""" - The function - self.create_image_urls() - or - Download().create_image_urls() - is not working correctly. - \n - {e}""") - - # Downloading data section - - def download_images(self) -> pd.DataFrame: - """summary_line - Keyword arguments: - argument download images - Return: pd.DataFrame - """ - try: - - print("Downloading Images") - new_data = { - "Path": [], - "XMin": [], - "YMin": [], - "XMax": [], - "YMax": [], - "ImageID": [], - "Url": [], - } - image_id_iter = tqdm((self.download_url_data["ImageID"])) - old_ourl = "" - for img_url, xmin, ymin, xmax, ymax, ourl in zip( - image_id_iter, - self.download_url_data["XMin"], - self.download_url_data["YMin"], - self.download_url_data["XMax"], - self.download_url_data["YMax"], - self.download_url_data["OriginalURL"], - ): - try: - if ourl != old_ourl: - old_ourl = ourl - image_id_iter.set_description(f"{self.idx}") - self.idx += 1 - new_data["Path"].append(f"{self.idx}.png") - new_data["XMin"].append(xmin) - new_data["YMin"].append(ymin) - new_data["XMax"].append(xmax) - new_data["YMax"].append(ymax) - new_data["Url"].append(ourl) - new_data["ImageID"].append(img_url) - except Exception as e: - print(e) - data = pd.DataFrame(new_data) - data.to_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv", - index=False, - ) - data.to_json( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.json", - ) - print("Downloaded Images") - return new_data - except Exception as e: - raise ValueError(f""" - The function - self.download_images() - or - Download().download_images() - is not working correctly. - \n - {e}""") - - def download(self) -> bool: - """summary_line - Keyword arguments: - argument: This is the funtion which uses - all of the funtions of this class - and combines it and download and - does all of the work. - Return: bool - """ - try: - - print("Intializing Downloading Proccess") - self.create_imageids() - self.create_bbox() - self.create_image_urls() - self.download_images() - - print("Finished Download Process") - except Exception as e: - raise ValueError(f""" - The function self.download() - or - Download().download() - is not working correctly. - \n - {e}""") diff --git a/.history/ML/Model/modelling/detectron2_20220429065528.py b/.history/ML/Model/modelling/detectron2_20220429065528.py deleted file mode 100644 index 6191bdf4..00000000 --- a/.history/ML/Model/modelling/detectron2_20220429065528.py +++ /dev/null @@ -1,533 +0,0 @@ -from Model import * -from Model.metrics import Metrics -from PIL import Image - -models = [ - "fast_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "retinanet_R_50_FPN_1x.py", - "retinanet_R_50_FPN_1x.yaml", - "retinanet_R_50_FPN_3x.yaml", - "rpn_R_50_C4_1x.yaml", - "rpn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", -] -max_iters = [50, 100, 125, 250, 500, 1000, 2000, 2500, 5000] -base_lrs = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001] -ims_per_batchs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -batch_size_per_images = [8, 16, 32, 64, 128, 256, 512] - - -class Detectron2: - """This class helps anyone to train a detectron2 model for this project easily so anyone can train this model.""" - - def __init__( - self, - base_lr: float = 0.00025, - labels: list = None, - max_iter: int = 250, - eval_period: int = 250, - ims_per_batch: int = 2, - batch_size_per_image: int = 128, - score_thresh_test: float = 0.625, - model: str = "COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml", - name: str = "baseline", - create_target_and_preds: int = 29, - test_sample_size=625, - data=pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv" - ), - ) -> None: - """ - - __init__ = initialize and get all of the params need - ------------------------------------------------------- - - base_lr = the base learning rate of the model which will allow the optimizer optimze better - - data = the data to create the dataset in detectron2 data format - - the data will be saved in ./data.npy file - - labels = labels of the dataset - - max_iter = no. of epochs or how many times the model needs to go through the data - - eval_period = step by step amount of iters that the model will be tested - - ims_per_batch = Number of Images that is in a Batch - - batch_size_per_image = Batch size for every image - - score_thresh_test = how much sure does the model be to show the predictions - - model = the model from the detectron2 model_zoo - - name = name of the wandb log - - create_target_and_preds = testing image - """ - self.metrics = Metrics() - if labels is None: - labels = ["Find-Card"] - self.remove_files_in_output() - self.data = data - self.data_other = pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv" - ) - self.labels = labels - self.devices = ["cpu", "cuda"] - self.tests = { - "models": [ - "fast_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "retinanet_R_50_FPN_1x.py", - "retinanet_R_50_FPN_1x.yaml", - "retinanet_R_50_FPN_3x.yaml", - "rpn_R_50_C4_1x.yaml", - "rpn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", - ], - "max_iters": [50, 100, 125, 250, 500, 1000, 2000, 2500, 5000], - "base_lrs": [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001], - "ims_per_batchs": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "batch_size_per_images": [8, 16, 32, 64, 128, 256, 512], - } # Tests for Param Tunning - try: - DatasetCatalog.register( - "data", self.load_data) # Registering the training data - MetadataCatalog.get("data").set( - thing_classes=self.labels) # Adding the labels - self.metadata = MetadataCatalog.get("data") # Getting the metadata - DatasetCatalog.register( - "test", - lambda: self.load_data(test=True)) # Registering the test data - MetadataCatalog.get("test").set( - thing_classes=self.labels) # Adding the labels - self.metadata_test = MetadataCatalog.get( - "test") # Getting the metadata - except Exception as e: - self.metadata = MetadataCatalog.get("data") # Getting the metadata - self.metadata_test = MetadataCatalog.get( - "test") # Getting the metadata - self.BASE_LR = base_lr - self.MAX_ITER = max_iter - self.EVAL_PERIOD = eval_period - self.IMS_PER_BATCH = ims_per_batch - self.BATCH_SIZE_PER_IMAGE = batch_size_per_image - self.SCORE_THRESH_TEST = score_thresh_test - self.model = model - self.NAME = name - self.cfg = self.create_cfg() # Creating the model config - self.create_target_and_preds_iter = create_target_and_preds - self.test_sample_size = test_sample_size - self.config = ({ - "BASE_LR": self.BASE_LR, - "MAX_ITER": self.MAX_ITER, - "EVAL_PERIOD": self.EVAL_PERIOD, - "IMS_PER_BATCH": self.IMS_PER_BATCH, - "BATCH_SIZE_PER_IMAGE": self.BATCH_SIZE_PER_IMAGE, - "SCORE_THRESH_TEST": self.SCORE_THRESH_TEST, - "MODEL": self.model, - "NAME": self.NAME, - }, ) - self.remove_files_in_output() - - @staticmethod - def remove_files_in_output() -> None: - """- remove_files_in_output - remove all of the file in ./output/""" - files_to_remove = os.listdir( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/output/" - ) # Get the files in the directory - try: - files_to_remove.remove("test_coco_format.json") - except Exception as e: - print(e) - for file_to_remove in tqdm( - files_to_remove): # Iter over the files in the directory - os.remove( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/output/{file_to_remove}" - ) # Delete the iter file - - def test(self, data_idx: int = 50) -> list: - """ - - test - croping and creating a box around the img xmin,ymin, xmax, ymax - ----------------------------------------------------- - - data_idx - the data index which is needed to be visualized - """ - info = self.data_other.iloc[data_idx] # getting the info of the index - img = cv2.imread( - f'/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{info["Path"]}' - ) # reading the img - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + - info["Path"]).shape[: - 2] # getting the height and width of the image - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) # getting the xmin,ymin, xmax, ymax - xmin = round(xmin * width) # converting it to real xmin - xmax = round(xmax * width) # converting it to real xmax - ymin = round(ymin * height) # converting it to real ymin - ymax = round(ymax * height) # converting it to real ymax - # The above is needed becuase open images gives their datasets xmin,ymin,xmax,ymax in a different way - x = xmin - y = ymin - w = xmax - xmin - h = ymax - ymin - x, y, w, h = round(x), round(y), round(w), round(h) - roi = img[y:y + h, x:x + w] # crop the image - cv2.rectangle(img, (x, y), (x + w, y + h), (200, 0, 0), - 10) # draw box around the bbox - return [img, roi] - - def load_data(self, test: bool = False) -> list: - """ - - load_data - loading the data in the detectron2 data format - ------------------------------------- - - test - if the return is supposed to be a test sample or not - - Defalt = False and type = bool - """ - if test is True and "data.npy" in os.listdir("./Model/save/"): - self.data = np.load( - "./Model/save/data.npy", allow_pickle=True - ) # Loading already saved detectron2 format file - self.data = self.data[:self.test_sample_size] - return self.data - # if "data.npy" in os.listdir("./Model/save/"): - # self.data = np.load("./Model/save/data.npy", allow_pickle=True) - # return self.data - new_data = [] - pil_issues_idx = 0 - data_iter = tqdm(range(len(self.data))) - for idx in data_iter: # iter over the data - try: - data_iter.set_description(f"{pil_issues_idx}/{idx}") - record = {} - info = self.data.iloc[idx] - Image.open( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]).shape[:2] - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) - xmin = round(xmin * width) - xmax = round(xmax * width) - ymin = round(ymin * height) - ymax = round(ymax * height) - record["file_name"] = ( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - record["height"] = height - record["width"] = width - objs = [{ - "bbox": [xmin, ymin, xmax, ymax], - "bbox_mode": BoxMode.XYXY_ABS, - "category_id": 0, - }] - record["image_id"] = idx - record["annotations"] = objs - new_data.append(record) - except OSError: - pil_issues_idx = pil_issues_idx + 1 - print(len(new_data)) - if test is True: - return new_data[:self.test_sample_size] - return new_data - - def save(self, **kwargs: dict) -> None: - """ - - save - it save the object with the {name}-{wandb-name}.pt and .pth - ---------------------------------------------------- - - **kwargs - like Model().save(a="b") - """ - torch.cuda.empty_cache() - files_and_object = kwargs - for files_and_object_key, files_and_object_val in tqdm( - zip(files_and_object.keys(), files_and_object.values()) - ): # iterate over the file and object - torch.save( - files_and_object_val, - f"./Model/models/{files_and_object_key}-{self.NAME}.pt", - ) # Save the file in .pt - torch.save( - files_and_object_val, - f"./Model/models/{files_and_object_key}-{self.NAME}.pth", - ) # Save the file in .pth - torch.cuda.empty_cache() - - def create_cfg(self) -> CfgNode: - """ - - create_cfg - create the config of the model - - other params - https://github.com/facebookresearch/detectron2/blob/main/detectron2/config/defaults.py - """ - torch.cuda.empty_cache() - cfg = get_cfg() # Creating a new cfg - cfg.merge_from_file(model_zoo.get_config_file( - self.model)) # Add the model - cfg.DATASETS.TRAIN = ("data", ) # adding train DataSet - cfg.DATASETS.TEST = () - cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url( - self.model) # Adding the weights - cfg.SOLVER.MAX_ITER = self.MAX_ITER # Set Max iter - cfg.TEST.EVAL_PERIOD = self.EVAL_PERIOD # Set Eval Period - cfg.SOLVER.BASE_LR = self.BASE_LR # Set Base LR - cfg.SOLVER.STEPS = [] # Set Steps - cfg.SOLVER.IMS_PER_BATCH = self.IMS_PER_BATCH # Set IMS_PER_BATCH - cfg.MODEL.ROI_HEADS.NUM_CLASSES = len( - self.labels) # Set len(self.labels) - cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = ( - self.BATCH_SIZE_PER_IMAGE) # Set Batch_Size_Per_Image - cfg.OUTPUT_DIR = "./Model/output" - torch.cuda.empty_cache() - return cfg - - def __train(self, ) -> DefaultTrainer: - """ - - __train - trains the cfg - this is used by Model.train() this is kind of the under function - """ - torch.cuda.empty_cache() - trainer = DefaultTrainer(self.cfg) # Train the cfg (Config) - torch.cuda.empty_cache() - # Resume the model or load a new model - trainer.resume_or_load(resume=False) - torch.cuda.empty_cache() - trainer.train() # training the model - torch.cuda.empty_cache() - return trainer - - def create_predictor(self) -> DefaultPredictor: - """- create_predictor - create the predictor to predict images""" - torch.cuda.empty_cache() - self.cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = ( - self.SCORE_THRESH_TEST) # Setting SCORE_THRESH_TEST - self.cfg.MODEL.WEIGHTS = ( - "./Model/output/model_final.pth" # The saved weights of the model - ) - predictor = DefaultPredictor(self.cfg) # Creating predictor - torch.cuda.empty_cache() - return predictor - - # Detectron2 - - def evaluation_detectron2(self, predictor): - metrics_coco = self.create_coco_eval_detectron2(predictor) - metrics_file = self.metrics_file_to_dict_detectron2() - test_images = self.predict_test_images_detectron2(predictor) - ( - preds, - target, - x, - y, - w, - h, - xmin, - ymin, - xmax, - ymax, - height, - width, - ) = self.create_target_and_preds_detectron2(predictor) - metrics = self.metrics.metrics(preds, target) - return { - "Metrics": metrics, - "metrics_coco": metrics_coco, - "metrics_file": metrics_file, - "test_images": test_images, - } - - def create_coco_eval_detectron2(self, - predictor: DefaultPredictor, - metadata: str = "test") -> dict: - """ - - create_coco_eval_detectron2 - create COCO Evaluator and tests it - ------------------------------- - - predictor - to create the evaluator - """ - torch.cuda.empty_cache() - evaluator = COCOEvaluator( - metadata, output_dir="./Model/output/") # Create evaluator - val_loader = build_detection_test_loader( - self.cfg, metadata) # Create data loader - metrics = inference_on_dataset( - predictor.model, val_loader, - evaluator) # Test the data with the evaluator - torch.cuda.empty_cache() - return metrics - - @staticmethod - def metrics_file_to_dict_detectron2() -> list: - """- metrics_file_to_dict_detectron2 - in ./output/metrics.json it logs the metrics of the model""" - new_logs = [] - try: - logs = open("./Model/output/metrics.json", "r").read().split("\n") - for log in tqdm(range(len(logs))): # uterate over the logs - try: - res = ast.literal_eval( - logs[log] - ) # convert str ("{'test':'test'}") to dict ({"test":"test"}) - new_logs.append(res) - except: - pass - return new_logs - except: - return new_logs - - def predict_test_images_detectron2(self, - predictor: DefaultPredictor) -> list: - """- predict_test_images_detectron2 - predict test images""" - imgs = [] - torch.cuda.empty_cache() - for img in tqdm( - os.listdir( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/" - )[:5]): # iterate over the test images - v = Visualizer( - cv2.imread( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}" - )[:, :, ::-1], - metadata=self.metadata, - ) - v = v.draw_instance_predictions( - predictor( - cv2.imread( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}" - ))["instances"].to("cpu")) # Draw pred boxes - v = v.get_image()[:, :, ::-1] - plt.figure(figsize=(24, 12)) - plt.imshow(v) # plot the image - plt.savefig( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/preds/{img}" - ) - plt.close() - imgs.append([ - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}", - v, - ]) - torch.cuda.empty_cache() - return imgs - - def create_target_and_preds_detectron2( - self, predictor: DefaultPredictor) -> tuple: - """- create_target_and_preds_detectron2 - create the target and predictions""" - info = self.data.iloc[self.create_target_and_preds_iter] - img = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]).shape[:2] - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) - xmin = round(xmin * width) - xmax = round(xmax * width) - ymin = round(ymin * height) - ymax = round(ymax * height) - x = xmin - y = ymin - w = xmax - xmin - h = ymax - ymin - preds = predictor(img) - if (len(preds["instances"].__dict__["_fields"] - ["pred_boxes"].__dict__["tensor"]) <= 0): - print(preds["instances"].__dict__["_fields"] - ["pred_boxes"].__dict__["tensor"]) - preds["instances"].__dict__["_fields"]["pred_boxes"].__dict__[ - "tensor"] = torch.tensor([[1, 1, 1, 1]]) - target = torch.tensor([xmin, ymin, xmax, ymax]) - return (preds, target, x, y, w, h, xmin, ymin, xmax, ymax, height, - width) - - def train(self, PROJECT_NAME, param) -> dict: - """- train - trains the model""" - torch.cuda.empty_cache() - wandb.init( - project=PROJECT_NAME, - name=str(self.NAME), - sync_tensorboard=True, - config=param, - ) - trainer = self.__train() - predictor = self.create_predictor() - metrics = self.evaluation_detectron2(predictor) - wandb.log(metrics["metrics_coco"]) - for metric_file in metrics["metrics_file"]: - wandb.log(metric_file) - for test_img in metrics["test_images"]: - wandb.log({test_img[0]: wandb.Image(cv2.imread(test_img[0]))}) - wandb.log(metrics["Metrics"]) - try: - self.save( - metrics_coco=metrics["metrics_coco"], - metrics_file=metrics["metrics_file"], - metrics=metrics["Metrics"], - ) - except Exception as e: - print(e) - wandb.finish() - return { - "trainer": trainer, - "predictor": predictor, - "metrics_coco": metrics["metrics_coco"], - "metrics_file": metrics["metrics_file"], - # "test_images": metrics["test_images"], - "metrics": metrics["Metrics"], - } - - def __str__(self) -> str: - return f""" - BASE_LR={self.BASE_LR} - \n - MAX_ITER={self.MAX_ITER} - \n - EVAL_PERIOD={self.EVAL_PERIOD} - \n - IMS_PER_BATCH={self.IMS_PER_BATCH} - \n - BATCH_SIZE_PER_IMAGE={self.BATCH_SIZE_PER_IMAGE} - \n - SCORE_THRESH_TEST={self.SCORE_THRESH_TEST} - \n - MODEL={self.model} - \n - NAME={self.NAME} - \n - Detectron2 Model - """ - - def __repr__(self) -> str: - return f""" - BASE_LR={self.BASE_LR} - \n - MAX_ITER={self.MAX_ITER} - \n - EVAL_PERIOD={self.EVAL_PERIOD} - \n - IMS_PER_BATCH={self.IMS_PER_BATCH} - \n - BATCH_SIZE_PER_IMAGE={self.BATCH_SIZE_PER_IMAGE} - \n - SCORE_THRESH_TEST={self.SCORE_THRESH_TEST} - \n - MODEL={self.model} - \n - NAME={self.NAME} - \n - Detectron2 Model - """ diff --git a/.history/ML/Model/modelling/detectron2_20220509224846.py b/.history/ML/Model/modelling/detectron2_20220509224846.py deleted file mode 100644 index 622f1cd1..00000000 --- a/.history/ML/Model/modelling/detectron2_20220509224846.py +++ /dev/null @@ -1,530 +0,0 @@ -from Model import * -from Model.metrics import Metrics -from PIL import Image - -models = [ - "fast_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "retinanet_R_50_FPN_1x.py", - "retinanet_R_50_FPN_1x.yaml", - "retinanet_R_50_FPN_3x.yaml", - "rpn_R_50_C4_1x.yaml", - "rpn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", -] -max_iters = [50, 100, 125, 250, 500, 1000, 2000, 2500, 5000] -base_lrs = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001] -ims_per_batchs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -batch_size_per_images = [8, 16, 32, 64, 128, 256, 512] - - -class Detectron2: - """This class helps anyone to train a detectron2 model for this project easily so anyone can train this model.""" - - def __init__( - self, - base_lr: float = 0.00025, - labels: list = None, - max_iter: int = 250, - eval_period: int = 250, - ims_per_batch: int = 2, - batch_size_per_image: int = 128, - score_thresh_test: float = 0.625, - model: str = "COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml", - name: str = "baseline", - create_target_and_preds: int = 29, - test_sample_size=625, - data=pd.read_csv( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/save/Data.csv" - ), - ) -> None: - """ - - __init__ = initialize and get all of the params need - ------------------------------------------------------- - - base_lr = the base learning rate of the model which will allow the optimizer optimze better - - data = the data to create the dataset in detectron2 data format - - the data will be saved in ./data.npy file - - labels = labels of the dataset - - max_iter = no. of epochs or how many times the model needs to go through the data - - eval_period = step by step amount of iters that the model will be tested - - ims_per_batch = Number of Images that is in a Batch - - batch_size_per_image = Batch size for every image - - score_thresh_test = how much sure does the model be to show the predictions - - model = the model from the detectron2 model_zoo - - name = name of the wandb log - - create_target_and_preds = testing image - """ - self.metrics = Metrics() - if labels is None: - labels = ["Find-Card"] - self.remove_files_in_output() - self.data = data - self.labels = labels - self.devices = ["cpu", "cuda"] - self.tests = { - "models": [ - "fast_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "retinanet_R_50_FPN_1x.py", - "retinanet_R_50_FPN_1x.yaml", - "retinanet_R_50_FPN_3x.yaml", - "rpn_R_50_C4_1x.yaml", - "rpn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", - ], - "max_iters": [50, 100, 125, 250, 500, 1000, 2000, 2500, 5000], - "base_lrs": [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001], - "ims_per_batchs": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "batch_size_per_images": [8, 16, 32, 64, 128, 256, 512], - } # Tests for Param Tunning - try: - DatasetCatalog.register( - "data", self.load_data) # Registering the training data - MetadataCatalog.get("data").set( - thing_classes=self.labels) # Adding the labels - self.metadata = MetadataCatalog.get("data") # Getting the metadata - DatasetCatalog.register( - "test", - lambda: self.load_data(test=True)) # Registering the test data - MetadataCatalog.get("test").set( - thing_classes=self.labels) # Adding the labels - self.metadata_test = MetadataCatalog.get( - "test") # Getting the metadata - except Exception as e: - self.metadata = MetadataCatalog.get("data") # Getting the metadata - self.metadata_test = MetadataCatalog.get( - "test") # Getting the metadata - self.BASE_LR = base_lr - self.MAX_ITER = max_iter - self.EVAL_PERIOD = eval_period - self.IMS_PER_BATCH = ims_per_batch - self.BATCH_SIZE_PER_IMAGE = batch_size_per_image - self.SCORE_THRESH_TEST = score_thresh_test - self.model = model - self.NAME = name - self.cfg = self.create_cfg() # Creating the model config - self.create_target_and_preds_iter = create_target_and_preds - self.test_sample_size = test_sample_size - self.config = ({ - "BASE_LR": self.BASE_LR, - "MAX_ITER": self.MAX_ITER, - "EVAL_PERIOD": self.EVAL_PERIOD, - "IMS_PER_BATCH": self.IMS_PER_BATCH, - "BATCH_SIZE_PER_IMAGE": self.BATCH_SIZE_PER_IMAGE, - "SCORE_THRESH_TEST": self.SCORE_THRESH_TEST, - "MODEL": self.model, - "NAME": self.NAME, - }, ) - self.remove_files_in_output() - - @staticmethod - def remove_files_in_output() -> None: - """- remove_files_in_output - remove all of the file in ./output/""" - files_to_remove = os.listdir( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/output/" - ) # Get the files in the directory - try: - files_to_remove.remove("test_coco_format.json") - except Exception as e: - print(e) - for file_to_remove in tqdm( - files_to_remove): # Iter over the files in the directory - os.remove( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/output/{file_to_remove}" - ) # Delete the iter file - - def test(self, data_idx: int = 50) -> list: - """ - - test - croping and creating a box around the img xmin,ymin, xmax, ymax - ----------------------------------------------------- - - data_idx - the data index which is needed to be visualized - """ - info = self.data_other.iloc[data_idx] # getting the info of the index - img = cv2.imread( - f'/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{info["Path"]}' - ) # reading the img - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + - info["Path"]).shape[: - 2] # getting the height and width of the image - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) # getting the xmin,ymin, xmax, ymax - xmin = round(xmin * width) # converting it to real xmin - xmax = round(xmax * width) # converting it to real xmax - ymin = round(ymin * height) # converting it to real ymin - ymax = round(ymax * height) # converting it to real ymax - # The above is needed becuase open images gives their datasets xmin,ymin,xmax,ymax in a different way - x = xmin - y = ymin - w = xmax - xmin - h = ymax - ymin - x, y, w, h = round(x), round(y), round(w), round(h) - roi = img[y:y + h, x:x + w] # crop the image - cv2.rectangle(img, (x, y), (x + w, y + h), (200, 0, 0), - 10) # draw box around the bbox - return [img, roi] - - def load_data(self, test: bool = False) -> list: - """ - - load_data - loading the data in the detectron2 data format - ------------------------------------- - - test - if the return is supposed to be a test sample or not - - Defalt = False and type = bool - """ - if test is True and "data.npy" in os.listdir("./Model/save/"): - self.data = np.load( - "./Model/save/data.npy", allow_pickle=True - ) # Loading already saved detectron2 format file - self.data = self.data[:self.test_sample_size] - return self.data - # if "data.npy" in os.listdir("./Model/save/"): - # self.data = np.load("./Model/save/data.npy", allow_pickle=True) - # return self.data - new_data = [] - pil_issues_idx = 0 - data_iter = tqdm(range(len(self.data))) - for idx in data_iter: # iter over the data - try: - data_iter.set_description(f"{pil_issues_idx}/{idx}") - record = {} - info = self.data.iloc[idx] - Image.open( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]).shape[:2] - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) - xmin = round(xmin * width) - xmax = round(xmax * width) - ymin = round(ymin * height) - ymax = round(ymax * height) - record["file_name"] = ( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - record["height"] = height - record["width"] = width - objs = [{ - "bbox": [xmin, ymin, xmax, ymax], - "bbox_mode": BoxMode.XYXY_ABS, - "category_id": 0, - }] - record["image_id"] = idx - record["annotations"] = objs - new_data.append(record) - except OSError: - pil_issues_idx = pil_issues_idx + 1 - print(len(new_data)) - if test is True: - return new_data[:self.test_sample_size] - return new_data - - def save(self, **kwargs: dict) -> None: - """ - - save - it save the object with the {name}-{wandb-name}.pt and .pth - ---------------------------------------------------- - - **kwargs - like Model().save(a="b") - """ - torch.cuda.empty_cache() - files_and_object = kwargs - for files_and_object_key, files_and_object_val in tqdm( - zip(files_and_object.keys(), files_and_object.values()) - ): # iterate over the file and object - torch.save( - files_and_object_val, - f"./Model/models/{files_and_object_key}-{self.NAME}.pt", - ) # Save the file in .pt - torch.save( - files_and_object_val, - f"./Model/models/{files_and_object_key}-{self.NAME}.pth", - ) # Save the file in .pth - torch.cuda.empty_cache() - - def create_cfg(self) -> CfgNode: - """ - - create_cfg - create the config of the model - - other params - https://github.com/facebookresearch/detectron2/blob/main/detectron2/config/defaults.py - """ - torch.cuda.empty_cache() - cfg = get_cfg() # Creating a new cfg - cfg.merge_from_file(model_zoo.get_config_file( - self.model)) # Add the model - cfg.DATASETS.TRAIN = ("data", ) # adding train DataSet - cfg.DATASETS.TEST = () - cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url( - self.model) # Adding the weights - cfg.SOLVER.MAX_ITER = self.MAX_ITER # Set Max iter - cfg.TEST.EVAL_PERIOD = self.EVAL_PERIOD # Set Eval Period - cfg.SOLVER.BASE_LR = self.BASE_LR # Set Base LR - cfg.SOLVER.STEPS = [] # Set Steps - cfg.SOLVER.IMS_PER_BATCH = self.IMS_PER_BATCH # Set IMS_PER_BATCH - cfg.MODEL.ROI_HEADS.NUM_CLASSES = len( - self.labels) # Set len(self.labels) - cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = ( - self.BATCH_SIZE_PER_IMAGE) # Set Batch_Size_Per_Image - cfg.OUTPUT_DIR = "./Model/output" - torch.cuda.empty_cache() - return cfg - - def __train(self, ) -> DefaultTrainer: - """ - - __train - trains the cfg - this is used by Model.train() this is kind of the under function - """ - torch.cuda.empty_cache() - trainer = DefaultTrainer(self.cfg) # Train the cfg (Config) - torch.cuda.empty_cache() - # Resume the model or load a new model - trainer.resume_or_load(resume=False) - torch.cuda.empty_cache() - trainer.train() # training the model - torch.cuda.empty_cache() - return trainer - - def create_predictor(self) -> DefaultPredictor: - """- create_predictor - create the predictor to predict images""" - torch.cuda.empty_cache() - self.cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = ( - self.SCORE_THRESH_TEST) # Setting SCORE_THRESH_TEST - self.cfg.MODEL.WEIGHTS = ( - "./Model/output/model_final.pth" # The saved weights of the model - ) - predictor = DefaultPredictor(self.cfg) # Creating predictor - torch.cuda.empty_cache() - return predictor - - # Detectron2 - - def evaluation_detectron2(self, predictor): - metrics_coco = self.create_coco_eval_detectron2(predictor) - metrics_file = self.metrics_file_to_dict_detectron2() - test_images = self.predict_test_images_detectron2(predictor) - ( - preds, - target, - x, - y, - w, - h, - xmin, - ymin, - xmax, - ymax, - height, - width, - ) = self.create_target_and_preds_detectron2(predictor) - metrics = self.metrics.metrics(preds, target) - return { - "Metrics": metrics, - "metrics_coco": metrics_coco, - "metrics_file": metrics_file, - "test_images": test_images, - } - - def create_coco_eval_detectron2(self, - predictor: DefaultPredictor, - metadata: str = "test") -> dict: - """ - - create_coco_eval_detectron2 - create COCO Evaluator and tests it - ------------------------------- - - predictor - to create the evaluator - """ - torch.cuda.empty_cache() - evaluator = COCOEvaluator( - metadata, output_dir="./Model/output/") # Create evaluator - val_loader = build_detection_test_loader( - self.cfg, metadata) # Create data loader - metrics = inference_on_dataset( - predictor.model, val_loader, - evaluator) # Test the data with the evaluator - torch.cuda.empty_cache() - return metrics - - @staticmethod - def metrics_file_to_dict_detectron2() -> list: - """- metrics_file_to_dict_detectron2 - in ./output/metrics.json it logs the metrics of the model""" - new_logs = [] - try: - logs = open("./Model/output/metrics.json", "r").read().split("\n") - for log in tqdm(range(len(logs))): # uterate over the logs - try: - res = ast.literal_eval( - logs[log] - ) # convert str ("{'test':'test'}") to dict ({"test":"test"}) - new_logs.append(res) - except: - pass - return new_logs - except: - return new_logs - - def predict_test_images_detectron2(self, - predictor: DefaultPredictor) -> list: - """- predict_test_images_detectron2 - predict test images""" - imgs = [] - torch.cuda.empty_cache() - for img in tqdm( - os.listdir( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/" - )[:5]): # iterate over the test images - v = Visualizer( - cv2.imread( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}" - )[:, :, ::-1], - metadata=self.metadata, - ) - v = v.draw_instance_predictions( - predictor( - cv2.imread( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}" - ))["instances"].to("cpu")) # Draw pred boxes - v = v.get_image()[:, :, ::-1] - plt.figure(figsize=(24, 12)) - plt.imshow(v) # plot the image - plt.savefig( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/preds/{img}" - ) - plt.close() - imgs.append([ - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}", - v, - ]) - torch.cuda.empty_cache() - return imgs - - def create_target_and_preds_detectron2( - self, predictor: DefaultPredictor) -> tuple: - """- create_target_and_preds_detectron2 - create the target and predictions""" - info = self.data.iloc[self.create_target_and_preds_iter] - img = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]).shape[:2] - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) - xmin = round(xmin * width) - xmax = round(xmax * width) - ymin = round(ymin * height) - ymax = round(ymax * height) - x = xmin - y = ymin - w = xmax - xmin - h = ymax - ymin - preds = predictor(img) - if (len(preds["instances"].__dict__["_fields"] - ["pred_boxes"].__dict__["tensor"]) <= 0): - print(preds["instances"].__dict__["_fields"] - ["pred_boxes"].__dict__["tensor"]) - preds["instances"].__dict__["_fields"]["pred_boxes"].__dict__[ - "tensor"] = torch.tensor([[1, 1, 1, 1]]) - target = torch.tensor([xmin, ymin, xmax, ymax]) - return (preds, target, x, y, w, h, xmin, ymin, xmax, ymax, height, - width) - - def train(self, PROJECT_NAME, param) -> dict: - """- train - trains the model""" - torch.cuda.empty_cache() - wandb.init( - project=PROJECT_NAME, - name=str(self.NAME), - sync_tensorboard=True, - config=param, - ) - trainer = self.__train() - predictor = self.create_predictor() - metrics = self.evaluation_detectron2(predictor) - wandb.log(metrics["metrics_coco"]) - for metric_file in metrics["metrics_file"]: - wandb.log(metric_file) - for test_img in metrics["test_images"]: - wandb.log({test_img[0]: wandb.Image(cv2.imread(test_img[0]))}) - wandb.log(metrics["Metrics"]) - try: - self.save( - metrics_coco=metrics["metrics_coco"], - metrics_file=metrics["metrics_file"], - metrics=metrics["Metrics"], - ) - except Exception as e: - print(e) - wandb.finish() - return { - "trainer": trainer, - "predictor": predictor, - "metrics_coco": metrics["metrics_coco"], - "metrics_file": metrics["metrics_file"], - # "test_images": metrics["test_images"], - "metrics": metrics["Metrics"], - } - - def __str__(self) -> str: - return f""" - BASE_LR={self.BASE_LR} - \n - MAX_ITER={self.MAX_ITER} - \n - EVAL_PERIOD={self.EVAL_PERIOD} - \n - IMS_PER_BATCH={self.IMS_PER_BATCH} - \n - BATCH_SIZE_PER_IMAGE={self.BATCH_SIZE_PER_IMAGE} - \n - SCORE_THRESH_TEST={self.SCORE_THRESH_TEST} - \n - MODEL={self.model} - \n - NAME={self.NAME} - \n - Detectron2 Model - """ - - def __repr__(self) -> str: - return f""" - BASE_LR={self.BASE_LR} - \n - MAX_ITER={self.MAX_ITER} - \n - EVAL_PERIOD={self.EVAL_PERIOD} - \n - IMS_PER_BATCH={self.IMS_PER_BATCH} - \n - BATCH_SIZE_PER_IMAGE={self.BATCH_SIZE_PER_IMAGE} - \n - SCORE_THRESH_TEST={self.SCORE_THRESH_TEST} - \n - MODEL={self.model} - \n - NAME={self.NAME} - \n - Detectron2 Model - """ diff --git a/.history/ML/Model/modelling/detectron2_20220509224849.py b/.history/ML/Model/modelling/detectron2_20220509224849.py deleted file mode 100644 index 5d309153..00000000 --- a/.history/ML/Model/modelling/detectron2_20220509224849.py +++ /dev/null @@ -1,527 +0,0 @@ -from Model import * -from Model.metrics import Metrics -from PIL import Image - -models = [ - "fast_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "retinanet_R_50_FPN_1x.py", - "retinanet_R_50_FPN_1x.yaml", - "retinanet_R_50_FPN_3x.yaml", - "rpn_R_50_C4_1x.yaml", - "rpn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", -] -max_iters = [50, 100, 125, 250, 500, 1000, 2000, 2500, 5000] -base_lrs = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001] -ims_per_batchs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -batch_size_per_images = [8, 16, 32, 64, 128, 256, 512] - - -class Detectron2: - """This class helps anyone to train a detectron2 model for this project easily so anyone can train this model.""" - - def __init__( - self, - base_lr: float = 0.00025, - labels: list = None, - max_iter: int = 250, - eval_period: int = 250, - ims_per_batch: int = 2, - batch_size_per_image: int = 128, - score_thresh_test: float = 0.625, - model: str = "COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml", - name: str = "baseline", - create_target_and_preds: int = 29, - test_sample_size=625, - ) -> None: - """ - - __init__ = initialize and get all of the params need - ------------------------------------------------------- - - base_lr = the base learning rate of the model which will allow the optimizer optimze better - - data = the data to create the dataset in detectron2 data format - - the data will be saved in ./data.npy file - - labels = labels of the dataset - - max_iter = no. of epochs or how many times the model needs to go through the data - - eval_period = step by step amount of iters that the model will be tested - - ims_per_batch = Number of Images that is in a Batch - - batch_size_per_image = Batch size for every image - - score_thresh_test = how much sure does the model be to show the predictions - - model = the model from the detectron2 model_zoo - - name = name of the wandb log - - create_target_and_preds = testing image - """ - self.metrics = Metrics() - if labels is None: - labels = ["Find-Card"] - self.remove_files_in_output() - self.data = data - self.labels = labels - self.devices = ["cpu", "cuda"] - self.tests = { - "models": [ - "fast_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "retinanet_R_50_FPN_1x.py", - "retinanet_R_50_FPN_1x.yaml", - "retinanet_R_50_FPN_3x.yaml", - "rpn_R_50_C4_1x.yaml", - "rpn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", - ], - "max_iters": [50, 100, 125, 250, 500, 1000, 2000, 2500, 5000], - "base_lrs": [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001], - "ims_per_batchs": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "batch_size_per_images": [8, 16, 32, 64, 128, 256, 512], - } # Tests for Param Tunning - try: - DatasetCatalog.register( - "data", self.load_data) # Registering the training data - MetadataCatalog.get("data").set( - thing_classes=self.labels) # Adding the labels - self.metadata = MetadataCatalog.get("data") # Getting the metadata - DatasetCatalog.register( - "test", - lambda: self.load_data(test=True)) # Registering the test data - MetadataCatalog.get("test").set( - thing_classes=self.labels) # Adding the labels - self.metadata_test = MetadataCatalog.get( - "test") # Getting the metadata - except Exception as e: - self.metadata = MetadataCatalog.get("data") # Getting the metadata - self.metadata_test = MetadataCatalog.get( - "test") # Getting the metadata - self.BASE_LR = base_lr - self.MAX_ITER = max_iter - self.EVAL_PERIOD = eval_period - self.IMS_PER_BATCH = ims_per_batch - self.BATCH_SIZE_PER_IMAGE = batch_size_per_image - self.SCORE_THRESH_TEST = score_thresh_test - self.model = model - self.NAME = name - self.cfg = self.create_cfg() # Creating the model config - self.create_target_and_preds_iter = create_target_and_preds - self.test_sample_size = test_sample_size - self.config = ({ - "BASE_LR": self.BASE_LR, - "MAX_ITER": self.MAX_ITER, - "EVAL_PERIOD": self.EVAL_PERIOD, - "IMS_PER_BATCH": self.IMS_PER_BATCH, - "BATCH_SIZE_PER_IMAGE": self.BATCH_SIZE_PER_IMAGE, - "SCORE_THRESH_TEST": self.SCORE_THRESH_TEST, - "MODEL": self.model, - "NAME": self.NAME, - }, ) - self.remove_files_in_output() - - @staticmethod - def remove_files_in_output() -> None: - """- remove_files_in_output - remove all of the file in ./output/""" - files_to_remove = os.listdir( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/output/" - ) # Get the files in the directory - try: - files_to_remove.remove("test_coco_format.json") - except Exception as e: - print(e) - for file_to_remove in tqdm( - files_to_remove): # Iter over the files in the directory - os.remove( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/output/{file_to_remove}" - ) # Delete the iter file - - def test(self, data_idx: int = 50) -> list: - """ - - test - croping and creating a box around the img xmin,ymin, xmax, ymax - ----------------------------------------------------- - - data_idx - the data index which is needed to be visualized - """ - info = self.data_other.iloc[data_idx] # getting the info of the index - img = cv2.imread( - f'/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/{info["Path"]}' - ) # reading the img - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + - info["Path"]).shape[: - 2] # getting the height and width of the image - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) # getting the xmin,ymin, xmax, ymax - xmin = round(xmin * width) # converting it to real xmin - xmax = round(xmax * width) # converting it to real xmax - ymin = round(ymin * height) # converting it to real ymin - ymax = round(ymax * height) # converting it to real ymax - # The above is needed becuase open images gives their datasets xmin,ymin,xmax,ymax in a different way - x = xmin - y = ymin - w = xmax - xmin - h = ymax - ymin - x, y, w, h = round(x), round(y), round(w), round(h) - roi = img[y:y + h, x:x + w] # crop the image - cv2.rectangle(img, (x, y), (x + w, y + h), (200, 0, 0), - 10) # draw box around the bbox - return [img, roi] - - def load_data(self, test: bool = False) -> list: - """ - - load_data - loading the data in the detectron2 data format - ------------------------------------- - - test - if the return is supposed to be a test sample or not - - Defalt = False and type = bool - """ - if test is True and "data.npy" in os.listdir("./Model/save/"): - self.data = np.load( - "./Model/save/data.npy", allow_pickle=True - ) # Loading already saved detectron2 format file - self.data = self.data[:self.test_sample_size] - return self.data - # if "data.npy" in os.listdir("./Model/save/"): - # self.data = np.load("./Model/save/data.npy", allow_pickle=True) - # return self.data - new_data = [] - pil_issues_idx = 0 - data_iter = tqdm(range(len(self.data))) - for idx in data_iter: # iter over the data - try: - data_iter.set_description(f"{pil_issues_idx}/{idx}") - record = {} - info = self.data.iloc[idx] - Image.open( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]).shape[:2] - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) - xmin = round(xmin * width) - xmax = round(xmax * width) - ymin = round(ymin * height) - ymax = round(ymax * height) - record["file_name"] = ( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - record["height"] = height - record["width"] = width - objs = [{ - "bbox": [xmin, ymin, xmax, ymax], - "bbox_mode": BoxMode.XYXY_ABS, - "category_id": 0, - }] - record["image_id"] = idx - record["annotations"] = objs - new_data.append(record) - except OSError: - pil_issues_idx = pil_issues_idx + 1 - print(len(new_data)) - if test is True: - return new_data[:self.test_sample_size] - return new_data - - def save(self, **kwargs: dict) -> None: - """ - - save - it save the object with the {name}-{wandb-name}.pt and .pth - ---------------------------------------------------- - - **kwargs - like Model().save(a="b") - """ - torch.cuda.empty_cache() - files_and_object = kwargs - for files_and_object_key, files_and_object_val in tqdm( - zip(files_and_object.keys(), files_and_object.values()) - ): # iterate over the file and object - torch.save( - files_and_object_val, - f"./Model/models/{files_and_object_key}-{self.NAME}.pt", - ) # Save the file in .pt - torch.save( - files_and_object_val, - f"./Model/models/{files_and_object_key}-{self.NAME}.pth", - ) # Save the file in .pth - torch.cuda.empty_cache() - - def create_cfg(self) -> CfgNode: - """ - - create_cfg - create the config of the model - - other params - https://github.com/facebookresearch/detectron2/blob/main/detectron2/config/defaults.py - """ - torch.cuda.empty_cache() - cfg = get_cfg() # Creating a new cfg - cfg.merge_from_file(model_zoo.get_config_file( - self.model)) # Add the model - cfg.DATASETS.TRAIN = ("data", ) # adding train DataSet - cfg.DATASETS.TEST = () - cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url( - self.model) # Adding the weights - cfg.SOLVER.MAX_ITER = self.MAX_ITER # Set Max iter - cfg.TEST.EVAL_PERIOD = self.EVAL_PERIOD # Set Eval Period - cfg.SOLVER.BASE_LR = self.BASE_LR # Set Base LR - cfg.SOLVER.STEPS = [] # Set Steps - cfg.SOLVER.IMS_PER_BATCH = self.IMS_PER_BATCH # Set IMS_PER_BATCH - cfg.MODEL.ROI_HEADS.NUM_CLASSES = len( - self.labels) # Set len(self.labels) - cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = ( - self.BATCH_SIZE_PER_IMAGE) # Set Batch_Size_Per_Image - cfg.OUTPUT_DIR = "./Model/output" - torch.cuda.empty_cache() - return cfg - - def __train(self, ) -> DefaultTrainer: - """ - - __train - trains the cfg - this is used by Model.train() this is kind of the under function - """ - torch.cuda.empty_cache() - trainer = DefaultTrainer(self.cfg) # Train the cfg (Config) - torch.cuda.empty_cache() - # Resume the model or load a new model - trainer.resume_or_load(resume=False) - torch.cuda.empty_cache() - trainer.train() # training the model - torch.cuda.empty_cache() - return trainer - - def create_predictor(self) -> DefaultPredictor: - """- create_predictor - create the predictor to predict images""" - torch.cuda.empty_cache() - self.cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = ( - self.SCORE_THRESH_TEST) # Setting SCORE_THRESH_TEST - self.cfg.MODEL.WEIGHTS = ( - "./Model/output/model_final.pth" # The saved weights of the model - ) - predictor = DefaultPredictor(self.cfg) # Creating predictor - torch.cuda.empty_cache() - return predictor - - # Detectron2 - - def evaluation_detectron2(self, predictor): - metrics_coco = self.create_coco_eval_detectron2(predictor) - metrics_file = self.metrics_file_to_dict_detectron2() - test_images = self.predict_test_images_detectron2(predictor) - ( - preds, - target, - x, - y, - w, - h, - xmin, - ymin, - xmax, - ymax, - height, - width, - ) = self.create_target_and_preds_detectron2(predictor) - metrics = self.metrics.metrics(preds, target) - return { - "Metrics": metrics, - "metrics_coco": metrics_coco, - "metrics_file": metrics_file, - "test_images": test_images, - } - - def create_coco_eval_detectron2(self, - predictor: DefaultPredictor, - metadata: str = "test") -> dict: - """ - - create_coco_eval_detectron2 - create COCO Evaluator and tests it - ------------------------------- - - predictor - to create the evaluator - """ - torch.cuda.empty_cache() - evaluator = COCOEvaluator( - metadata, output_dir="./Model/output/") # Create evaluator - val_loader = build_detection_test_loader( - self.cfg, metadata) # Create data loader - metrics = inference_on_dataset( - predictor.model, val_loader, - evaluator) # Test the data with the evaluator - torch.cuda.empty_cache() - return metrics - - @staticmethod - def metrics_file_to_dict_detectron2() -> list: - """- metrics_file_to_dict_detectron2 - in ./output/metrics.json it logs the metrics of the model""" - new_logs = [] - try: - logs = open("./Model/output/metrics.json", "r").read().split("\n") - for log in tqdm(range(len(logs))): # uterate over the logs - try: - res = ast.literal_eval( - logs[log] - ) # convert str ("{'test':'test'}") to dict ({"test":"test"}) - new_logs.append(res) - except: - pass - return new_logs - except: - return new_logs - - def predict_test_images_detectron2(self, - predictor: DefaultPredictor) -> list: - """- predict_test_images_detectron2 - predict test images""" - imgs = [] - torch.cuda.empty_cache() - for img in tqdm( - os.listdir( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/" - )[:5]): # iterate over the test images - v = Visualizer( - cv2.imread( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}" - )[:, :, ::-1], - metadata=self.metadata, - ) - v = v.draw_instance_predictions( - predictor( - cv2.imread( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}" - ))["instances"].to("cpu")) # Draw pred boxes - v = v.get_image()[:, :, ::-1] - plt.figure(figsize=(24, 12)) - plt.imshow(v) # plot the image - plt.savefig( - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/preds/{img}" - ) - plt.close() - imgs.append([ - f"/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/test_images/{img}", - v, - ]) - torch.cuda.empty_cache() - return imgs - - def create_target_and_preds_detectron2( - self, predictor: DefaultPredictor) -> tuple: - """- create_target_and_preds_detectron2 - create the target and predictions""" - info = self.data.iloc[self.create_target_and_preds_iter] - img = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]) - height, width = cv2.imread( - "/media/indika/Sync/Programmer-RD-AI/Programming/Projects/Python/Rest-Api/Car-Object-Detection-REST-API/Find-Card/ML/Model/dataset/Img/" - + info["Path"]).shape[:2] - xmin, ymin, xmax, ymax = ( - info["XMin"], - info["YMin"], - info["XMax"], - info["YMax"], - ) - xmin = round(xmin * width) - xmax = round(xmax * width) - ymin = round(ymin * height) - ymax = round(ymax * height) - x = xmin - y = ymin - w = xmax - xmin - h = ymax - ymin - preds = predictor(img) - if (len(preds["instances"].__dict__["_fields"] - ["pred_boxes"].__dict__["tensor"]) <= 0): - print(preds["instances"].__dict__["_fields"] - ["pred_boxes"].__dict__["tensor"]) - preds["instances"].__dict__["_fields"]["pred_boxes"].__dict__[ - "tensor"] = torch.tensor([[1, 1, 1, 1]]) - target = torch.tensor([xmin, ymin, xmax, ymax]) - return (preds, target, x, y, w, h, xmin, ymin, xmax, ymax, height, - width) - - def train(self, PROJECT_NAME, param) -> dict: - """- train - trains the model""" - torch.cuda.empty_cache() - wandb.init( - project=PROJECT_NAME, - name=str(self.NAME), - sync_tensorboard=True, - config=param, - ) - trainer = self.__train() - predictor = self.create_predictor() - metrics = self.evaluation_detectron2(predictor) - wandb.log(metrics["metrics_coco"]) - for metric_file in metrics["metrics_file"]: - wandb.log(metric_file) - for test_img in metrics["test_images"]: - wandb.log({test_img[0]: wandb.Image(cv2.imread(test_img[0]))}) - wandb.log(metrics["Metrics"]) - try: - self.save( - metrics_coco=metrics["metrics_coco"], - metrics_file=metrics["metrics_file"], - metrics=metrics["Metrics"], - ) - except Exception as e: - print(e) - wandb.finish() - return { - "trainer": trainer, - "predictor": predictor, - "metrics_coco": metrics["metrics_coco"], - "metrics_file": metrics["metrics_file"], - # "test_images": metrics["test_images"], - "metrics": metrics["Metrics"], - } - - def __str__(self) -> str: - return f""" - BASE_LR={self.BASE_LR} - \n - MAX_ITER={self.MAX_ITER} - \n - EVAL_PERIOD={self.EVAL_PERIOD} - \n - IMS_PER_BATCH={self.IMS_PER_BATCH} - \n - BATCH_SIZE_PER_IMAGE={self.BATCH_SIZE_PER_IMAGE} - \n - SCORE_THRESH_TEST={self.SCORE_THRESH_TEST} - \n - MODEL={self.model} - \n - NAME={self.NAME} - \n - Detectron2 Model - """ - - def __repr__(self) -> str: - return f""" - BASE_LR={self.BASE_LR} - \n - MAX_ITER={self.MAX_ITER} - \n - EVAL_PERIOD={self.EVAL_PERIOD} - \n - IMS_PER_BATCH={self.IMS_PER_BATCH} - \n - BATCH_SIZE_PER_IMAGE={self.BATCH_SIZE_PER_IMAGE} - \n - SCORE_THRESH_TEST={self.SCORE_THRESH_TEST} - \n - MODEL={self.model} - \n - NAME={self.NAME} - \n - Detectron2 Model - """ diff --git a/.history/ML/run_20220313204243.py b/.history/ML/run_20220313204243.py deleted file mode 100644 index eeb9a582..00000000 --- a/.history/ML/run_20220313204243.py +++ /dev/null @@ -1,23 +0,0 @@ -from Model import * -from Model.param_tunning import * - -# Params -params = { - "models": [ - "faster_rcnn_R_50_C4_1x.yaml", - "faster_rcnn_R_50_C4_3x.yaml", - "faster_rcnn_R_50_DC5_1x.yaml", - "faster_rcnn_R_50_DC5_3x.yaml", - "faster_rcnn_R_50_FPN_1x.yaml", - "faster_rcnn_R_50_FPN_3x.yaml", - "faster_rcnn_R_101_DC5_3x.yaml", - "faster_rcnn_R_101_FPN_3x.yaml", - "faster_rcnn_X_101_32x8d_FPN_3x.yaml", - ], - "max_iters": [500], - "base_lrs": [0.0025, 0.00025, 0.000025, 0.0000025], - "batch_size_per_images": [16, 32, 64, 128, 256], - "ims_per_batchs": [1, 2, 5, 10], -} -pr = Param_Tunning() -pr.tune(params) diff --git a/.history/ML/run_20220509224632.py b/.history/ML/run_20220509224632.py deleted file mode 100644 index 8ebe2782..00000000 --- a/.history/ML/run_20220509224632.py +++ /dev/null @@ -1,2 +0,0 @@ -from Model import * -from Model.param_tunning import * diff --git a/.history/ML/run_20220509224639.py b/.history/ML/run_20220509224639.py deleted file mode 100644 index 2fb68e48..00000000 --- a/.history/ML/run_20220509224639.py +++ /dev/null @@ -1,5 +0,0 @@ -from Model import * -from Model.param_tunning import * - -d = Download() -d.download() diff --git a/.history/README_20220510101058.md b/.history/README_20220510101058.md deleted file mode 100644 index 0ba1b8c6..00000000 --- a/.history/README_20220510101058.md +++ /dev/null @@ -1,6 +0,0 @@ -I have used alot of Kaggle Datasets and I want to help the community also. So I hope this helps at least one person. -:) -This is data from the oldest facebooks stocks. :) -The data has been taken from https://finance.yahoo.com/ -I will update as frequently as possible. - diff --git a/.history/README_20220515114046.md b/.history/README_20220515114046.md deleted file mode 100644 index b3db496f..00000000 --- a/.history/README_20220515114046.md +++ /dev/null @@ -1,5 +0,0 @@ -I have used alot of Kaggle Datasets and I want to help the community also. So I hope this helps at least one person. -:) -The data has been taken from Open Images V6 -This is a subset of Open Images. -I will update as frequently as possible. diff --git a/.history/README_20220515114052.md b/.history/README_20220515114052.md deleted file mode 100644 index fa41fef2..00000000 --- a/.history/README_20220515114052.md +++ /dev/null @@ -1,6 +0,0 @@ -I have used alot of Kaggle Datasets and I want to help the community also. So I hope this helps at least one person. -:) -The data has been taken from Open Images V6 -Subset of Anaimls. -This is a subset of Open Images. -I will update as frequently as possible.