From b49b4b360bb9f7a7216ed2303ed511b87d2bbfea Mon Sep 17 00:00:00 2001 From: "Darrell Malone Jr." Date: Sun, 3 Nov 2024 21:25:32 -0600 Subject: [PATCH] Update requirements and Dockerfile to fix issues (#414) with Docker build. - remove Pandas - remove psycopg2 - update typing-extensions for Pydantic --- backend/Dockerfile | 14 +++++++++++++- backend/database/__init__.py | 2 +- backend/database/core.py | 10 ++++------ requirements/_core.in | 3 +-- requirements/dev_unix.in | 4 +--- requirements/dev_unix.txt | 34 +++++----------------------------- requirements/dev_windows.in | 2 -- requirements/dev_windows.txt | 35 +++++------------------------------ requirements/prod.in | 2 -- requirements/prod.txt | 34 +++++----------------------------- 10 files changed, 35 insertions(+), 105 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 150f76026..b2400dc2b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,18 @@ FROM python:3-slim-bookworm AS base -RUN apt-get update && apt-get install curl -y && apt-get install g++ libpq-dev gcc python3-dev -y +# Install dependencies +RUN apt-get update -y && apt-get install -y \ + curl \ + g++ \ + libpq-dev \ + gcc \ + python3-dev \ + libffi-dev + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.12.1/wait /wait RUN chmod +x /wait diff --git a/backend/database/__init__.py b/backend/database/__init__.py index 13e8c71b8..e79d55865 100644 --- a/backend/database/__init__.py +++ b/backend/database/__init__.py @@ -1,5 +1,5 @@ # flake8: noqa: F401 -from backend.database.core import db_cli, execute_query +from backend.database.core import db_cli # Neo4j / NeoModel related imports from neomodel import config as neo_config diff --git a/backend/database/core.py b/backend/database/core.py index e108d1c5a..949d220b1 100644 --- a/backend/database/core.py +++ b/backend/database/core.py @@ -8,12 +8,12 @@ from typing import Optional import click -import pandas as pd from flask import current_app from flask.cli import AppGroup, with_appcontext from werkzeug.utils import secure_filename from neomodel import install_all_labels from neo4j import Driver +from typing import List, Dict, Any from ..utils import dev_only @@ -23,10 +23,10 @@ ) -def execute_query(filename: str) -> Optional[pd.DataFrame]: +def execute_query(filename: str) -> Optional[List[Dict[str, Any]]]: """Run a Cypher query from a file using the provided Neo4j connection. - It returns a Pandas DataFrame if the query yields results; otherwise, + It returns a list of dictionaries if the query yields results; otherwise, it returns None. """ # Read the query from the file @@ -45,9 +45,7 @@ def execute_query(filename: str) -> Optional[pd.DataFrame]: if records: # Convert Neo4j records to a list of dictionaries data = [record.data() for record in records] - # Create a DataFrame - df = pd.DataFrame(data) - return df + return data else: return None diff --git a/requirements/_core.in b/requirements/_core.in index c7e021148..90a2dfb12 100644 --- a/requirements/_core.in +++ b/requirements/_core.in @@ -5,9 +5,9 @@ celery flake8 flask gunicorn -pandas>=2.2.2 pip-tools pydantic==2.9.2 +typing-extensions>=4.12.2 # Required for pydantic pytest pytest-env pytest-cov @@ -32,6 +32,5 @@ jupyter mixpanel ua-parser ujson -testcontainers neo4j neomodel==5.3.3 diff --git a/requirements/dev_unix.in b/requirements/dev_unix.in index 14d65f94d..8608a8462 100644 --- a/requirements/dev_unix.in +++ b/requirements/dev_unix.in @@ -1,3 +1 @@ --r _core.in -psycopg2-binary -psycopg-binary \ No newline at end of file +-r _core.in \ No newline at end of file diff --git a/requirements/dev_unix.txt b/requirements/dev_unix.txt index d6acc6fd4..b7d0b6500 100644 --- a/requirements/dev_unix.txt +++ b/requirements/dev_unix.txt @@ -12,8 +12,6 @@ anyio==4.3.0 # via # httpx # jupyter-server -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -94,8 +92,6 @@ defusedxml==0.7.1 # via nbconvert dnspython==2.6.1 # via email-validator -docker==7.1.0 - # via testcontainers email-validator==2.1.1 # via -r requirements/_core.in et-xmlfile==1.1.0 @@ -280,9 +276,7 @@ notebook-shim==0.2.4 # jupyterlab # notebook numpy==1.26.4 - # via - # -r requirements/_core.in - # pandas + # via -r requirements/_core.in openpyxl==3.0.9 # via -r requirements/_core.in overrides==7.7.0 @@ -300,8 +294,6 @@ packaging==24.0 # pytest # qtconsole # qtpy -pandas==2.2.2 - # via -r requirements/_core.in pandocfilters==1.5.1 # via nbconvert parso==0.8.3 @@ -329,10 +321,6 @@ prompt-toolkit==3.0.43 # jupyter-console psutil==5.9.8 # via ipykernel -psycopg-binary==3.1.18 - # via -r requirements/dev_unix.in -psycopg2-binary==2.9.9 - # via -r requirements/dev_unix.in ptyprocess==0.7.0 # via # pexpect @@ -381,15 +369,12 @@ python-dateutil==2.9.0 # botocore # celery # jupyter-client - # pandas python-dotenv==1.0.1 # via -r requirements/_core.in python-json-logger==2.0.7 # via jupyter-events pytz==2024.1 - # via - # neo4j - # pandas + # via neo4j pyyaml==6.0.1 # via # -r requirements/_core.in @@ -413,7 +398,6 @@ referencing==0.34.0 requests==2.32.3 # via # -r requirements/_core.in - # docker # jupyterlab-server # mixpanel rfc3339-validator==0.1.4 @@ -455,8 +439,6 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.8.1 - # via -r requirements/_core.in tinycss2==1.2.1 # via nbconvert tornado==6.4 @@ -486,15 +468,13 @@ traitlets==5.14.2 # qtconsole types-python-dateutil==2.9.0.20240316 # via arrow -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via + # -r requirements/_core.in # pydantic # pydantic-core - # testcontainers tzdata==2024.1 - # via - # celery - # pandas + # via celery ua-parser==0.18.0 # via -r requirements/_core.in ujson==5.10.0 @@ -504,10 +484,8 @@ uri-template==1.3.0 urllib3==1.26.18 # via # botocore - # docker # mixpanel # requests - # testcontainers vine==5.1.0 # via # amqp @@ -531,8 +509,6 @@ wheel==0.43.0 # via pip-tools widgetsnbextension==4.0.10 # via ipywidgets -wrapt==1.16.0 - # via testcontainers wtforms==2.3.3 # via # -r requirements/_core.in diff --git a/requirements/dev_windows.in b/requirements/dev_windows.in index 09e46a2a9..89ea095e1 100644 --- a/requirements/dev_windows.in +++ b/requirements/dev_windows.in @@ -1,3 +1 @@ -r _core.in -psycopg2 -psycopg diff --git a/requirements/dev_windows.txt b/requirements/dev_windows.txt index e5a62aded..021b33dd5 100644 --- a/requirements/dev_windows.txt +++ b/requirements/dev_windows.txt @@ -12,8 +12,6 @@ anyio==4.3.0 # via # httpx # jupyter-server -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -94,8 +92,6 @@ defusedxml==0.7.1 # via nbconvert dnspython==2.6.1 # via email-validator -docker==7.1.0 - # via testcontainers email-validator==2.1.1 # via -r requirements/_core.in et-xmlfile==1.1.0 @@ -280,9 +276,7 @@ notebook-shim==0.2.4 # jupyterlab # notebook numpy==1.26.4 - # via - # -r requirements/_core.in - # pandas + # via -r requirements/_core.in openpyxl==3.0.9 # via -r requirements/_core.in overrides==7.7.0 @@ -300,8 +294,6 @@ packaging==24.0 # pytest # qtconsole # qtpy -pandas==2.2.2 - # via -r requirements/_core.in pandocfilters==1.5.1 # via nbconvert parso==0.8.3 @@ -329,10 +321,6 @@ prompt-toolkit==3.0.43 # jupyter-console psutil==5.9.8 # via ipykernel -psycopg==3.1.18 - # via -r requirements/dev_windows.in -psycopg2==2.9.9 - # via -r requirements/dev_windows.in ptyprocess==0.7.0 # via # pexpect @@ -381,15 +369,12 @@ python-dateutil==2.9.0 # botocore # celery # jupyter-client - # pandas python-dotenv==1.0.1 # via -r requirements/_core.in python-json-logger==2.0.7 # via jupyter-events pytz==2024.1 - # via - # neo4j - # pandas + # via neo4j pyyaml==6.0.1 # via # -r requirements/_core.in @@ -413,7 +398,6 @@ referencing==0.34.0 requests==2.32.3 # via # -r requirements/_core.in - # docker # jupyterlab-server # mixpanel rfc3339-validator==0.1.4 @@ -455,8 +439,6 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.8.1 - # via -r requirements/_core.in tinycss2==1.2.1 # via nbconvert tornado==6.4 @@ -486,16 +468,13 @@ traitlets==5.14.2 # qtconsole types-python-dateutil==2.9.0.20240316 # via arrow -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via - # psycopg + # -r requirements/_core.in # pydantic # pydantic-core - # testcontainers tzdata==2024.1 - # via - # celery - # pandas + # via celery ua-parser==0.18.0 # via -r requirements/_core.in ujson==5.10.0 @@ -505,10 +484,8 @@ uri-template==1.3.0 urllib3==1.26.18 # via # botocore - # docker # mixpanel # requests - # testcontainers vine==5.1.0 # via # amqp @@ -532,8 +509,6 @@ wheel==0.43.0 # via pip-tools widgetsnbextension==4.0.10 # via ipywidgets -wrapt==1.16.0 - # via testcontainers wtforms==2.3.3 # via # -r requirements/_core.in diff --git a/requirements/prod.in b/requirements/prod.in index 304ffd583..89ea095e1 100644 --- a/requirements/prod.in +++ b/requirements/prod.in @@ -1,3 +1 @@ -r _core.in -psycopg2-binary==2.9.9 -psycopg-binary==3.1.18 diff --git a/requirements/prod.txt b/requirements/prod.txt index c2b239800..83251fa64 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -12,8 +12,6 @@ anyio==4.3.0 # via # httpx # jupyter-server -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -94,8 +92,6 @@ defusedxml==0.7.1 # via nbconvert dnspython==2.6.1 # via email-validator -docker==7.1.0 - # via testcontainers email-validator==2.1.1 # via -r requirements/_core.in et-xmlfile==1.1.0 @@ -280,9 +276,7 @@ notebook-shim==0.2.4 # jupyterlab # notebook numpy==1.26.4 - # via - # -r requirements/_core.in - # pandas + # via -r requirements/_core.in openpyxl==3.0.9 # via -r requirements/_core.in overrides==7.7.0 @@ -300,8 +294,6 @@ packaging==24.0 # pytest # qtconsole # qtpy -pandas==2.2.2 - # via -r requirements/_core.in pandocfilters==1.5.1 # via nbconvert parso==0.8.3 @@ -329,10 +321,6 @@ prompt-toolkit==3.0.43 # jupyter-console psutil==5.9.8 # via ipykernel -psycopg-binary==3.1.18 - # via -r requirements/prod.in -psycopg2-binary==2.9.9 - # via -r requirements/prod.in ptyprocess==0.7.0 # via # pexpect @@ -381,15 +369,12 @@ python-dateutil==2.9.0 # botocore # celery # jupyter-client - # pandas python-dotenv==1.0.1 # via -r requirements/_core.in python-json-logger==2.0.7 # via jupyter-events pytz==2024.1 - # via - # neo4j - # pandas + # via neo4j pyyaml==6.0.1 # via # -r requirements/_core.in @@ -413,7 +398,6 @@ referencing==0.34.0 requests==2.32.3 # via # -r requirements/_core.in - # docker # jupyterlab-server # mixpanel rfc3339-validator==0.1.4 @@ -455,8 +439,6 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.8.1 - # via -r requirements/_core.in tinycss2==1.2.1 # via nbconvert tornado==6.4 @@ -486,15 +468,13 @@ traitlets==5.14.2 # qtconsole types-python-dateutil==2.9.0.20240316 # via arrow -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via + # -r requirements/_core.in # pydantic # pydantic-core - # testcontainers tzdata==2024.1 - # via - # celery - # pandas + # via celery ua-parser==0.18.0 # via -r requirements/_core.in ujson==5.10.0 @@ -504,10 +484,8 @@ uri-template==1.3.0 urllib3==1.26.18 # via # botocore - # docker # mixpanel # requests - # testcontainers vine==5.1.0 # via # amqp @@ -531,8 +509,6 @@ wheel==0.43.0 # via pip-tools widgetsnbextension==4.0.10 # via ipywidgets -wrapt==1.16.0 - # via testcontainers wtforms==2.3.3 # via # -r requirements/_core.in