diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f9d74bc..655361a 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -28,7 +28,8 @@ jobs: WHATSAPP_BUSINESS_PHONE_NUMBER_ID: ${{ secrets.WHATSAPP_BUSINESS_PHONE_NUMBER_ID }} WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER: ${{ secrets.WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER }} WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK: ${{ secrets.WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK }} - + PYTHONPATH: src + container: python:3.10 services: postgres: diff --git a/main_file.py b/main_file.py deleted file mode 100644 index 26c5e01..0000000 --- a/main_file.py +++ /dev/null @@ -1,9 +0,0 @@ -import typer - -from agents.ansari import Ansari -from config import get_settings -from presenters.file_presenter import FilePresenter - -if __name__ == "__main__": - ansari = Ansari(get_settings()) - typer.run(FilePresenter(ansari).present) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..87d730c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "ansari-backend" +version = "0.1.0" +description = "Ansari is an AI assistant to enhance understanding and practice of Islam." +authors = [ + { name = "Ansari Project", email = "feedback@ansari.chat" } +] +requires-python = ">=3.8" + +[project.urls] +Homepage = "https://github.com/ansari-project/ansari-backend" +Documentation = "https://github.com/ansari-project/ansari-backend" +Source = "https://github.com/ansari-project/ansari-backend" +Tracker = "https://github.com/ansari-project/ansari-backend/issues" + +[tool.ruff] +line-length = 88 +lint.select = ["E", "F", "W"] +lint.ignore = ["E501"] \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..ba62aa3 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + asyncio: mark a test as asyncio \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ea849d5..67b366b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ bcrypt +build discord.py diskcache fastapi @@ -14,10 +15,13 @@ psycopg2-binary pydantic_settings pyislam pyjwt +pytest-asyncio rich sendgrid +setuptools tenacity tiktoken typer uvicorn +wheel zxcvbn \ No newline at end of file diff --git a/src/ansari/__init__.py b/src/ansari/__init__.py new file mode 100644 index 0000000..32fdc7d --- /dev/null +++ b/src/ansari/__init__.py @@ -0,0 +1,4 @@ +# This file marks the directory as a Python package. +from .config import Settings, get_settings + +__all__ = ["Settings", "get_settings"] diff --git a/src/ansari/agents/__init__.py b/src/ansari/agents/__init__.py new file mode 100644 index 0000000..a582aea --- /dev/null +++ b/src/ansari/agents/__init__.py @@ -0,0 +1,4 @@ +from .ansari import Ansari +from .ansari_workflow import AnsariWorkflow + +__all__ = ["Ansari", "AnsariWorkflow"] diff --git a/agents/ansari.py b/src/ansari/agents/ansari.py similarity index 97% rename from agents/ansari.py rename to src/ansari/agents/ansari.py index 6b30f1e..eb4ddfa 100644 --- a/agents/ansari.py +++ b/src/ansari/agents/ansari.py @@ -2,8 +2,6 @@ import json import logging import os -import re -import sys import time import traceback from datetime import date, datetime @@ -12,11 +10,11 @@ import litellm from langfuse.decorators import langfuse_context, observe -from config import get_settings -from tools.search_hadith import SearchHadith -from tools.search_vectara import SearchVectara -from tools.search_quran import SearchQuran -from util.prompt_mgr import PromptMgr +from ansari.config import get_settings +from ansari.tools.search_hadith import SearchHadith +from ansari.tools.search_vectara import SearchVectara +from ansari.tools.search_quran import SearchQuran +from ansari.util.prompt_mgr import PromptMgr logger = logging.getLogger(__name__ + ".Ansari") logging_level = get_settings().LOGGING_LEVEL.upper() @@ -49,7 +47,7 @@ def __init__(self, settings, message_logger=None, json_format=False): sm.get_tool_name(): sm, } self.model = settings.MODEL - self.pm = PromptMgr() + self.pm = PromptMgr(src_dir=settings.PROMPT_PATH) self.sys_msg = self.pm.bind(settings.SYSTEM_PROMPT_FILE_NAME).render() self.tools = [ x.get_tool_description() for x in self.tool_name_to_instance.values() diff --git a/agents/ansari_workflow.py b/src/ansari/agents/ansari_workflow.py similarity index 97% rename from agents/ansari_workflow.py rename to src/ansari/agents/ansari_workflow.py index a79662d..2bfc113 100644 --- a/agents/ansari_workflow.py +++ b/src/ansari/agents/ansari_workflow.py @@ -6,10 +6,10 @@ import litellm -from tools.search_hadith import SearchHadith -from tools.search_vectara import SearchVectara -from tools.search_quran import SearchQuran -from util.prompt_mgr import PromptMgr +from ansari.tools.search_hadith import SearchHadith +from ansari.tools.search_vectara import SearchVectara +from ansari.tools.search_quran import SearchQuran +from ansari.util.prompt_mgr import PromptMgr logger = logging.getLogger(__name__ + ".AnsariWorkflow") diff --git a/ansari_db.py b/src/ansari/ansari_db.py similarity index 99% rename from ansari_db.py rename to src/ansari/ansari_db.py index 9650b01..bfb8641 100644 --- a/ansari_db.py +++ b/src/ansari/ansari_db.py @@ -2,7 +2,7 @@ import logging from contextlib import contextmanager from datetime import datetime, timedelta, timezone - +from typing import Union import bcrypt import jwt import psycopg2 @@ -10,7 +10,7 @@ from fastapi import HTTPException, Request from jwt import ExpiredSignatureError, InvalidTokenError -from config import Settings, get_settings +from ansari.config import Settings, get_settings logger = logging.getLogger(__name__) logging_level = get_settings().LOGGING_LEVEL.upper() @@ -588,7 +588,7 @@ def store_quran_answer(self, surah: int, ayah: int, question: str, ansari_answer ) conn.commit() - def get_quran_answer(self, surah: int, ayah: int, question: str) -> str | None: + def get_quran_answer(self, surah: int, ayah: int, question: str) -> Union[str, None]: """ Retrieve the stored answer for a given surah, ayah, and question. @@ -612,7 +612,6 @@ def get_quran_answer(self, surah: int, ayah: int, question: str) -> str | None: """ cur.execute(select_cmd, (surah, ayah, question)) result = cur.fetchone() - if result: return result[0] else: diff --git a/agents/__init__.py b/src/ansari/app/__init__.py similarity index 100% rename from agents/__init__.py rename to src/ansari/app/__init__.py diff --git a/main_api.py b/src/ansari/app/main_api.py similarity index 97% rename from main_api.py rename to src/ansari/app/main_api.py index 8f62faf..3b5e951 100644 --- a/main_api.py +++ b/src/ansari/app/main_api.py @@ -1,5 +1,6 @@ import logging import os +from typing import Union import uuid import psycopg2 @@ -16,12 +17,12 @@ from sendgrid.helpers.mail import Mail from zxcvbn import zxcvbn -from agents.ansari import Ansari -from agents.ansari_workflow import AnsariWorkflow -from ansari_db import AnsariDB, MessageLogger -from config import Settings, get_settings -from main_whatsapp import router as whatsapp_router -from presenters.api_presenter import ApiPresenter +from ansari.agents import Ansari +from ansari.agents import AnsariWorkflow +from ansari.ansari_db import AnsariDB, MessageLogger +from ansari.config import Settings, get_settings +from ansari.app.main_whatsapp import router as whatsapp_router +from ansari.presenters.api_presenter import ApiPresenter logger = logging.getLogger(__name__) logging_level = get_settings().LOGGING_LEVEL.upper() @@ -36,7 +37,6 @@ def main(): add_app_middleware() - def add_app_middleware(): app.add_middleware( CORSMiddleware, @@ -46,8 +46,7 @@ def add_app_middleware(): allow_headers=["*"], ) -main() - +main() db = AnsariDB(get_settings()) ansari = Ansari(get_settings()) @@ -689,7 +688,7 @@ class AyahQuestionRequest(BaseModel): surah: int ayah: int question: str - augment_question: bool | None = False + augment_question: Union[bool,None] = False apikey: str @app.post("/api/v2/ayah") @@ -703,6 +702,7 @@ async def answer_ayah_question( try: # Create AnsariWorkflow instance + logging.debug("Creating AnsariWorkflow instance for {req.surah}:{req.ayah}") ansari_workflow = AnsariWorkflow(settings) ayah_id = req.surah*1000 + req.ayah @@ -750,6 +750,6 @@ async def answer_ayah_question( db.store_quran_answer(req.surah, req.ayah, req.question, ansari_answer) return {"response": ansari_answer} - except Exception as e: - logger.error(f"Error in answer_ayah_question: {str(e)}") + except Exception: + logger.error("Error in answer_ayah_question", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") diff --git a/main_discord.py b/src/ansari/app/main_discord.py similarity index 79% rename from main_discord.py rename to src/ansari/app/main_discord.py index fd25faf..1b90744 100644 --- a/main_discord.py +++ b/src/ansari/app/main_discord.py @@ -1,5 +1,5 @@ -from agents.ansari import Ansari -from config import get_settings +from ansari.agents import Ansari +from ansari.config import get_settings from presenters.discord_presenter import DiscordPresenter # This work involves 3 agents, with Ansari as primary. diff --git a/src/ansari/app/main_file.py b/src/ansari/app/main_file.py new file mode 100644 index 0000000..6008630 --- /dev/null +++ b/src/ansari/app/main_file.py @@ -0,0 +1,9 @@ +import typer + +from ansari.agents import Ansari +from ansari.config import get_settings +from ansari.presenters.file_presenter import FilePresenter + +if __name__ == "__main__": + ansari = Ansari(get_settings()) + typer.run(FilePresenter(ansari).present) diff --git a/main_gradio.py b/src/ansari/app/main_gradio.py similarity index 60% rename from main_gradio.py rename to src/ansari/app/main_gradio.py index b36443b..01283c3 100644 --- a/main_gradio.py +++ b/src/ansari/app/main_gradio.py @@ -1,6 +1,6 @@ -from agents.ansari import Ansari -from config import get_settings -from presenters.gradio_presenter import GradioPresenter +from ansari.agents import Ansari +from ansari.config import get_settings +from ansari.presenters.gradio_presenter import GradioPresenter if __name__ == "__main__": agent = Ansari(get_settings()) diff --git a/main_stdio.py b/src/ansari/app/main_stdio.py similarity index 58% rename from main_stdio.py rename to src/ansari/app/main_stdio.py index 32053b6..e724559 100644 --- a/main_stdio.py +++ b/src/ansari/app/main_stdio.py @@ -1,8 +1,8 @@ import logging -from agents.ansari import Ansari -from config import get_settings -from presenters.stdio_presenter import StdioPresenter +from ansari.agents import Ansari +from ansari.config import get_settings +from ansari.presenters.stdio_presenter import StdioPresenter if __name__ == "__main__": logging.basicConfig(level=logging.INFO) diff --git a/main_whatsapp.py b/src/ansari/app/main_whatsapp.py similarity index 96% rename from main_whatsapp.py rename to src/ansari/app/main_whatsapp.py index e7e8ded..d93d44d 100644 --- a/main_whatsapp.py +++ b/src/ansari/app/main_whatsapp.py @@ -4,9 +4,9 @@ from fastapi import APIRouter, HTTPException, Request from fastapi.responses import HTMLResponse -from agents.ansari import Ansari -from config import get_settings -from presenters.whatsapp_presenter import WhatsAppPresenter +from ansari.agents import Ansari +from ansari.config import get_settings +from ansari.presenters.whatsapp_presenter import WhatsAppPresenter # Initialize logging logger = logging.getLogger(__name__) diff --git a/config.py b/src/ansari/config.py similarity index 89% rename from config.py rename to src/ansari/config.py index eb1426c..f27def1 100644 --- a/config.py +++ b/src/ansari/config.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from functools import lru_cache from typing import Literal, Optional, Union @@ -33,6 +34,15 @@ class Settings(BaseSettings): missing="ignore", ) + def get_resource_path(filename): + # Get the directory of the current script + script_dir = Path(__file__).resolve() + # Construct the path to the resources directory + resources_dir = script_dir.parent / 'resources' + # Construct the full path to the resource file + path = resources_dir / filename + return path + DATABASE_URL: PostgresDsn = Field( default="postgresql://postgres:password@localhost:5432/ansari" ) @@ -82,12 +92,12 @@ class Settings(BaseSettings): TAFSIR_FN_NAME: str = Field(default="search_tafsir") TAFSIR_FN_DESCRIPTION: str = Field( default=""" - Queries Tafsir Ibn Kathir (the renowned Qur'anic exegesis) for relevant - interpretations and explanations. You call this function when you need to - provide authoritative Qur'anic commentary and understanding based on Ibn - Kathir's work. Regardless of the language used in the original conversation, - you will translate the query into English before searching the tafsir. The - function returns a list of **potentially** relevant matches, which may include + Queries Tafsir Ibn Kathir (the renowned Qur'anic exegesis) for relevant + interpretations and explanations. You call this function when you need to + provide authoritative Qur'anic commentary and understanding based on Ibn + Kathir's work. Regardless of the language used in the original conversation, + you will translate the query into English before searching the tafsir. The + function returns a list of **potentially** relevant matches, which may include multiple passages of interpretation and analysis. """ ) @@ -114,13 +124,15 @@ class Settings(BaseSettings): WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER: Optional[SecretStr] = Field(default=None) WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK: Optional[SecretStr] = Field(default=None) - template_dir: DirectoryPath = Field(default="resources/templates") + template_dir: DirectoryPath = Field(default=get_resource_path("templates")) diskcache_dir: str = Field(default="diskcache_dir") MODEL: str = Field(default="gpt-4o") MAX_TOOL_TRIES: int = Field(default=3) MAX_FAILURES: int = Field(default=1) SYSTEM_PROMPT_FILE_NAME: str = Field(default="system_msg_tool") + PROMPT_PATH: str = Field(default=str(get_resource_path("prompts"))) + LOGGING_LEVEL: str = Field(default="INFO") diff --git a/presenters/api_presenter.py b/src/ansari/presenters/api_presenter.py similarity index 100% rename from presenters/api_presenter.py rename to src/ansari/presenters/api_presenter.py diff --git a/presenters/discord_presenter.py b/src/ansari/presenters/discord_presenter.py similarity index 100% rename from presenters/discord_presenter.py rename to src/ansari/presenters/discord_presenter.py diff --git a/presenters/file_presenter.py b/src/ansari/presenters/file_presenter.py similarity index 100% rename from presenters/file_presenter.py rename to src/ansari/presenters/file_presenter.py diff --git a/presenters/gradio_presenter.py b/src/ansari/presenters/gradio_presenter.py similarity index 100% rename from presenters/gradio_presenter.py rename to src/ansari/presenters/gradio_presenter.py diff --git a/presenters/stdio_presenter.py b/src/ansari/presenters/stdio_presenter.py similarity index 100% rename from presenters/stdio_presenter.py rename to src/ansari/presenters/stdio_presenter.py diff --git a/presenters/whatsapp_presenter.py b/src/ansari/presenters/whatsapp_presenter.py similarity index 99% rename from presenters/whatsapp_presenter.py rename to src/ansari/presenters/whatsapp_presenter.py index e5660d7..2a852a2 100644 --- a/presenters/whatsapp_presenter.py +++ b/src/ansari/presenters/whatsapp_presenter.py @@ -6,7 +6,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from config import get_settings +from ansari.config import get_settings # Initialize logging logger = logging.getLogger(__name__) diff --git a/resources/prompts/greeting.txt b/src/ansari/resources/prompts/greeting.txt similarity index 100% rename from resources/prompts/greeting.txt rename to src/ansari/resources/prompts/greeting.txt diff --git a/resources/prompts/news.txt b/src/ansari/resources/prompts/news.txt similarity index 100% rename from resources/prompts/news.txt rename to src/ansari/resources/prompts/news.txt diff --git a/resources/prompts/system_msg_tool.txt b/src/ansari/resources/prompts/system_msg_tool.txt similarity index 100% rename from resources/prompts/system_msg_tool.txt rename to src/ansari/resources/prompts/system_msg_tool.txt diff --git a/resources/structure_of_api_responses/meta_whatsapp_api_structure_of_a_reply_msg_status.json b/src/ansari/resources/structure_of_api_responses/meta_whatsapp_api_structure_of_a_reply_msg_status.json similarity index 100% rename from resources/structure_of_api_responses/meta_whatsapp_api_structure_of_a_reply_msg_status.json rename to src/ansari/resources/structure_of_api_responses/meta_whatsapp_api_structure_of_a_reply_msg_status.json diff --git a/resources/structure_of_api_responses/meta_whatsapp_structure_of_a_user_incoming_msg.json b/src/ansari/resources/structure_of_api_responses/meta_whatsapp_structure_of_a_user_incoming_msg.json similarity index 100% rename from resources/structure_of_api_responses/meta_whatsapp_structure_of_a_user_incoming_msg.json rename to src/ansari/resources/structure_of_api_responses/meta_whatsapp_structure_of_a_user_incoming_msg.json diff --git a/resources/templates/ask_question.txt b/src/ansari/resources/templates/ask_question.txt similarity index 100% rename from resources/templates/ask_question.txt rename to src/ansari/resources/templates/ask_question.txt diff --git a/resources/templates/password_reset.html b/src/ansari/resources/templates/password_reset.html similarity index 100% rename from resources/templates/password_reset.html rename to src/ansari/resources/templates/password_reset.html diff --git a/tools/__init__.py b/src/ansari/tools/__init__.py similarity index 100% rename from tools/__init__.py rename to src/ansari/tools/__init__.py diff --git a/tools/search_hadith.py b/src/ansari/tools/search_hadith.py similarity index 100% rename from tools/search_hadith.py rename to src/ansari/tools/search_hadith.py diff --git a/tools/search_mawsuah.py b/src/ansari/tools/search_mawsuah.py similarity index 100% rename from tools/search_mawsuah.py rename to src/ansari/tools/search_mawsuah.py diff --git a/tools/search_quran.py b/src/ansari/tools/search_quran.py similarity index 100% rename from tools/search_quran.py rename to src/ansari/tools/search_quran.py diff --git a/tools/search_vectara.py b/src/ansari/tools/search_vectara.py similarity index 99% rename from tools/search_vectara.py rename to src/ansari/tools/search_vectara.py index 846695e..1927b1b 100644 --- a/tools/search_vectara.py +++ b/src/ansari/tools/search_vectara.py @@ -1,4 +1,5 @@ import json +import logging import requests diff --git a/src/ansari/util/__init__.py b/src/ansari/util/__init__.py new file mode 100644 index 0000000..6a99ca3 --- /dev/null +++ b/src/ansari/util/__init__.py @@ -0,0 +1,4 @@ +# This file makes the 'util' directory a package. +from .prompt_mgr import PromptMgr + +__all__ = ["PromptMgr"] diff --git a/util/prompt_mgr.py b/src/ansari/util/prompt_mgr.py similarity index 63% rename from util/prompt_mgr.py rename to src/ansari/util/prompt_mgr.py index 6c37133..c1c1c87 100644 --- a/util/prompt_mgr.py +++ b/src/ansari/util/prompt_mgr.py @@ -1,6 +1,7 @@ from typing import Union from pydantic import BaseModel +from pathlib import Path class Prompt(BaseModel): @@ -16,7 +17,16 @@ def render(self, **kwargs) -> str: class PromptMgr: - def __init__(self, hot_reload: bool = True, src_dir: str = "resources/prompts"): + def get_resource_path(filename): + # Get the directory of the current script + script_dir = Path(__file__).resolve() + # Construct the path to the resources directory + resources_dir = script_dir.parent.parent / 'resources' + # Construct the full path to the resource file + path = resources_dir / filename + return path + + def __init__(self, hot_reload: bool = True, src_dir: str = str(get_resource_path("prompts"))): """Creates a prompt manager. Args: diff --git a/tests/test_answer_quality.py b/tests/test_answer_quality.py index 0ff969d..eb0564c 100644 --- a/tests/test_answer_quality.py +++ b/tests/test_answer_quality.py @@ -6,8 +6,8 @@ import pytest from jinja2 import Environment, FileSystemLoader -from agents.ansari import Ansari -from config import get_settings +from ansari.agents import Ansari +from ansari.config import get_settings LOGGER = logging.getLogger(__name__) logging_level = get_settings().LOGGING_LEVEL.upper() @@ -23,7 +23,7 @@ @pytest.fixture(scope="module") def data(): - tenv = Environment(loader=FileSystemLoader("resources/templates/")) + tenv = Environment(loader=FileSystemLoader("src/ansari/resources/templates/")) q_temp = tenv.get_template("ask_question.txt") df = pd.read_csv("tests/batik-v1-en.csv") cache = {} diff --git a/tests/test_main_api.py b/tests/test_main_api.py index 2316bbc..8836880 100644 --- a/tests/test_main_api.py +++ b/tests/test_main_api.py @@ -5,12 +5,16 @@ import pytest from fastapi.testclient import TestClient -from main_api import app -from config import get_settings -from ansari_db import AnsariDB +from ansari.app.main_api import app +from ansari.config import get_settings +from ansari.ansari_db import AnsariDB client = TestClient(app) +# Configure logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + # Test data valid_email_base = "test@example.com" valid_password = "StrongPassword123!"