From 2079972dbdfce9e584a085757c3b2928bb6f19d8 Mon Sep 17 00:00:00 2001 From: Boo Date: Thu, 18 Jul 2024 02:36:38 +0400 Subject: [PATCH] add balance cache --- README.md | 2 +- api/app/routes/balance.py | 5 +- api/app/services/balance.py | 32 ++++---- api/app/services/transaction.py | 23 +++++- api/tests/test_balance.py | 40 ---------- api/tests/test_balance_cache.py | 97 +++++++++++++++++++++++++ docker-compose.dev.yml | 4 + docker-compose.yml | 2 +- secrets.env | 2 - ui/app/templates/transaction/add.jinja2 | 2 + 10 files changed, 145 insertions(+), 64 deletions(-) create mode 100644 api/tests/test_balance_cache.py delete mode 100644 secrets.env diff --git a/README.md b/README.md index 1308e72..db8e061 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ pytest - [x] tags - [x] transactions - [x] balances -- [ ] balance cache +- [x] balance cache - [x] date range search - [ ] recurrent payments - [ ] donation categories diff --git a/api/app/routes/balance.py b/api/app/routes/balance.py index 29b3bab..88811ae 100644 --- a/api/app/routes/balance.py +++ b/api/app/routes/balance.py @@ -14,10 +14,7 @@ @balance_router.get("/{entity_id}", response_model=BalanceSchema) def get_balance( entity_id: int, - specific_date: datetime | None = None, balance_service: BalanceService = Depends(), actor_entity: Entity = Depends(get_entity_from_token), ): - return balance_service.get_balances( - entity_id=entity_id, specific_date=specific_date - ) + return balance_service.get_balances(entity_id=entity_id) diff --git a/api/app/services/balance.py b/api/app/services/balance.py index 0d7d182..dd925fa 100644 --- a/api/app/services/balance.py +++ b/api/app/services/balance.py @@ -13,6 +13,8 @@ class BalanceService: + _cache = {} + def __init__( self, db: Session = Depends(get_db), @@ -21,13 +23,24 @@ def __init__( self.db = db self.entity_service = entity_service - def get_balances( - self, entity_id: int, specific_date: datetime | None = None - ) -> BalanceSchema: + def invalidate_cache_entry(self, entity_id: int): + self._cache.pop(entity_id, None) + + def get_balances(self, entity_id: int) -> BalanceSchema: """ Calculates the current balances for a given entity across all currencies. Only confirmed transactions are considered. """ + if entity_id in self._cache: + return self._cache[entity_id] + else: + result = self._get_balances(entity_id) + self._cache[entity_id] = result + return result + + def _get_balances( + self, entity_id: int + ) -> BalanceSchema: # Check that entity exists self.entity_service.get(entity_id) @@ -51,15 +64,6 @@ def sum_transactions(confirmed: bool) -> dict[str, Decimal]: Transaction.confirmed == confirmed, ) - # Date limiter — don't count transactions after this date - if specific_date is not None: - credit_query = credit_query.filter( - Transaction.created_at <= specific_date - ) - debit_query = debit_query.filter( - Transaction.created_at <= specific_date - ) - # Group sums by currency credit_query = credit_query.group_by(Transaction.currency) debit_query = debit_query.group_by(Transaction.currency) @@ -86,7 +90,9 @@ def sum_transactions(confirmed: bool) -> dict[str, Decimal]: return total_by_currency - return BalanceSchema( + result = BalanceSchema( confirmed=sum_transactions(confirmed=True), non_confirmed=sum_transactions(confirmed=False), ) + self._cache[entity_id] = result + return result diff --git a/api/app/services/transaction.py b/api/app/services/transaction.py index 5775416..5cbb624 100644 --- a/api/app/services/transaction.py +++ b/api/app/services/transaction.py @@ -1,17 +1,26 @@ """Transaction service""" +from fastapi import Depends +from app.schemas.base import BaseUpdateSchema +from app.db import get_db +from app.services.tag import TagService +from app.services.balance import BalanceService from app.models.entity import Entity from app.models.transaction import Transaction -from app.schemas.transaction import TransactionFiltersSchema +from app.schemas.transaction import TransactionCreateSchema, TransactionFiltersSchema, TransactionUpdateSchema from app.services.base import BaseService from app.services.mixins.taggable_mixin import TaggableServiceMixin from sqlalchemy import or_ -from sqlalchemy.orm import Query +from sqlalchemy.orm import Query, Session class TransactionService(TaggableServiceMixin[Transaction], BaseService[Transaction]): model = Transaction + def __init__(self, db: Session = Depends(get_db), balance_service: BalanceService = Depends()): + self.db = db + self._balance_service = balance_service + def _apply_filters( self, query: Query[Transaction], filters: TransactionFiltersSchema ) -> Query[Transaction]: @@ -41,5 +50,13 @@ def _apply_filters( query = query.filter(self.model.confirmed == filters.confirmed) return query - def create(self, schema, actor_entity: Entity) -> Transaction: + def create(self, schema: TransactionCreateSchema, actor_entity: Entity) -> Transaction: + self._balance_service.invalidate_cache_entry(schema.from_entity_id) + self._balance_service.invalidate_cache_entry(schema.to_entity_id) return super().create(schema, overrides={"actor_entity_id": actor_entity.id}) + + def update(self, obj_id: int, update_schema: TransactionUpdateSchema, overrides: dict = {}) -> Transaction: + tx = self.get(obj_id) + self._balance_service.invalidate_cache_entry(tx.from_entity_id) + self._balance_service.invalidate_cache_entry(tx.to_entity_id) + return super().update(obj_id, update_schema, overrides) \ No newline at end of file diff --git a/api/tests/test_balance.py b/api/tests/test_balance.py index e9955a3..111ab20 100644 --- a/api/tests/test_balance.py +++ b/api/tests/test_balance.py @@ -71,43 +71,3 @@ def test_entity_transactions_balance( balance_b = Decimal(response.json()["confirmed"]["usd"]) assert balance_b == Decimal("100") - # Make second transaction to verify that specific_date works correctly - time.sleep(1) - - transaction_data = { - "from_entity_id": entity_a_id, - "to_entity_id": entity_b_id, - "amount": "100.00", - "currency": "usd", - } - response = test_app.post( - "/transactions/", json=transaction_data, headers={"x-token": token_a} - ) - transaction_created_at = response.json()["created_at"] - transaction_id = response.json()["id"] - - # Confirm the transaction - response = test_app.patch( - f"/transactions/{transaction_id}", - json={"confirmed": True}, - headers={"x-token": token_a}, - ) - assert response.status_code == status.HTTP_200_OK - - # Check balance before the transaction - check_date = datetime.fromisoformat(transaction_created_at) - timedelta( - seconds=1 - ) - response = test_app.get( - f"/balances/{entity_a_id}?specific_date={check_date.isoformat()}", - headers={"x-token": token_a}, - ) - balance_a = Decimal(response.json()["confirmed"]["usd"]) - assert balance_a == Decimal("-100") - - # Check balance after the transaction (just in case) - response = test_app.get( - f"/balances/{entity_a_id}", headers={"x-token": token_a} - ) - balance_a = Decimal(response.json()["confirmed"]["usd"]) - assert balance_a == Decimal("-200") diff --git a/api/tests/test_balance_cache.py b/api/tests/test_balance_cache.py new file mode 100644 index 0000000..67220eb --- /dev/null +++ b/api/tests/test_balance_cache.py @@ -0,0 +1,97 @@ +import pytest +from datetime import datetime, timedelta +from decimal import Decimal +from fastapi import status +from fastapi.testclient import TestClient +import time + +class TestBalanceServiceCache: + """Test cache functionality in BalanceService""" + + def test_cache_functionality(self, test_app: TestClient, token_factory, token): + # Create entities + response = test_app.post( + "/entities", json={"name": "Entity A"}, headers={"x-token": token} + ) + entity_a_id = response.json()["id"] + token_a = token_factory(entity_a_id) + + response = test_app.post( + "/entities", json={"name": "Entity B"}, headers={"x-token": token} + ) + entity_b_id = response.json()["id"] + token_b = token_factory(entity_b_id) + + response = test_app.post( + "/entities", json={"name": "Entity C"}, headers={"x-token": token} + ) + entity_c_id = response.json()["id"] + token_c = token_factory(entity_c_id) + + # Create transactions between entities + transactions = [ + {"from_entity_id": entity_a_id, "to_entity_id": entity_b_id, "amount": "100.00", "currency": "usd"}, + {"from_entity_id": entity_a_id, "to_entity_id": entity_c_id, "amount": "200.00", "currency": "usd"}, + {"from_entity_id": entity_b_id, "to_entity_id": entity_c_id, "amount": "150.00", "currency": "usd"}, + ] + + for transaction_data in transactions: + response = test_app.post( + "/transactions/", json=transaction_data, headers={"x-token": token_factory(transaction_data['from_entity_id'])} + ) + assert response.status_code == status.HTTP_200_OK + transaction_id = response.json()["id"] + + # Confirm the transaction + response = test_app.patch( + f"/transactions/{transaction_id}", + json={"confirmed": True}, + headers={"x-token": token_factory(transaction_data['from_entity_id'])}, + ) + assert response.status_code == status.HTTP_200_OK + + # Get initial balances using the cached method + response = test_app.get(f"/balances/{entity_a_id}", headers={"x-token": token_a}) + cached_balance_a = response.json() + response = test_app.get(f"/balances/{entity_b_id}", headers={"x-token": token_b}) + cached_balance_b = response.json() + response = test_app.get(f"/balances/{entity_c_id}", headers={"x-token": token_c}) + cached_balance_c = response.json() + + # Expected balances after transactions + expected_balance_a = {"confirmed": {"usd": "-300.00"}, "non_confirmed": {}} + expected_balance_b = {"confirmed": {"usd": "-50.00"}, "non_confirmed": {}} + expected_balance_c = {"confirmed": {"usd": "350.00"}, "non_confirmed": {}} + + assert cached_balance_a == expected_balance_a + assert cached_balance_b == expected_balance_b + assert cached_balance_c == expected_balance_c + + # Wait to simulate cache usage and check balances again + time.sleep(1) + response = test_app.get(f"/balances/{entity_a_id}", headers={"x-token": token_a}) + cached_balance_a_again = response.json() + response = test_app.get(f"/balances/{entity_b_id}", headers={"x-token": token_b}) + cached_balance_b_again = response.json() + response = test_app.get(f"/balances/{entity_c_id}", headers={"x-token": token_c}) + cached_balance_c_again = response.json() + + assert cached_balance_a_again == cached_balance_a + assert cached_balance_b_again == cached_balance_b + assert cached_balance_c_again == cached_balance_c + + # Invalidate the cache and check balances again + test_app.get(f"/balances/{entity_a_id}/invalidate", headers={"x-token": token_a}) + test_app.get(f"/balances/{entity_b_id}/invalidate", headers={"x-token": token_b}) + test_app.get(f"/balances/{entity_c_id}/invalidate", headers={"x-token": token_c}) + + response = test_app.get(f"/balances/{entity_a_id}", headers={"x-token": token_a}) + uncached_balance_a = response.json() + response = test_app.get(f"/balances/{entity_b_id}", headers={"x-token": token_b}) + uncached_balance_b = response.json() + response = test_app.get(f"/balances/{entity_c_id}", headers={"x-token": token_c}) + uncached_balance_c = response.json() + + assert uncached_balance_a == expected_balance_a + assert uncached_balance_b == expected_balance_b + assert uncached_balance_c == expected_balance_c diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 950ade3..94c3190 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,9 +2,13 @@ services: api: volumes: - ./api:/opt/api + - api-data:/opt/data command: --reload ui: volumes: - ./ui:/opt/ui command: --reload + +volumes: + api-data: diff --git a/docker-compose.yml b/docker-compose.yml index bd4f98b..7ce466e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: ui: build: ui ports: - - "5000:5000" + - "5001:5000" env_file: - secrets.env diff --git a/secrets.env b/secrets.env deleted file mode 100644 index 98863d1..0000000 --- a/secrets.env +++ /dev/null @@ -1,2 +0,0 @@ -export REFINANCE_SECRET_KEY=0000-1111-2222-3333-4444-5555 -export REFINANCE_TELEGRAM_BOT_API_TOKEN=0000000:abcdef0123456789 diff --git a/ui/app/templates/transaction/add.jinja2 b/ui/app/templates/transaction/add.jinja2 index b0e88de..1ec1f4e 100644 --- a/ui/app/templates/transaction/add.jinja2 +++ b/ui/app/templates/transaction/add.jinja2 @@ -10,6 +10,7 @@ {{ form.from_entity_name.label }} @@ -24,6 +25,7 @@ {{ form.to_entity_name.label }}