From c1b4161e75dd9ebe8c4e1fc7ded3eb77de12e4e5 Mon Sep 17 00:00:00 2001 From: ErikKalkoken Date: Mon, 5 Jun 2023 13:33:07 +0200 Subject: [PATCH 1/2] Add methods to TOC and other doc improvements --- docs/_static/custom.css | 5 + docs/api.rst | 6 +- docs/extensions.rst | 20 ++++ docs/index.rst | 13 ++- src/aiodiskqueue/engines/base.py | 18 +++- src/aiodiskqueue/engines/dbm.py | 164 ++++++++++++++--------------- src/aiodiskqueue/engines/simple.py | 8 +- src/aiodiskqueue/engines/sqlite.py | 4 +- src/aiodiskqueue/queues.py | 6 +- 9 files changed, 138 insertions(+), 106 deletions(-) create mode 100644 docs/extensions.rst diff --git a/docs/_static/custom.css b/docs/_static/custom.css index adc8faa..1ceb572 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,3 +1,8 @@ .logo { overflow-wrap: normal; } + +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} diff --git a/docs/api.rst b/docs/api.rst index aa00196..d91a42c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -4,6 +4,7 @@ API Reference =============== +Complete reference of the public API. Queues ====== @@ -14,8 +15,3 @@ Exceptions ========== .. automodule:: aiodiskqueue.exceptions - -Storage Engines -=============== - -.. automodule:: aiodiskqueue.engines diff --git a/docs/extensions.rst b/docs/extensions.rst new file mode 100644 index 0000000..5b11a9f --- /dev/null +++ b/docs/extensions.rst @@ -0,0 +1,20 @@ +.. currentmodule:: aiodiskqueue + +=============== +Customization +=============== + +Storage Engines +=============== + +aiodiskqueue uses the DbmEngine as default, but you can also select a different storage engine. + +Or you can create your own storage engine by inheriting from :class:`.FifoStorageEngine`. + +.. automodule:: aiodiskqueue.engines.dbm + +.. automodule:: aiodiskqueue.engines.simple + +.. automodule:: aiodiskqueue.engines.sqlite + +.. automodule:: aiodiskqueue.engines.base diff --git a/docs/index.rst b/docs/index.rst index 518359c..3128870 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,10 +7,17 @@ .. toctree:: :hidden: - :maxdepth: 2 + :maxdepth: 4 api + extensions + + +Contens +------- .. toctree:: - :hidden: - :maxdepth: 1 + :maxdepth: 2 + + api + extensions diff --git a/src/aiodiskqueue/engines/base.py b/src/aiodiskqueue/engines/base.py index cf62acf..bd46045 100644 --- a/src/aiodiskqueue/engines/base.py +++ b/src/aiodiskqueue/engines/base.py @@ -1,4 +1,4 @@ -"""Engines for storing the queues on disk.""" +"""Base class for storage engines.""" import logging from abc import ABC, abstractmethod @@ -8,7 +8,7 @@ logger = logging.getLogger("aiodiskqueue") -class _FifoStorageEngine(ABC): +class FifoStorageEngine(ABC): """Base class for all storage engines implementing a FIFO queue.""" def __init__(self, data_path: Path) -> None: @@ -16,11 +16,17 @@ def __init__(self, data_path: Path) -> None: @abstractmethod async def initialize(self) -> List[Any]: - """Initialize data file.""" + """Initialize data file. + + :meta private: + """ @abstractmethod async def fetch_all(self) -> List[Any]: - """Return all items in data file.""" + """Return all items in data file. + + :meta private: + """ @abstractmethod async def add_item(self, item: Any): @@ -29,6 +35,8 @@ async def add_item(self, item: Any): Args: item: Item to be appended items: All items including the one to be appended + + :meta private: """ @abstractmethod @@ -38,4 +46,6 @@ async def remove_item(self): Args: item: Item to be removed items: All items not including the one to be removed + + :meta private: """ diff --git a/src/aiodiskqueue/engines/dbm.py b/src/aiodiskqueue/engines/dbm.py index e80cee2..0e2ecf6 100644 --- a/src/aiodiskqueue/engines/dbm.py +++ b/src/aiodiskqueue/engines/dbm.py @@ -1,4 +1,4 @@ -"""Engines for storing the queues on disk.""" +"""Engines for storing the queues with DBM.""" import dbm import logging @@ -6,97 +6,91 @@ from pathlib import Path from typing import Any, List, Optional, Union -try: - import aiodbm -except ImportError: - has_aiodbm = False -else: - has_aiodbm = True +import aiodbm -from .base import _FifoStorageEngine +from .base import FifoStorageEngine logger = logging.getLogger("aiodiskqueue") -if has_aiodbm: - - class DbmEngine(_FifoStorageEngine): - """A queue storage engine using DBM.""" - - def __init__(self, data_path: Path) -> None: - super().__init__(data_path) - self._data_path_2 = str(data_path.absolute()) - - HEAD_ID_KEY = "head_id" - TAIL_ID_KEY = "tail_id" - - async def initialize(self): - async with aiodbm.open(self._data_path_2, "c") as db: - await db.set("dummy", "test") - await db.delete("dummy") - - async def fetch_all(self) -> List[Any]: - try: - async with aiodbm.open(self._data_path_2, "r") as db: - head_id = await self._get_obj(db, self.HEAD_ID_KEY) - tail_id = await self._get_obj(db, self.TAIL_ID_KEY) - if not head_id or not tail_id: - return [] - - items = [] - for item_id in range(head_id, tail_id + 1): - item_key = self._make_item_key(item_id) - item = await self._get_obj(db, item_key) - items.append(item) - except dbm.error: - items = [] - return items +class DbmEngine(FifoStorageEngine): + """A queue storage engine using DBM.""" - async def add_item(self, item: Any): - async with aiodbm.open(self._data_path_2, "w") as db: - tail_id = await self._get_obj(db, self.TAIL_ID_KEY) - if tail_id: - item_id = tail_id + 1 - is_first = False - else: - item_id = 1 - is_first = True + def __init__(self, data_path: Path) -> None: + super().__init__(data_path) + self._data_path_2 = str(data_path.absolute()) - await self._set_obj(db, self._make_item_key(item_id), item) - await self._set_obj(db, self.TAIL_ID_KEY, item_id) + _HEAD_ID_KEY = "head_id" + _TAIL_ID_KEY = "tail_id" - if is_first: - await self._set_obj(db, self.HEAD_ID_KEY, item_id) + async def initialize(self): + async with aiodbm.open(self._data_path_2, "c") as db: + await db.set("dummy", "test") + await db.delete("dummy") - async def remove_item(self): - async with aiodbm.open(self._data_path_2, "w") as db: - head_id = await self._get_obj(db, self.HEAD_ID_KEY) - tail_id = await self._get_obj(db, self.TAIL_ID_KEY) + async def fetch_all(self) -> List[Any]: + try: + async with aiodbm.open(self._data_path_2, "r") as db: + head_id = await self._get_obj(db, self._HEAD_ID_KEY) + tail_id = await self._get_obj(db, self._TAIL_ID_KEY) if not head_id or not tail_id: - raise ValueError("Nothing to remove from an empty database") - item_key = self._make_item_key(head_id) - await db.delete(item_key) - - if head_id != tail_id: - # there are items left - await self._set_obj(db, self.HEAD_ID_KEY, head_id + 1) - else: - # was last item - await db.delete(self.HEAD_ID_KEY) - await db.delete(self.TAIL_ID_KEY) - - @staticmethod - def _make_item_key(item_id: int) -> str: - return f"item-{item_id}" - - @staticmethod - async def _get_obj(db, key: Union[str, bytes]) -> Optional[Any]: - data = await db.get(key) - if not data: - return None - return pickle.loads(data) - - @staticmethod - async def _set_obj(db, key: Union[str, bytes], item: Any): - data = pickle.dumps(item) - await db.set(key, data) + return [] + + items = [] + for item_id in range(head_id, tail_id + 1): + item_key = self._make_item_key(item_id) + item = await self._get_obj(db, item_key) + items.append(item) + except dbm.error: + items = [] + + return items + + async def add_item(self, item: Any): + async with aiodbm.open(self._data_path_2, "w") as db: + tail_id = await self._get_obj(db, self._TAIL_ID_KEY) + if tail_id: + item_id = tail_id + 1 + is_first = False + else: + item_id = 1 + is_first = True + + await self._set_obj(db, self._make_item_key(item_id), item) + await self._set_obj(db, self._TAIL_ID_KEY, item_id) + + if is_first: + await self._set_obj(db, self._HEAD_ID_KEY, item_id) + + async def remove_item(self): + async with aiodbm.open(self._data_path_2, "w") as db: + head_id = await self._get_obj(db, self._HEAD_ID_KEY) + tail_id = await self._get_obj(db, self._TAIL_ID_KEY) + if not head_id or not tail_id: + raise ValueError("Nothing to remove from an empty database") + item_key = self._make_item_key(head_id) + await db.delete(item_key) + + if head_id != tail_id: + # there are items left + await self._set_obj(db, self._HEAD_ID_KEY, head_id + 1) + else: + # was last item + await db.delete(self._HEAD_ID_KEY) + await db.delete(self._TAIL_ID_KEY) + + @staticmethod + def _make_item_key(item_id: int) -> str: + return f"item-{item_id}" + + @staticmethod + async def _get_obj(db, key: Union[str, bytes]) -> Optional[Any]: + data = await db.get(key) + if not data: + return None + return pickle.loads(data) + + @staticmethod + async def _set_obj(db, key: Union[str, bytes], item: Any): + data = pickle.dumps(item) + await db.set(key, data) diff --git a/src/aiodiskqueue/engines/simple.py b/src/aiodiskqueue/engines/simple.py index 8fa4002..81ae0ec 100644 --- a/src/aiodiskqueue/engines/simple.py +++ b/src/aiodiskqueue/engines/simple.py @@ -1,4 +1,4 @@ -"""Engines for storing the queues on disk.""" +"""Engines for storing the queues in flat files.""" import io import logging @@ -8,12 +8,12 @@ import aiofiles import aiofiles.os -from .base import _FifoStorageEngine +from .base import FifoStorageEngine logger = logging.getLogger("aiodiskqueue") -class PickledList(_FifoStorageEngine): +class PickledList(FifoStorageEngine): """This engine stores items as one singular pickled list of items.""" async def initialize(self): @@ -54,7 +54,7 @@ async def _save_all_items(self, items: List[Any]): logger.debug("Wrote queue with %d items: %s", len(items), self._data_path) -class PickleSequence(_FifoStorageEngine): +class PickleSequence(FifoStorageEngine): """This engine stores items as a sequence of single pickles.""" async def initialize(self): diff --git a/src/aiodiskqueue/engines/sqlite.py b/src/aiodiskqueue/engines/sqlite.py index 936b35b..97209e3 100644 --- a/src/aiodiskqueue/engines/sqlite.py +++ b/src/aiodiskqueue/engines/sqlite.py @@ -12,13 +12,13 @@ else: has_aiosqlite = True -from .base import _FifoStorageEngine +from .base import FifoStorageEngine logger = logging.getLogger("aiodiskqueue") if has_aiosqlite: - class SqliteEngine(_FifoStorageEngine): + class SqliteEngine(FifoStorageEngine): """A queue storage engine using Sqlite.""" async def initialize(self): diff --git a/src/aiodiskqueue/queues.py b/src/aiodiskqueue/queues.py index 1528055..31fd013 100644 --- a/src/aiodiskqueue/queues.py +++ b/src/aiodiskqueue/queues.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any, Union -from aiodiskqueue.engines.base import _FifoStorageEngine +from aiodiskqueue.engines.base import FifoStorageEngine from aiodiskqueue.engines.dbm import DbmEngine from aiodiskqueue.exceptions import QueueEmpty, QueueFull from aiodiskqueue.utils import NoDirectInstantiation @@ -29,7 +29,7 @@ def __init__( data_path: Path, maxsize: int, queue: list, - storage_engine: _FifoStorageEngine, + storage_engine: FifoStorageEngine, ) -> None: """Direct instantiation would break the persistance feature and has therefore been disabled. @@ -208,7 +208,7 @@ async def create( if not cls_storage_engine: cls_storage_engine = DbmEngine else: - if not issubclass(cls_storage_engine, _FifoStorageEngine): + if not issubclass(cls_storage_engine, FifoStorageEngine): raise TypeError("Invalid storage engine") storage_engine = cls_storage_engine(data_path) queue = await storage_engine.fetch_all() From 4e8a076aa8eb21bfa378c4161fecf1f4cbf62443 Mon Sep 17 00:00:00 2001 From: ErikKalkoken Date: Mon, 5 Jun 2023 13:38:09 +0200 Subject: [PATCH 2/2] Add CHANGELOG and bump version --- CHANGELOG.md | 20 ++++++++++++++++++++ src/aiodiskqueue/__init__.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ee51218 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [Unreleased] - yyyy-mm-dd + +### Added + +### Changed + +### Fixed + +## [0.1.0] - 2023-06-05 + +### Added + +- Initial release diff --git a/src/aiodiskqueue/__init__.py b/src/aiodiskqueue/__init__.py index 387ae1a..2fe0d5d 100644 --- a/src/aiodiskqueue/__init__.py +++ b/src/aiodiskqueue/__init__.py @@ -4,7 +4,7 @@ from aiodiskqueue.exceptions import QueueEmpty, QueueFull from aiodiskqueue.queues import Queue -__version__ = "0.1.0b8" +__version__ = "0.1.0" __all__ = ["engines", "Queue", "QueueEmpty", "QueueFull"]