From 3187bc9da3db9b36104eaaa36c0a82dd8961f367 Mon Sep 17 00:00:00 2001 From: Maxwell Flitton Date: Fri, 17 Jan 2025 12:30:19 +0000 Subject: [PATCH] refactoring setup.py config for pip install (#145) --- .gitignore | 1 + docker-compose.yml | 11 ++++++- pyproject.toml | 2 +- scripts/download_snapshots.sh | 15 ++++++++++ setup.py | 21 ++++++++++--- surrealdb/data/types/datetime.py | 8 ++--- surrealdb/data/types/duration.py | 3 +- tests/builds/big_data_dockerfile | 17 +++++++++++ tests/integration/async/test_batch.py | 41 ++++++++++++++++++++++++++ tests/scripts/build_big_data_server.sh | 17 +++++++++++ tests/unit/cbor_types/test_datetime.py | 4 ++- 11 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 scripts/download_snapshots.sh create mode 100644 tests/builds/big_data_dockerfile create mode 100644 tests/integration/async/test_batch.py create mode 100644 tests/scripts/build_big_data_server.sh diff --git a/.gitignore b/.gitignore index 293663e7..e1536619 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,4 @@ clog/ manifest/ .mypy_cache/ .ruff_cache/ +tests/db_snapshots/ diff --git a/docker-compose.yml b/docker-compose.yml index 2f3b664f..0b478b99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,16 @@ services: - SURREAL_PASS=root - SURREAL_LOG=trace ports: - - 8000:8000 + - 8300:8000 + + surrealdb_big_data: + image: surrealdb-big-data +# environment: +# - SURREAL_USER=root +# - SURREAL_PASS=root +# - SURREAL_LOG=trace + ports: + - 9000:8000 surrealdb_200: image: surrealdb/surrealdb:v2.0.0 diff --git a/pyproject.toml b/pyproject.toml index 142c8fb0..f5223978 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ ] [build-system] -requires = ["setuptools", "wheel", "setuptools-rust"] +requires = ["setuptools", "wheel"] [tool.poetry.dependencies] python = "^3.10" diff --git a/scripts/download_snapshots.sh b/scripts/download_snapshots.sh new file mode 100644 index 00000000..21f477f8 --- /dev/null +++ b/scripts/download_snapshots.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# navigate to directory +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +cd $SCRIPTPATH + +cd .. +cd tests + +if [ -d "./db_snapshots" ]; then + echo "DB snapshots are already present" + rm -rf ./db_snapshots +fi + +dockpack pull -i maxwellflitton/surrealdb-data -d ./db_snapshots diff --git a/setup.py b/setup.py index 53bde2d5..d79b3c68 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ import pathlib from setuptools import setup -from setuptools_rust import Binding, RustExtension with open("README.md", "r") as fh: @@ -21,15 +20,29 @@ long_description=long_description, long_description_content_type="text/markdown", version=version, - rust_extensions=[RustExtension("surrealdb.rust_surrealdb", binding=Binding.PyO3)], packages=[ "surrealdb", - "surrealdb.execution_mixins", - "surrealdb.async_execution_mixins" + "surrealdb.data", + "surrealdb.data.types" ], package_data={ "surrealdb": ["binaries/*"], }, + install_requires=[ + "cbor2==5.6.5", + "certifi==2024.12.14", + "charset-normalizer==3.4.0", + "idna==3.10", + "pytz==2024.2", + "requests==2.32.3", + "semantic-version==2.10.0", + "tomli==2.2.1", + "types-pytz==2024.2.0.20241003", + "types-requests==2.32.0.20241016", + "typing_extensions==4.12.2", + "urllib3==2.2.3", + "websockets==14.1", + ], # rust extensions are not zip safe, just like C-extensions. zip_safe=False, include_package_data=True diff --git a/surrealdb/data/types/datetime.py b/surrealdb/data/types/datetime.py index 30a4b022..53526d96 100644 --- a/surrealdb/data/types/datetime.py +++ b/surrealdb/data/types/datetime.py @@ -1,17 +1,17 @@ -import pytz # type: ignore - from dataclasses import dataclass from datetime import datetime -from math import floor from typing import Tuple +import pytz # type: ignore +from math import floor + @dataclass class DateTimeCompact: timestamp: int = 0 # nanoseconds @staticmethod - def parse(seconds: int, nanoseconds: int): + def parse(seconds: int, nanoseconds: int) -> "DateTimeCompact": return DateTimeCompact(nanoseconds + (seconds * pow(10, 9))) def get_seconds_and_nano(self) -> Tuple[int, int]: diff --git a/surrealdb/data/types/duration.py b/surrealdb/data/types/duration.py index 756e3771..09b94025 100644 --- a/surrealdb/data/types/duration.py +++ b/surrealdb/data/types/duration.py @@ -1,7 +1,8 @@ from dataclasses import dataclass -from math import floor from typing import Tuple +from math import floor + @dataclass class Duration: diff --git a/tests/builds/big_data_dockerfile b/tests/builds/big_data_dockerfile new file mode 100644 index 00000000..ffebe019 --- /dev/null +++ b/tests/builds/big_data_dockerfile @@ -0,0 +1,17 @@ +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y libpq-dev +RUN apt-get install -y curl +RUN apt-get install libssl-dev -y +RUN apt-get install tree -y +RUN apt-get install -y --no-install-recommends unzip bash +RUN curl -sSL https://raw.githubusercontent.com/maxwellflitton/surrealdb-backup-cli/main/scripts/install.sh | bash + +COPY ./db_snapshots/big_data_snapshot/package/ ./big_data_snapshot/ +RUN ./surrealdb_backup unpack -d ./unpacked_data/ -t ./big_data_snapshot/ +RUN rm ./unpacked_data/LOCK +RUN curl -sSf https://install.surrealdb.com | sh + +EXPOSE 8000 + +CMD ["surreal", "start", "--bind", "0.0.0.0:8000", "--allow-all", "--user", "root", "--pass", "root", "rocksdb:./unpacked_data"] diff --git a/tests/integration/async/test_batch.py b/tests/integration/async/test_batch.py new file mode 100644 index 00000000..dcac1084 --- /dev/null +++ b/tests/integration/async/test_batch.py @@ -0,0 +1,41 @@ +from typing import List +from unittest import TestCase, main + +from surrealdb import SurrealDB, RecordID +from tests.integration.connection_params import TestConnectionParams +import asyncio +import websockets + + +class TestBatch(TestCase): + + def setUp(self) -> None: + self.params = TestConnectionParams() + self.db = SurrealDB(self.params.url) + self.queries: List[str] = [] + + self.db.connect() + self.db.use(self.params.namespace, self.params.database) + self.db.sign_in("root", "root") + + # self.query = """ + # CREATE |product:1000000| CONTENT { + # name: rand::enum('Cruiser Hoodie', 'Surreal T-Shirt'), + # colours: [rand::string(10), rand::string(10),], + # price: rand::int(20, 60), + # time: { + # created_at: rand::time(1611847404, 1706455404), + # updated_at: rand::time(1651155804, 1716906204) + # } + # }; + # """ + # self.db.query(query=self.query) + + def tearDown(self) -> None: + pass + + def test_batch(self): + print("test_batch") + +if __name__ == '__main__': + main() diff --git a/tests/scripts/build_big_data_server.sh b/tests/scripts/build_big_data_server.sh new file mode 100644 index 00000000..bd46d5f0 --- /dev/null +++ b/tests/scripts/build_big_data_server.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# navigate to directory +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +cd $SCRIPTPATH + +cd .. + +if [ -d "./db_snapshots/big_data_snapshot" ]; then + echo "removing the big data snapshot" + rm -rf ./db_snapshots/big_data_snapshot +fi + +dockpack pull -i maxwellflitton/surrealdb-data -d ./db_snapshots/big_data_snapshot +rm ./db_snapshots/big_data_snapshot/package/LOCK + +docker build -f ./builds/big_data_dockerfile -t surrealdb-big-data . diff --git a/tests/unit/cbor_types/test_datetime.py b/tests/unit/cbor_types/test_datetime.py index 8bd284d1..ec3f4249 100644 --- a/tests/unit/cbor_types/test_datetime.py +++ b/tests/unit/cbor_types/test_datetime.py @@ -1,4 +1,4 @@ -from unittest import TestCase +from unittest import TestCase, main from surrealdb.data.types.datetime import DateTimeCompact from surrealdb.data.cbor import encode, decode @@ -21,3 +21,5 @@ def test_datetime(self): self.assertEqual(decoded.get_date_time(), '2024-12-12T09:00:58.083988Z') +if __name__ == '__main__': + main()