feat/refactor: migrate to gunicorn/uvicorn

winstxnhdw · Feb 26, 2024 · e54dd71 · e54dd71
1 parent f9d1bc2
commit e54dd71
Show file tree

Hide file tree

Showing 16 changed files with 229 additions and 149 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -5,6 +5,7 @@
 *.pyc
 Dockerfile*
 
+cache/
 .github/
 tests/
 .vscode/

diff --git a/Caddyfile b/Caddyfile
@@ -1,4 +1,5 @@
 {
+	order rate_limit before basicauth
 	order cache before rewrite
 	cache
 }
@@ -7,16 +8,17 @@
 	cache {
 		allowed_http_verbs POST
 		ttl 24h
-		timeout {
-			backend 60s
+	}
+
+	rate_limit {
+		zone dynamic_example {
+			key {remote_host}
+			events {$EVENTS_PER_WINDOW}
+			window 30s
 		}
 	}
 
 	handle_path /api/* {
-		reverse_proxy http://localhost:{$SERVER_PORT} {
-			transport http {
-				versions h2c
-			}
-		}
+		reverse_proxy http://localhost:{$SERVER_PORT}
 	}
 }
diff --git a/Dockerfile b/Dockerfile
@@ -6,5 +6,6 @@ ENV APP_PORT 7860
 ENV OMP_NUM_THREADS 8
 ENV CT2_USE_EXPERIMENTAL_PACKED_GEMM 1
 ENV CT2_FORCE_CPU_ISA AVX512
+ENV EVENTS_PER_WINDOW 5
 
 EXPOSE $APP_PORT
diff --git a/Dockerfile.build b/Dockerfile.build
@@ -1,4 +1,4 @@
-FROM python:3.11.8-slim as python-builder
+FROM python:slim as python-builder
 
 ENV POETRY_VIRTUALENVS_CREATE false
 ENV POETRY_HOME /opt/poetry
@@ -16,14 +16,18 @@ RUN poetry install --without=dev --no-root
 
 FROM caddy:builder-alpine as caddy-builder
 
-RUN xcaddy build --with github.com/caddyserver/cache-handler
+RUN xcaddy build \
+    --with github.com/caddyserver/cache-handler \
+    --with github.com/mholt/caddy-ratelimit
 
 
-FROM python:3.11.8-slim
+FROM python:slim
 
 ENV HOME /home/user
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
+ENV SERVER_PORT 5000
+ENV EVENTS_PER_WINDOW 100000
 
 RUN useradd -m -u 1000 user
 
@@ -33,7 +37,7 @@ WORKDIR $HOME/app
 
 COPY --chown=user --from=caddy-builder  /usr/bin/caddy /usr/bin/caddy
 COPY --chown=user --from=python-builder /usr/local/bin /usr/local/bin
-COPY --chown=user --from=python-builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
+COPY --chown=user --from=python-builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
 COPY --chown=user . $HOME/app
 
 CMD ["supervisord"]
diff --git a/README.md b/README.md
@@ -24,3 +24,18 @@ curl -N 'https://winstxnhdw-llm-api.hf.space/api/v1/generate' \
          "instruction": "What is the capital of Japan?"
       }'
 ```
+
+## Development
+
+First, install the required dependencies for your editor with the following.
+
+```bash
+poetry install
+```
+
+Now, you can access the Swagger UI at [localhost:7860/api/docs](http://localhost:7860/api/docs) after spinning the server up locally with the following.
+
+```bash
+docker build -f Dockerfile.build -t llm-api .
+docker run --rm -e APP_PORT=7860 -p 7860:7860 llm-api
+```
diff --git a/gunicorn.conf.py b/gunicorn.conf.py
@@ -0,0 +1,13 @@
+# pylint: skip-file
+
+from server import initialise
+from server.config import Config
+
+wsgi_app = f'{initialise.__module__}:{initialise.__name__}()'
+reload = False
+accesslog = '-'
+preload_app = True
+bind = f'0.0.0.0:{Config.server_port}'
+workers = Config.worker_count
+worker_class = 'uvicorn.workers.UvicornWorker'
+timeout = 500
diff --git a/main.py b/main.py
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -31,16 +31,18 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.11"
 typing-extensions = "^4.9.0"
-hypercorn = "^0.16.0"
-uvloop = "^0.19.0"
 fastapi = "^0.109.2"
 pydantic-settings = "^2.2.1"
+uvloop = "^0.19.0"
+gunicorn = "^21.2.0"
+uvicorn = "^0.27.1"
+httptools = "^0.6.1"
+supervisor = "^4.2.5"
 ctranslate2 = "^4.0.0"
-transformers = "^4.37.2"
+transformers = "^4.38.1"
 sentencepiece = "^0.2.0"
 protobuf = "^4.25.3"
 jinja2 = "^3.1.3"
-supervisor = "^4.2.5"
 
 [tool.poetry.group.dev.dependencies]
 pyright = "^1.1.351"