Merge pull request #184 from mobiusml/gpu_tests

Run Tests with GPU

.github/workflows/run_tests_with_gpu.yml

@@ -0,0 +1,33 @@
+name: Run Tests with GPU
+
+on:
+  workflow_dispatch: # Allows for manual triggering
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set SSH permissions
+      run: |
+        mkdir -p ~/.ssh
+        chmod 700 ~/.ssh
+        sudo chown $USER:$USER ~/.ssh
+
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+
+    - name: Install and configure dstack
+      run: |
+        pip install dstack
+        dstack config --url https://sky.dstack.ai --project ${{ secrets.DSTACK_PROJECT }} --token ${{ secrets.DSTACK_TOKEN }}
+        dstack init
+    
+    - name: Run tests with GPU
+      run: |
+        HF_TOKEN=${{ secrets.HF_TOKEN }} dstack apply -f tests.dstack.yml --force -y

pg_ctl.sh

@@ -1,10 +1,13 @@
 #!/bin/bash
 
+# Get the PostgreSQL major version (e.g., 14)
+PG_VERSION=$(psql --version | awk '{print $3}' | cut -d '.' -f 1)
+
 if [ "$2" = "--pgdata" ]; then
     # Recursively change parent directories permissions so that pg_ctl can access the necessary files as non-root.
     f=$(dirname "$3")
     while [[ $f != / ]]; do chmod o+rwx "$f"; f=$(dirname "$f"); done;
 fi
 
 # Execute pg_ctl as postgres instead of root.
-sudo -u postgres "/usr/lib/postgresql/16/bin/pg_ctl" "$@"
+sudo -u postgres "/usr/lib/postgresql/$PG_VERSION/bin/pg_ctl" "$@"

test_volume.dstack.yml

@@ -0,0 +1,8 @@
+type: volume
+
+name: test-models-cache
+
+backend: runpod
+region: EU-SE-1
+
+size: 150GB

tests.dstack.yml

@@ -0,0 +1,32 @@
+type: task
+
+name: aana-tests
+
+backends: [runpod]
+
+image: nvidia/cuda:12.1.0-cudnn8-devel-ubuntu22.04
+
+env:
+  - HF_TOKEN
+
+commands:
+  - apt-get update
+  - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+  - apt-get install -y libgl1 libglib2.0-0 ffmpeg python3 python3-dev postgresql sudo
+  - locale-gen en_US.UTF-8
+  - export LANG="en_US.UTF-8" LANGUAGE="en_US:en" LC_ALL="en_US.UTF-8"
+  - curl -sSL https://install.python-poetry.org | python3 -
+  - export PATH=$PATH:/root/.local/bin
+  - poetry install
+  - HF_HUB_CACHE="/models_cache" CUDA_VISIBLE_DEVICES="0" poetry run pytest -vv
+
+volumes:
+  - name: test-models-cache
+    path: /models_cache
+
+max_price: 1.0
+
+resources:
+  cpu: 9..
+  memory: 32GB..
+  gpu: 40GB..