ci-cd.yml

name: CI/CD

on:
  pull_request:
  workflow_dispatch:
  schedule:
      - cron: "37 9 * * *"

jobs:
  test:
    name: Memorization Test
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        config: [
          "pythia",
          "codellama",
          "llama-3",
          "mistral-memorize",
          ]
    env:
      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
      MODAL_ENVIRONMENT: ci-llm-finetuning
      ALLOW_WANDB: "true"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install Modal
        run: |
          python -m pip install --upgrade pip
          pip install modal pyyaml pandas

      - name: Prep config and data for CI
        run: |
          python ci/prep_for_ci.py --config=config/${{ matrix.config }}.yml --data=data/sqlqa.subsample.jsonl
          echo `wc -l data/sqlqa.subsample.jsonl | awk '{print $1}'` lines in test data
          echo Config:
          cat config/${{ matrix.config }}.yml

      - name: Run training job on Modal
        run: |
          modal run src.train --config=config/${{ matrix.config }}.yml --data=data/sqlqa.subsample.jsonl

      - name: Check training results
        run: |
          python ci/check_loss.py

      - name: Check inference results
        run: |
          python ci/check_inference.py