ci.yml

# -*- coding: utf-8 -*-
#
# This file is part of HEPData.
# Copyright (C) 2020 CERN.
#
# HEPData is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# HEPData is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with HEPData; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

name: Continuous Integration


on:
  push:
  pull_request:
    branches: [ main ]
  release:
    types: [ published ]
  schedule:
    - cron:  '35 1 * * *'


concurrency:
  group: ${{github.event_name}}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:

  test:

    runs-on: ubuntu-latest

    permissions: write-all

    env:
      node-version: '18'

    strategy:

      matrix:
        postgres-version: [ 12, 13 ]
        os-version: [ '2.7.0' ]
        python-version: [ '3.7', '3.8', '3.9' ]
        exclude:
          - postgres-version: ${{ github.event.act && 13 }}
          - python-version: ${{ github.event.act && '3.7' }}
          - python-version: ${{ github.event.act && '3.8' }}

    # Service containers to run with `runner-job`
    services:

      postgres:
        image: postgres:${{ matrix.postgres-version }}
        # Provide the password for postgres
        env:
          POSTGRES_USER: hepdata
          POSTGRES_PASSWORD: hepdata
          POSTGRES_DB: hepdata_test
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

      redis:
        image: redis
        ports:
          - "6379:6379"

      opensearch:
        image: opensearchproject/opensearch:${{ matrix.os-version }}
        env:
          node.name: os01
          cluster.name: hepdata
          cluster.initial_master_nodes: os01
          OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
          plugins.security.disabled: true
        ports:
        - "9200:9200"
        - "9600:9600"

    steps:

    # Services are not currently supported by act (https://github.com/nektos/act#services)
    - name: Set up services for act
      if: ${{ env.ACT }}
      run: |
        docker run --name act-postgres -e POSTGRES_USER=hepdata -e POSTGRES_PASSWORD=hepdata -e POSTGRES_DB=hepdata_test -p 5432:5432 --rm -d postgres:${{ matrix.postgres-version }}
        docker run --name act-redis --rm -d -p 6379:6379 redis
        docker run --name act-opensearch --rm -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "plugins.security.disabled=true" opensearchproject/opensearch:${{ matrix.os-version }}

    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Set up node
      uses: actions/setup-node@v3
      with:
        node-version: ${{ env.node-version }}
    - name: Cache pip modules
      uses: actions/cache@v3
      if: ${{ !env.ACT }}
      env:
        cache-name: cache-pip-modules
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-test-${{ env.cache-name }}-
    - name: Cache node modules
      uses: actions/cache@v3
      if: ${{ !env.ACT }}
      env:
        cache-name: cache-node-modules
      with:
        path: ~/.npm
        key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/bundles.py') }}
        restore-keys: |
          ${{ runner.os }}-test-${{ env.cache-name }}-
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip setuptools py
        python -m pip install twine wheel coveralls requirements-builder
        python -m pip install --force-reinstall -r requirements.txt
        python -m pip install -e .[tests]
    - name: Initialise hepdata
      run: |
        hepdata db init
        hepdata db create
        hepdata utils reindex -rc True
    - name: Setup npm and build assets
      run: |
        hepdata collect -v
        hepdata webpack buildall
        cp hepdata/config_local.gh.py hepdata/config_local.py
    - name: Run tests
      env:
        COVERAGE_FILE: '.coverage_func'
      run: |
        py.test -vv tests/*_test.py
    - name: Setup Sauce Connect
      uses: saucelabs/sauce-connect-action@v2
      if: startsWith(matrix.postgres-version, '12') && startsWith(matrix.python-version, '3.9')
      with:
        username: ${{ secrets.SAUCE_USERNAME }}
        accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
        region: eu-central
        tunnelName: ${{ github.run_id }}
        scVersion: 4.9.0
        verbose: true
    - name: Run end-to-end tests
      if: startsWith(matrix.postgres-version, '12') && startsWith(matrix.python-version, '3.9')
      env:
        SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
        SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
        COVERAGE_FILE: '.coverage_e2e'
      run: |
        if [[ -n ${{ secrets.SAUCE_USERNAME }} && -n ${{ secrets.SAUCE_ACCESS_KEY}} ]]; then py.test -vv tests/e2e; fi
    - name: Run coveralls
      if: ${{ startsWith(matrix.postgres-version, '12') && startsWith(matrix.python-version, '3.9') && !env.ACT }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        COVERALLS_SERVICE_NAME: github
      run: |
        coverage combine .coverage_func .coverage_e2e
        coveralls
    - name: Build docs
      run: |
        python -m pip install -e .[docs]
        cd docs && make html
    - uses: actions/upload-artifact@v3
      if: ${{ failure() && !env.ACT }}
      with:
        name: sauce-connect-log
        path: ${{ env.SAUCE_CONNECT_DIR_IN_HOST }}/sauce-connect.log
    - name: Stop service containers for act
      if: ${{ always() && env.ACT }}
      run: |
        docker stop act-postgres act-redis act-opensearch

  deploy:
    needs: test
    if: contains(github.ref, 'main') || github.event_name == 'release'
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Deploy to DockerHub
      env:
        CI_TAG: ${{ github.event.release.tag_name }}
        DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
        DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
      run: |
        ./build-and-deploy.sh