saucelabs-UI-tests.yml

name: Sauce Labs UI Tests
on:
  # Schedule the UI tests so we can see in Sentry how the duration of transactions
  # changes over time.
  schedule:
    - cron: '0 0 * * *'
  push:
    branches:
      - master

  pull_request:
    paths:
      - 'Sources/**'
      - 'Tests/**'
      - '.github/workflows/saucelabs-UI-tests.yml'
      - 'fastlane/**'
      - '.sauce/config.yml'
      - 'scripts/set-device-tests-environment.patch'
      - 'scripts/ci-select-xcode.sh'

jobs:
  build-ui-tests:
    name: Build UITests with Xcode ${{matrix.xcode}}
    runs-on: ${{matrix.runs-on}}
    strategy:
      matrix:
        include:
          - runs-on: macos-11
            xcode: '12.5.1'

          - runs-on: macos-12
            xcode: '13.4.1'

    steps:
      - uses: actions/checkout@v4
      - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}
      - name: Install SentryCli
        run: brew install getsentry/tools/sentry-cli
      - run: git apply ./scripts/set-device-tests-environment.patch
      - name: Cache iOS-Swift App and dSYM build products
        id: ios-swift-cache
        uses: actions/cache@v3
        with:
          path: |
            DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
            DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
          key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/iOS-Swift/**') }}-Xcode-${{ matrix.xcode }}
      - name: Cache iOS-Swift UI Test Runner App build product
        id: ios-swift-uitest-runner-cache
        uses: actions/cache@v3
        with:
          path: |
            DerivedData/Build/Products/Debug-iphoneos/iOS-SwiftUITests-Runner.app
          key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/iOS-SwiftUITests/**') }}-Xcode-${{ matrix.xcode }}
      - run: fastlane build_ios_swift_for_tests
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - run: fastlane build_ios_swift_ui_test
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Upload dSYMs
        run: |
          sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
      - name: Archiving DerivedData
        uses: actions/upload-artifact@v3
        with:
          name: DerivedData-Xcode-${{matrix.xcode}}
          path: |
            **/Debug-iphoneos/iOS-Swift.app
            **/Debug-iphoneos/iOS-SwiftUITests-Runner.app

  run-ui-tests-with-sauce:
    name: Run UI Tests for iOS ${{ matrix.suite }} on Sauce Labs
    runs-on: ubuntu-latest
    needs: build-ui-tests
    strategy:
      fail-fast: false
      matrix:
        include:
          - xcode: '13.4.1'
            suite: 'iOS-15'

          # We want to test the frame tracker at 120 fps
          - xcode: '13.4.1'
            suite: 'iPhone-Pro'

          - xcode: '13.4.1'
            suite: 'iOS-14'

          - xcode: '13.4.1'
            suite: 'iOS-13'

          # iOS 12 has a failing test that we need to fix https://github.com/getsentry/sentry-cocoa/issues/1566
          # iOS 11 keeps timing out and we don't know how to fix it.
          - xcode: '12.5.1'
            suite: 'iOS-10'

    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v3
        with:
          name: DerivedData-Xcode-${{matrix.xcode}}

      - run: npm install -g saucectl@0.107.2

      # As Sauce Labs is a bit flaky we retry 5 times
      - name: Run Tests in SauceLab
        env:
          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
        run: for i in {1..5}; do saucectl run --select-suite ${{ matrix.suite }} && break ; done