evergreen.yml

# Evergreen Project Config

# When a task that used to pass starts to fail
# Go through all versions that may have been skipped to detect
# when the task started failing
stepback: true

# Mark a failure as a system/bootstrap failure (purple box) rather then a task
# failure by default.
# Actual testing tasks are marked with `type: test`
command_type: system

# Protect ourself against rogue test case, or curl gone wild, that runs forever
# 12 minutes is the longest we'll ever run
exec_timeout_secs: 3600 # 12 minutes is the longest we'll ever run

# What to do when evergreen hits the timeout (`post:` tasks are run automatically)
timeout:
  - command: shell.exec
    params:
      script: |
        ls -la

functions:
  "build connector":
    - command: shell.exec
      params:
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          if $UPDATE_PLUGIN_VERSION; then
            sed -i "s/\[Version = \"0.0.0\"\]/\[Version = \"${RELEASE_VERSION}\"\]/g" \
              connector/MongoDBAtlasODBC.pq
            sed -i "s/<connector-version>/${RELEASE_VERSION}/g" \
              connector/MongoDBAtlasODBC.pq
          fi
          tools/MakePQX.exe compile connector/
          tools/MakePQX.exe pack -mz connector/bin/AnyCPU/Debug/connector.mez -t MongoDBAtlasODBC.pqx

  "fetch packaged connector":
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC.pqx
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC.pqx
        bucket: mciuploads

  "fetch signed connector":
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC-signed.pqx
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC-signed.pqx
        bucket: mciuploads

  "fetch powerbi source":
    - command: git.get_project
      params:
        directory: mongo-powerbi-connector

  "generate expansions":
    - command: shell.exec
      params:
        silent: true
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |

          if [[ "${triggered_by_git_tag}" != "" ]]; then
            export RELEASE_VERSION=$(echo ${triggered_by_git_tag} | sed s/v//)
            export UPDATE_PLUGIN_VERSION=true
          else
            export UPDATE_PLUGIN_VERSION=false
            export RELEASE_VERSION="0.0.0-$(git describe --always)"
          fi

          cat <<EOT > expansions.yml
          RELEASE_VERSION: "$RELEASE_VERSION"
          UPDATE_PLUGIN_VERSION: "$UPDATE_PLUGIN_VERSION"
          RELEASE_PQX_PATH: "mongo-powerbi-connector/MongoDBAtlasODBC-$RELEASE_VERSION.pqx"
          prepare_shell: |
            set -o errexit
            export PATH="$PATH"
            export RELEASE_VERSION="$RELEASE_VERSION"
            export RELEASE_PQX_PATH="$RELEASE_PQX_PATH"
            export UPDATE_PLUGIN_VERSION="$UPDATE_PLUGIN_VERSION"
            export ADF_TEST_LOCAL_USER="${adf_test_local_user}"
            export ADF_TEST_LOCAL_PWD="${adf_test_local_pwd}"
            export ADF_TEST_LOCAL_AUTH_DB="${adf_test_local_auth_db}"
            export ADF_TEST_LOCAL_HOST="${adf_test_local_host}"
            export MDB_TEST_LOCAL_PORT="${mdb_test_local_port}"
          EOT
    - command: expansions.update
      params:
        file: mongo-powerbi-connector/expansions.yml

  "generate github token":
    command: github.generate_token
    params:
      owner: 10gen
      repo: mongohouse
      expansion_name: github_token

  "install mongoimport tool":
    - command: shell.exec
      params:
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          MONGO_TOOLS_VERSION=100.6.1
          curl -L -o mongodb-tools.zip \
            https://fastdl.mongodb.org/tools/db/mongodb-database-tools-windows-x86_64-$MONGO_TOOLS_VERSION.zip
          unzip -jn mongodb-tools.zip
          chmod +x mongoimport.exe

  "install power query tools":
    - command: shell.exec
      params:
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          curl  -L -o tools.zip https://www.nuget.org/api/v2/package/Microsoft.PowerQuery.SdkTools/
          unzip -n tools.zip
          chmod -R +x tools/*

  "run integration test":
    - command: shell.exec
      type: test
      params:
        env:
          GITHUB_TOKEN: "${github_token}"
        add_expansions_to_env: true
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          set +e

          # Start local ADF
          ./resources/run_adf.sh start &&

          # Import dataset to MongoDB
          ./mongoimport.exe --uri="mongodb://$ADF_TEST_LOCAL_HOST:$MDB_TEST_LOCAL_PORT/supplies" \
            --drop resources/integration_test/testdata/sales.json &&
          ./mongoimport.exe --uri="mongodb://$ADF_TEST_LOCAL_HOST:$MDB_TEST_LOCAL_PORT/integration_test" \
            --drop resources/integration_test/testdata/complex_types.json &&

          MONGOSHELL=$(find ./local_adf/ | grep mongosh.exe | head -1) &&

          # Generate schema on imported data
          $MONGOSHELL -u $ADF_TEST_LOCAL_USER --password $ADF_TEST_LOCAL_PWD --authenticationDatabase \
            $ADF_TEST_LOCAL_AUTH_DB $ADF_TEST_LOCAL_HOST/admin \
            --eval 'db.runCommand({sqlGenerateSchema: 1,
            sampleNamespaces: ["integration_test.complex_types", "supplies.sales"], setSchemas: true})' &&

          # Set credentials for the queryFiles
          echo $(eval echo $(cat resources/credentials.json )) | tools/PQTest.exe set-credential \
            --extension MongoDBAtlasODBC.pqx \
            --queryFile connector/MongoDBAtlasODBC.datasource-test.query.pq &&
          # Until the SDK is updated (October-November 2023, we can't run these tests)
          # echo $(eval echo $(cat resources/credentials.json )) | tools/PQTest.exe set-credential \
          #   --extension MongoDBAtlasODBC.pqx \
          #   --queryFile connector/MongoDBAtlasODBC.nativequery-test.query.pq &&

          # Run Integration Tests
          echo -e "\nRunning MongoDBAtlasODBC.datasource-test.query.pq" &&
          test_output=`tools/PQTest.exe run-test --extension MongoDBAtlasODBC.pqx \
            --queryFile connector/MongoDBAtlasODBC.datasource-test.query.pq --prettyPrint` 
          # Until the SDK is updated (October-November 2023, we can't run these tests)
          # echo "Running MongoDBAtlasODBC.nativequery-test.query.pq" &&
          # test_output+=`tools/PQTest.exe run-test --extension MongoDBAtlasODBC.pqx \
          #   --queryFile connector/MongoDBAtlasODBC.nativequery-test.query.pq --prettyPrint`
          EXITCODE=$?
          echo "$test_output"
          echo $test_output > MongoDBAtlasODBC.integration-test.log
          # Check for "100% success rate" and "All X Passed"
          if echo "$test_output" | grep -q "100% success rate" && echo "$test_output" | grep -q "All .* Passed"; then
            echo "All tests passed successfully."
          else
            echo "Tests failed or unexpected output. Check the log for details."
            EXITCODE=1
          fi

          # Stop local ADF, always run this to keep task from hanging
          ./resources/run_adf.sh stop
          exit $EXITCODE

  "setup ODBC driver on Windows":
    - command: shell.exec
      params:
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          curl -O https://translators-connectors-releases.s3.amazonaws.com/mongosql-odbc-driver/windows/snapshot/release/atsql.dll
          chmod 777 ./atsql.dll
          sed -i 's@%DRIVER_DLL_PATH%@'"$(echo "$(cygpath -w $(pwd))" | sed s',\\,\\\\\\\\,g')"'@' resources/odbc/setupDSN.reg
          reg import "resources\odbc\setupDSN.reg"
          echo "----- Registry entries after setup ----"
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\ODBC Drivers" -s 2> /dev/null || true
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\ODBC\ODBCINST.INI\ODBC Drivers" -s 2> /dev/null || true
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\ODBC\ODBCINST.INI\MongoDB Atlas SQL ODBC Driver" -s 2>/dev/null || true
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\MongoDB Atlas SQL ODBC Driver" -s 2> /dev/null || true
          echo "-------------------------"

  "clean up ODBC driver on Windows":
    - command: shell.exec
      params:
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          reg import "resources\odbc\cleanup_driver.reg"
          echo "----- Registry entries after clean-up----"
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\ODBC Drivers" -s 2> /dev/null || true
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\ODBC\ODBCINST.INI\ODBC Drivers" -s 2> /dev/null || true
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\ODBC\ODBCINST.INI\MongoDB Atlas SQL ODBC Driver" -s 2>/dev/null || true
          reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\MongoDB Atlas SQL ODBC Driver" -s 2> /dev/null || true
          echo "-------------------------"

  "sign connector":
    - command: shell.exec
      type: system
      params:
        silent: true
        working_dir: mongo-powerbi-connector
        env:
          SELF_SIGNED_PWD: "${self_signed_pwd}"
          SELF_SIGNED_CERT: "${self_signed_cert}"
          SELF_SIGNED_KEY: "${self_signed_key}"
        script: |
          ${prepare_shell}
          # The redaction of evergreen variables in the evergreen code is using go's replaceAll in a
          # way that replaces new lines with spaces. If they fix that, these sed commands will probably still
          # work, but will become unnecessary. In each case, we need to keep it from sticking new
          # lines in "BEGIN CERTIFICATE" (etc) because openssl is very very particular about
          # newlines and spaces in key and cert files.
          echo -ne $SELF_SIGNED_CERT | sed 's| CERT|CERT|g' | sed 's| |\n|g' | sed 's|CERT| CERT|g' > cert2 > mdb-self-signed-certificate.crt
          echo -ne $SELF_SIGNED_KEY | sed 's| PRIVATE |PRIVATE|g' | sed 's| |\n|g' | sed 's|PRIVATE| PRIVATE |g' > mdb-self-signed-key.pem
          # Convert certificate to pfx format, used by MakePQX.exe
          openssl pkcs12 -export -out mdb-self-signed-certificate.pfx -inkey mdb-self-signed-key.pem -in mdb-self-signed-certificate.crt -password pass:$SELF_SIGNED_PWD
          tools/MakePQX.exe sign MongoDBAtlasODBC.pqx --certificate  mdb-self-signed-certificate.pfx --password $SELF_SIGNED_PWD
          tools/MakePQX.exe verify MongoDBAtlasODBC.pqx
          cp MongoDBAtlasODBC.pqx MongoDBAtlasODBC-signed.pqx

  "upload packaged connector":
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC.pqx
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC.pqx
        bucket: mciuploads
        permissions: public-read
        content_type: application/octet-stream
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: mongo-powerbi-connector/connector/bin/AnyCPU/Debug/connector.mez
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC.mez
        bucket: mciuploads
        permissions: public-read
        content_type: application/octet-stream
        display_name: MongoDBAtlasODBC.mez

  "upload release":
    - command: s3.put
      params:
        aws_key: ${release_aws_key}
        aws_secret: ${release_aws_secret}
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC-signed.pqx
        remote_file: ${RELEASE_PQX_PATH}
        bucket: translators-connectors-releases
        permissions: public-read
        display_name: MongoDBAtlasODBC-${RELEASE_VERSION}.pqx
        content_type: application/octet-stream
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC.mez
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC.mez
        bucket: mciuploads
    - command: s3.put
      params:
        aws_key: ${release_aws_key}
        aws_secret: ${release_aws_secret}
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC.mez
        remote_file: mongo-powerbi-connector/MongoDBAtlasODBC-${RELEASE_VERSION}.mez
        bucket: translators-connectors-releases
        permissions: public-read
        content_type: application/octet-stream
        display_name: MongoDBAtlasODBC-${RELEASE_VERSION}.mez

  "upload signed connector":
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC-signed.pqx
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC-signed.pqx
        bucket: mciuploads
        permissions: public-read
        content_type: application/octet-stream

  "upload integration test logs":
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: mongo-powerbi-connector/MongoDBAtlasODBC.integration-test.log
        remote_file: mongo-powerbi-connector/artifacts/${version_id}/${build_variant}/MongoDBAtlasODBC.integration-test.log
        bucket: mciuploads
        permissions: public-read
        content_type: application/octet-stream

  "update download center feed":
    - command: shell.exec
      params:
        shell: bash
        working_dir: mongo-powerbi-connector
        script: |
          ${prepare_shell}
          if [[ $(git rev-parse --abbrev-ref HEAD) != 'main' ]]; then
             exit 1
          fi
          sed -i 's@{RELEASE_VERSION}@'${RELEASE_VERSION}'@' mongo-powerbi-connector-downloads_template.json
          sed -i 's@{RELEASE_PQX_PATH}@'${RELEASE_PQX_PATH}'@' mongo-powerbi-connector-downloads_template.json
          echo "-------------------------"
          cat mongo-powerbi-connector-downloads_template.json
          echo "-------------------------"
    - command: s3.put
      params:
        aws_key: ${release_aws_key}
        aws_secret: ${release_aws_secret}
        local_file: mongo-powerbi-connector/mongo-powerbi-connector-downloads_template.json
        remote_file: mongo-powerbi-connector/mongo-powerbi-connector-downloads.json
        bucket: translators-connectors-releases
        permissions: public-read
        content_type: application/json

pre:
  - func: "fetch powerbi source"
  - func: "generate expansions"
  - func: "install power query tools"

tasks:
  - name: build
    commands:
      - func: "build connector"
      - func: "upload packaged connector"

  - name: integration-test
    depends_on:
      - name: build
    commands:
      - func: "install mongoimport tool"
      - func: "generate github token"
      - func: "run integration test"

  - name: sign
    depends_on:
      - name: build
      - name: integration-test
    commands:
      - func: "fetch packaged connector"
      - func: "install power query tools"
      - func: "sign connector"
      - func: "upload signed connector"

  - name: release
    git_tag_only: true
    depends_on:
      - name: sign
    commands:
      - func: "fetch signed connector"
      - func: "upload release"

  - name: test-release
    patch_only: true
    depends_on:
      - name: sign
    commands:
      - func: "fetch signed connector"
      - func: "upload release"

  - name: download-center-update
    git_tag_only: true
    depends_on:
      - name: release
    commands:
      - func: "update download center feed"

task_groups:
  - name: test-integration-group
    setup_group_can_fail_task: true
    setup_group:
      - func: "fetch powerbi source"
      - func: "generate expansions"
      - func: "fetch packaged connector"
      - func: "install power query tools"
      - func: "setup ODBC driver on Windows"
    teardown_group:
      - func: "clean up ODBC driver on Windows"
      - func: "upload integration test logs"
    tasks:
      - integration-test

buildvariants:
  - name: windows-64
    display_name: "Windows"
    tasks:
      - name: build
        run_on: [windows-64-vs2019-large]
      - name: test-integration-group
        run_on: [windows-64-vs2019-large]
      - name: sign
        run_on: [windows-64-vs2015-large]
      - name: release
        run_on: [windows-64-vs2019-large]
      - name: test-release
        run_on: [windows-64-vs2019-large]
      - name: download-center-update
        run_on: [windows-64-vs2019-large]