diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 6ab8b080b..8432d2f92 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -15,7 +15,7 @@ body: attributes: label: Singer SDK Version description: Version of the library you are using - placeholder: "0.42.1" + placeholder: "0.44.1" validations: required: true - type: checkboxes diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 2647d95ca..b41620bf9 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,3 +3,9 @@ contact_links: - name: Meltano Community url: https://meltano.com/slack about: Join us on Slack. +- name: Start a discussion + url: https://github.com/meltano/sdk/discussions/new + about: Start a GitHub discussion. +- name: Singer SDK Documentation + url: https://sdk.meltano.com + about: Learn more about the Singer SDK. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8625a873a..5ea6b4257 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,7 +17,7 @@ updates: - "patch" versioning-strategy: increase-if-necessary - package-ecosystem: pip - directory: "/.github/workflows" + directory: "/.github/workflows/resources" schedule: interval: weekly time: "12:00" diff --git a/.github/workflows/api-changes.yml b/.github/workflows/api-changes.yml index 108d5b6f0..7f18d056f 100644 --- a/.github/workflows/api-changes.yml +++ b/.github/workflows/api-changes.yml @@ -34,7 +34,7 @@ jobs: - name: Install tools env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | python -Im pip install -U pip pipx install griffe nox diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 23d2f79eb..1c592cc21 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -1,4 +1,4 @@ -name: codspeed +name: Performance Testing with CodSpeed 🐇 on: push: @@ -24,6 +24,13 @@ on: # performance analysis in order to generate initial data. workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + jobs: benchmarks: runs-on: ubuntu-latest diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt deleted file mode 100644 index 9c097dc1a..000000000 --- a/.github/workflows/constraints.txt +++ /dev/null @@ -1,5 +0,0 @@ -griffe~=1.5 -pip==24.3.1 -poetry==1.8.4 -pre-commit==4.0.1 -nox==2024.10.9 diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 90454b8dd..21b08bff0 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -30,36 +30,32 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Upgrade pip - env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt - run: | - pip install pip - pip --version + + - uses: astral-sh/setup-uv@v5 + with: + version: ">=0.4.30" - name: Install Poetry env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + UV_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | - pipx install poetry + uv tool install poetry poetry --version - uses: actions/setup-python@v5 with: python-version: 3.x - - uses: astral-sh/setup-uv@v4 - with: - version: ">=0.4.30" - - name: Install pre-commit + env: + UV_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | uv tool install --with=pre-commit-uv pre-commit pre-commit --version - name: Install Nox env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + UV_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | uv tool install nox nox --version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cd706cb6..0239b2c03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,7 @@ jobs: with: name: Packages path: dist - - uses: actions/attest-build-provenance@v1 + - uses: actions/attest-build-provenance@v2 id: attest with: subject-path: "./dist/singer_sdk*" @@ -65,7 +65,7 @@ jobs: name: Packages path: dist - name: Publish - uses: pypa/gh-action-pypi-publish@v1.12.2 + uses: pypa/gh-action-pypi-publish@v1.12.4 upload-to-release: name: Upload files to release diff --git a/.github/workflows/resources/requirements.txt b/.github/workflows/resources/requirements.txt new file mode 100644 index 000000000..8a61ee357 --- /dev/null +++ b/.github/workflows/resources/requirements.txt @@ -0,0 +1,5 @@ +griffe~=1.5 +pip==25.0 +poetry==2.0.1 +pre-commit==4.1.0 +nox==2024.10.9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f946d098..1f4f3de2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,6 @@ on: pull_request: types: [opened, synchronize, reopened] paths: - - "cookiecutter/**" - "samples/**" - "singer_sdk/**" - "tests/**" @@ -12,13 +11,12 @@ on: - "poetry.lock" - "pyproject.toml" - ".github/workflows/test.yml" - - ".github/workflows/constraints.txt" + - ".github/workflows/resources/requirements.txt" push: branches: - main - v* paths: - - "cookiecutter/**" - "samples/**" - "singer_sdk/**" - "tests/**" @@ -26,7 +24,7 @@ on: - "poetry.lock" - "pyproject.toml" - ".github/workflows/test.yml" - - ".github/workflows/constraints.txt" + - ".github/workflows/resources/requirements.txt" workflow_dispatch: inputs: {} @@ -73,14 +71,14 @@ jobs: - name: Upgrade pip env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | pip install pip pip --version - name: Install Nox env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | pipx install 'nox[uv]' nox --version @@ -128,14 +126,14 @@ jobs: - name: Upgrade pip env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | pip install pip pip --version - name: Install Nox env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | pipx install 'nox[uv]' nox --version @@ -161,7 +159,7 @@ jobs: - name: Upgrade pip env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | pip install pip pip --version @@ -173,7 +171,7 @@ jobs: - name: Install Nox env: - PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt + PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt run: | pipx install 'nox[uv]' nox --version diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index ab81bb067..c4f025393 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -51,7 +51,7 @@ jobs: - name: Bump version id: cz-bump - uses: commitizen-tools/commitizen-action@0.22.0 + uses: commitizen-tools/commitizen-action@0.23.1 with: increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }} prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b45f70d1..be583b662 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,14 +37,14 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.30.0 + rev: 0.31.0 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.1 + rev: v0.9.3 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] @@ -59,6 +59,6 @@ repos: )$ - repo: https://github.com/python-poetry/poetry - rev: 1.8.0 + rev: 2.0.1 hooks: - id: poetry-check diff --git a/.readthedocs.yml b/.readthedocs.yml index 19438e79e..277036810 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-24.04 tools: - python: "3.12" + python: "3.13" sphinx: builder: html diff --git a/CHANGELOG.md b/CHANGELOG.md index 78d17d1e8..df61b344e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,76 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.44.1 (2025-01-29) + +### 🐛 Fixes + +- [#2847](https://github.com/meltano/sdk/issues/2847) Update Cookiecutter templates +- [#2844](https://github.com/meltano/sdk/issues/2844) Avoid writing an empty state -- _**Thanks @joaopamaral!**_ +- [#2843](https://github.com/meltano/sdk/issues/2843) Use a SQLAlchemy to generate an insert statement + +## v0.44.0 (2025-01-23) + +### ✨ New + +- [#2830](https://github.com/meltano/sdk/issues/2830) Allow developers to mark stream schema and settings fields as deprecated +- [#2829](https://github.com/meltano/sdk/issues/2829) Support a `x-sql-datatype` JSON Schema annotation to let targets customize SQL type handling +- [#2819](https://github.com/meltano/sdk/issues/2819) Add SHA256 encryption method to inline stream maps -- _**Thanks @ben-schulz-mh!**_ + +### 📦 Packaging changes + +- [#2407](https://github.com/meltano/sdk/issues/2407) Use Poetry support for PEP 621 +- [#2822](https://github.com/meltano/sdk/issues/2822) Make paramiko and transitive SSH dependencies optional +- [#2821](https://github.com/meltano/sdk/issues/2821) Require urllib3 < 2 on Python < 3.10 + +## v0.43.1 (2024-12-10) + +### 🐛 Fixes + +- [#2807](https://github.com/meltano/sdk/issues/2807) Allow developers to set `RESTStream.http_method` + +## v0.43.0 (2024-12-10) + +### ✨ New + +- [#2482](https://github.com/meltano/sdk/issues/2482) Allow SQL tap developers to auto-skip certain schemas from discovery +- [#2784](https://github.com/meltano/sdk/issues/2784) Added a new built-in setting `activate_version` for targets to optionally disable processing of `ACTIVATE_VERSION` messages +- [#2780](https://github.com/meltano/sdk/issues/2780) Numeric values are now parsed as `decimal.Decimal` in REST and GraphQL stream responses +- [#2775](https://github.com/meltano/sdk/issues/2775) Log a stream's bookmark (if it's avaiable) when its sync starts +- [#2703](https://github.com/meltano/sdk/issues/2703) Targets now emit record count from the built-in batch file processor +- [#2774](https://github.com/meltano/sdk/issues/2774) Accept a `maxLength` limit for VARCHARs +- [#2769](https://github.com/meltano/sdk/issues/2769) Add `versioning-strategy` to dependabot config of Cookiecutter templates +- [#2765](https://github.com/meltano/sdk/issues/2765) The last received Singer message is now logged when the target fails +- [#2762](https://github.com/meltano/sdk/issues/2762) Support other content-types in REST streams + +### 🐛 Fixes + +- [#2790](https://github.com/meltano/sdk/issues/2790) Ensure the required global folder tap settings are merged into the concrete implementation settings +- [#2785](https://github.com/meltano/sdk/issues/2785) Use FS-specific `listdir` in folder tap +- [#2778](https://github.com/meltano/sdk/issues/2778) The path of the offending field is now printed for config validation errors +- [#2770](https://github.com/meltano/sdk/issues/2770) Respect standard Singer stream metadata `table-key-properties`, `replication-key` and `forced-replication-method` +- [#2755](https://github.com/meltano/sdk/issues/2755) Safely compare UUID replication keys with state bookmarks -- _**Thanks @nikzavada!**_ + +### ⚙️ Under the Hood + +- [#2805](https://github.com/meltano/sdk/issues/2805) Rename setting `activate_version` to `process_activate_version_messages` +- [#2788](https://github.com/meltano/sdk/issues/2788) Fail early if input files to `--catalog` or `--state` do not exist +- [#2781](https://github.com/meltano/sdk/issues/2781) Added a class method to instantiate `SQLToJSONSchema` from the tap configuration +- [#2566](https://github.com/meltano/sdk/issues/2566) Standardize on JSON Schema Draft 2020-12 to validate stream schemas +- [#2751](https://github.com/meltano/sdk/issues/2751) Dropped support for Python 3.8 + +### ⚡ Performance Improvements + +- [#2793](https://github.com/meltano/sdk/issues/2793) Improved discovery performance for SQL taps + +### 📚 Documentation Improvements + +- [#2796](https://github.com/meltano/sdk/issues/2796) Document how to configure nested stream maps values with environment variables in Meltano + +### 📦 Packaging changes + +- [#2797](https://github.com/meltano/sdk/issues/2797) SQL taps now require SQLAlchemy 2.0+ + ## v0.42.1 (2024-11-11) ### 🐛 Fixes diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml index 179acf18a..ca63a4234 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml @@ -10,11 +10,12 @@ permissions: jobs: build: runs-on: ubuntu-latest + outputs: + version: {{ '${{ steps.baipp.outputs.package_version }}' }} steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: hynek/build-and-inspect-python-package@v2 + id: baipp publish: name: Publish to PyPI @@ -23,7 +24,13 @@ jobs: ## TODO: optionally provide the name of the environment for the trusted ## publisher on PyPI ## https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment - # environment: pypi + # environment: + # name: pypi + {%- if cookiecutter.variant != "None (Skip)" %} + # url: https://pypi.org/project/{{cookiecutter.variant}}-{{cookiecutter.mapper_id}}/{{ '${{ steps.baipp.outputs.package_version }}' }} + {%- else %} + # url: https://pypi.org/project/{{cookiecutter.mapper_id}}/{{ '${{ steps.baipp.outputs.package_version }}' }} + {%- endif %} if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v4 @@ -42,4 +49,4 @@ jobs: - name: Publish ## TODO: create a trusted publisher on PyPI ## https://docs.pypi.org/trusted-publishers/ - uses: pypa/gh-action-pypi-publish@v1.12.2 + uses: pypa/gh-action-pypi-publish@v1.12.3 diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml index e456b9896..876efb216 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml @@ -18,19 +18,19 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.31.0 hooks: - id: check-dependabot - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.3 + rev: v0.9.3 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.13.0 + rev: v1.14.1 hooks: - id: mypy diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md index ded365fb2..8f24517b3 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md @@ -6,11 +6,11 @@ Built with the [Meltano Mapper SDK](https://sdk.meltano.com) for Singer Mappers.