Merge branch 'main' into edgarrmondragon/feat/singer-decimal

meltano · Jan 30, 2025 · 584d3b5 · 584d3b5
2 parents 9df828b + 1ea1520
commit 584d3b5
Show file tree

Hide file tree

Showing 108 changed files with 2,356 additions and 1,418 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.42.1"
+      placeholder: "0.44.1"
     validations:
       required: true
   - type: checkboxes

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -3,3 +3,9 @@ contact_links:
 - name: Meltano Community
   url: https://meltano.com/slack
   about: Join us on Slack.
+- name: Start a discussion
+  url: https://github.com/meltano/sdk/discussions/new
+  about: Start a GitHub discussion.
+- name: Singer SDK Documentation
+  url: https://sdk.meltano.com
+  about: Learn more about the Singer SDK.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -17,7 +17,7 @@ updates:
           - "patch"
     versioning-strategy: increase-if-necessary
   - package-ecosystem: pip
-    directory: "/.github/workflows"
+    directory: "/.github/workflows/resources"
     schedule:
       interval: weekly
       time: "12:00"

diff --git a/.github/workflows/api-changes.yml b/.github/workflows/api-changes.yml
@@ -34,7 +34,7 @@ jobs:
 
     - name: Install tools
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         python -Im pip install -U pip
         pipx install griffe nox

diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -1,4 +1,4 @@
-name: codspeed
+name: Performance Testing with CodSpeed 🐇
 
 on:
   push:
@@ -24,6 +24,13 @@ on:
   # performance analysis in order to generate initial data.
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1"
+
 jobs:
   benchmarks:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml
@@ -30,36 +30,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Upgrade pip
-      env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
-      run: |
-        pip install pip
-        pip --version
+
+    - uses: astral-sh/setup-uv@v5
+      with:
+        version: ">=0.4.30"
 
     - name: Install Poetry
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        UV_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
-        pipx install poetry
+        uv tool install poetry
         poetry --version
 
     - uses: actions/setup-python@v5
       with:
         python-version: 3.x
 
-    - uses: astral-sh/setup-uv@v4
-      with:
-        version: ">=0.4.30"
-
     - name: Install pre-commit
+      env:
+        UV_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         uv tool install --with=pre-commit-uv pre-commit
         pre-commit --version
 
     - name: Install Nox
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        UV_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         uv tool install nox
         nox --version

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -40,7 +40,7 @@ jobs:
       with:
         name: Packages
         path: dist
-    - uses: actions/attest-build-provenance@v1
+    - uses: actions/attest-build-provenance@v2
       id: attest
       with:
         subject-path: "./dist/singer_sdk*"
@@ -65,7 +65,7 @@ jobs:
         name: Packages
         path: dist
     - name: Publish
-      uses: pypa/[email protected].2
+      uses: pypa/[email protected].4
 
   upload-to-release:
     name: Upload files to release

diff --git a/.github/workflows/resources/requirements.txt b/.github/workflows/resources/requirements.txt
@@ -0,0 +1,5 @@
+griffe~=1.5
+pip==25.0
+poetry==2.0.1
+pre-commit==4.1.0
+nox==2024.10.9
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -4,29 +4,27 @@ on:
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
-      - "cookiecutter/**"
       - "samples/**"
       - "singer_sdk/**"
       - "tests/**"
       - "noxfile.py"
       - "poetry.lock"
       - "pyproject.toml"
       - ".github/workflows/test.yml"
-      - ".github/workflows/constraints.txt"
+      - ".github/workflows/resources/requirements.txt"
   push:
     branches:
       - main
       - v*
     paths:
-      - "cookiecutter/**"
       - "samples/**"
       - "singer_sdk/**"
       - "tests/**"
       - "noxfile.py"
       - "poetry.lock"
       - "pyproject.toml"
       - ".github/workflows/test.yml"
-      - ".github/workflows/constraints.txt"
+      - ".github/workflows/resources/requirements.txt"
   workflow_dispatch:
     inputs: {}
 
@@ -73,14 +71,14 @@ jobs:
 
     - name: Upgrade pip
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         pip install pip
         pip --version
 
     - name: Install Nox
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         pipx install 'nox[uv]'
         nox --version
@@ -128,14 +126,14 @@ jobs:
 
     - name: Upgrade pip
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         pip install pip
         pip --version
 
     - name: Install Nox
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         pipx install 'nox[uv]'
         nox --version
@@ -161,7 +159,7 @@ jobs:
 
     - name: Upgrade pip
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         pip install pip
         pip --version
@@ -173,7 +171,7 @@ jobs:
 
     - name: Install Nox
       env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/resources/requirements.txt
       run: |
         pipx install 'nox[uv]'
         nox --version

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -51,7 +51,7 @@ jobs:
 
     - name: Bump version
       id: cz-bump
-      uses: commitizen-tools/commitizen-action@0.22.0
+      uses: commitizen-tools/commitizen-action@0.23.1
       with:
         increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }}
         prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -37,14 +37,14 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.30.0
+  rev: 0.31.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.1
+  rev: v0.9.3
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
@@ -59,6 +59,6 @@ repos:
       )$
 
 - repo: https://github.com/python-poetry/poetry
-  rev: 1.8.0
+  rev: 2.0.1
   hooks:
   - id: poetry-check
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-24.04
   tools:
-    python: "3.12"
+    python: "3.13"
 
 sphinx:
   builder: html

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,76 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.44.1 (2025-01-29)
+
+### 🐛 Fixes
+
+- [#2847](https://github.com/meltano/sdk/issues/2847) Update Cookiecutter templates
+- [#2844](https://github.com/meltano/sdk/issues/2844) Avoid writing an empty state -- _**Thanks @joaopamaral!**_
+- [#2843](https://github.com/meltano/sdk/issues/2843) Use a SQLAlchemy to generate an insert statement
+
+## v0.44.0 (2025-01-23)
+
+### ✨ New
+
+- [#2830](https://github.com/meltano/sdk/issues/2830) Allow developers to mark stream schema and settings fields as deprecated
+- [#2829](https://github.com/meltano/sdk/issues/2829) Support a `x-sql-datatype` JSON Schema annotation to let targets customize SQL type handling
+- [#2819](https://github.com/meltano/sdk/issues/2819) Add SHA256 encryption method to inline stream maps -- _**Thanks @ben-schulz-mh!**_
+
+### 📦 Packaging changes
+
+- [#2407](https://github.com/meltano/sdk/issues/2407) Use Poetry support for PEP 621
+- [#2822](https://github.com/meltano/sdk/issues/2822) Make paramiko and transitive SSH dependencies optional
+- [#2821](https://github.com/meltano/sdk/issues/2821) Require urllib3 < 2 on Python < 3.10
+
+## v0.43.1 (2024-12-10)
+
+### 🐛 Fixes
+
+- [#2807](https://github.com/meltano/sdk/issues/2807) Allow developers to set `RESTStream.http_method`
+
+## v0.43.0 (2024-12-10)
+
+### ✨ New
+
+- [#2482](https://github.com/meltano/sdk/issues/2482) Allow SQL tap developers to auto-skip certain schemas from discovery
+- [#2784](https://github.com/meltano/sdk/issues/2784) Added a new built-in setting `activate_version` for targets to optionally disable processing of `ACTIVATE_VERSION` messages
+- [#2780](https://github.com/meltano/sdk/issues/2780) Numeric values are now parsed as `decimal.Decimal` in REST and GraphQL stream responses
+- [#2775](https://github.com/meltano/sdk/issues/2775) Log a stream's bookmark (if it's avaiable) when its sync starts
+- [#2703](https://github.com/meltano/sdk/issues/2703) Targets now emit record count from the built-in batch file processor
+- [#2774](https://github.com/meltano/sdk/issues/2774) Accept a `maxLength` limit for VARCHARs
+- [#2769](https://github.com/meltano/sdk/issues/2769) Add `versioning-strategy` to dependabot config of Cookiecutter templates
+- [#2765](https://github.com/meltano/sdk/issues/2765) The last received Singer message is now logged when the target fails
+- [#2762](https://github.com/meltano/sdk/issues/2762) Support other content-types in REST streams
+
+### 🐛 Fixes
+
+- [#2790](https://github.com/meltano/sdk/issues/2790) Ensure the required global folder tap settings are merged into the concrete implementation settings
+- [#2785](https://github.com/meltano/sdk/issues/2785) Use FS-specific `listdir` in folder tap
+- [#2778](https://github.com/meltano/sdk/issues/2778) The path of the offending field is now printed for config validation errors
+- [#2770](https://github.com/meltano/sdk/issues/2770) Respect standard Singer stream metadata `table-key-properties`, `replication-key` and `forced-replication-method`
+- [#2755](https://github.com/meltano/sdk/issues/2755) Safely compare UUID replication keys with state bookmarks -- _**Thanks @nikzavada!**_
+
+### ⚙️ Under the Hood
+
+- [#2805](https://github.com/meltano/sdk/issues/2805) Rename setting `activate_version` to `process_activate_version_messages`
+- [#2788](https://github.com/meltano/sdk/issues/2788) Fail early if input files to `--catalog` or `--state` do not exist
+- [#2781](https://github.com/meltano/sdk/issues/2781) Added a class method to instantiate `SQLToJSONSchema` from the tap configuration
+- [#2566](https://github.com/meltano/sdk/issues/2566) Standardize on JSON Schema Draft 2020-12 to validate stream schemas
+- [#2751](https://github.com/meltano/sdk/issues/2751) Dropped support for Python 3.8
+
+### ⚡ Performance Improvements
+
+- [#2793](https://github.com/meltano/sdk/issues/2793) Improved discovery performance for SQL taps
+
+### 📚 Documentation Improvements
+
+- [#2796](https://github.com/meltano/sdk/issues/2796) Document how to configure nested stream maps values with environment variables in Meltano
+
+### 📦 Packaging changes
+
+- [#2797](https://github.com/meltano/sdk/issues/2797) SQL taps now require SQLAlchemy 2.0+
+
 ## v0.42.1 (2024-11-11)
 
 ### 🐛 Fixes

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml
@@ -10,11 +10,12 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
+    outputs:
+      version: {{ '${{ steps.baipp.outputs.package_version }}' }}
     steps:
     - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
     - uses: hynek/build-and-inspect-python-package@v2
+      id: baipp
 
   publish:
     name: Publish to PyPI
@@ -23,7 +24,13 @@ jobs:
     ## TODO: optionally provide the name of the environment for the trusted
     ## publisher on PyPI
     ## https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment
-    # environment: pypi
+    # environment:
+    #   name: pypi
+    {%- if cookiecutter.variant != "None (Skip)" %}
+    #   url: https://pypi.org/project/{{cookiecutter.variant}}-{{cookiecutter.mapper_id}}/{{ '${{ steps.baipp.outputs.package_version }}' }}
+    {%- else %}
+    #   url: https://pypi.org/project/{{cookiecutter.mapper_id}}/{{ '${{ steps.baipp.outputs.package_version }}' }}
+    {%- endif %}
     if: startsWith(github.ref, 'refs/tags/')
     steps:
     - uses: actions/download-artifact@v4
@@ -42,4 +49,4 @@ jobs:
     - name: Publish
       ## TODO: create a trusted publisher on PyPI
       ## https://docs.pypi.org/trusted-publishers/
-      uses: pypa/[email protected].2
+      uses: pypa/[email protected].3
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
@@ -18,19 +18,19 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.4
+  rev: 0.31.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.3
+  rev: v0.9.3
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.13.0
+  rev: v1.14.1
   hooks:
   - id: mypy