pyproject.toml

[tool.poetry]
name = "meltano"
version = "3.5.0a3"
description = "Meltano is your CLI for ELT+: Open Source, Flexible, and Scalable. Move, transform, and test your data with confidence using a streamlined data engineering workflow you’ll love."
authors = ["Meltano <hello@meltano.com>"]
license = "MIT"
readme = "README.md"
repository = "https://github.com/meltano/meltano"
classifiers = [
  "Development Status :: 5 - Production/Stable",
  "Programming Language :: Python :: 3 :: Only",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
  "License :: OSI Approved :: MIT License",
  "Operating System :: OS Independent",
]
packages = [
  { include = "meltano", from = "src", format = "wheel" },
  { include = "src", format = "sdist" },
  { include = "tests", format = "sdist"},
]
keywords = [
  "Meltano",
  "ELT",
  "Data integration",
  "singer-io",
  "dbt",
]
documentation = "https://docs.meltano.com"
homepage = "https://meltano.com"

[tool.poetry.urls]
"Changelog" = "https://github.com/meltano/meltano/blob/main/CHANGELOG.md"
"Issue Tracker" = "https://github.com/meltano/meltano/issues"
"Slack" = "https://meltano.com/slack"
"Twitter" = "https://twitter.com/meltanodata/"
"Youtube" = "https://www.youtube.com/meltano"

[tool.poetry.dependencies]
aiodocker = "^0.22.0"
alembic = "^1.13.2"
atomicwrites = "^1.2.1"
azure-common = {version = "^1.1.28", optional = true}
azure-core = {version = "^1.30.2", optional = true}
azure-identity = {version = "^1.17.1", optional = true}
azure-storage-blob = {version = "^12.20.0", optional = true}
boto3 = {version = "^1.34.136", optional = true}
check-jsonschema = "^0.28.6"
click = "^8.1"
click-default-group = "^1.2.4"
click-didyoumean = "^0.3.1"
croniter = "^2.0.5"
fasteners = "^0.19"
flatten-dict = "^0"
google-cloud-storage = {version = ">=1.31.0", optional = true}
importlib-resources = "^6.4.0"
jinja2 = "^3.1.4"
jsonschema = "^4.22"
packaging = "^24.1"
platformdirs = "^4.2.2"
psutil = "^6.0.0"
psycopg = {version = "^3.2.1", extras = ["binary"], optional = true}
psycopg2-binary = {version="^2.9.9", optional=true}
pyjwt = "^2.6.0"
pymssql = {version = "^2.3.0", optional = true}
python = ">=3.8,<3.13"
python-dateutil = "^2.9.0"
python-dotenv = "^1.0.1"
python-slugify = "^8.0.4"
python-ulid = "^1.1.0"
pyyaml = "^6.0.1"
questionary = "^2.0.1"
requests = "^2.32.3"
rich = "^13.7.1"
"ruamel.yaml" = "^0.18.6"
setuptools = {version = "^70.1.1", python = ">=3.12"}
smart-open = "^7.0.4"
snowplow-tracker = "^1.0.2"
sqlalchemy = "^2.0.31"
structlog = "^24.2.0"
tabulate = "^0.9.0"
typing-extensions = "^4.12.2"
tzlocal = "^5.2"
# Compatibility issues with boto: https://github.com/boto/botocore/pull/3034
urllib3 = "<2"
uv = { version = ">=0.1.24,<0.3", optional = true }
virtualenv = "^20.26.3"
yaspin = "^2.3.0"

[tool.poetry.extras]
azure = ["azure-storage-blob", "azure-common", "azure-core", "azure-identity"]
gcs = ["google-cloud-storage"]
mssql = ["pymssql"]
postgres = ["psycopg"]
psycopg2 = ["psycopg2-binary"]
s3 = ["boto3"]
uv = ["uv"]

[tool.poetry.group.dev.dependencies]
backoff = "^2.1.2"
black = "^24.4.2"
boto3-stubs = {extras = ["essential"], version = "1.34.136"}
bumpversion = "^0.6.0"
colorama = "^0.4.4"
coverage = {extras = ["toml"], version = ">=7.3.2,!=7.3.3"}
hypothesis = "^6.104.2"
mock = "^5.0.2"
moto = "^5.0.10"
mypy = "^1.10.1"
pre-commit = "^3.5.0"
pytest = "^8.2.2"
pytest-asyncio = "^0.23.7"
pytest-cov = "^5.0.0"
pytest-docker = "^3.1"
pytest-httpserver = "^1.0.10"
pytest-order = "^1.2"
pytest-randomly = "^3.12"
pytest-rerunfailures = "^14.0"
pytest-structlog = "^1.0"
pytest-xdist = "^3.6"
requests-mock = "^1.12.1"
setproctitle = "^1.3" # Used by pytest-xdist to aid with handling resource intensive processes.
time-machine = "^2.14.2"
types-croniter = "^2.0.0"
types-jsonschema = "^4.22.0.20240610"
types-psutil = "^6.0.0.20240621"
types-python-dateutil = "^2.9.0.20240316"
types-python-slugify = "^8.0.2.20240310"
types-pyyaml = "^6.0.12.20240311"
types-requests = "^2.31.0"
types-tabulate = "^0.9.0.20240106"

[tool.poetry.scripts]
meltano = "meltano.cli:main"

[tool.black]
line-length = 88

[tool.flakeheaven]
accept_encodings = "utf-8"
baseline = "flakeheaven-baseline"
docstring_style = "google"
doctests = true
enable_extensions = "G"
exclude = [
  ".nox/",
  ".venv",
  "venv",
  "*.md",
  "src/meltano/migrations/",
]
format = "colored"
inline_quotes = "double"
max_complexity = 6
max_imports = 20
max_line_length = 88
show_source = true
statistics = true

[tool.flakeheaven.plugins]
"flake8-*" = [
  "+*",
]
"flake8-isort" = [
  "-*",
]
"flake8-commas" = [
  "-*",
]
"flake8-bandit" = [
  "-*",
]
"flake8-eradicate" = [
  "-*", # Covered by Ruff
]
"flake8-string-format" = [
  "-P101", # Allow format strings with unindex parameters (because the pyupgrade pre-commit hook enforces their usage)
]
"flake8-rst-docstrings" = [
  "-*",
]
mccabe = [
  "+*",
]
pycodestyle = [
  "-*", # Covered by Ruff
]
pyflakes = [
  "+*",
]
"wemake-python-styleguide" = [
  "+*",
  "-WPS100",
  "-WPS110", # Allow blacklisted variable names that can be clear enough in method context
  "-WPS111",
  "-WPS112",
  "-WPS115", # Allow upper-case constants in Enum subclasses
  "-WPS201",
  "-WPS202", # Allow many module members
  "-WPS211", # Allow "too many"
  "-WPS212", # Ignore 'too many return statements' - noqa parse inconsistent between py38 vs prior
  "-WPS220",
  "-WPS221",
  "-WPS226",
  "-WPS229",
  "-WPS231",
  "-WPS232",
  "-WPS237",
  "-WPS300", # Allow local folder imports (import .module)
  "-WPS305", # Allow f-strings
  "-WPS306", # Ignore missing base class, required by pyupgrade: https://github.com/asottile/pyupgrade#rewrites-class-declaration
  "-WPS309", # Allow reversed compare order
  "-WPS316",
  "-WPS317", # Allow "incorrect multi-line parameters", since Black forces them to be "incorrect"
  "-WPS323", # Allow % formatting on strings, for logging
  "-WPS324", # Allow inconsistent returns - permitted because of too many false positives
  "-WPS326", # Allow syntax-level (process when first parsed, rather than at every run) string concatenation
  "-WPS332", # Allow assignment expressions (walrus operator :=)
  "-WPS337",
  "-WPS338", # Allow 'incorrect' order of methods in a class
  "-WPS348", # Allow `.` at beginning of line to accommodate Black formatting of multiline chained function calls
  "-WPS352",
  "-WPS354", # Allow consecutive yield expressions
  "-WPS402", # Allow "overuse" of noqa comments.
  "-WPS404",
  "-WPS407",
  "-WPS410", # Allow metadata variable (__all__)
  "-WPS412",
  "-WPS414", # Allow unpacking assignments
  "-WPS420", # Allow `pass` keywords in `except` bodies to prevent falling through to another `except`
  "-WPS429",
  "-WPS430",
  "-WPS431", # Allow nested classes
  "-WPS433",
  "-WPS440", # Allow block variable overlap, e.g. multiple for-loops that use the same name for their loop variables
  "-WPS441", # Allow control variables to be used after their block, because this rule has too many false-positives
  "-WPS454",
  "-WPS458", # Import name collisions - disabled because it raises many false positives
  "-WPS461", # Allow all inline ignores
  "-WPS501",
  "-WPS507",
  "-WPS509", # Allow "incorrect" ternary expressions
  "-WPS528", # Allow iterating over keys in a dict, then using them to access values
  "-WPS529", # Allow use of `dict_object[key]` (i.e. `dict.__getitem__`) instead of `dict.get`
  "-WPS600", # Allow subclassing a builtin, such as 'str' (used for serializable Enums)
  "-WPS602", # Allow using @staticmethod
  "-WPS615", # Allow "unpythonic" getters and setters
]

[tool.flakeheaven.exceptions."*tests/*"]
# Don't require docstrings in tests
"flake8-darglint" = [
  "-DAR101",
  "-DAR201",
  "-DAR301",
]
"flake8-docstrings" = [
  "-D100",
  "-D101",
  "-D102",
  "-D103",
  "-D104",
  "-D105",
  "-D107",
]
"flake8-string-format" = [
  "-P103", # String does contain unindexed parameters
]
"wemake-python-styleguide" = [
  "-WPS114", # Allow underscored number name pattern (e.g. tap_covid_19)
  "-WPS204", # Don't warn on overused expressions in test methods
  "-WPS211", # Don't warn on too many arguments in test methods
  "-WPS210", # Don't warn on too many local variables in test methods
  "-WPS218", # Don't warn on too many `assert` statements in test methods
  "-WPS213", # Found too many expressions
  "-WPS214", # Don't warn on too many methods in test modules
  "-WPS217", # Don't warn about there being too many await expressions
  "-WPS226", # Don't warn on too many string constants in test methods
  "-WPS316", # Context manager with too many assignments
  "-WPS442", # Allow outer scope name shadowing for fixtures
  "-WPS430", # Allow nested functions in test methods
  "-WPS432", # Allow magic numbers
  "-WPS437", # Allow protected attribute usage in test methods
  "-WPS201", # Found module with too many imports
  "-WPS425", # Found boolean non-keyword argument
  "-WPS118", # Found too long name
  "-WPS342", # Avoid false positive for implicit raw strings
]

[tool.flakeheaven.exceptions."!*tests/*"]
"flake8-debugger" = [
  "-T100",
]

[tool.flakeheaven.exceptions."scripts/*"]
"wemake-python-styleguide" = [
  "-WPS421", # Allow for print function calls
]

[tool.flakeheaven.exceptions."*/cli/*"]
"flake8-docstrings" = [
  "-D301", # Allow backslashes in dosctrings
]
"flake8-darglint" = [
  "-*", # Don't require docstrings in click commands
]
"wemake-python-styleguide" = [
  "-WPS210", # Allow many local variables, since these aid in string building
  "-WPS211", # Allow many arguments, since these often represent CLI arguments
  "-WPS213", # Allow many expressions, since string building requires them
  "-WPS216", # Allow many decorators, since `click` uses them to define the CLI
  "-WPS202", # Allow many module members
  "-WPS204", # Allow 'overuse' of expressions (e.g. ctx.obj["-settings"])
]

[tool.flakeheaven.exceptions."src/meltano/migrations/versions/*"]
"flake8-docstrings" = [
  "-D101", # Don't class docstrings in migrations
  "-D103", # Don't require function docstrings in migrations
  "-D400", # Don't require periods in the first lines of dosctrings
]
"wemake-python-styleguide" = [
  "-WPS102", # Allow module names starting with numbers
  "-WPS118", # Allow long filenames
  "-WPS204", # Allow expression "-overuse"
  "-WPS226", # Allow string constant "-overuse"
  "-WPS432", # Allow "-magic" numbers
]

[tool.flakeheaven.exceptions."src/meltano/core/job/job.py"]
"wemake-python-styleguide" = [
  "-WPS601", # Class attributes can be used as instance attributes in sqlalchemy.declarative model
]

[tool.flakeheaven.exceptions."src/meltano/core/project_files.py"]
"wemake-python-styleguide" = [
  "-WPS226", # Dict manipulation requires lots of string constant references
]

[tool.flakeheaven.exceptions."src/meltano/core/project_init_service.py"]
"wemake-python-styleguide" = [
  "-WPS226", # Found string constant over-use: blue > 3
  "-WPS213", # Found too many expressions: 27 > 9
]

[tool.flakeheaven.exceptions."src/**/__init__.py"]
pyflakes = [
  "-F401", # Allow for using __init__.py to promote imports to the module namespace
]

[tool.flakeheaven.exceptions."tests/**/__init__.py"]
pyflakes = [
  "-F401", # Allow for using __init__.py to promote imports to the module namespace
]
# Don't require docstrings in tests
"flake8-darglint" = [
  "-DAR101",
  "-DAR201",
  "-DAR301",
]
"flake8-docstrings" = [
  "-D100",
  "-D101",
  "-D102",
  "-D103",
  "-D104",
]

[tool.flakeheaven.exceptions."src/meltano/__init__.py"]
"wemake-python-styleguide" = [
  "-WPS412", # Found `__init__.py` module with logic
  "-WPS410", # Found wrong metadata variable
]

[tool.flakeheaven.exceptions."src/meltano/core/block/extract_load.py"]
"wemake-python-styleguide" = [
  "-WPS201", # Found module with too many imports: 21 > 20
]

[tool.flakeheaven.exceptions."src/meltano/core/logging/utils.py"]
"wemake-python-styleguide" = [
  "-WPS100", # Found wrong module name
]

[tool.flakeheaven.exceptions."src/meltano/core/cli_messages.py"]
# Special exceptions for cli magic
"wemake-python-styleguide" = [
  "-WPS114",
  "-WPS360",
]
pycodestyle = [
  "-W291",
  "-W293",
]

[tool.flakeheaven.exceptions."src/meltano/core/settings_store.py"]
"wemake-python-styleguide" = [
  "-WPS204", # Found overused expression: allow settings store implementations to all return vars with the same names
]

[tool.flakeheaven.exceptions."src/meltano/core/utils/__init__.py"]
"flake8-docstrings" = [
  "-D103", # Many of these utility functions are descriptively named and self-documenting
]

[tool.flakeheaven.exceptions."src/meltano/core/state_store/*"]
"wemake-python-styleguide" = [
  "-WPS428", # Ignore ... in ABC
]

[tool.flakeheaven.exceptions."src/meltano/core/state_store/__init__.py"]
"wemake-python-styleguide" = [
  "-WPS320", # Ignore multi-line function annotations
]

[tool.pytest.ini_options]
addopts = "--cov=meltano --cov=tests --doctest-modules --order-scope=module -ra -n auto --dist=loadfile"
asyncio_mode = "auto"
log_cli = 1
log_cli_format = "%(message)s"
log_cli_level = "CRITICAL"
log_file = "pytest.log"
log_file_date_format = "%Y-%m-%d %H:%M:%S"
log_file_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
log_file_level = "DEBUG"
markers = [
  "backend(type): backend specific test",
  "meta: meta test",
  "concurrent: test requires true concurrency",
  "slow: slow test",
]
testpaths = [
  "tests/",
]

[tool.coverage.run]
branch = true
concurrency = [
  "multiprocessing",
  "thread",
]
parallel = true
relative_files = true  # This allows coverage to be measured in Windows
source = [
  "meltano",
  "tests",
]
omit = [
  "src/meltano/migrations/env.py",
  "src/meltano/cli/interactive/*",
]

[tool.coverage.paths]
source = [
  "src/meltano/",
  "**/site-packages/meltano/",
  "**/site-packages/meltano*/meltano/",
]

[tool.coverage.report]
exclude_also = [
  '''if (t\.)?TYPE_CHECKING:''',
  "\\.\\.\\.",
]
precision = 2
show_missing = true
skip_covered = true

[tool.commitizen]
name = "cz_version_bump"
prerelease_offset = 1
tag_format = "v$major.$minor.$patch$prerelease"
changelog_merge_prerelease = true
version = "3.5.0a3"
version_files = [
  "pyproject.toml:^version =",
  "src/meltano/__init__.py:^__version__ =",
  'docs/package.json:^  "version":',
]

[tool.mypy]
incremental = false # Disabled until https://github.com/python/mypy/issues/12664 is resolved
files = [
  "src/meltano/",
]
exclude = [
  "src/meltano/migrations/",
]
warn_unused_configs = true
warn_unused_ignores = true

# The following whitelist is used to allow for incremental adoption
# of MyPy. Modules should be removed from this whitelist as and when
# their respective type errors have been addressed. No new modules
# should be added to this whitelist.
[[tool.mypy.overrides]]
ignore_errors = true
module = [
  "meltano.cli.add",
  "meltano.cli.install",
  "meltano.cli.interactive.*",
  "meltano.cli.invoke",
  "meltano.cli.state",
  # Way too many modules at the root of meltano.core to tackle them all at once  # so listing them individually here.
  "meltano.core.db",
  "meltano.core.elt_context",
  "meltano.core.meltano_invoker",
  "meltano.core.migration_service",
  "meltano.core.models",
  "meltano.core.plugin_install_service",
  "meltano.core.plugin_invoker",
  "meltano.core.plugin_test_service",
  "meltano.core.project_add_service",
  "meltano.core.project_files",
  "meltano.core.project_plugins_service",
  "meltano.core.project_settings_service",
  "meltano.core.select_service",
  "meltano.core.settings_store",
  "meltano.core.sqlalchemy",
  "meltano.core.task_sets_service",
  "meltano.core.transform_add_service",
  # Meltano Core submodules
  "meltano.core.behavior.*",
  "meltano.core.block.*",
  "meltano.core.job.*",
  "meltano.core.plugin.airflow",
  "meltano.core.plugin.base",
  "meltano.core.plugin.command",
  "meltano.core.plugin.config_service",
  "meltano.core.plugin.dbt.*",
  "meltano.core.plugin.factory",
  "meltano.core.plugin.file",
  "meltano.core.plugin.meltano_file",
  "meltano.core.plugin.project_plugin",
  "meltano.core.plugin.settings_service",
  "meltano.core.plugin.singer.base",
  "meltano.core.plugin.singer.catalog",
  "meltano.core.plugin.singer.mapper",
  "meltano.core.plugin.singer.target",
  "meltano.core.runner.*",
  "meltano.core.tracking.*",
  "meltano.core.utils.*",
]

[[tool.mypy.overrides]]
ignore_missing_imports = true
module = [
  "click_default_group.*",  # https://github.com/click-contrib/click-default-group/issues/26
  "click_didyoumean.*",     # Unreleased, https://github.com/click-contrib/click-didyoumean/commit/048b275077382e5a80bdf05b02830013eb37ddd1
]

[build-system]
requires = ["poetry-core==1.7.0"]
build-backend = "poetry.core.masonry.api"

[tool.ruff]
line-length = 88
target-version = "py38"

[tool.ruff.lint]
exclude = [
  "src/meltano/migrations/*",
]
ignore = [
  "ANN101", # Missing type annotation for `self` in method
  "ANN102", # Missing type annotation for `cls` in class method
  "N818",   # Allow Exceptions to have suffix 'Exception' rather than 'Error'
  "PT004",  # Add leading underscore to fixtures that do not return anything
  "UP026",  # Replace `mock` import with `unittest.mock` - remove once Python 3.7 support is dropped
  "S310",   # Allow `urllib.open`
  "S603",   # Allow `subprocess.run(..., shell=False)`
  "COM812", # Handled by the Ruff formatter
  "ISC001", # Handled by the Ruff formatter
]
select = [
  "A",   # flake8-builtins
  "ARG", # flake8-unused-arguments
  "B",   # flake8-bugbear
  "C4",  # flake8-comprehensions
  "COM", # flake8-commas
  "DTZ", # flake8-datetimez
  "E",   # pycodestyle (error)
  "EM",   # flake8-errmsg
  "ERA", # flake8-eradicate
  "F",   # pyflakes
  "G",   # flake8-logging-format
  "I",   # isort
  "ISC", # flake8-implicit-str-concat
  "N",   # pep8-naming
  "PIE", # flake8-pie
  "PT",  # flake8-pytest-style
  "Q",   # flake8-quotes
  "RET", # flake8-return
  "RSE", # flake8-raise
  "S",   # flake8-bandit
  "SIM", # flake8-simplify
  "T10", # flake8-debugger
  "T20", # flake8-print
  "TID", # flake8-tidy-imports
  "UP",  # pyupgrade
  "W",   # pycodestyle (warning)
  "TID", # flake8-tidy-imports
  "TCH", # flake8-type-checking
]
unfixable = [
  "ERA001", # Don't remove commented-out code
]

[tool.ruff.lint.per-file-ignores]
"src/meltano/**/__init__.py" = [
  "F401", # Permit unused imports in `__init__.py` files
]
"tests/**" = [
  "S101", # Allow 'assert' in tests
]

[tool.ruff.lint.flake8-annotations]
allow-star-arg-any = true
mypy-init-return = true
suppress-dummy-args = true

[tool.ruff.lint.flake8-pytest-style]
parametrize-values-type = "tuple"

[tool.ruff.lint.isort]
known-first-party = ["asserts", "fixtures", "meltano"]
required-imports = ["from __future__ import annotations"]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.flake8-tidy-imports.banned-api]
"logging.getLogger".msg = "Use `structlog.stdlib.get_logger` instead"
"logging.critical".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.fatal".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.error".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.exception".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.warning".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.warn".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.info".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.debug".msg = "Create a logger with `structlog.stdlib.get_logger`"
"logging.log".msg = "Create a logger with `structlog.stdlib.get_logger`"
"meltano.cli.cli.command".msg = "Use `click.command` and `meltano.cli.cli.add_command` instead"
"meltano.cli.cli.group".msg = "Use `click.group` and `meltano.cli.cli.add_command` instead"