pyproject.toml

[build-system]
requires = ["flit_core ~=3.2"]
build-backend = "flit_core.buildapi"

[project]
name = "apache-airflow-provider-transfers"
dynamic = ["version"]
description = """This project contains the Universal Transfer Operator which can transfer all the data
that could be read from the source Dataset into the destination Dataset.
From a DAG author standpoint, all transfers would be performed through the invocation of only the
Universal Transfer Operator."""

authors = [
    { name = "Astronomer", email = "humans@astronomer.io" },
]
readme = "README.md"
license = { file = "LICENSE" }

requires-python = ">=3.7"
dependencies = [
    "apache-airflow>=2.0",
    "attrs>=20.0",
    "pandas>=1.3.4,<2.0.0", # Pinning it to <2.0.0 to avoid breaking changes
    "pyarrow",
    "python-frontmatter",
    "smart-open",
    "SQLAlchemy>=1.3.18",
    "apache-airflow-providers-common-sql"
]

keywords = ["airflow", "provider", "astronomer", "sql", "decorator", "task flow", "elt", "etl", "dag"]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "License :: OSI Approved :: Apache Software License",
    "Topic :: Database",
    "Framework :: Apache Airflow",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.7",
    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
]

[project.optional-dependencies]
tests = [
    "pytest>=6.0",
    "pytest-split",
    "pytest-dotenv",
    "requests-mock",
    "pytest-cov",
    "pytest-describe",
    "types-requests",
    "mypy",
    "sqlalchemy-stubs", # Change when sqlalchemy is upgraded https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html
]
google = [
    "protobuf<=3.20", # Google bigquery client require protobuf <= 3.20.0. We can remove the limitation when this limitation is removed
    "apache-airflow-providers-google>=6.4.0",
    "sqlalchemy-bigquery>=1.3.0",
    "smart-open[gcs]>=5.2.1"
]
snowflake = [
    "apache-airflow-providers-snowflake",
    "snowflake-sqlalchemy>=1.2.0",
    "snowflake-connector-python[pandas]<3.0.0",
    # pinning snowflake-connector-python[pandas]<3.0.0 due to a conflict in snowflake-connector-python/pyarrow/google
    # packages and pandas-gbq/google packages which is forcing pandas-gbq of version 0.13.2 installed, which is not
    # compatible with pandas 1.5.3
]

amazon = [
    "apache-airflow-providers-amazon>=5.0.0",
    "s3fs",
    "smart-open[s3]>=5.2.1",
]

openlineage = ["openlineage-airflow>=0.17.0"]

fivetran = ["airflow-provider-fivetran>=1.1.3"]

all = [
    "apache-airflow-providers-amazon",
    "apache-airflow-providers-google>=6.4.0",
    "apache-airflow-providers-snowflake",
    "smart-open[all]>=5.2.1",
    "snowflake-connector-python[pandas]<3.0.0",
    # pinning snowflake-connector-python[pandas]<3.0.0 due to a conflict in snowflake-connector-python/pyarrow/google
    # packages and pandas-gbq/google packages which is forcing pandas-gbq of version 0.13.2 installed, which is not
    # compatible with pandas 1.5.3
    "snowflake-sqlalchemy>=1.2.0",
    "sqlalchemy-bigquery>=1.3.0",
    "s3fs",
    "protobuf<=3.20", # Google bigquery client require protobuf <= 3.20.0. We can remove the limitation when this limitation is removed
    "openlineage-airflow>=0.17.0",
    "airflow-provider-fivetran>=1.1.3",
    "apache-airflow-providers-sftp"
]
doc = [
    "myst-parser>=0.17",
    "sphinx>=4.4.0",
    "sphinx-autoapi>=2.0.0",
    "sphinx-rtd-theme"
]
sftp = [
    "apache-airflow-providers-sftp",
    "smart-open[ssh]>=5.2.1",
]

[project.urls]
Home = "https://astronomer.io/"
Source = "https://github.com/astronomer/astro-sdk/universal_transfer_operator"

[project.entry-points.apache_airflow_provider]
provider_info = "universal_transfer_operator.__init__:get_provider_info"

[tool.pytest.ini_options]
minversion = "6.0"
addopts = "--durations=30 --durations-min=1.0"
env_files = [".env"]
testpaths = ["tests"]
markers = [
    "integration"
]


[tool.flit.module]
name = "universal_transfer_operator"

[tool.mypy]
color_output = true
#disallow_any_generics = true
#disallow_incomplete_defs = true
#disallow_untyped_defs = true
files = ["src/universal_transfer_operator"]
no_implicit_optional = true
pretty = true
strict_equality = true
show_error_codes = true
show_error_context = true
warn_redundant_casts = true
warn_return_any = true
warn_unused_configs = true
ignore_missing_imports = true

[tool.black]
line-length = 110
target-version = ['py37', 'py38', 'py39', 'py310']

[tool.ruff]
line-length = 120

# Enable Pyflakes `E` and `F` codes by default.
extend-select = [
    "W",    # pycodestyle warnings
    "I",    # isort
    "C90",  # Complexity
#    "B",    # flake8-bugbear
    "C",    # flake8-comprehensions
#    "ANN",  # flake8-comprehensions
    "ISC",  # flake8-implicit-str-concat
    "T10",  # flake8-debugger
    "A",    # flake8-builtins
    "UP",   # pyupgrade
]
extend-ignore = ["A002"]

# Exclude a variety of commonly ignored directories.
extend-exclude = [
    "__pycache__",
    "docs/source/conf.py",
]

target-version = "py37"
fix = true

[per-file-ignores]
"src/universal_transfer_operator/__init__.py" = ["F401"]
"src/universal_transfer_operator/integrations/hooks/fivetran.py" = ["C901"]


[mccabe]
max-complexity = 6

[isort]
combine-as-imports = true
known-first-party = ["universal_transfer_operator", "tests", "tests_integration"]