From 5c8c3ec166421d84dea87ab72dc8c3ef689113ef Mon Sep 17 00:00:00 2001 From: airflow-oss-bot <airflow-oss-bot@astronomer.io> Date: Tue, 5 Mar 2024 00:19:59 +0000 Subject: [PATCH 1/5] Updating setup.cfg with RC provider packages on https://github.com/apache/airflow/issues/37890 --- python-sdk/pyproject.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/python-sdk/pyproject.toml b/python-sdk/pyproject.toml index adb648706..7ddba7d1e 100644 --- a/python-sdk/pyproject.toml +++ b/python-sdk/pyproject.toml @@ -55,7 +55,7 @@ tests = [ ] google = [ "protobuf", - "apache-airflow-providers-google>=6.4.0", + "apache-airflow-providers-google==10.16.0rc1", "sqlalchemy-bigquery>=1.3.0", "smart-open[gcs]>=5.2.1" ] @@ -65,15 +65,15 @@ snowflake = [ "snowflake-connector-python[pandas]" ] postgres = [ - "apache-airflow-providers-postgres", + "apache-airflow-providers-postgres==5.10.2rc1", ] amazon = [ - "apache-airflow-providers-amazon>=5.0.0", + "apache-airflow-providers-amazon==8.19.0rc1", "s3fs", "smart-open[s3]>=5.2.1", ] azure = [ - "apache-airflow-providers-microsoft-azure", + "apache-airflow-providers-microsoft-azure==9.0.1rc1", "azure-storage-blob", "smart-open[azure]>=5.2.1", ] @@ -86,7 +86,7 @@ ftp = [ "smart-open>=5.2.1", ] openlineage = [ - "apache-airflow-providers-openlineage>=1.4.0", + "apache-airflow-providers-openlineage==1.6.0rc1", ] databricks = [ @@ -101,7 +101,7 @@ mssql = [ ] mysql = [ - "apache-airflow-providers-mysql", + "apache-airflow-providers-mysql==5.5.4rc1", ] duckdb = [ @@ -109,10 +109,10 @@ duckdb = [ ] all = [ - "apache-airflow-providers-amazon", - "apache-airflow-providers-google>=6.4.0", + "apache-airflow-providers-amazon==8.19.0rc1", + "apache-airflow-providers-google==10.16.0rc1", "apache-airflow-providers-ftp", - "apache-airflow-providers-postgres", + "apache-airflow-providers-postgres==5.10.2rc1", "apache-airflow-providers-snowflake", "apache-airflow-providers-sftp", "smart-open[all]>=5.2.1", @@ -125,12 +125,12 @@ all = [ "databricks-sql-connector<2.9.0", "s3fs", "protobuf", - "apache-airflow-providers-openlineage>=1.4.0", - "apache-airflow-providers-microsoft-azure", + "apache-airflow-providers-openlineage==1.6.0rc1", + "apache-airflow-providers-microsoft-azure==9.0.1rc1", "azure-storage-blob", "apache-airflow-providers-microsoft-mssql>=3.2", "airflow-provider-duckdb>=0.0.2", - "apache-airflow-providers-mysql" + "apache-airflow-providers-mysql==5.5.4rc1", ] doc = [ "myst-parser>=0.17", From 2f12bb4d4887b8d9668fa6681eb6e7dfb20478a7 Mon Sep 17 00:00:00 2001 From: Pankaj <pankaj.singh@astronomer.io> Date: Wed, 6 Mar 2024 17:45:39 +0530 Subject: [PATCH 2/5] Fix test --- .github/workflows/ci-python-sdk.yaml | 1 + python-sdk/noxfile.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-python-sdk.yaml b/.github/workflows/ci-python-sdk.yaml index 595cbf2e7..1e6e1e3c9 100644 --- a/.github/workflows/ci-python-sdk.yaml +++ b/.github/workflows/ci-python-sdk.yaml @@ -51,6 +51,7 @@ env: SFTP_PASSWORD: ${{ secrets.SFTP_PASSWORD }} AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES: "airflow.* astro.*" AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }} AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} AIRFLOW_VAR_FOO: templated_file_name diff --git a/python-sdk/noxfile.py b/python-sdk/noxfile.py index d49473d41..bc722f494 100644 --- a/python-sdk/noxfile.py +++ b/python-sdk/noxfile.py @@ -25,7 +25,7 @@ def test(session: nox.Session, airflow) -> None: """Run both unit and integration tests.""" env = { "AIRFLOW_HOME": f"~/airflow-{airflow}-python-{session.python}", - "AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES": "airflow\\.* astro\\.*", + "AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES": "airflow.* astro.*", } session.install(f"apache-airflow~={airflow}") From e4d342ffaa4a7def8f05fea6632525adf8b24430 Mon Sep 17 00:00:00 2001 From: Pankaj <pankaj.singh@astronomer.io> Date: Wed, 6 Mar 2024 20:14:33 +0530 Subject: [PATCH 3/5] Fix regx --- python-sdk/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-sdk/noxfile.py b/python-sdk/noxfile.py index bc722f494..c725582b8 100644 --- a/python-sdk/noxfile.py +++ b/python-sdk/noxfile.py @@ -73,7 +73,7 @@ def test_examples_by_dependency(session: nox.Session, extras): env = { "AIRFLOW_HOME": "~/airflow-latest-python-latest", - "AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES": "airflow\\.* astro\\.*", + "AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES": "airflow.* astro.*", } session.install("-e", f".[{pypi_deps}]") From 89445da9284a53c0113e5576a969ae237c3068fe Mon Sep 17 00:00:00 2001 From: Pankaj <pankaj.singh@astronomer.io> Date: Wed, 6 Mar 2024 20:34:00 +0530 Subject: [PATCH 4/5] Fix example --- python-sdk/example_dags/example_dataframe_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-sdk/example_dags/example_dataframe_api.py b/python-sdk/example_dags/example_dataframe_api.py index aea1722ba..5215735a4 100644 --- a/python-sdk/example_dags/example_dataframe_api.py +++ b/python-sdk/example_dags/example_dataframe_api.py @@ -60,7 +60,7 @@ def find_worst_covid_month(dfs: List[pd.DataFrame]): """ res = {} for covid_month_data in dfs: - if ALLOWED_DESERIALIZATION_CLASSES == "airflow\\.* astro\\.*": + if ALLOWED_DESERIALIZATION_CLASSES == "airflow.* astro.*": covid_month = datetime.fromtimestamp(covid_month_data.Date_YMD.iloc[0] / 1e3).strftime("%Y-%m") else: covid_month = covid_month_data.Date_YMD.iloc[0].__format__("%Y-%m") From d2a23e543ae3ba7ed328bac13bf45bcc2ddba434 Mon Sep 17 00:00:00 2001 From: Pankaj <pankaj.singh@astronomer.io> Date: Wed, 6 Mar 2024 21:25:14 +0530 Subject: [PATCH 5/5] pin smart open --- python-sdk/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-sdk/pyproject.toml b/python-sdk/pyproject.toml index 7ddba7d1e..ade28c34e 100644 --- a/python-sdk/pyproject.toml +++ b/python-sdk/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "pandas<2.2.0", # Versions 2.2.0 seems to be breaking our tests, so pin it to this version temporarily. "pyarrow", "python-frontmatter", - "smart-open", + "smart-open<7.0.0", "SQLAlchemy>=1.3.18", "Flask-Session<0.6.0" # This release breaking our tests, let's pin it as a temporary workaround ]