From 6e665c439ad7413a7e074a262419e50d551ba904 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 26 Dec 2024 17:06:29 +0900 Subject: [PATCH] feat(AIR302): argument appbuilder is now removed in BaseAuthManager and its subclasses --- .../test/fixtures/airflow/AIR302_args.py | 19 +- .../src/rules/airflow/rules/removal_in_3.rs | 38 ++ ...airflow__tests__AIR302_AIR302_args.py.snap | 327 +++++++++--------- 3 files changed, 221 insertions(+), 163 deletions(-) diff --git a/crates/ruff_linter/resources/test/fixtures/airflow/AIR302_args.py b/crates/ruff_linter/resources/test/fixtures/airflow/AIR302_args.py index 1ed4516b112a8a..6955295b799f5a 100644 --- a/crates/ruff_linter/resources/test/fixtures/airflow/AIR302_args.py +++ b/crates/ruff_linter/resources/test/fixtures/airflow/AIR302_args.py @@ -6,6 +6,7 @@ from airflow.providers.amazon.aws.log.s3_task_handler import S3TaskHandler from airflow.providers.apache.hdfs.log.hdfs_task_handler import HdfsTaskHandler from airflow.providers.elasticsearch.log.es_task_handler import ElasticsearchTaskHandler +from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager from airflow.providers.google.cloud.log.gcs_task_handler import GCSTaskHandler from airflow.providers.standard.operators import datetime, trigger_dagrun from airflow.providers.standard.sensors import weekday @@ -51,7 +52,9 @@ def decorator_deprecated_operator_args(): trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator( task_id="trigger_dagrun_op1", execution_date="2024-12-04" ) - trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") + trigger_dagrun_op2 = TriggerDagRunOperator( + task_id="trigger_dagrun_op2", execution_date="2024-12-04" + ) branch_dt_op = datetime.BranchDateTimeOperator( task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 @@ -62,10 +65,16 @@ def decorator_deprecated_operator_args(): sla=timedelta(seconds=10), ) - dof_task_sensor = weekday.DayOfWeekSensor(task_id="dof_task_sensor", use_task_execution_day=True) - dof_task_sensor2 = DayOfWeekSensor(task_id="dof_task_sensor2", use_task_execution_day=True) + dof_task_sensor = weekday.DayOfWeekSensor( + task_id="dof_task_sensor", use_task_execution_day=True + ) + dof_task_sensor2 = DayOfWeekSensor( + task_id="dof_task_sensor2", use_task_execution_day=True + ) - bdow_op = weekday.BranchDayOfWeekOperator(task_id="bdow_op", use_task_execution_day=True) + bdow_op = weekday.BranchDayOfWeekOperator( + task_id="bdow_op", use_task_execution_day=True + ) bdow_op2 = BranchDayOfWeekOperator(task_id="bdow_op2", use_task_execution_day=True) trigger_dagrun_op >> trigger_dagrun_op2 @@ -79,3 +88,5 @@ def decorator_deprecated_operator_args(): HdfsTaskHandler(filename_template="/tmp/test") ElasticsearchTaskHandler(filename_template="/tmp/test") GCSTaskHandler(filename_template="/tmp/test") + +FabAuthManager(None) diff --git a/crates/ruff_linter/src/rules/airflow/rules/removal_in_3.rs b/crates/ruff_linter/src/rules/airflow/rules/removal_in_3.rs index 5dbfbbfc3d6f90..063a0f38ae23ba 100644 --- a/crates/ruff_linter/src/rules/airflow/rules/removal_in_3.rs +++ b/crates/ruff_linter/src/rules/airflow/rules/removal_in_3.rs @@ -35,6 +35,30 @@ fn is_airflow_secret_backend(segments: &[&str]) -> bool { } } +fn is_airflow_auth_manager(segments: &[&str]) -> bool { + match segments { + ["airflow", "auth", "manager", rest @ ..] => { + if let Some(last_element) = rest.last() { + last_element.ends_with("AuthManager") + } else { + false + } + } + + ["airflow", "providers", rest @ ..] => { + if let (Some(pos), Some(last_element)) = + (rest.iter().position(|&s| s == "auth_manager"), rest.last()) + { + pos + 1 < rest.len() && last_element.ends_with("AuthManager") + } else { + false + } + } + + _ => false, + } +} + fn is_airflow_hook(qualname: &QualifiedName) -> bool { match qualname.segments() { ["airflow", "hooks", rest @ ..] => { @@ -217,6 +241,20 @@ fn removed_argument(checker: &mut Checker, qualname: &QualifiedName, arguments: None::<&str>, )); } + _ if is_airflow_auth_manager(qualname.segments()) => { + if !arguments.is_empty() { + checker.diagnostics.push(Diagnostic::new( + Airflow3Removal { + // deprecated: (*arguments).to_string(), + deprecated: "appbuilder".to_string(), + replacement: Replacement::Message( + "The constructor takes no parameter now.", + ), + }, + arguments.range(), + )); + } + } _ if is_airflow_task_handler(qualname.segments()) => { checker.diagnostics.extend(diagnostic_for_argument( arguments, diff --git a/crates/ruff_linter/src/rules/airflow/snapshots/ruff_linter__rules__airflow__tests__AIR302_AIR302_args.py.snap b/crates/ruff_linter/src/rules/airflow/snapshots/ruff_linter__rules__airflow__tests__AIR302_AIR302_args.py.snap index 6352746dedb098..39a4da150e591e 100644 --- a/crates/ruff_linter/src/rules/airflow/snapshots/ruff_linter__rules__airflow__tests__AIR302_AIR302_args.py.snap +++ b/crates/ruff_linter/src/rules/airflow/snapshots/ruff_linter__rules__airflow__tests__AIR302_AIR302_args.py.snap @@ -2,242 +2,251 @@ source: crates/ruff_linter/src/rules/airflow/mod.rs snapshot_kind: text --- -AIR302_args.py:17:39: AIR302 [*] `schedule_interval` is removed in Airflow 3.0 +AIR302_args.py:18:39: AIR302 [*] `schedule_interval` is removed in Airflow 3.0 | -15 | DAG(dag_id="class_schedule", schedule="@hourly") -16 | -17 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") +16 | DAG(dag_id="class_schedule", schedule="@hourly") +17 | +18 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") | ^^^^^^^^^^^^^^^^^ AIR302 -18 | -19 | DAG(dag_id="class_timetable", timetable=NullTimetable()) +19 | +20 | DAG(dag_id="class_timetable", timetable=NullTimetable()) | = help: Use `schedule` instead ℹ Safe fix -14 14 | -15 15 | DAG(dag_id="class_schedule", schedule="@hourly") -16 16 | -17 |-DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") - 17 |+DAG(dag_id="class_schedule_interval", schedule="@hourly") -18 18 | -19 19 | DAG(dag_id="class_timetable", timetable=NullTimetable()) -20 20 | - -AIR302_args.py:19:31: AIR302 [*] `timetable` is removed in Airflow 3.0 - | -17 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") -18 | -19 | DAG(dag_id="class_timetable", timetable=NullTimetable()) +15 15 | +16 16 | DAG(dag_id="class_schedule", schedule="@hourly") +17 17 | +18 |-DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") + 18 |+DAG(dag_id="class_schedule_interval", schedule="@hourly") +19 19 | +20 20 | DAG(dag_id="class_timetable", timetable=NullTimetable()) +21 21 | + +AIR302_args.py:20:31: AIR302 [*] `timetable` is removed in Airflow 3.0 + | +18 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") +19 | +20 | DAG(dag_id="class_timetable", timetable=NullTimetable()) | ^^^^^^^^^ AIR302 | = help: Use `schedule` instead ℹ Safe fix -16 16 | -17 17 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") -18 18 | -19 |-DAG(dag_id="class_timetable", timetable=NullTimetable()) - 19 |+DAG(dag_id="class_timetable", schedule=NullTimetable()) -20 20 | +17 17 | +18 18 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly") +19 19 | +20 |-DAG(dag_id="class_timetable", timetable=NullTimetable()) + 20 |+DAG(dag_id="class_timetable", schedule=NullTimetable()) 21 21 | -22 22 | def sla_callback(*arg, **kwargs): +22 22 | +23 23 | def sla_callback(*arg, **kwargs): -AIR302_args.py:26:34: AIR302 `sla_miss_callback` is removed in Airflow 3.0 +AIR302_args.py:27:34: AIR302 `sla_miss_callback` is removed in Airflow 3.0 | -26 | DAG(dag_id="class_sla_callback", sla_miss_callback=sla_callback) +27 | DAG(dag_id="class_sla_callback", sla_miss_callback=sla_callback) | ^^^^^^^^^^^^^^^^^ AIR302 | -AIR302_args.py:34:6: AIR302 [*] `schedule_interval` is removed in Airflow 3.0 +AIR302_args.py:35:6: AIR302 [*] `schedule_interval` is removed in Airflow 3.0 | -34 | @dag(schedule_interval="0 * * * *") +35 | @dag(schedule_interval="0 * * * *") | ^^^^^^^^^^^^^^^^^ AIR302 -35 | def decorator_schedule_interval(): -36 | pass +36 | def decorator_schedule_interval(): +37 | pass | = help: Use `schedule` instead ℹ Safe fix -31 31 | pass -32 32 | +32 32 | pass 33 33 | -34 |-@dag(schedule_interval="0 * * * *") - 34 |+@dag(schedule="0 * * * *") -35 35 | def decorator_schedule_interval(): -36 36 | pass -37 37 | +34 34 | +35 |-@dag(schedule_interval="0 * * * *") + 35 |+@dag(schedule="0 * * * *") +36 36 | def decorator_schedule_interval(): +37 37 | pass +38 38 | -AIR302_args.py:39:6: AIR302 [*] `timetable` is removed in Airflow 3.0 +AIR302_args.py:40:6: AIR302 [*] `timetable` is removed in Airflow 3.0 | -39 | @dag(timetable=NullTimetable()) +40 | @dag(timetable=NullTimetable()) | ^^^^^^^^^ AIR302 -40 | def decorator_timetable(): -41 | pass +41 | def decorator_timetable(): +42 | pass | = help: Use `schedule` instead ℹ Safe fix -36 36 | pass -37 37 | +37 37 | pass 38 38 | -39 |-@dag(timetable=NullTimetable()) - 39 |+@dag(schedule=NullTimetable()) -40 40 | def decorator_timetable(): -41 41 | pass -42 42 | +39 39 | +40 |-@dag(timetable=NullTimetable()) + 40 |+@dag(schedule=NullTimetable()) +41 41 | def decorator_timetable(): +42 42 | pass +43 43 | -AIR302_args.py:44:6: AIR302 `sla_miss_callback` is removed in Airflow 3.0 +AIR302_args.py:45:6: AIR302 `sla_miss_callback` is removed in Airflow 3.0 | -44 | @dag(sla_miss_callback=sla_callback) +45 | @dag(sla_miss_callback=sla_callback) | ^^^^^^^^^^^^^^^^^ AIR302 -45 | def decorator_sla_callback(): -46 | pass +46 | def decorator_sla_callback(): +47 | pass | -AIR302_args.py:52:39: AIR302 [*] `execution_date` is removed in Airflow 3.0 +AIR302_args.py:53:39: AIR302 [*] `execution_date` is removed in Airflow 3.0 | -50 | def decorator_deprecated_operator_args(): -51 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator( -52 | task_id="trigger_dagrun_op1", execution_date="2024-12-04" +51 | def decorator_deprecated_operator_args(): +52 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator( +53 | task_id="trigger_dagrun_op1", execution_date="2024-12-04" | ^^^^^^^^^^^^^^ AIR302 -53 | ) -54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") +54 | ) +55 | trigger_dagrun_op2 = TriggerDagRunOperator( | = help: Use `logical_date` instead ℹ Safe fix -49 49 | @dag() -50 50 | def decorator_deprecated_operator_args(): -51 51 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator( -52 |- task_id="trigger_dagrun_op1", execution_date="2024-12-04" - 52 |+ task_id="trigger_dagrun_op1", logical_date="2024-12-04" -53 53 | ) -54 54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") -55 55 | - -AIR302_args.py:54:78: AIR302 [*] `execution_date` is removed in Airflow 3.0 - | -52 | task_id="trigger_dagrun_op1", execution_date="2024-12-04" -53 | ) -54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") - | ^^^^^^^^^^^^^^ AIR302 -55 | -56 | branch_dt_op = datetime.BranchDateTimeOperator( +50 50 | @dag() +51 51 | def decorator_deprecated_operator_args(): +52 52 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator( +53 |- task_id="trigger_dagrun_op1", execution_date="2024-12-04" + 53 |+ task_id="trigger_dagrun_op1", logical_date="2024-12-04" +54 54 | ) +55 55 | trigger_dagrun_op2 = TriggerDagRunOperator( +56 56 | task_id="trigger_dagrun_op2", execution_date="2024-12-04" + +AIR302_args.py:56:39: AIR302 [*] `execution_date` is removed in Airflow 3.0 + | +54 | ) +55 | trigger_dagrun_op2 = TriggerDagRunOperator( +56 | task_id="trigger_dagrun_op2", execution_date="2024-12-04" + | ^^^^^^^^^^^^^^ AIR302 +57 | ) | = help: Use `logical_date` instead ℹ Safe fix -51 51 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator( -52 52 | task_id="trigger_dagrun_op1", execution_date="2024-12-04" -53 53 | ) -54 |- trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") - 54 |+ trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", logical_date="2024-12-04") -55 55 | -56 56 | branch_dt_op = datetime.BranchDateTimeOperator( -57 57 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 - -AIR302_args.py:57:33: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0 - | -56 | branch_dt_op = datetime.BranchDateTimeOperator( -57 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 +53 53 | task_id="trigger_dagrun_op1", execution_date="2024-12-04" +54 54 | ) +55 55 | trigger_dagrun_op2 = TriggerDagRunOperator( +56 |- task_id="trigger_dagrun_op2", execution_date="2024-12-04" + 56 |+ task_id="trigger_dagrun_op2", logical_date="2024-12-04" +57 57 | ) +58 58 | +59 59 | branch_dt_op = datetime.BranchDateTimeOperator( + +AIR302_args.py:60:33: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0 + | +59 | branch_dt_op = datetime.BranchDateTimeOperator( +60 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 | ^^^^^^^^^^^^^^^^^^^^^^ AIR302 -58 | ) -59 | branch_dt_op2 = BranchDateTimeOperator( +61 | ) +62 | branch_dt_op2 = BranchDateTimeOperator( | = help: Use `use_task_logical_date` instead ℹ Safe fix -54 54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") -55 55 | -56 56 | branch_dt_op = datetime.BranchDateTimeOperator( -57 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 - 57 |+ task_id="branch_dt_op", use_task_logical_date=True, task_concurrency=5 -58 58 | ) -59 59 | branch_dt_op2 = BranchDateTimeOperator( -60 60 | task_id="branch_dt_op2", - -AIR302_args.py:57:62: AIR302 [*] `task_concurrency` is removed in Airflow 3.0 - | -56 | branch_dt_op = datetime.BranchDateTimeOperator( -57 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 +57 57 | ) +58 58 | +59 59 | branch_dt_op = datetime.BranchDateTimeOperator( +60 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 + 60 |+ task_id="branch_dt_op", use_task_logical_date=True, task_concurrency=5 +61 61 | ) +62 62 | branch_dt_op2 = BranchDateTimeOperator( +63 63 | task_id="branch_dt_op2", + +AIR302_args.py:60:62: AIR302 [*] `task_concurrency` is removed in Airflow 3.0 + | +59 | branch_dt_op = datetime.BranchDateTimeOperator( +60 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 | ^^^^^^^^^^^^^^^^ AIR302 -58 | ) -59 | branch_dt_op2 = BranchDateTimeOperator( +61 | ) +62 | branch_dt_op2 = BranchDateTimeOperator( | = help: Use `max_active_tis_per_dag` instead ℹ Safe fix -54 54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04") -55 55 | -56 56 | branch_dt_op = datetime.BranchDateTimeOperator( -57 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 - 57 |+ task_id="branch_dt_op", use_task_execution_day=True, max_active_tis_per_dag=5 -58 58 | ) -59 59 | branch_dt_op2 = BranchDateTimeOperator( -60 60 | task_id="branch_dt_op2", - -AIR302_args.py:61:9: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0 - | -59 | branch_dt_op2 = BranchDateTimeOperator( -60 | task_id="branch_dt_op2", -61 | use_task_execution_day=True, +57 57 | ) +58 58 | +59 59 | branch_dt_op = datetime.BranchDateTimeOperator( +60 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5 + 60 |+ task_id="branch_dt_op", use_task_execution_day=True, max_active_tis_per_dag=5 +61 61 | ) +62 62 | branch_dt_op2 = BranchDateTimeOperator( +63 63 | task_id="branch_dt_op2", + +AIR302_args.py:64:9: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0 + | +62 | branch_dt_op2 = BranchDateTimeOperator( +63 | task_id="branch_dt_op2", +64 | use_task_execution_day=True, | ^^^^^^^^^^^^^^^^^^^^^^ AIR302 -62 | sla=timedelta(seconds=10), -63 | ) +65 | sla=timedelta(seconds=10), +66 | ) | = help: Use `use_task_logical_date` instead ℹ Safe fix -58 58 | ) -59 59 | branch_dt_op2 = BranchDateTimeOperator( -60 60 | task_id="branch_dt_op2", -61 |- use_task_execution_day=True, - 61 |+ use_task_logical_date=True, -62 62 | sla=timedelta(seconds=10), -63 63 | ) -64 64 | - -AIR302_args.py:62:9: AIR302 `sla` is removed in Airflow 3.0 - | -60 | task_id="branch_dt_op2", -61 | use_task_execution_day=True, -62 | sla=timedelta(seconds=10), +61 61 | ) +62 62 | branch_dt_op2 = BranchDateTimeOperator( +63 63 | task_id="branch_dt_op2", +64 |- use_task_execution_day=True, + 64 |+ use_task_logical_date=True, +65 65 | sla=timedelta(seconds=10), +66 66 | ) +67 67 | + +AIR302_args.py:65:9: AIR302 `sla` is removed in Airflow 3.0 + | +63 | task_id="branch_dt_op2", +64 | use_task_execution_day=True, +65 | sla=timedelta(seconds=10), | ^^^ AIR302 -63 | ) +66 | ) | -AIR302_args.py:78:15: AIR302 `filename_template` is removed in Airflow 3.0 +AIR302_args.py:87:15: AIR302 `filename_template` is removed in Airflow 3.0 | -77 | # deprecated filename_template arugment in FileTaskHandler -78 | S3TaskHandler(filename_template="/tmp/test") +86 | # deprecated filename_template arugment in FileTaskHandler +87 | S3TaskHandler(filename_template="/tmp/test") | ^^^^^^^^^^^^^^^^^ AIR302 -79 | HdfsTaskHandler(filename_template="/tmp/test") -80 | ElasticsearchTaskHandler(filename_template="/tmp/test") +88 | HdfsTaskHandler(filename_template="/tmp/test") +89 | ElasticsearchTaskHandler(filename_template="/tmp/test") | -AIR302_args.py:79:17: AIR302 `filename_template` is removed in Airflow 3.0 +AIR302_args.py:88:17: AIR302 `filename_template` is removed in Airflow 3.0 | -77 | # deprecated filename_template arugment in FileTaskHandler -78 | S3TaskHandler(filename_template="/tmp/test") -79 | HdfsTaskHandler(filename_template="/tmp/test") +86 | # deprecated filename_template arugment in FileTaskHandler +87 | S3TaskHandler(filename_template="/tmp/test") +88 | HdfsTaskHandler(filename_template="/tmp/test") | ^^^^^^^^^^^^^^^^^ AIR302 -80 | ElasticsearchTaskHandler(filename_template="/tmp/test") -81 | GCSTaskHandler(filename_template="/tmp/test") +89 | ElasticsearchTaskHandler(filename_template="/tmp/test") +90 | GCSTaskHandler(filename_template="/tmp/test") | -AIR302_args.py:80:26: AIR302 `filename_template` is removed in Airflow 3.0 +AIR302_args.py:89:26: AIR302 `filename_template` is removed in Airflow 3.0 | -78 | S3TaskHandler(filename_template="/tmp/test") -79 | HdfsTaskHandler(filename_template="/tmp/test") -80 | ElasticsearchTaskHandler(filename_template="/tmp/test") +87 | S3TaskHandler(filename_template="/tmp/test") +88 | HdfsTaskHandler(filename_template="/tmp/test") +89 | ElasticsearchTaskHandler(filename_template="/tmp/test") | ^^^^^^^^^^^^^^^^^ AIR302 -81 | GCSTaskHandler(filename_template="/tmp/test") +90 | GCSTaskHandler(filename_template="/tmp/test") | -AIR302_args.py:81:16: AIR302 `filename_template` is removed in Airflow 3.0 +AIR302_args.py:90:16: AIR302 `filename_template` is removed in Airflow 3.0 | -79 | HdfsTaskHandler(filename_template="/tmp/test") -80 | ElasticsearchTaskHandler(filename_template="/tmp/test") -81 | GCSTaskHandler(filename_template="/tmp/test") +88 | HdfsTaskHandler(filename_template="/tmp/test") +89 | ElasticsearchTaskHandler(filename_template="/tmp/test") +90 | GCSTaskHandler(filename_template="/tmp/test") | ^^^^^^^^^^^^^^^^^ AIR302 +91 | +92 | FabAuthManager(None) + | + +AIR302_args.py:92:15: AIR302 `appbuilder` is removed in Airflow 3.0; The constructor takes no parameter now. + | +90 | GCSTaskHandler(filename_template="/tmp/test") +91 | +92 | FabAuthManager(None) + | ^^^^^^ AIR302 |