Merge branch-25.08 into main (NVIDIA#13284)

NvTimLiu · web-flow · commit 4da10eeab224 · 2025-08-09T15:59:59.000+08:00
Merge branch-25.08 into main

Note: merge this PR with **Create a merge commit to merge**
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,5 @@
 # Change log
-Generated on 2025-08-08
+Generated on 2025-08-09
 
 ## Release 25.08
 
@@ -37,6 +37,7 @@ Generated on 2025-08-08
 ### Bugs Fixed
 |||
 |:---|:---|
+|[#13279](https://github.com/NVIDIA/spark-rapids/issues/13279)|[BUG] Integration test failures with Delta|
 |[#13265](https://github.com/NVIDIA/spark-rapids/issues/13265)|[BUG] test failed Expected error IllegalArgumentException/ArrayIndexOutOfBoundsException|
 |[#13254](https://github.com/NVIDIA/spark-rapids/issues/13254)|[BUG] delta_lake_time_travel_test.py::test_time_travel_on_non_existing_table failed Expected error 'AnalysisException' did not appear|
 |[#13049](https://github.com/NVIDIA/spark-rapids/issues/13049)|[BUG] hash_aggregate_test tests FAILED on [DATABRICKS]|
@@ -128,6 +129,8 @@ Generated on 2025-08-08
 ### PRs
 |||
 |:---|:---|
+|[#13280](https://github.com/NVIDIA/spark-rapids/pull/13280)|Fix fallback test params for Delta MergeCommand, UpdateCommand, and DeleteCommand|
+|[#13258](https://github.com/NVIDIA/spark-rapids/pull/13258)|Update changelog for the v25.08 release [skip ci]|
 |[#13257](https://github.com/NVIDIA/spark-rapids/pull/13257)|Update dependency version JNI, private, hybrid to 25.08.0|
 |[#13123](https://github.com/NVIDIA/spark-rapids/pull/13123)|Enable MERGE, UPDATE, DELETE by default for Delta 3.3.0|
 |[#13267](https://github.com/NVIDIA/spark-rapids/pull/13267)|Use pytest.ExceptionInfo to create the same exception text as pytest raises|
diff --git a/integration_tests/src/main/python/delta_lake_delete_test.py b/integration_tests/src/main/python/delta_lake_delete_test.py
@@ -19,7 +19,7 @@
 from delta_lake_utils import *
 from marks import *
 from spark_session import is_before_spark_320, is_databricks_runtime, supports_delta_lake_deletion_vectors, \
-    with_cpu_session, with_gpu_session, is_databricks143_or_later
+    with_cpu_session, with_gpu_session, is_before_spark_353
 
 delta_delete_enabled_conf = copy_and_update(delta_writes_enabled_conf,
                                             {"spark.rapids.sql.command.DeleteCommand": "true",
@@ -63,16 +63,19 @@ def checker(data_path, do_delete):
     delta_sql_delete_test(spark_tmp_path, use_cdf, dest_table_func, delete_sql, checker, enable_deletion_vectors,
                           partition_columns)
 
+fallback_test_params = [{"spark.rapids.sql.format.delta.write.enabled": "false"},
+                        {"spark.rapids.sql.format.parquet.enabled": "false"},
+                        {"spark.rapids.sql.format.parquet.write.enabled": "false"},
+                        {"spark.rapids.sql.command.DeleteCommand": "false"},
+                        ]
+if is_before_spark_353():
+    # DeleteCommand is disabled by default before Spark 3.5.3
+    fallback_test_params.append(delta_writes_enabled_conf)
+
 @allow_non_gpu("ExecutedCommandExec", *delta_meta_allow)
 @delta_lake
 @ignore_order
-@pytest.mark.parametrize("disable_conf",
-                          [{"spark.rapids.sql.format.delta.write.enabled": "false"},
-                           {"spark.rapids.sql.format.parquet.enabled": "false"},
-                           {"spark.rapids.sql.format.parquet.write.enabled": "false"},
-                           {"spark.rapids.sql.command.DeleteCommand": "false"},
-                           delta_writes_enabled_conf  # Test disabled by default
-                           ], ids=idfn)
+@pytest.mark.parametrize("disable_conf", fallback_test_params, ids=idfn)
 @pytest.mark.skipif(is_before_spark_320(), reason="Delta Lake writes are not supported before Spark 3.2.x")
 @pytest.mark.parametrize("enable_deletion_vectors", deletion_vector_values, ids=idfn)
 def test_delta_delete_disabled_fallback(spark_tmp_path, disable_conf, enable_deletion_vectors):
diff --git a/integration_tests/src/main/python/delta_lake_merge_test.py b/integration_tests/src/main/python/delta_lake_merge_test.py
@@ -24,17 +24,22 @@
                                            {"spark.rapids.sql.command.MergeIntoCommand": "true",
                                             "spark.rapids.sql.command.MergeIntoCommandEdge": "true"})
 
+fallback_test_params = [{"spark.rapids.sql.format.delta.write.enabled": "false"},
+                        {"spark.rapids.sql.format.parquet.enabled": "false"},
+                        {"spark.rapids.sql.format.parquet.write.enabled": "false"},
+                        {"spark.rapids.sql.command.MergeIntoCommand": "false"},
+                        ]
+if is_before_spark_353():
+    # MergeCommand is disabled by default before Spark 3.5.3.
+    # In Spark 3.5.3 and later, MergeCommand is enabled by default, but may not run on GPU yet
+    # because of https://github.com/NVIDIA/spark-rapids/issues/8042.
+    # See https://github.com/NVIDIA/spark-rapids/issues/13021#issuecomment-3166724473 for details.
+    fallback_test_params.append(delta_writes_enabled_conf)
 
 @allow_non_gpu(delta_write_fallback_allow, *delta_meta_allow)
 @delta_lake
 @ignore_order
-@pytest.mark.parametrize("disable_conf",
-                         [{"spark.rapids.sql.format.delta.write.enabled": "false"},
-                          {"spark.rapids.sql.format.parquet.enabled": "false"},
-                          {"spark.rapids.sql.format.parquet.write.enabled": "false"},
-                          {"spark.rapids.sql.command.MergeIntoCommand": "false"},
-                          delta_writes_enabled_conf  # Test disabled by default
-                         ], ids=idfn)
+@pytest.mark.parametrize("disable_conf", fallback_test_params, ids=idfn)
 @pytest.mark.skipif(is_before_spark_320(), reason="Delta Lake writes are not supported before Spark 3.2.x")
 @pytest.mark.parametrize("enable_deletion_vectors", deletion_vector_values_with_350DB143_xfail_reasons(
                             enabled_xfail_reason='https://github.com/NVIDIA/spark-rapids/issues/12042'), ids=idfn)
diff --git a/integration_tests/src/main/python/delta_lake_update_test.py b/integration_tests/src/main/python/delta_lake_update_test.py
@@ -82,15 +82,18 @@ def write_func(spark, path):
     assert_gpu_fallback_write(write_func, read_delta_path, data_path,
                               "ExecutedCommandExec", delta_update_enabled_conf)
 
+fallback_test_params = [{"spark.rapids.sql.format.delta.write.enabled": "false"},
+                        {"spark.rapids.sql.format.parquet.write.enabled": "false"},
+                        {"spark.rapids.sql.command.UpdateCommand": "false"},
+                        ]
+if is_before_spark_353():
+    # UpdateCommand is disabled by default before Spark 3.5.3
+    fallback_test_params.append(delta_writes_enabled_conf)
+
 @allow_non_gpu(delta_write_fallback_allow, *delta_meta_allow)
 @delta_lake
 @ignore_order
-@pytest.mark.parametrize("disable_conf",
-                         [{"spark.rapids.sql.format.delta.write.enabled": "false"},
-                          {"spark.rapids.sql.format.parquet.write.enabled": "false"},
-                          {"spark.rapids.sql.command.UpdateCommand": "false"},
-                          delta_writes_enabled_conf  # Test disabled by default
-                          ], ids=idfn)
+@pytest.mark.parametrize("disable_conf", fallback_test_params, ids=idfn)
 @pytest.mark.skipif(is_before_spark_320(), reason="Delta Lake writes are not supported before Spark 3.2.x")
 @pytest.mark.parametrize("enable_deletion_vector", deletion_vector_values_with_350DB143_xfail_reasons(
                             enabled_xfail_reason='https://github.com/NVIDIA/spark-rapids/issues/12042'), ids=idfn)
