diff --git a/spark_on_k8s/airflow/operators.py b/spark_on_k8s/airflow/operators.py index c4a9f24..cbaef13 100644 --- a/spark_on_k8s/airflow/operators.py +++ b/spark_on_k8s/airflow/operators.py @@ -407,6 +407,8 @@ def execute(self, context: Context): raise AirflowException(f"The job finished with status: {app_status}") def execute_complete(self, context: Context, event: dict, **kwargs): + self.namespace = event["namespace"] + self._driver_pod_name = event["pod_name"] if self.app_waiter == "log": from spark_on_k8s.utils.app_manager import SparkAppManager