From f64913277d6f4ee4d8c4c9d8228db19f98105561 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 21 Jul 2025 17:48:11 +0800 Subject: [PATCH 1/2] Fix spark.kubernetes.file.upload.path permission --- .../apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 3b84f6946d8..05a09e0293e 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -296,7 +296,8 @@ class SparkProcessBuilder( fs = path.getFileSystem(hadoopConf) if (!fs.exists(path)) { info(s"Try creating $KUBERNETES_FILE_UPLOAD_PATH: $uploadPath") - fs.mkdirs(path, KUBERNETES_UPLOAD_PATH_PERMISSION) + // SPARK-30860: use the class method to avoid the umask causing permission issues + FileSystem.mkdirs(fs, path, KUBERNETES_UPLOAD_PATH_PERMISSION) } } catch { case ioe: IOException => @@ -410,7 +411,8 @@ object SparkProcessBuilder { final val INTERNAL_RESOURCE = "spark-internal" final val KUBERNETES_FILE_UPLOAD_PATH = "spark.kubernetes.file.upload.path" - final val KUBERNETES_UPLOAD_PATH_PERMISSION = new FsPermission(Integer.parseInt("777", 8).toShort) + final val KUBERNETES_UPLOAD_PATH_PERMISSION = + FsPermission.createImmutable(Integer.parseInt("777", 8).toShort) final val YEAR_FMT = DateTimeFormatter.ofPattern("yyyy") final val MONTH_FMT = DateTimeFormatter.ofPattern("MM") From 7527060ac1ffad896bcc0fedc42e30f003dacb67 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 21 Jul 2025 18:02:41 +0800 Subject: [PATCH 2/2] fix --- .../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 0b5852fab54..2a17d182eff 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -117,11 +117,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite override def beforeAll(): Unit = { super.beforeAll() val fs = FileSystem.get(getHadoopConf) - fs.mkdirs( + FileSystem.mkdirs( + fs, new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) fs.setPermission(new Path("/"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) - fs.setPermission(new Path("/spark"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) fs.copyFromLocalFile(new Path(driverTemplate.getPath), new Path("/spark/driver.yml")) }