[SPARK-49206][CORE][UI] Add Environment Variables table to Master EnvironmentPage

### What changes were proposed in this pull request?

This PR aims to add `Environment Variables` table to Master `EnvironmentPage` via a new configuration, `spark.master.ui.visibleEnvVarPrefixes`.

### Why are the changes needed?

To allow users to expose and show the environment variables of Spark Master.

### Does this PR introduce _any_ user-facing change?

Yes, but this is a new table on `Spark Master` UI's `EnvironmentPage`.

### How was this patch tested?

Pass the CIs with newly added test case.

**DEFAULT**
```
$ sbin/start-master.sh
```
![Screenshot 2024-08-12 at 00 53 26](https://github.com/user-attachments/assets/b7929536-a25d-4bd5-876d-908ed7403b92)

**Expose `AWS_`**
```
$ AWS_CA_BUNDLE=/tmp/root-ca.pem \
  AWS_ENDPOINT_URL=https://s3express-usw2-az1.us-west-2.amazonaws.com \
  SPARK_MASTER_OPTS="-Dspark.master.ui.visibleEnvVarPrefixes=AWS_" \
  sbin/start-master.sh
```
![Screenshot 2024-08-12 at 01 05 25](https://github.com/user-attachments/assets/50a57ba1-8ed8-4827-ad51-b303da09a663)

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes apache#47714 from dongjoon-hyun/SPARK-49206.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: yangjie01 <[email protected]>

Author: dongjoon-hyun

core/src/main/resources/org/apache/spark/ui/static/webui.js

@@ -75,6 +75,7 @@ $(function() {
   collapseTablePageLoad('collapse-aggregated-systemProperties','aggregated-systemProperties');
   collapseTablePageLoad('collapse-aggregated-metricsProperties','aggregated-metricsProperties');
   collapseTablePageLoad('collapse-aggregated-classpathEntries','aggregated-classpathEntries');
+  collapseTablePageLoad('collapse-aggregated-environmentVariables','aggregated-environmentVariables');
   collapseTablePageLoad('collapse-aggregated-activeJobs','aggregated-activeJobs');
   collapseTablePageLoad('collapse-aggregated-completedJobs','aggregated-completedJobs');
   collapseTablePageLoad('collapse-aggregated-failedJobs','aggregated-failedJobs');

core/src/main/scala/org/apache/spark/deploy/master/ui/EnvironmentPage.scala

@@ -17,12 +17,14 @@
 
 package org.apache.spark.deploy.master.ui
 
+import scala.jdk.CollectionConverters._
 import scala.xml.Node
 
 import jakarta.servlet.http.HttpServletRequest
 
 import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.internal.config.UI.MASTER_UI_VISIBLE_ENV_VAR_PREFIXES
 import org.apache.spark.ui._
 import org.apache.spark.util.Utils
 
@@ -39,6 +41,9 @@ private[ui] class EnvironmentPage(
     val systemProperties = Utils.redact(conf, details("System Properties")).sorted
     val metricsProperties = Utils.redact(conf, details("Metrics Properties")).sorted
     val classpathEntries = details("Classpath Entries").sorted
+    val prefixes = conf.get(MASTER_UI_VISIBLE_ENV_VAR_PREFIXES)
+    val environmentVariables = System.getenv().asScala
+      .filter { case (k, _) => prefixes.exists(k.startsWith(_)) }.toSeq.sorted
 
     val runtimeInformationTable = UIUtils.listingTable(propertyHeader, propertyRow,
       jvmInformation, fixedWidth = true, headerClasses = headerClasses)
@@ -52,6 +57,8 @@ private[ui] class EnvironmentPage(
       metricsProperties, fixedWidth = true, headerClasses = headerClasses)
     val classpathEntriesTable = UIUtils.listingTable(classPathHeader, classPathRow,
       classpathEntries, fixedWidth = true, headerClasses = headerClasses)
+    val environmentVariablesTable = UIUtils.listingTable(propertyHeader, propertyRow,
+      environmentVariables, fixedWidth = true, headerClasses = headerClasses)
 
     val content =
       <div>
@@ -124,6 +131,17 @@ private[ui] class EnvironmentPage(
         <div class="aggregated-classpathEntries collapsible-table collapsed">
           {classpathEntriesTable}
         </div>
+        <span class="collapse-aggregated-environmentVariables collapse-table"
+            onClick="collapseTable('collapse-aggregated-environmentVariables',
+            'aggregated-environmentVariables')">
+          <h4>
+            <span class="collapse-table-arrow arrow-closed"></span>
+            <a>Environment Variables</a>
+          </h4>
+        </span>
+        <div class="aggregated-environmentVariables collapsible-table collapsed">
+          {environmentVariablesTable}
+        </div>
       </span>
     UIUtils.basicSparkPage(request, content, "Environment")
   }

core/src/main/scala/org/apache/spark/internal/config/UI.scala

@@ -251,6 +251,13 @@ private[spark] object UI {
     .stringConf
     .createOptional
 
+  val MASTER_UI_VISIBLE_ENV_VAR_PREFIXES = ConfigBuilder("spark.master.ui.visibleEnvVarPrefixes")
+    .doc("Comma-separated list of key-prefix strings to show environment variables")
+    .version("4.0.0")
+    .stringConf
+    .toSequence
+    .createWithDefault(Seq.empty[String])
+
   val UI_SQL_GROUP_SUB_EXECUTION_ENABLED = ConfigBuilder("spark.ui.groupSQLSubExecutionEnabled")
     .doc("Whether to group sub executions together in SQL UI when they belong to the same " +
       "root execution")

core/src/test/scala/org/apache/spark/deploy/master/ui/ReadOnlyMasterWebUISuite.scala

@@ -26,6 +26,7 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.master._
 import org.apache.spark.deploy.master.ui.MasterWebUISuite._
 import org.apache.spark.internal.config.DECOMMISSION_ENABLED
+import org.apache.spark.internal.config.UI.MASTER_UI_VISIBLE_ENV_VAR_PREFIXES
 import org.apache.spark.internal.config.UI.UI_KILL_ENABLED
 import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}
 import org.apache.spark.util.Utils
@@ -35,6 +36,7 @@ class ReadOnlyMasterWebUISuite extends SparkFunSuite {
   val conf = new SparkConf()
     .set(UI_KILL_ENABLED, false)
     .set(DECOMMISSION_ENABLED, false)
+    .set(MASTER_UI_VISIBLE_ENV_VAR_PREFIXES.key, "SPARK_SCALA_")
   val securityMgr = new SecurityManager(conf)
   val rpcEnv = mock(classOf[RpcEnv])
   val master = mock(classOf[Master])
@@ -86,4 +88,13 @@ class ReadOnlyMasterWebUISuite extends SparkFunSuite {
     assert(result.contains("Spark Properties"))
     assert(result.contains("Hadoop Properties"))
   }
+
+  test("SPARK-49206: Add 'Environment Variables' table to Master 'EnvironmentPage'") {
+    val url = s"http://${Utils.localHostNameForURI()}:${masterWebUI.boundPort}/environment/"
+    val conn = sendHttpRequest(url, "GET", "")
+    assert(conn.getResponseCode === SC_OK)
+    val result = Source.fromInputStream(conn.getInputStream).mkString
+    assert(result.contains("Environment Variables"))
+    assert(result.contains("<tr><td>SPARK_SCALA_VERSION</td><td>2.1"))
+  }
 }