From 96b585db2fcad4af14efea2b32c17f6fa5591eee Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Fri, 5 Apr 2024 12:01:50 +0000 Subject: [PATCH 1/3] add help for launching notebooks in any pyspark or sparkr application --- .../tools-integrations/jupyter-lab.md | 38 +++++++++++++++++++ .../ocean-spark/tools-integrations/vs-code.md | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/docs/ocean-spark/tools-integrations/jupyter-lab.md create mode 100644 src/docs/ocean-spark/tools-integrations/vs-code.md diff --git a/src/docs/ocean-spark/tools-integrations/jupyter-lab.md b/src/docs/ocean-spark/tools-integrations/jupyter-lab.md new file mode 100644 index 0000000000..c4ecac8d5c --- /dev/null +++ b/src/docs/ocean-spark/tools-integrations/jupyter-lab.md @@ -0,0 +1,38 @@ +# Jupyter Lab + +Install and launch JupyterLab server on any Spark application image with Python installed. + +## Configuration + +To start a Spark application with SparkConnect server, either run the mainClass SparkConnectServer or enable the SparkConnect plugin. Using the Spark Connect plugin, the application can run other tasks or services while enabling Spark Connect. + +### Launch JupyterLab using the JupyterLab main class + +```json +"mainClass": "com.netapp.spark.JupyterLab", +"deps": { + "packages": ["com.netapp.spark:jupyter-plugin:1.2.8"], + "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] +}, +``` + +### Launch JupyterLab using the spark plugin + +```json +"sparkConf": { + "spark.plugins": "com.netapp.spark.JupyterPlugin", + "spark.jupyter.work.dir": "/opt/spark/work-dir" +}, +"deps": { + "packages": ["com.netapp.spark:jupyter-plugin:1.2.8"], + "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] +}, +``` + +## Access + +Access the JupyterLab server from a URL in the following format + +``` +https://console.spotinst.com/api/ocean/spark/cluster/osc-mycluster/app/spark-myapp/notebook/ +``` \ No newline at end of file diff --git a/src/docs/ocean-spark/tools-integrations/vs-code.md b/src/docs/ocean-spark/tools-integrations/vs-code.md new file mode 100644 index 0000000000..ef77826e10 --- /dev/null +++ b/src/docs/ocean-spark/tools-integrations/vs-code.md @@ -0,0 +1,38 @@ +# VS Code Server + +Install and launch VS Code server on any Spark application image with Python installed. + +## Configuration + +To start a Spark application with SparkConnect server, either run the mainClass SparkConnectServer or enable the SparkConnect plugin. Using the Spark Connect plugin, the application can run other tasks or services while enabling Spark Connect. + +### Launch VS Code Server using the VSCodeServer main class + +```json +"mainClass": "com.netapp.spark.VSCodeServer", +"deps": { + "packages": ["com.netapp.spark:vscode-plugin:1.2.8"], + "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] +}, +``` + +### Launch VS Code Server using the spark plugin + +```json +"sparkConf": { + "spark.plugins": "com.netapp.spark.SparkCodeServerPlugin", + "spark.jupyter.work.dir": "/opt/spark/work-dir" +}, +"deps": { + "packages": ["com.netapp.spark:jupyter-plugin:1.2.8"], + "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] +}, +``` + +## Access + +Access the JupyterLab server from a URL in the following format + +``` +https://console.spotinst.com/api/ocean/spark/cluster/osc-mycluster/app/spark-myapp/code/ +``` \ No newline at end of file From 03fa858b1a6ded8dcab69f3a7870ba5f2f36d874 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Fri, 5 Apr 2024 12:32:03 +0000 Subject: [PATCH 2/3] update plugin version --- src/docs/ocean-spark/tools-integrations/jupyter-lab.md | 4 ++-- src/docs/ocean-spark/tools-integrations/vs-code.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/docs/ocean-spark/tools-integrations/jupyter-lab.md b/src/docs/ocean-spark/tools-integrations/jupyter-lab.md index c4ecac8d5c..29dee7b39b 100644 --- a/src/docs/ocean-spark/tools-integrations/jupyter-lab.md +++ b/src/docs/ocean-spark/tools-integrations/jupyter-lab.md @@ -11,7 +11,7 @@ To start a Spark application with SparkConnect server, either run the mainClass ```json "mainClass": "com.netapp.spark.JupyterLab", "deps": { - "packages": ["com.netapp.spark:jupyter-plugin:1.2.8"], + "packages": ["com.netapp.spark:jupyter-plugin:1.3.0"], "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] }, ``` @@ -24,7 +24,7 @@ To start a Spark application with SparkConnect server, either run the mainClass "spark.jupyter.work.dir": "/opt/spark/work-dir" }, "deps": { - "packages": ["com.netapp.spark:jupyter-plugin:1.2.8"], + "packages": ["com.netapp.spark:jupyter-plugin:1.3.0"], "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] }, ``` diff --git a/src/docs/ocean-spark/tools-integrations/vs-code.md b/src/docs/ocean-spark/tools-integrations/vs-code.md index ef77826e10..e1990848e5 100644 --- a/src/docs/ocean-spark/tools-integrations/vs-code.md +++ b/src/docs/ocean-spark/tools-integrations/vs-code.md @@ -11,7 +11,7 @@ To start a Spark application with SparkConnect server, either run the mainClass ```json "mainClass": "com.netapp.spark.VSCodeServer", "deps": { - "packages": ["com.netapp.spark:vscode-plugin:1.2.8"], + "packages": ["com.netapp.spark:vscode-plugin:1.3.0"], "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] }, ``` @@ -24,7 +24,7 @@ To start a Spark application with SparkConnect server, either run the mainClass "spark.jupyter.work.dir": "/opt/spark/work-dir" }, "deps": { - "packages": ["com.netapp.spark:jupyter-plugin:1.2.8"], + "packages": ["com.netapp.spark:jupyter-plugin:1.3.0"], "repositories": ["https://us-central1-maven.pkg.dev/ocean-spark/ocean-spark-adapters"] }, ``` From dff54c41abcc01c2bb3ca07bea3f3ef37a5e650d Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Fri, 5 Apr 2024 14:34:55 +0000 Subject: [PATCH 3/3] added notebook help to sidebar --- src/docs/ocean-spark/_sidebar.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/docs/ocean-spark/_sidebar.md b/src/docs/ocean-spark/_sidebar.md index 7a525c4e93..21761ebedc 100644 --- a/src/docs/ocean-spark/_sidebar.md +++ b/src/docs/ocean-spark/_sidebar.md @@ -29,6 +29,8 @@ - [Run Apps from Airflow](ocean-spark/tools-integrations/run-apps-from-airflow) - [Spark Connect](ocean-spark/tools-integrations/spark-connect) - [JDBC](ocean-spark/tools-integrations/jdbc) + - [JupyterLab](ocean-spark/tools-integrations/jupyter-lab) + - [VS Code](ocean-spark/tools-integrations/vs-code) - [Hive Metastore](ocean-spark/tools-integrations/hive-metastore) - [AWS Glue Data Catalog](ocean-spark/tools-integrations/aws-glue-catalog) - [Docker Images Release Notes](ocean-spark/docker-images-release-notes/)