[query] fully define mill compile dependencies

Author: ehigham

Makefile

@@ -57,8 +57,10 @@ pylint-hailtop:
 	$(PYTHON) -m pylint --rcfile pylintrc hail/python/hailtop --score=n
 
 .PHONY: check-hail
+check-hail: export HAIL_BUILD_MODE=Dev
+check-hail: HAIL_DEPLOY_TARGET='Generic(2.13,3.5.3)'
 check-hail: check-hail-fast pylint-hailtop
-	cd hail && HAIL_BUILD_MODE=Dev SCALA_VERSION=2.13 $(MILL) $(MILLOPTS) hail[].__.checkFormat + hail[].__.fix --check
+	cd hail && $(MILL) $(MILLOPTS) hail[].__.checkFormat + hail[].__.fix --check
 
 .PHONY: check-batch
 check-batch: check-batch-fast pylint-batch

build.yaml

@@ -773,7 +773,7 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=Release
+      export HAIL_DEPLOY_TARGET=dataproc-2.3.x HAIL_BUILD_MODE=Release MILLOPTS='--no-daemon'
       time retry sh mill --no-daemon --version
 
       # We've encountered the following sporadic error in CI between `mill`
@@ -785,7 +785,7 @@ steps:
       # This was likely from running mill in server mode; mill main exits before
       # the jar has finished being written.
       time retry make shadowJar
-      if [ ! -f out/hail/2.12/assembly.dest/out.jar ]; then
+      if [ ! -f "out/hail/${HAIL_DEPLOY_TARGET}/assembly.dest/out.jar" ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -851,12 +851,12 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
+      export HAIL_DEPLOY_TARGET=dataproc-2.3.x HAIL_BUILD_MODE=CI MILLOPTS='--no-daemon'
       time retry sh mill --no-daemon --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar
-      if [ ! -f out/hail/2.12/assembly.dest/out.jar ]; then
+      if [ ! -f "out/hail/${HAIL_DEPLOY_TARGET}/assembly.dest/out.jar" ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -884,12 +884,12 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
+      export HAIL_DEPLOY_TARGET=dataproc-2.3.x MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
       time retry sh mill --no-daemon --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowTestJar
-      if [ ! -f out/hail/2.12/test/assembly.dest/out.jar ]; then
+      if [ ! -f "out/hail/${HAIL_DEPLOY_TARGET}/test/assembly.dest/out.jar" ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -944,12 +944,13 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_BUILD_MODE=Release MILLOPTS='--no-daemon'
+      export SPARK_VERSION="3.3.1" SCALA_VERSION="2.12.13"
+      export HAIL_DEPLOY_TARGET=hdinsight-5.1 HAIL_BUILD_MODE=Release MILLOPTS='--no-daemon'
       time retry sh mill --no-daemon --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar
-      if [ ! -f out/hail/2.12/assembly.dest/out.jar ]; then
+      if [ ! -f "out/hail/${HAIL_DEPLOY_TARGET}/assembly.dest/out.jar" ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -1081,7 +1082,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/dataproc-2.3.x/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /splits.tar.gz
         to: /io/splits.tar.gz
@@ -1701,8 +1702,8 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      HAIL_BUILD_MODE='CI'
-      time retry sh mill --no-daemon hail[2.13].__.compile
+      export HAIL_BUILD_MODE='CI' SCALA_VERSION=2.13 SPARK_VERSION=3.5.3
+      time retry sh mill --no-daemon 'hail[].__.compile'
     inputs:
       - from: /repo
         to: /io/repo
@@ -2573,7 +2574,7 @@ steps:
           valueFrom: default_ns.name
         mountPath: /batch-gsa-key
     inputs:
-      - from: /derived/release/hail/out/hail/2.12/assembly.dest/out.jar
+      - from: /derived/release/hail/out/hail/dataproc-2.3.x/assembly.dest/out.jar
         to: /io/hail.jar
       - from: /repo/hail/env/REVISION
         to: /io/REVISION
@@ -3499,7 +3500,7 @@ steps:
       # #14452 - must build wheel with release `hailtop/hailctl/deploy.yaml`
       # use `-o` to prevent `mill` from rebuilding the "SHADOW_JAR"
       make upload-artifacts HAIL_BUILD_MODE=Release DEPLOY_REMOTE=origin \
-        -o out/hail/2.12/assembly.dest/out.jar
+        -o shadowJar
 
       echo Setting arguments and invoking release.sh.
 
@@ -3524,8 +3525,8 @@ steps:
     inputs:
       - from: /repo
         to: /io/repo
-      - from: /derived/release/hail/out/hail/2.12/assembly.dest/out.jar
-        to: /io/repo/hail/out/hail/2.12/assembly.dest/out.jar
+      - from: /derived/release/hail/out/hail/dataproc-2.3.x/assembly.dest/out.jar
+        to: /io/repo/hail/out/hail/dataproc-2.3.x/assembly.dest/out.jar
       - from: /azure-wheel
         to: /io/azure-wheel
       - from: /www.tar.gz
@@ -3769,7 +3770,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/dataproc-2.3.x/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /repo/hail/testng-fs.xml
         to: /io/testng-fs.xml
@@ -3811,7 +3812,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/dataproc-2.3.x/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /repo/hail/testng-services.xml
         to: /io/testng-services.xml

hail/.scalafix.conf

@@ -31,5 +31,6 @@ RemoveUnused {
 }
 
 ForbiddenSymbol.symbols = [
+  "org.sfl4j",
   "scala.collection.compat.immutable.ArraySeq"
 ]

hail/Makefile

@@ -19,8 +19,7 @@ HAIL_PATCH_VERSION := 137
 HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION)
 HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION)
 
-SCALA_VERSION ?= 2.12
-SPARK_VERSION ?= 3.5.3
+HAIL_DEPLOY_TARGET ?= dataproc-2.3.x
 
 include env_var.mk
 
@@ -77,7 +76,7 @@ mill-clean:
 # be recomputed.
 FORCE:
 
-SHADOW_JAR := out/hail/$(SCALA_VERSION)/assembly.dest/out.jar
+SHADOW_JAR := out/hail/$(HAIL_DEPLOY_TARGET)/assembly.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(SHADOW_JAR): native-lib-prebuilt
 endif
@@ -87,14 +86,14 @@ $(SHADOW_JAR): FORCE
 .PHONY: shadowJar
 shadowJar: $(SHADOW_JAR)
 
-NON_SHADOW_JAR := out/hail/$(SCALA_VERSION)/jar.dest/out.jar
+NON_SHADOW_JAR := out/hail/$(HAIL_DEPLOY_TARGET)/jar.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(NON_SHADOW_JAR): native-lib-prebuilt
 endif
 $(NON_SHADOW_JAR): FORCE
 	$(MILL) $(MILLOPTS) hail[].jar
 
-SHADOW_TEST_JAR := out/hail/$(SCALA_VERSION)/test/assembly.dest/out.jar
+SHADOW_TEST_JAR := out/hail/$(HAIL_DEPLOY_TARGET)/test/assembly.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(SHADOW_TEST_JAR): native-lib-prebuilt
 endif
@@ -104,7 +103,7 @@ $(SHADOW_TEST_JAR): FORCE
 .PHONY: shadowTestJar
 shadowTestJar: $(SHADOW_TEST_JAR)
 
-EXTRA_CLASSPATH := out/hail/$(SCALA_VERSION)/writeRunClasspath.dest/runClasspath
+EXTRA_CLASSPATH := out/hail/$(HAIL_DEPLOY_TARGET)/writeRunClasspath.dest/runClasspath
 $(EXTRA_CLASSPATH): FORCE
 	$(MILL) $(MILLOPTS) hail[].writeRunClasspath