Build rules_kotlin using rules_kotlin (#1081)

* Build rules_kotlin using rules_kotlin

* Fix non-bzlmod, format

* Add reflect for js.

* Remove unused variable

* remove dev dependency arguments, clean cache

* Restore lint, remove hacks

* Fix module formatting

* Fix ktlint

* Compose classpath issue -- is it the preloader?

* toolchain with reflect needed?

* Ensure the bootstrap repos are configured

* isolated bootstrap

* Remove rogue file

* Apply formatting

* Add release MODULE, clean up unused imports

* Fix lexical order

* Learning portable sed

* The journy of sed continues

* variations.

* pipes are not my friend

* pipes are not my friend

* to perl!

* Adjust rbe java settings

* Fix format

* Disable bazel 6 android, as the rules_java are not compatible

* downgrade rules java

* Set java toolchains

* fix lint error

* buildifier, again

* [ci] Attempt to prebuild and override bazel 6 example

* Remove unused in .bazelrc

* clean up presubmit

* Fix module formatting

* Remove clean from CI

* Fix module formatting

* Remove lock

* remove rules_cc and buildkite

* Restore buildkite

* Restore rules_cc

* Use bzlmod with ubuntu

* Revert "Use bzlmod with ubuntu"

This reverts commit 339a9c0.

Author: restingbull

.bazelci/presubmit.yml

@@ -31,6 +31,11 @@ tasks:
     build_targets:
       - "//:rules_kotlin_release"
   rbe_ubuntu1604:
+    build_flags:
+      - "--enable_bzlmod=false"
+      - "--config=rbe"
+      - "--host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11"
+      - "--java_toolchain=@bazel_tools//tools/jdk:toolchain_java11"
     test_targets:
       - "--"
       - "//src/test/kotlin/io/bazel/kotlin/builder:builder_tests"
@@ -42,6 +47,7 @@ tasks:
       # Override the default worker strategy for remote builds (worker strategy
       # cannot be used with remote builds)
       - "--strategy=KotlinCompile=remote"
+      - "--config=rbe"
 # TODO: Enable these tests once this example is building against the release rules_kotlin binary
 #  example-android-5.x:
 #    name: "Example - Android using Bazel 5.x"

.bazelrc

@@ -1,5 +1,8 @@
 common --enable_bzlmod=true
 
+common:rbe --java_runtime_version=11
+common:rbe --tool_java_runtime_version=11
+
 build --strategy=KotlinCompile=worker
 build --test_output=all
 build --verbose_failures

.gitignore

@@ -6,3 +6,4 @@
 **/.DS_Store
 trace.profile.gz
 .idea
+MODULE.bazel.lock

BUILD

@@ -61,12 +61,12 @@ test_suite(
 release_archive(
     name = "rules_kotlin_release",
     srcs = [
-        "MODULE.bazel",
         "WORKSPACE.bzlmod",
     ],
     src_map = {
         "BUILD.release.bazel": "BUILD.bazel",
         "WORKSPACE.release.bazel": "WORKSPACE",
+        "MODULE.release.bazel": "MODULE.bazel",
     },
     deps = [
         "//kotlin:pkg",

MODULE.bazel

@@ -18,9 +18,25 @@ use_repo(
     "com_github_google_ksp",
     "com_github_jetbrains_kotlin",
     "com_github_pinterest_ktlint",
+    "released_rules_kotlin",
     "rules_android",
 )
 
+# Once the released rules_koltin is defined, configure it.
+rules_kotlin_bootstrap_extensions = use_extension(
+    "//src/main/starlark/core/repositories:bzlmod_bootstrap.bzl",
+    "rules_kotlin_bootstrap_extensions",
+)
+use_repo(
+    rules_kotlin_bootstrap_extensions,
+    "kt_java_stub_template",
+    "released_com_github_google_ksp",
+    "released_com_github_jetbrains_kotlin",
+)
+
+register_toolchains("@released_rules_kotlin//kotlin/internal:default_toolchain")
+
+# Back to the regularly scheduled configuration.
 register_toolchains("//kotlin/internal:default_toolchain")
 
 # TODO(bencodes) We should be able to remove this once rules_android has rolled out official Bzlmod support

MODULE.release.bazel

@@ -0,0 +1,32 @@
+module(
+    name = "rules_kotlin",
+    version = "1.9.0",
+    repo_name = "rules_kotlin",
+)
+
+bazel_dep(name = "platforms", version = "0.0.6")
+bazel_dep(name = "bazel_skylib", version = "1.4.2")
+bazel_dep(name = "rules_java", version = "7.2.0")
+bazel_dep(name = "rules_python", version = "0.23.1")
+bazel_dep(name = "rules_cc", version = "0.0.8")
+
+rules_kotlin_extensions = use_extension(
+    "//src/main/starlark/core/repositories:bzlmod_setup.bzl",
+    "rules_kotlin_extensions",
+)
+use_repo(
+    rules_kotlin_extensions,
+    "buildkite_config",
+    "com_github_google_ksp",
+    "com_github_jetbrains_kotlin",
+    "com_github_pinterest_ktlint",
+    "rules_android",
+)
+
+register_toolchains("//kotlin/internal:default_toolchain")
+
+# TODO(bencodes) We should be able to remove this once rules_android has rolled out official Bzlmod support
+remote_android_extensions = use_extension("@bazel_tools//tools/android:android_extensions.bzl", "remote_android_tools_extensions")
+use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools")
+
+bazel_dep(name = "rules_proto", version = "5.3.0-21.7")

scripts/reflow_skylark

@@ -18,6 +18,6 @@
 
 buildifier --warnings=-confusing-name,-constant-glob,-duplicated-name,-function-docstring,-function-docstring-args,-function-docstring-header,-module-docstring,-name-conventions,-no-effect,-constant-glob,-provider-params,-print,-rule-impl-return,-bzl-visibility,-unnamed-macro,-uninitialized,-unreachable -lint warn $(find . -type f \
     -iname "*.bzl" -or \
-    -name "*.bazel" -or \
+    -name "*.bazel"  -and -not -name "MODULE.*" -or \
     -name "WORKSPACE"
 )

src/main/kotlin/BUILD.release.bazel

@@ -42,6 +42,7 @@ java_binary(
         "//kotlin/compiler:jvm-abi-gen",
         "//kotlin/compiler:kotlin-annotation-processing",
         "//kotlin/compiler:kotlin-compiler",
+        "//kotlin/compiler:kotlin-reflect",
         "//kotlin/compiler:symbol-processing-api",
         "//kotlin/compiler:symbol-processing-cmdline",
         "//src/main/kotlin/io/bazel/kotlin/compiler",
@@ -56,6 +57,7 @@ java_binary(
         "-D@rules_kotlin...compiler=$(rlocationpath //src/main/kotlin/io/bazel/kotlin/compiler)",
         "-D@com_github_google_ksp...symbol-processing-api=$(rlocationpath //kotlin/compiler:symbol-processing-api)",
         "-D@com_github_google_ksp...symbol-processing-cmdline=$(rlocationpath //kotlin/compiler:symbol-processing-cmdline)",
+        "-D@rules_kotlin..kotlin.compiler.kotlin-reflect=$(rlocationpath //kotlin/compiler:kotlin-reflect)",
         "-XX:-MaxFDLimit",
     ],
     main_class = "io.bazel.kotlin.builder.cmd.Build",

src/main/kotlin/bootstrap.bzl

@@ -11,115 +11,30 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-load("@rules_java//java:defs.bzl", "java_binary", "java_import")
-load("//kotlin:jvm.bzl", _for_ide = "kt_jvm_library")
+load("@released_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
+load("@rules_java//java:defs.bzl", "java_binary")
 load("//kotlin:lint.bzl", _ktlint_fix = "ktlint_fix", _ktlint_test = "ktlint_test")
 load("//third_party:jarjar.bzl", "jar_jar")
 
-_BOOTSTRAP_LIB_ARGS = ["-jvm-target", "1.8"]
-
-def _resolve_dep_label(d):
-    if d.startswith("///src/main/kotlin/io/bazel/kotlin") and not d.endswith("_for_ide"):
-        _, _, target = d.rpartition(":")
-        if target == None:
-            # untested
-            return d + "_for_ide"
-        else:
-            _, _, target = d.rpartition("/")
-            return d + ":" + target + "_for_ide"
-    else:
-        return d
-
-def kt_bootstrap_library(name, srcs, visibility = [], deps = [], neverlink_deps = [], runtime_deps = []):
+def kt_bootstrap_library(name, deps = [], neverlink_deps = [], srcs = [], visibility = [], **kwargs):
     """
     Simple compilation of a kotlin library using a non-persistent worker. The target is a JavaInfo provider.
 
-    The target is tagged `"no-ide"` as intellij can't compile it. A seperate private target is created which is suffixed
-    with `_for_ide`. If the dep is under the package `//src/main/kotlin/io/bazel/kotlin/builder/...` then it will be
-    added to the `_for_ide` target by adding a `_for_ide` prefix.
-
     deps: the dependenices, the are setup as runtime_deps of the library.
-    neverlink_deps: deps that won't be linked. These deps are added to the `"for_ide"` target.
+    neverlink_deps: deps that won't be linked.
     """
-    jar_label = name + "_jar"
-    dep_label = name + "_deps"
-    native.filegroup(
-        name = dep_label,
-        srcs = deps + neverlink_deps,
-        tags = ["no-ide"],
-        visibility = ["//visibility:private"],
+    kt_jvm_library(
+        name = "%s_neverlink" % name,
+        exports = neverlink_deps,
+        neverlink = True,
     )
-    command = """
-function join_by { local IFS="$$1"; shift; echo "$$*"; }
-case "$$(uname -s)" in
-    CYGWIN*|MINGW32*|MSYS*)
-        SEP=";"
-        ;;
-    *)
-        SEP=":"
-        ;;
-esac
-NAME=%s
-CP="%s"
-ARGS="%s"
-
-CMD="$(JAVA) -Xmx256M -Xms32M -noverify \
-      -cp $(location //kotlin/compiler:kotlin-preloader) org.jetbrains.kotlin.preloading.Preloader \
-      -cp $(location //kotlin/compiler:kotlin-compiler) org.jetbrains.kotlin.cli.jvm.K2JVMCompiler \
-      $$CP -d $(@D)/$${NAME}_temp.jar $${ARGS} $(SRCS)"
 
-$$CMD
-
-case "$(location @bazel_tools//tools/jdk:singlejar)" in
-    *.jar)
-        SJ="$(JAVA) -jar $(location @bazel_tools//tools/jdk:singlejar)"
-        ;;
-    *)
-        SJ="$(location @bazel_tools//tools/jdk:singlejar)"
-        ;;
-esac
-
-$$SJ \
-    --normalize \
-    --compression \
-    --sources $(@D)/$${NAME}_temp.jar \
-    --output $(OUTS)
-
-rm $(@D)/$${NAME}_temp.jar
-""" % (name, "-cp $$(join_by $$SEP $(locations :%s)) " % dep_label if deps + neverlink_deps else "", " ".join(_BOOTSTRAP_LIB_ARGS))
-    native.genrule(
-        name = jar_label,
-        tools = [
-            "@com_github_jetbrains_kotlin//:home",
-            "//kotlin/compiler:kotlin-preloader",
-            "//kotlin/compiler:kotlin-compiler",
-            "@bazel_tools//tools/jdk:singlejar",
-            dep_label,
-        ],
-        srcs = srcs,
-        outs = [name + ".jar"],
-        tags = ["no-ide"],
-        visibility = ["//visibility:private"],
-        toolchains = [
-            "@bazel_tools//tools/jdk:current_host_java_runtime",
-        ],
-        cmd = command,
-    )
-    java_import(
+    kt_jvm_library(
         name = name,
-        jars = [jar_label],
-        tags = ["no-ide"],
-        runtime_deps = deps + runtime_deps,
-        visibility = visibility,
-    )
-
-    # hsyed todo this part of the graph should not be wired up outside of development.
-    _for_ide(
-        name = name + "_for_ide",
         srcs = srcs,
-        neverlink = 1,
-        deps = [_resolve_dep_label(d) for d in deps] + neverlink_deps,
-        visibility = ["//visibility:private"],
+        visibility = visibility,
+        deps = deps + ["%s_neverlink" % name],
+        **kwargs
     )
 
     _ktlint_test(
@@ -136,6 +51,7 @@ rm $(@D)/$${NAME}_temp.jar
         visibility = ["//visibility:private"],
         config = "//:ktlint_editorconfig",
         tags = ["no-ide", "ktlint"],
+        **kwargs
     )
 
 def kt_bootstrap_binary(

src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel

@@ -18,11 +18,12 @@ kt_bootstrap_binary(
         "//kotlin/compiler:jvm-abi-gen",
         "//kotlin/compiler:kotlin-annotation-processing",
         "//kotlin/compiler:kotlin-compiler",
+        "//kotlin/compiler:kotlin-reflect",
         "//kotlin/compiler:symbol-processing-api",
         "//kotlin/compiler:symbol-processing-cmdline",
         "//src/main/kotlin:jdeps-gen",
         "//src/main/kotlin:skip-code-gen",
-        "//src/main/kotlin/io/bazel/kotlin/compiler",
+        "//src/main/kotlin/io/bazel/kotlin/compiler:compiler.jar",
         "@com_github_jetbrains_kotlin//:home",
     ],
     jvm_flags = [
@@ -31,9 +32,10 @@ kt_bootstrap_binary(
         "-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath //kotlin/compiler:kotlin-annotation-processing)",
         "-D@rules_kotlin...jdeps-gen=$(rlocationpath //src/main/kotlin:jdeps-gen)",
         "-D@rules_kotlin...skip-code-gen=$(rlocationpath //src/main/kotlin:skip-code-gen)",
-        "-D@rules_kotlin...compiler=$(rlocationpath //src/main/kotlin/io/bazel/kotlin/compiler)",
+        "-D@rules_kotlin...compiler=$(rlocationpath //src/main/kotlin/io/bazel/kotlin/compiler:compiler.jar)",
         "-D@com_github_google_ksp...symbol-processing-api=$(rlocationpath //kotlin/compiler:symbol-processing-api)",
         "-D@com_github_google_ksp...symbol-processing-cmdline=$(rlocationpath //kotlin/compiler:symbol-processing-cmdline)",
+        "-D@rules_kotlin..kotlin.compiler.kotlin-reflect=$(rlocationpath //kotlin/compiler:kotlin-reflect)",
         "-XX:-MaxFDLimit",
     ],
     main_class = "io.bazel.kotlin.builder.cmd.Build",