Merge pull request #226 from lrytz/gh-upload

add ghUpload for publishing to GitHub releases

Author: lrytz

.travis.yml

@@ -22,7 +22,8 @@ install:
   - sudo apt-get install -y rpm
 
 language: scala
-jdk: openjdk8
+# java 11+ needed for sttp
+jdk: openjdk21
 
 env:
   global:

admin/build.ps1

@@ -34,9 +34,12 @@ if ($env:APPVEYOR_FORCED_BUILD -eq 'true') {
   ensureVersion
   clearIvyCache
   if ($env:mode -eq 'release') {
-    echo "Running a release for $env:version"
-    $repositoriesFile="$env:APPVEYOR_BUILD_FOLDER\conf\repositories"
-    & cmd /c "sbt ""-Dsbt.override.build.repos=true"" ""-Dsbt.repository.config=$repositoriesFile"" ""-Dproject.version=$env:version"" ""show fullResolvers"" clean update s3Upload" '2>&1'
+    if ($env:version -match '-bin-' -or $env:version -match '-pre-') {
+      & cmd /c "sbt ""-Dproject.version=$env:version"" clean update ""show s3Upload/mappings""" '2>&1'
+    } else {
+      echo "Running a release for $env:version"
+      & cmd /c "sbt ""-Dproject.version=$env:version"" clean update ghUpload" '2>&1'
+    }
     checkExit
   } else {
     echo "Unknown mode: '$env:mode'"

admin/build.sh

@@ -90,15 +90,20 @@ if [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then
     setupSSH
     . scripts/jobs/release/website/update-api
   elif [[ "$mode" == "release" ]]; then
-    echo "Running a release for $version"
     triggerMsiRelease
-    repositoriesFile="$TRAVIS_BUILD_DIR/conf/repositories"
-    # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version
-    sbt -Dsbt.log.noformat=true \
-      -Dsbt.override.build.repos=true -Dsbt.repository.config="$repositoriesFile" \
-      -Dproject.version=$version \
-      "show fullResolvers" clean update s3Upload
-    triggerSmoketest
+    if [[ "$version" =~ -bin- || "$version" =~ -pre- ]]; then
+      # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version
+      sbt -Dsbt.log.noformat=true \
+        -Dproject.version=$version \
+        clean update "show s3Upload/mappings"
+    else
+      echo "Running a release for $version"
+      # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version
+      sbt -Dsbt.log.noformat=true \
+        -Dproject.version=$version \
+        clean update ghUpload
+      triggerSmoketest
+    fi
   else
     echo "Unknown build mode: '$mode'"
     exit 1

appveyor.yml

@@ -6,10 +6,11 @@ branches:
     - 2.12.x
     - 2.13.x
 
-image: Visual Studio 2015
+image: Visual Studio 2019
 
 environment:
-  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
+  # java 11+ needed for sttp
+  JAVA_HOME: C:\Program Files\Java\jdk21
   AWS_ACCESS_KEY_ID:
     secure: X1Ix1soRBDMtfbi8IFNPOggDP2XquhW+uKcJ+XC0kiM=
   AWS_SECRET_ACCESS_KEY:
@@ -18,7 +19,7 @@ environment:
     secure: nfWNfbyzNQwdg1eWHZX93XIJLoAhFOr1fR8+L86s7a3jdo/HydEZ8TyEKsPipQ+/
 
 install:
-  - cmd: choco install sbt --version=1.3.2 -ia "INSTALLDIR=""C:\sbt"""
+  - cmd: choco install sbt --version=1.10.11 -ia "INSTALLDIR=""C:\sbt"""
   - cmd: SET PATH=C:\sbt\bin;%JAVA_HOME%\bin;%PATH%
 
 build_script:

build.sbt

@@ -1,4 +1,6 @@
-import ScalaDist.upload
+import ScalaDist.{s3Upload, ghUpload}
+
+resolvers += "scala-integration" at "https://scala-ci.typesafe.com/artifactory/scala-integration/"
 
 // so we don't require a native git install
 useJGit
@@ -22,9 +24,9 @@ Versioning.settings
 // are known/understood, at scala/scala-dist#171
 scalaVersion := version.value
 
-upload / mappings := Seq()
+s3Upload / mappings := Seq()
 
-upload := {
+s3Upload := {
   import com.amazonaws.services.s3.AmazonS3ClientBuilder
   import com.amazonaws.services.s3.model.PutObjectRequest
   import com.amazonaws.regions.Regions
@@ -33,12 +35,50 @@ upload := {
   val client = AmazonS3ClientBuilder.standard.withRegion(Regions.US_EAST_1).build
 
   val log = streams.value.log
-    (upload / mappings).value map { case (file, key) =>
+    (s3Upload / mappings).value map { case (file, key) =>
     log.info("Uploading "+ file.getAbsolutePath() +" as "+ key)
     client.putObject(new PutObjectRequest("downloads.typesafe.com", key, file))
   }
 }
 
+ghUpload := {
+  import sttp.client3._
+  import _root_.io.circe._, _root_.io.circe.parser._
+
+  val log = streams.value.log
+  val ghRelease = s"v${(Universal / version).value}"
+
+  val token = sys.env.getOrElse("GITHUB_OAUTH_TOKEN", throw new MessageOnlyException("GITHUB_OAUTH_TOKEN missing"))
+
+  val backend = HttpURLConnectionBackend()
+
+  val rRes = basicRequest
+    .get(uri"https://api.github.com/repos/scala/scala/releases/tags/$ghRelease")
+    .header("Accept", "application/vnd.github+json")
+    .header("Authorization", s"Bearer $token")
+    .header("X-GitHub-Api-Version", "2022-11-28")
+    .send(backend)
+  val releaseId = rRes.body.flatMap(parse).getOrElse(Json.Null).hcursor.downField("id").as[Int].getOrElse(
+    throw new MessageOnlyException(s"Release not found: $ghRelease"))
+
+  (s3Upload / mappings).value map { case (file, _) =>
+    log.info(s"Uploading ${file.getAbsolutePath} as ${file.getName} to https://github.com/scala/scala/releases/tag/$ghRelease")
+
+    // https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset
+    val request = basicRequest
+      .post(uri"https://uploads.github.com/repos/scala/scala/releases/${releaseId}/assets?name=${file.getName}")
+      .contentType("application/octet-stream")
+      .header("Accept", "application/vnd.github+json")
+      .header("Authorization", s"Bearer $token")
+      .header("X-GitHub-Api-Version", "2022-11-28")
+      .body(file)
+
+    val response = request.send(backend)
+    if (response.code.code != 201)
+      throw new MessageOnlyException(s"Upload failed: status=${response.code}\n${response.body}")
+  }
+}
+
 ScalaDist.settings
 
 Docs.settings

conf/repositories

@@ -10,7 +10,8 @@ import com.amazonaws.services.s3.model.PutObjectResult
 
 // can't call it Universal -- that's taken by the packager
 object ScalaDist {
-  val upload=TaskKey[Seq[PutObjectResult]]("s3-upload","Uploads files to an S3 bucket.")
+  val s3Upload = TaskKey[Seq[PutObjectResult]]("s3Upload","Uploads files to an S3 bucket.")
+  val ghUpload = TaskKey[Seq[Unit]]("ghUpload","Uploads files to GitHub Releases.")
 
   def createMappingsWith(deps: Seq[(sbt.librarymanagement.ConfigRef, ModuleID, Artifact, File)],
                          distMappingGen: (ModuleID, Artifact, File) => Seq[(File, String)]): Seq[(File, String)] =
@@ -21,30 +22,30 @@ object ScalaDist {
       case _ => Seq()
     }
 
-  // used to make s3-upload upload the file produced by fileTask to the path scala/$version/${file.name}
+  // used to make s3Upload upload the file produced by fileTask to the path scala/$version/${file.name}
   private def uploadMapping(fileTask: TaskKey[File]) = Def.task {
     val file = fileTask.value
     file -> s"scala/${version.value}/${file.getName}"
   }
 
-  // make it so that s3-upload will upload the msi when we're running on windows, and everything else when we're on linux
-  // s3-upload thus depends on the package tasks listed below
+  // make it so that s3Upload will upload the msi when we're running on windows, and everything else when we're on linux
+  // s3Upload thus depends on the package tasks listed below
   def platformSettings =
     if (sys.props("os.name").toLowerCase(java.util.Locale.US) contains "windows")
-      Wix.settings :+ (upload / mappings += uploadMapping(Windows / packageBin).value)
+      Wix.settings :+ (s3Upload / mappings += uploadMapping(Windows / packageBin).value)
     else Unix.settings ++ Seq(
-      upload / mappings += uploadMapping(Universal / packageBin).value,
-      upload / mappings += uploadMapping(Universal / packageZipTarball).value,
-      upload / mappings += uploadMapping(UniversalDocs / packageBin).value,
-      upload / mappings += uploadMapping(UniversalDocs / packageZipTarball).value,
-      upload / mappings += uploadMapping(UniversalDocs / packageXzTarball).value,
-      upload / mappings += uploadMapping(Rpm / packageBin).value,
+      s3Upload / mappings += uploadMapping(Universal / packageBin).value,
+      s3Upload / mappings += uploadMapping(Universal / packageZipTarball).value,
+      s3Upload / mappings += uploadMapping(UniversalDocs / packageBin).value,
+      s3Upload / mappings += uploadMapping(UniversalDocs / packageZipTarball).value,
+      s3Upload / mappings += uploadMapping(UniversalDocs / packageXzTarball).value,
+       s3Upload / mappings += uploadMapping(Rpm / packageBin).value,
       // Debian needs special handling because the value sbt-native-packager
       // gives us for `Debian / packageBin` (coming from the archiveFilename
       // method) includes the debian version and arch information,
       // which we historically have not included.  I don't see a way to
       // override the filename on disk, so we re-map at upload time
-      upload / mappings += Def.task {
+      s3Upload / mappings += Def.task {
         (Debian / packageBin).value ->
           s"scala/${version.value}/${(Debian / name).value}-${version.value}.deb"
       }.value
@@ -65,13 +66,6 @@ object ScalaDist {
       // create lib directory by resolving scala-dist's dependencies
       // to populate the rest of the distribution, explode scala-dist artifact itself
       Universal / mappings ++= createMappingsWith(update.value.toSeq, universalMappings),
-
-      // work around regression in sbt-native-packager 1.0.5 where
-      // these tasks invoke `tar` without any flags at all.  the issue
-      // was fixed in 1.1.0, so this could be revisited when we upgrade
-      UniversalDocs / packageZipTarball / universalArchiveOptions := Seq("--force-local", "-pcvf"),
-      UniversalDocs / packageXzTarball  / universalArchiveOptions := Seq("--force-local", "-pcvf")
-
     )
 
   // private lazy val onWindows = System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows")

project/ScalaDist.scala

@@ -1 +1 @@
-sbt.version=1.10.6
+sbt.version=1.10.11

project/build.properties

@@ -7,5 +7,11 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.8.1")
 
 libraryDependencies += "com.amazonaws" % "aws-java-sdk-s3" % "1.12.5"
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.1")
+libraryDependencies += "com.softwaremill.sttp.client3" %% "core" % "3.11.0"
+libraryDependencies ++= Seq(
+  "io.circe" %% "circe-core",
+  "io.circe" %% "circe-generic",
+  "io.circe" %% "circe-parser"
+).map(_ % "0.14.13")
 
+addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.1")

project/plugins.sbt

@@ -14,7 +14,7 @@ function travis_fold_end() {
 function postCommitStatus() {
   if [[ "$scala_sha" != "" ]]; then
     local jsonTemplate='{ "state": "%s", "target_url": "%s", "description": "%s", "context": "%s"}'
-    local json=$(printf "$jsonTemplate" "$1" "https://travis-ci.com/scala/scala-dist/builds/$TRAVIS_BUILD_ID" "$1" "travis/scala-dist/$version/$mode")
+    local json=$(printf "$jsonTemplate" "$1" "https://app.travis-ci.com/github/scala/scala-dist/builds/$TRAVIS_BUILD_ID" "$1" "travis/scala-dist/$version/$mode")
     [[ -z "$GITHUB_OAUTH_TOKEN" ]] && (echo "Missing environment variable GITHUB_OAUTH_TOKEN!"; exit 1)
     TMPFILE=$(mktemp -t curl.XXXXXXXXXX)