Merge pull request #36 from allenai/sbt-codeartifact

Sbt codeartifact

Author: bbstilson

.circleci/config.yml

@@ -5,7 +5,7 @@ jobs:
     docker:
       - image: openjdk:8
     environment:
-      SBT_VERSION: 1.2.8
+      SBT_VERSION: 1.4.9
     steps:
       - run: echo 'export ARTIFACT_BUILD=$CIRCLE_PROJECT_REPONAME-$CIRCLE_BUILD_NUM.zip' >> $BASH_ENV
       - run:
@@ -25,7 +25,7 @@ jobs:
           command: cat /dev/null | sbt clean
       - run:
           name: Test package (only integration tests)
-          command: cat /dev/null | sbt +test
+          command: cat /dev/null | sbt +it:test
       - run:
           name: Check formatting
           command: cat /dev/null | sbt scalafmtCheckAll

.gitignore

@@ -4,3 +4,9 @@
 .settings
 bin/
 target/
+.bloop/
+.bsp/
+.metals/
+.vscode/
+project/metals.sbt
+project/project/

README.md

@@ -1,25 +1,15 @@
-datastore
-======
+# datastore
 
 **Boss**: Dirk
 
 A thin wrapper over S3 that stores and retrieves immutable data in S3 and supports versioning.
 
 The real READMEs are [here for the API](datastore/README.md) and [here for the CLI](datastore-cli/README.md).
 
-### Releasing
+## Releasing
 
-This project releases to BinTray.  To make a release:
+This project releases to CodeArtifact. To make a release, simply run:
 
-1. Pull the latest code on the master branch that you want to release
-1. Edit `build.sbt` to remove "-SNAPSHOT" from the current version
-1. Create a pull request if desired or push to master if you are only changing the version
-1. Tag the release `git tag -a vX.Y.Z -m "Release X.Y.Z"` replacing X.Y.Z with the correct version
-1. Push the tag back to origin `git push origin vX.Y.Z`
-1. Release the build on Bintray `sbt +publish` (the "+" is required to cross-compile)
-1. Verify publication [on bintray.com](https://bintray.com/allenai/maven)
-1. Bump the version in `build.sbt` on master (and push!) with X.Y.Z+1-SNAPSHOT (e.g., 2.5.1
--SNAPSHOT after releasing 2.5.0)
-
-If you make a mistake you can rollback the release with `sbt bintrayUnpublish` and retag the
- version to a different commit as necessary.
+```bash
+sbt release
+```

build.sbt

@@ -1,63 +1,13 @@
-import Dependencies._
-
-lazy val scala211 = "2.11.12"
-lazy val scala212 = "2.12.9"
-lazy val scala213 = "2.13.0" // Not supported yet (collections changes required in common)
-lazy val supportedScalaVersions = List(scala212, scala211)
-
-ThisBuild / organization := "org.allenai.datastore"
-ThisBuild / scalaVersion := scala212
-ThisBuild / version      := "2.0.0"
-
-lazy val projectSettings = Seq(
-  crossScalaVersions := supportedScalaVersions,
-  resolvers ++= Seq(Resolver.bintrayRepo("allenai", "maven")),
-  dependencyOverrides ++= Logging.loggingDependencyOverrides,
-  publishMavenStyle := true,
-  publishArtifact in Test := false,
-  pomIncludeRepository := { _ => false },
-  licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0.html")),
-  homepage := Some(url("https://github.com/allenai/datastore")),
-  scmInfo := Some(ScmInfo(
-    url("https://github.com/allenai/datastore"),
-    "https://github.com/allenai/datastore.git")),
-  pomExtra := (
-      <developers>
-        <developer>
-          <id>allenai-dev-role</id>
-          <name>Allen Institute for Artificial Intelligence</name>
-          <email>dev-role@allenai.org</email>
-        </developer>
-      </developers>),
-  bintrayPackage := s"${organization.value}:${name.value}_${scalaBinaryVersion.value}",
-  bintrayOrganization := Some("allenai"),
-  bintrayRepository := "maven"
-)
-
-inConfig(IntegrationTest)(org.scalafmt.sbt.ScalafmtPlugin.scalafmtConfigSettings)
-
-lazy val root = (project in file("."))
-    .aggregate(
-      datastore,
-      cli
-    )
-    .configs(IntegrationTest)
-    .settings(
-      crossScalaVersions := Nil,
-      publish / skip := true
-    )
-
-lazy val datastore = (project in file("datastore"))
-    .settings(
-      Defaults.itSettings,
-      projectSettings
-    )
-    .configs(IntegrationTest)
-
-lazy val cli = (project in file("datastore-cli"))
-    .settings(
-      Defaults.itSettings,
-      projectSettings
-    )
-    .dependsOn(datastore)
-    .configs(IntegrationTest)
+lazy val datastore = project
+  .in(file("datastore"))
+  .settings(Release.publishSettings)
+
+lazy val cli = project
+  .in(file("datastore-cli"))
+  .settings(Release.publishSettings)
+  .dependsOn(datastore)
+
+lazy val root = project
+  .in(file("."))
+  .aggregate(datastore, cli)
+  .settings(Release.noPublishSettings)

datastore-cli/src/main/scala/org/allenai/datastore/cli/DownloadApp.scala

@@ -13,11 +13,11 @@ object DownloadApp extends App {
   )
 
   val parser = new scopt.OptionParser[Options]("scopt") {
-    opt[Boolean]("assumeFile") action { (f, c) =>
+    opt[Boolean]("assumeFile").action { (f, c) =>
       c.copy(assumeFile = f)
     } text ("Assumes that the object in the datastore is a file.")
 
-    opt[Boolean]("assumeDirectory") action { (d, c) =>
+    opt[Boolean]("assumeDirectory").action { (d, c) =>
       c.copy(assumeDirectory = d)
     } text ("Assumes that the object in the datastore is a directory.")
 
@@ -34,19 +34,19 @@ object DownloadApp extends App {
         "whether the object in the datastore is a file or a directory."
     )
 
-    opt[String]('g', "group") required () action { (g, c) =>
+    opt[String]('g', "group").required().action { (g, c) =>
       c.copy(group = g)
     } text ("Group name of the object in the datastore")
 
-    opt[String]('n', "name") required () action { (n, c) =>
+    opt[String]('n', "name").required().action { (n, c) =>
       c.copy(name = n)
     } text ("Name of the object in the datastore")
 
-    opt[Int]('v', "version") required () action { (v, c) =>
+    opt[Int]('v', "version").required().action { (v, c) =>
       c.copy(version = v)
     } text ("Version number of the object in the datastore")
 
-    opt[String]('d', "datastore") action { (d, c) =>
+    opt[String]('d', "datastore").action { (d, c) =>
       c.copy(datastore = Some(Datastore(d)))
     } text (s"Datastore to use. Default is ${Datastore.defaultName}")
 

datastore-cli/src/main/scala/org/allenai/datastore/cli/UploadApp.scala

@@ -15,35 +15,35 @@ object UploadApp extends App {
   )
 
   val parser = new scopt.OptionParser[Options]("scopt") {
-    opt[File]('p', "path") required () action { (p, c) =>
+    opt[File]('p', "path").required().action { (p, c) =>
       c.copy(path = p)
     } text ("Path to the file or directory you want uploaded")
 
-    opt[String]('g', "group") required () action { (g, c) =>
+    opt[String]('g', "group").required().action { (g, c) =>
       c.copy(group = g)
     } text ("Group name to store the file or directory under")
 
-    opt[String]('n', "name") required () action { (n, c) =>
+    opt[String]('n', "name").required().action { (n, c) =>
       c.copy(name = n)
     } text ("Name to store the file or directory under")
 
-    opt[Int]('v', "version") required () action { (v, c) =>
+    opt[Int]('v', "version").required().action { (v, c) =>
       c.copy(version = v)
     } text ("Version number to store the file or directory under")
 
-    opt[String]('d', "datastore") action { (d, c) =>
+    opt[String]('d', "datastore").action { (d, c) =>
       c.copy(datastore = Some(Datastore(d)))
     } text (s"Datastore to use. Default is ${Datastore.defaultName}")
 
-    opt[Boolean]("overwrite") action { (_, c) =>
+    opt[Boolean]("overwrite").action { (_, c) =>
       c.copy(overwrite = true)
     } text ("Overwrite if the group, version, and name already exists")
 
     help("help")
   }
 
   Common.handleDatastoreExceptions {
-    parser.parse(args, Options()) foreach { config =>
+    parser.parse(args, Options()).foreach { config =>
       val datastore = config.datastore.getOrElse {
         Common.printDefaultDatastoreWarning()
         Datastore

datastore-cli/src/main/scala/org/allenai/datastore/cli/UrlApp.scala

@@ -13,11 +13,11 @@ object UrlApp extends App {
   )
 
   val parser = new scopt.OptionParser[Options]("scopt") {
-    opt[Boolean]("assumeFile") action { (f, c) =>
+    opt[Boolean]("assumeFile").action { (f, c) =>
       c.copy(assumeFile = f)
     } text ("Assumes that the object in the datastore is a file.")
 
-    opt[Boolean]("assumeDirectory") action { (d, c) =>
+    opt[Boolean]("assumeDirectory").action { (d, c) =>
       c.copy(assumeDirectory = d)
     } text ("Assumes that the object in the datastore is a directory.")
 
@@ -34,27 +34,27 @@ object UrlApp extends App {
         "whether the object in the datastore is a file or a directory."
     )
 
-    opt[String]('g', "group") required () action { (g, c) =>
+    opt[String]('g', "group").required().action { (g, c) =>
       c.copy(group = g)
     } text ("Group name of the object in the datastore")
 
-    opt[String]('n', "name") required () action { (n, c) =>
+    opt[String]('n', "name").required().action { (n, c) =>
       c.copy(name = n)
     } text ("Name of the object in the datastore")
 
-    opt[Int]('v', "version") required () action { (v, c) =>
+    opt[Int]('v', "version").required().action { (v, c) =>
       c.copy(version = v)
     } text ("Version number of the object in the datastore")
 
-    opt[String]('d', "datastore") action { (d, c) =>
+    opt[String]('d', "datastore").action { (d, c) =>
       c.copy(datastore = Some(Datastore(d)))
     } text (s"Datastore to use. Default is ${Datastore.defaultName}")
 
     help("help")
   }
 
   Common.handleDatastoreExceptions {
-    parser.parse(args, Options()) foreach { config =>
+    parser.parse(args, Options()).foreach { config =>
       val datastore = config.datastore.getOrElse {
         Common.printDefaultDatastoreWarning()
         Datastore

datastore/build.sbt

@@ -3,13 +3,14 @@ import Dependencies._
 name := "datastore"
 
 libraryDependencies ++= Seq(
-  awsJavaSdk exclude("commons-logging", "commons-logging"),
+  awsJavaSdk exclude ("commons-logging", "commons-logging"),
   commonsIO,
   allenAiCommon,
   allenAiTestkit % "test,it",
   Logging.logbackClassic,
   Logging.logbackCore,
   Logging.slf4jApi,
-  "org.slf4j" % "jcl-over-slf4j" % Logging.slf4jVersion)
+  "org.slf4j" % "jcl-over-slf4j" % Logging.slf4jVersion
+)
 
 fork in IntegrationTest := true

datastore/src/main/scala/org/allenai/datastore/Datastore.scala

@@ -19,6 +19,7 @@ import org.apache.commons.io.FileUtils
 import org.slf4j.LoggerFactory
 import ch.qos.logback.classic.Level
 
+import scala.collection.compat._
 import scala.collection.JavaConverters._
 import scala.util.control.NonFatal
 import scala.util.Random
@@ -684,7 +685,7 @@ class Datastore(val name: String, val s3: AmazonS3Client) extends Logging {
     getAllListings(listObjectsRequest)
       .flatMap(_.getCommonPrefixes.asScala)
       .map(_.stripSuffix("/"))
-      .to[collection.SortedSet]
+      .to(collection.SortedSet)
   }
 
   /** Lists all items in a group
@@ -706,7 +707,7 @@ class Datastore(val name: String, val s3: AmazonS3Client) extends Logging {
         .map { os =>
           Locator.fromKey(os.getKey)
         }
-        .to[collection.SortedSet]
+        .to(collection.SortedSet)
     }
 
   //

project/Dependencies.scala

@@ -1,8 +1,11 @@
 import sbt._
 
 object Dependencies {
-  val allenAiCommon = "org.allenai.common" %% "common-core" % "2.0.0"
-  val allenAiTestkit = "org.allenai.common" %% "common-testkit" % "2.0.0"
+  val scalaCollectionCompat = "org.scala-lang.modules" %% "scala-collection-compat" % "2.4.3"
+
+  val commonVersion = "2.3.1"
+  val allenAiCommon = "org.allenai.common" %% "common-core" % commonVersion
+  val allenAiTestkit = "org.allenai.common" %% "common-testkit" % commonVersion
 
   val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4"
   val scalaTest = "org.scalatest" %% "scalatest" % "3.0.8"
@@ -44,19 +47,19 @@ object Dependencies {
         // Exclude the transitive dependencies that might mess things up for us.
         // slf4j replaces log4j.
         (module
-            exclude ("log4j", "log4j")
-            exclude ("commons-logging", "commons-logging")
-            // We're using logback as the slf4j implementation, and we're providing it below.
-            exclude ("org.slf4j", "slf4j-log4j12")
-            exclude ("org.slf4j", "slf4j-jdk14")
-            exclude ("org.slf4j", "slf4j-jcl")
-            exclude ("org.slf4j", "slf4j-simple")
-            // We'll explicitly provide the logback version; this avoids having to do an override.
-            exclude ("ch.qos.logback", "logback-core")
-            exclude ("ch.qos.logback", "logback-classic")
-            // We add bridges explicitly as well
-            exclude ("org.slf4j", "log4j-over-slf4j")
-            exclude ("org.slf4j", "jcl-over-slf4j"))
+          exclude ("log4j", "log4j")
+          exclude ("commons-logging", "commons-logging")
+        // We're using logback as the slf4j implementation, and we're providing it below.
+          exclude ("org.slf4j", "slf4j-log4j12")
+          exclude ("org.slf4j", "slf4j-jdk14")
+          exclude ("org.slf4j", "slf4j-jcl")
+          exclude ("org.slf4j", "slf4j-simple")
+        // We'll explicitly provide the logback version; this avoids having to do an override.
+          exclude ("ch.qos.logback", "logback-core")
+          exclude ("ch.qos.logback", "logback-classic")
+        // We add bridges explicitly as well
+          exclude ("org.slf4j", "log4j-over-slf4j")
+          exclude ("org.slf4j", "jcl-over-slf4j"))
       }
     }
     // Now, add the logging libraries.

project/GlobalPlugin.scala

@@ -0,0 +1,23 @@
+import Dependencies._
+
+import sbt._
+import Keys._
+import codeartifact.CodeArtifactKeys
+
+object GlobalPlugin extends AutoPlugin {
+  override def trigger = allRequirements
+
+  override def projectConfigurations: Seq[Configuration] = Seq(IntegrationTest)
+
+  override def projectSettings =
+    Defaults.itSettings ++ inConfig(IntegrationTest)(
+      org.scalafmt.sbt.ScalafmtPlugin.scalafmtConfigSettings
+    ) ++ Seq(
+      organization := "org.allenai.datastore",
+      scalaVersion := ScalaVersions.SCALA_213,
+      CodeArtifactKeys.codeArtifactUrl := "https://org-allenai-s2-896129387501.d.codeartifact.us-west-2.amazonaws.com/maven/private",
+      fork in Test := true,
+      libraryDependencies += scalaCollectionCompat,
+      dependencyOverrides ++= Logging.loggingDependencyOverrides
+    )
+}