From 264ca8e2aca0683a779b4ecc57a7c161dce86d94 Mon Sep 17 00:00:00 2001 From: Eric Peters Date: Sat, 11 Sep 2021 14:35:30 -0700 Subject: [PATCH] * Add assemblyUnzipDirectory to AssemblyOption to use a different directory for unzipping jars * Add assemblyCacheDependency task to unzip jar dependencies as a separate task --- README.md | 19 + build.sbt | 3 +- src/main/contraband/AssemblyOption.contra | 36 -- src/main/scala/sbtassembly/Assembly.scala | 372 ++++++++++++++---- src/main/scala/sbtassembly/AssemblyKeys.scala | 35 +- .../scala/sbtassembly/AssemblyOption.scala | 52 ++- .../scala/sbtassembly/AssemblyPlugin.scala | 25 +- .../scala/sbtassembly/AssemblyUtils.scala | 123 ++++++ .../scala/sbtassembly/MergeStrategy.scala | 1 - src/sbt-test/caching/caching/build.sbt | 2 + .../caching/caching/project/plugins.sbt | 2 + src/sbt-test/caching/caching/test | 16 + src/sbt-test/caching/unzip/build.sbt | 67 ++++ .../caching/unzip/project/plugins.sbt | 9 + .../caching/unzip/src/main/scala/hello.scala | 8 + src/sbt-test/caching/unzip/test | 63 +++ 16 files changed, 672 insertions(+), 161 deletions(-) delete mode 100644 src/main/contraband/AssemblyOption.contra create mode 100644 src/sbt-test/caching/unzip/build.sbt create mode 100644 src/sbt-test/caching/unzip/project/plugins.sbt create mode 100644 src/sbt-test/caching/unzip/src/main/scala/hello.scala create mode 100644 src/sbt-test/caching/unzip/test diff --git a/README.md b/README.md index 9925e21f..2c09be77 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,25 @@ lazy val app = (project in file("app")) ) ``` + +### Unzip Caching + +When assembling an über artifact, that has many library dependencies, the unzip process can be very IO intensive. These unzipped directories are very suitable for CI systems to persist in between job runs. + +```scala +lazy val app = (project in file("app")) + .settings( + assemblyUnzipDirectory := Some(localCacheDirectory.value / "sbt-assembly" / "dependencies"), + assemblyCacheUnzip := true, // this is the default setting + assemblyCacheUseHardLinks := true, // this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO + // more settings here ... + ) +``` + +To populate the assemblyUnzipDirectory without a full assembly: + + sbt assemblyCacheDependency + Other Things ------------ diff --git a/build.sbt b/build.sbt index a50f18eb..dace209e 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ ThisBuild / crossScalaVersions := Seq(scala212) ThisBuild / scalaVersion := scala212 lazy val root = (project in file(".")) - .enablePlugins(SbtPlugin) // ContrabandPlugin + .enablePlugins(SbtPlugin) .settings(pomConsistency2021DraftSettings) .settings(nocomma { name := "sbt-assembly" @@ -21,7 +21,6 @@ lazy val root = (project in file(".")) case "2.12" => "1.2.8" } } - Compile / generateContrabands / sourceManaged := baseDirectory.value / "src" / "main" / "contraband-scala" }) ThisBuild / scmInfo := Some( diff --git a/src/main/contraband/AssemblyOption.contra b/src/main/contraband/AssemblyOption.contra deleted file mode 100644 index 0c7e39b8..00000000 --- a/src/main/contraband/AssemblyOption.contra +++ /dev/null @@ -1,36 +0,0 @@ -package sbtassembly -@target(Scala) - -type AssemblyOption { - assemblyDirectory: java.io.File @since("0.15.0") - - ## include compiled class files from itself or subprojects - includeBin: Boolean! = true @since("0.15.0") - - includeScala: Boolean! = true @since("0.15.0") - - ## include class files from external dependencies - includeDependency: Boolean! = true @since("0.15.0") - - excludedJars: sbt.Keys.Classpath! = raw"Nil" @since("0.15.0") - - excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile! = raw"sbtassembly.Assembly.defaultExcludedFiles" @since("0.15.0") - - mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy! = raw"sbtassembly.MergeStrategy.defaultMergeStrategy" @since("0.15.0") - - cacheOutput: Boolean! = true @since("0.15.0") - - cacheUnzip: Boolean! = true @since("0.15.0") - - appendContentHash: Boolean! = false @since("0.15.0") - - prependShellScript: sbtassembly.Assembly.SeqString @since("0.15.0") - - maxHashLength: Int @since("0.15.0") - - shadeRules: [com.eed3si9n.jarjarabrams.ShadeRule] @since("0.15.0") - - scalaVersion: String! = "" @since("0.15.0") - - level: sbt.Level.Value! = raw"sbt.Level.Info" @since("0.15.0") -} diff --git a/src/main/scala/sbtassembly/Assembly.scala b/src/main/scala/sbtassembly/Assembly.scala index d78b37ee..f925db24 100644 --- a/src/main/scala/sbtassembly/Assembly.scala +++ b/src/main/scala/sbtassembly/Assembly.scala @@ -3,20 +3,21 @@ package sbtassembly import sbt._ import Keys._ import Path.relativeTo + import java.security.MessageDigest -import java.io.{IOException, File} -import scala.collection.mutable +import java.io.{File, IOException} import Def.Initialize import PluginCompat._ import com.eed3si9n.jarjarabrams._ +import scala.collection.immutable.ListMap +import scala.collection.mutable +import scala.collection.parallel.immutable.ParVector + + object Assembly { import AssemblyPlugin.autoImport.{ Assembly => _, _ } - // used for contraband - type SeqFileToSeqFile = Seq[File] => Seq[File] - type SeqString = Seq[String] - private val scalaPre213Libraries = Vector( "scala-actors", "scala-compiler", @@ -43,8 +44,9 @@ object Assembly { import java.util.jar.{Attributes, Manifest} lazy val (ms: Vector[(File, String)], stratMapping: List[(String, MergeStrategy)]) = { - log.debug("Merging files...") + log.info("Merging files...") applyStrategies(mappings, ao.mergeStrategy, ao.assemblyDirectory.get, log) + // ao.assemblyUnzipDirectory.get } def makeJar(outPath: File): Unit = { import Package._ @@ -66,7 +68,7 @@ object Assembly { time = option.asInstanceOf[{def value: Option[Long]}].value } catch { case e: Throwable => - log.debug(e.toString) + log.info(e.toString) } } else { log.warn("Ignored unknown package option " + option) @@ -95,7 +97,7 @@ object Assembly { ) } catch { case e: Throwable => - log.debug(e.toString) + log.info(e.toString) } } ao.prependShellScript foreach { shellScript: Seq[String] => @@ -116,7 +118,7 @@ object Assembly { } } lazy val inputs = { - log.debug("Checking every *.class/*.jar file's SHA-1.") + log.info("Checking every *.class/*.jar file's SHA-1.") val rawHashBytes = (mappings.toVector.par flatMap { m => m.sourcePackage match { @@ -136,7 +138,7 @@ object Assembly { val cachedMakeJar = inputChanged(cacheDir / "assembly-inputs") { (inChanged, inputs: Seq[Byte]) => outputChanged(cacheDir / "assembly-outputs") { (outChanged, jar: PlainFileInfo) => if (inChanged) { - log.debug("SHA-1: " + bytesToString(inputs)) + log.info("SHA-1: " + bytesToString(inputs)) } // if if (inChanged || outChanged) makeJar(out) else log.info("Assembly up to date: " + jar.file) @@ -217,48 +219,54 @@ object Assembly { // even though fullClasspath includes deps, dependencyClasspath is needed to figure out // which jars exactly belong to the deps for packageDependency option. - def assembleMappings(classpath: Classpath, dependencies: Classpath, - ao: AssemblyOption, log: Logger): Vector[MappingSet] = { - val tempDir = ao.assemblyDirectory.get - if (!ao.cacheUnzip) IO.delete(tempDir) - if (!tempDir.exists) tempDir.mkdir() + def assembleMappings( + classpath: Classpath, + dependencies: Classpath, + ao: AssemblyOption, + log: Logger, + state: State + ): Vector[MappingSet] = { + val assemblyDir = ao.assemblyDirectory.get + val assemblyUnzipDir = ao.assemblyUnzipDirectory.getOrElse(assemblyDir) + val projectIdMsg: String = getProjectIdMsg(state) - val shadeRules = ao.shadeRules + if (!ao.cacheOutput && assemblyDir.exists) { + log.info(s"AssemblyOption.cacheOutput set to false, deleting assemblyDirectory: $assemblyDir for project: $projectIdMsg") + IO.delete(assemblyDir) + } - val (libs, dirs) = classpath.toVector.sortBy(_.data.getCanonicalPath).partition(c => ClasspathUtilities.isArchive(c.data)) + for { + unzipDir <- ao.assemblyUnzipDirectory + if !ao.cacheUnzip + if unzipDir.exists + } { + log.info(s"AssemblyOption.cacheUnzip set to false, deleting assemblyUnzipDirectory: $unzipDir for project: $projectIdMsg") + IO.delete(unzipDir) + } - val depLibs = dependencies.map(_.data).toSet.filter(ClasspathUtilities.isArchive) - val excludedJars = ao.excludedJars map {_.data} + if (!assemblyDir.exists) IO.createDirectory(assemblyDir) + if (!assemblyUnzipDir.exists) IO.createDirectory(assemblyUnzipDir) - val scalaLibraries = { - val scalaVersionParts = VersionNumber(ao.scalaVersion) - val isScala213AndLater = scalaVersionParts.numbers.length>=2 && scalaVersionParts._1.get>=2 && scalaVersionParts._2.get>=13 - if (isScala213AndLater) scala213AndLaterLibraries else scalaPre213Libraries - } + val (libsFiltered: Vector[Attributed[File]], dirs: Vector[Attributed[File]]) = getFilteredLibsAndDirs( + classpath = classpath, + dependencies = dependencies, + assemblyOption = ao + ) - val libsFiltered = libs flatMap { - case jar if excludedJars contains jar.data.asFile => None - case jar if isScalaLibraryFile(scalaLibraries, jar.data.asFile) => - if (ao.includeScala) Some(jar) else None - case jar if depLibs contains jar.data.asFile => - if (ao.includeDependency) Some(jar) else None - case jar => - if (ao.includeBin) Some(jar) else None - } - val dirRules = shadeRules.filter(_.isApplicableToCompiling) - val dirsFiltered = + val dirRules: Seq[ShadeRule] = ao.shadeRules.filter(_.isApplicableToCompiling) + val dirsFiltered: ParVector[File] = dirs.par flatMap { case dir => if (ao.includeBin) Some(dir) else None } map { dir => val hash = sha1name(dir.data) - IO.write(tempDir / (hash + "_dir.dir"), dir.data.getCanonicalPath, IO.utf8, false) - val dest = tempDir / (hash + "_dir") + IO.write(assemblyDir / (hash + "_dir.dir"), dir.data.getCanonicalPath, IO.utf8, false) + val dest = assemblyDir / (hash + "_dir") if (dest.exists) { IO.delete(dest) } - dest.mkdir() + IO.createDirectory(dest) IO.copyDirectory(dir.data, dest) if (dirRules.nonEmpty) { val mappings = ((dest ** (-DirectoryFilter)).get pair relativeTo(dest)) map { @@ -268,62 +276,85 @@ object Assembly { } dest } - val jarDirs = - (for(jar <- libsFiltered.par) yield { - val jarName = jar.data.asFile.getName - val jarRules = shadeRules - .filter(r => (r.isApplicableToAll || - jar.metadata.get(moduleID.key) - .map(m => ModuleCoordinate(m.organization, m.name, m.revision)) - .exists(r.isApplicableTo))) - val hash = sha1name(jar.data) + "_" + sha1content(jar.data) + "_" + sha1rules(jarRules) - val jarNamePath = tempDir / (hash + ".jarName") - val dest = tempDir / hash - // If the jar name path does not exist, or is not for this jar, unzip the jar - if (!ao.cacheUnzip || !jarNamePath.exists || IO.read(jarNamePath) != jar.data.getCanonicalPath ) - { - log.debug("Including: %s".format(jarName)) - IO.delete(dest) - dest.mkdir() - AssemblyUtils.unzip(jar.data, dest, log) - IO.delete(ao.excludedFiles(Seq(dest))) - if (jarRules.nonEmpty) { - val mappings = ((dest ** (-DirectoryFilter)).get pair relativeTo(dest)) map { - case (k, v) => k.toPath -> v - } - Shader.shadeDirectory(dirRules, dest.toPath, mappings, ao.level == Level.Debug) - } - - // Write the jarNamePath at the end to minimise the chance of having a - // corrupt cache if the user aborts the build midway through - IO.write(jarNamePath, jar.data.getCanonicalPath, IO.utf8, false) - } - else log.debug("Including from cache: %s".format(jarName)) - (dest, jar.data) - }) + val jarDirs: ParVector[(File, File)] = processDependencyJars( + libsFiltered, + ao, + isCacheOnly = false, + log, + state + ) - log.debug("Calculate mappings...") + log.info("Calculate mappings...") val base: Vector[File] = dirsFiltered.seq ++ (jarDirs map { _._1 }) - val excluded = (ao.excludedFiles(base) ++ base).toSet - val retval = (dirsFiltered map { d => MappingSet(None, AssemblyUtils.getMappings(d, excluded)) }).seq ++ + val excluded: Set[File] = (ao.excludedFiles(base) ++ base).toSet + val retval: Vector[MappingSet] = (dirsFiltered map { d => MappingSet(None, AssemblyUtils.getMappings(d, excluded)) }).seq ++ (jarDirs map { case (d, j) => MappingSet(Some(j), AssemblyUtils.getMappings(d, excluded)) }) - retval.toVector + retval + } + + def assemblyCacheDependency( + classpath: Classpath, + dependencies: Classpath, + assemblyOption: AssemblyOption, + log: Logger, + state: State + ): Boolean = { + if (!assemblyOption.cacheUnzip) sys.error("AssemblyOption.cacheUnzip must be true") + if (assemblyOption.assemblyUnzipDirectory.isEmpty) sys.error("AssemblyOption.assemblyUnzipDirectory must be supplied") + + val (libsFiltered: Vector[Attributed[File]], _) = getFilteredLibsAndDirs( + classpath = classpath, + dependencies = dependencies, + assemblyOption = assemblyOption + ) + + processDependencyJars(libsFiltered, assemblyOption, isCacheOnly = true, log, state) + + true } def assemblyTask(key: TaskKey[File]): Initialize[Task[File]] = Def.task { - val t = (test in key).value + // Run tests if enabled before assembly task + val _ = (test in key).value + val s = (streams in key).value Assembly( - (assemblyOutputPath in key).value, (assemblyOption in key).value, - (packageOptions in key).value, (assembledMappings in key).value, - s.cacheDirectory, s.log) + (assemblyOutputPath in key).value, + (assemblyOption in key).value, + (packageOptions in key).value, + (assembledMappings in key).value, + s.cacheDirectory, + s.log + ) } + def assembledMappingsTask(key: TaskKey[File]): Initialize[Task[Seq[MappingSet]]] = Def.task { val s = (streams in key).value assembleMappings( - (fullClasspath in assembly).value, (externalDependencyClasspath in assembly).value, - (assemblyOption in key).value, s.log) + (fullClasspath in assembly).value, + (externalDependencyClasspath in assembly).value, + (assemblyOption in key).value, + s.log, + state.value + ) + } + + def assemblyCacheDependencyTask(key: TaskKey[File]): Initialize[Task[Boolean]] = Def.task { + val s = (streams in key).value + val ao = (assemblyOption in key).value + val cp = (fullClasspath in assembly).value + val deps = (externalDependencyClasspath in assembly).value + val currentState = state.value + val projectIdMsg: String = getProjectIdMsg(currentState) + + if (!ao.cacheUnzip || ao.assemblyUnzipDirectory.isEmpty) { + if (!ao.cacheUnzip) s.log.warn(s"AssemblyOption.cacheUnzip must be true. Skipping unzip task for projectID: $projectIdMsg.") + if (ao.assemblyUnzipDirectory.isEmpty) s.log.warn(s"AssemblyOption.assemblyUnzipDirectory must be be supplied. Skipping cache unzip task for projectID: $projectIdMsg") + false + } else { + assemblyCacheDependency(classpath = cp, dependencies = deps, ao, s.log, currentState) + } } def isSystemJunkFile(fileName: String): Boolean = @@ -357,6 +388,183 @@ object Assembly { def isScalaLibraryFile(scalaLibraries: Vector[String], file: File): Boolean = scalaLibraries exists { x => file.getName startsWith x } + private[sbtassembly] def getProjectIdMsg(state: State): String = { + val project = Project.extract(state) + + val projectName = project.get(Keys.projectID).name + val currentRefProjectName = project.currentRef.project + + if (projectName != currentRefProjectName) s"$projectName/$currentRefProjectName" + else projectName + } + + private[sbtassembly] def processDependencyJars( + libsFiltered: Vector[Attributed[File]], + assemblyOption: AssemblyOption, + isCacheOnly: Boolean, + log: Logger, + state: State + ): ParVector[(File, File)] = { + val defaultAssemblyDir = assemblyOption.assemblyDirectory.get + val assemblyUnzipDir: File = assemblyOption.assemblyUnzipDirectory.getOrElse(defaultAssemblyDir) + val assemblyDir: Option[File] = if (isCacheOnly) None else Some(defaultAssemblyDir) + val isSameDir: Boolean = assemblyDir.exists{ _ == assemblyUnzipDir } + + if (!assemblyUnzipDir.exists) IO.createDirectory(assemblyUnzipDir) + if (assemblyDir.isDefined && !assemblyDir.get.exists) IO.createDirectory(assemblyDir.get) + + state.locked(assemblyUnzipDir / "sbt-assembly.lock") { + + val projectIdMsg: String = getProjectIdMsg(state) + + if (!assemblyUnzipDir.exists) IO.createDirectory(assemblyUnzipDir) + if (assemblyDir.isDefined && !assemblyDir.get.exists) IO.createDirectory(assemblyDir.get) + + val unzippingIntoMessage: String = if (isCacheOnly && !isSameDir) "unzip cache" else "output cache" + + val useHardLinks: Boolean = assemblyOption.cacheUseHardLinks && !isCacheOnly && { + if (isSameDir) { + log.warn(s"cacheUseHardLinks is enabled for project ($projectIdMsg), but assemblyUnzipDirectory is the same as assemblyDirectory ($assemblyUnzipDirectory)") + false + } else { + val isHardLinkSupported = AssemblyUtils.isHardLinkSupported(sourceDir = assemblyUnzipDir, destDir = assemblyDir.get) + if (!isHardLinkSupported) log.warn(s"cacheUseHardLinks is enabled for project ($projectIdMsg), but file system doesn't support hardlinks between from $assemblyUnzipDir to ${assemblyDir.get}") + isHardLinkSupported + } + } + + // Ensure we are not processing the same File twice, retain original ordering + val jarToAttributedFiles: ListMap[File, Vector[Attributed[File]]] = + libsFiltered + .foldLeft(ListMap.empty[File, Vector[Attributed[File]]]) { + case (lm, f) => + val canonicalJar = f.data.getCanonicalFile + lm.updated(canonicalJar, f +: lm.getOrElse(canonicalJar, Vector.empty)) + } + + for { + jar: File <- jarToAttributedFiles.keys.toVector.par + jarName = jar.asFile.getName + jarRules = assemblyOption.shadeRules + .filter { r => + r.isApplicableToAll || + jarToAttributedFiles.getOrElse(jar, Vector.empty) + .flatMap(_.metadata.get(moduleID.key)) + .map(m => ModuleCoordinate(m.organization, m.name, m.revision)) + .exists(r.isApplicableTo) + } + hash = sha1name(jar) + "_" + sha1content(jar) + "_" + sha1rules(jarRules) + jarOutputDir = (assemblyDir.getOrElse(assemblyUnzipDir) / hash).getCanonicalFile + } yield { + // TODO: Canonical path might be problematic if mount points inside docker are different + val jarNameFinalPath = new File(jarOutputDir + ".jarName") + + val jarNameCachePath = assemblyUnzipDir / (hash + ".jarName") + val jarCacheDir = assemblyUnzipDir / hash + + // If the jar name path does not exist, or is not for this jar, unzip the jar + if (!jarNameFinalPath.exists || IO.read(jarNameFinalPath) != jar.getCanonicalPath) { + log.info("Including: %s, for project: %s".format(jarName, projectIdMsg)) + + // Copy/Link from cache location if cache exists and is current + if (assemblyOption.cacheUnzip && + jarNameCachePath.exists && IO.read(jarNameCachePath) == jar.getCanonicalPath && + !jarNameFinalPath.exists + //(!jarNameFinalPath.exists || IO.read(jarNameFinalPath) != jar.getCanonicalPath) + ) { + if (useHardLinks) { + log.info("Creating hardlinks of %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = true) + } else { + log.info("Copying %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = false) + } + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = useHardLinks) + IO.delete(jarNameFinalPath) // write after merge/shade rules applied + // Unzip into cache dir and copy over + } else if (assemblyOption.cacheUnzip && jarNameFinalPath != jarNameCachePath) { + IO.delete(jarCacheDir) + IO.delete(jarOutputDir) + + IO.createDirectory(jarCacheDir) + IO.createDirectory(jarOutputDir) + + log.info("Unzipping %s into unzip cache: %s for project: %s".format(jarName, jarCacheDir, projectIdMsg)) + val files = AssemblyUtils.unzip(jar, jarCacheDir, log) + + // TODO: This is kind of a hack, but doing it seems to prevent a docker file system issue preventing + // FileNotFound exception after unzipping + files.foreach { f => + assert(f.exists(), s"File $f not found after unzipping $jar into $jarCacheDir!") + } + + if (useHardLinks) log.info("Creating hardlinks of %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + else log.info("Copying %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = useHardLinks) + // Don't use cache dir, just unzip to output cache + } else { + IO.delete(jarOutputDir) + IO.createDirectory(jarOutputDir) + log.info("Unzipping %s into %s: %s, for project: %s".format(jarName, unzippingIntoMessage, jarOutputDir, projectIdMsg)) + AssemblyUtils.unzip(jar, jarOutputDir, log) + } + + if (!isCacheOnly) { + IO.delete(assemblyOption.excludedFiles(Seq(jarOutputDir))) + if (jarRules.nonEmpty) { + val mappings = ((jarOutputDir ** (-DirectoryFilter)).get pair relativeTo(jarOutputDir)) map { + case (k, v) => k.toPath -> v + } + val dirRules: Seq[ShadeRule] = assemblyOption.shadeRules.filter(_.isApplicableToCompiling) + Shader.shadeDirectory(dirRules, jarOutputDir.toPath, mappings, assemblyOption.level == Level.Debug) + } + } + + // Write the jarNamePath at the end to minimise the chance of having a + // corrupt cache if the user aborts the build midway through + if (jarNameFinalPath != jarNameCachePath && !jarNameCachePath.exists) + IO.write(jarNameCachePath, jar.getCanonicalPath, IO.utf8, false) + + IO.write(jarNameFinalPath, jar.getCanonicalPath, IO.utf8, false) + } else { + if (isCacheOnly) log.info("Unzip cache of %s is up to date, for project: %s".format(jarName, projectIdMsg)) + else log.info("Including %s from output cache: %s, for project: %s".format(jarName, jarOutputDir, projectIdMsg)) + } + (jarOutputDir, jar) + } + } + } + + private[sbtassembly] def getFilteredLibsAndDirs( + classpath: Classpath, + dependencies: Classpath, + assemblyOption: AssemblyOption + ): (Vector[Attributed[File]], Vector[Attributed[File]]) = { + val (libs: Vector[Attributed[File]], dirs: Vector[Attributed[File]]) = + classpath.toVector.sortBy(_.data.getCanonicalPath).partition(c => ClasspathUtilities.isArchive(c.data)) + + val depLibs: Set[File] = dependencies.map(_.data).toSet.filter(ClasspathUtilities.isArchive) + val excludedJars: Seq[File] = assemblyOption.excludedJars map {_.data} + + val scalaLibraries: Vector[String] = { + val scalaVersionParts = VersionNumber(assemblyOption.scalaVersion) + val isScala213AndLater = scalaVersionParts.numbers.length>=2 && scalaVersionParts._1.get>=2 && scalaVersionParts._2.get>=13 + if (isScala213AndLater) scala213AndLaterLibraries else scalaPre213Libraries + } + + val libsFiltered: Vector[Attributed[File]] = libs flatMap { + case jar if excludedJars contains jar.data.asFile => None + case jar if isScalaLibraryFile(scalaLibraries, jar.data.asFile) => + if (assemblyOption.includeScala) Some(jar) else None + case jar if depLibs contains jar.data.asFile => + if (assemblyOption.includeDependency) Some(jar) else None + case jar => + if (assemblyOption.includeBin) Some(jar) else None + } + + (libsFiltered, dirs) + } + private[sbtassembly] def sha1 = MessageDigest.getInstance("SHA-1") private[sbtassembly] def sha1content(f: File): String = { Using.fileInputStream(f) { in => diff --git a/src/main/scala/sbtassembly/AssemblyKeys.scala b/src/main/scala/sbtassembly/AssemblyKeys.scala index f6e4c7fa..8f27bfec 100644 --- a/src/main/scala/sbtassembly/AssemblyKeys.scala +++ b/src/main/scala/sbtassembly/AssemblyKeys.scala @@ -5,23 +5,26 @@ import Keys._ import com.eed3si9n.jarjarabrams trait AssemblyKeys { - lazy val assembly = taskKey[File]("Builds a deployable über JAR") - lazy val assembleArtifact = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact") - lazy val assemblyOption = taskKey[AssemblyOption]("Configuration for making a deployable über JAR") - lazy val assembledMappings = taskKey[Seq[MappingSet]]("Keeps track of jar origins for each source") + lazy val assembly = taskKey[File]("Builds a deployable über JAR") + lazy val assembleArtifact = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact") + lazy val assemblyOption = taskKey[AssemblyOption]("Configuration for making a deployable über JAR") + lazy val assembledMappings = taskKey[Seq[MappingSet]]("Keeps track of jar origins for each source") + lazy val assemblyCacheDependency = taskKey[Boolean]("Caches the unzipped products of the dependency JAR files. Requires assemblyCacheUnzip (true) and AssemblyOption.assemblyUnzipCacheDir to be provided.") - lazy val assemblyPackageScala = taskKey[File]("Produces the Scala artifact") - lazy val assemblyPackageDependency = taskKey[File]("Produces the dependency artifact") - lazy val assemblyJarName = taskKey[String]("name of the über jar") - lazy val assemblyDefaultJarName = taskKey[String]("default name of the über jar") - lazy val assemblyOutputPath = taskKey[File]("output path of the über jar") - lazy val assemblyExcludedJars = taskKey[Classpath]("list of excluded jars") - lazy val assemblyMergeStrategy = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy") - lazy val assemblyShadeRules = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar") - lazy val assemblyAppendContentHash = settingKey[Boolean]("Appends SHA-1 fingerprint to the assembly file name") - lazy val assemblyMaxHashLength = settingKey[Int]("Length of SHA-1 fingerprint used for the assembly file name") - lazy val assemblyCacheUnzip = settingKey[Boolean]("Enables (true) or disables (false) cacheing the unzipped products of the dependency JAR files") - lazy val assemblyCacheOutput = settingKey[Boolean]("Enables (true) or disables (false) cacheing the output if the content has not changed") + lazy val assemblyUnzipDirectory = settingKey[Option[File]]("Specify a directory to unzip the products of dependency JAR files (e.g. assemblyUnzipDirectory := Some(localCacheDirectory.value / \"sbt-assembly\" / \"dependencies\"). Default None (uses default assembly directory).") + lazy val assemblyPackageScala = taskKey[File]("Produces the Scala artifact") + lazy val assemblyPackageDependency = taskKey[File]("Produces the dependency artifact") + lazy val assemblyJarName = taskKey[String]("name of the über jar") + lazy val assemblyDefaultJarName = taskKey[String]("default name of the über jar") + lazy val assemblyOutputPath = taskKey[File]("output path of the über jar") + lazy val assemblyExcludedJars = taskKey[Classpath]("list of excluded jars") + lazy val assemblyMergeStrategy = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy") + lazy val assemblyShadeRules = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar") + lazy val assemblyAppendContentHash = settingKey[Boolean]("Appends SHA-1 fingerprint to the assembly file name") + lazy val assemblyMaxHashLength = settingKey[Int]("Length of SHA-1 fingerprint used for the assembly file name") + lazy val assemblyCacheUnzip = settingKey[Boolean]("Enables (true) or disables (false) cacheing the unzipped products of the dependency JAR files") + lazy val assemblyCacheOutput = settingKey[Boolean]("Enables (true) or disables (false) cacheing the output if the content has not changed") + lazy val assemblyCacheUseHardLinks = settingKey[Boolean]("Experimental. Enables (true) or disables (false) using Files.createLink from the unzipped dependency cache to the assembly directory. Requires both paths to be on the same physical filesystem. Default false.") lazy val assemblyPrependShellScript = settingKey[Option[Seq[String]]]("A launch script to prepend to the über JAR") } diff --git a/src/main/scala/sbtassembly/AssemblyOption.scala b/src/main/scala/sbtassembly/AssemblyOption.scala index bdb1823a..0b3fcd60 100644 --- a/src/main/scala/sbtassembly/AssemblyOption.scala +++ b/src/main/scala/sbtassembly/AssemblyOption.scala @@ -10,39 +10,42 @@ package sbtassembly */ final class AssemblyOption private ( val assemblyDirectory: Option[java.io.File], + val assemblyUnzipDirectory: Option[java.io.File], val includeBin: Boolean, val includeScala: Boolean, val includeDependency: Boolean, val excludedJars: sbt.Keys.Classpath, - val excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, - val mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, + val excludedFiles: Seq[java.io.File] => Seq[java.io.File], + val mergeStrategy: String => MergeStrategy, val cacheOutput: Boolean, val cacheUnzip: Boolean, + val cacheUseHardLinks: Boolean, // Experimental, requires assemblyDirectory and assemblyUnzipDirectory to be on the same physical filesystem volume. val appendContentHash: Boolean, - val prependShellScript: Option[sbtassembly.Assembly.SeqString], + val prependShellScript: Option[Seq[String]], val maxHashLength: Option[Int], val shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], val scalaVersion: String, val level: sbt.Level.Value) extends Serializable { - private def this() = this(None, true, true, true, Nil, sbtassembly.Assembly.defaultExcludedFiles, sbtassembly.MergeStrategy.defaultMergeStrategy, true, true, false, None, None, Vector(), "", sbt.Level.Info) + private def this() = this(None, None, true, true, true, Nil, sbtassembly.Assembly.defaultExcludedFiles, sbtassembly.MergeStrategy.defaultMergeStrategy, true, true, false, false, None, None, Nil, "", sbt.Level.Info) override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { - case x: AssemblyOption => (this.assemblyDirectory == x.assemblyDirectory) && (this.includeBin == x.includeBin) && (this.includeScala == x.includeScala) && (this.includeDependency == x.includeDependency) && (this.excludedJars == x.excludedJars) && (this.excludedFiles == x.excludedFiles) && (this.mergeStrategy == x.mergeStrategy) && (this.cacheOutput == x.cacheOutput) && (this.cacheUnzip == x.cacheUnzip) && (this.appendContentHash == x.appendContentHash) && (this.prependShellScript == x.prependShellScript) && (this.maxHashLength == x.maxHashLength) && (this.shadeRules == x.shadeRules) && (this.scalaVersion == x.scalaVersion) && (this.level == x.level) + case x: AssemblyOption => (this.assemblyDirectory == x.assemblyDirectory) && (this.assemblyUnzipDirectory == x.assemblyUnzipDirectory) && (this.includeBin == x.includeBin) && (this.includeScala == x.includeScala) && (this.includeDependency == x.includeDependency) && (this.excludedJars == x.excludedJars) && (this.excludedFiles == x.excludedFiles) && (this.mergeStrategy == x.mergeStrategy) && (this.cacheOutput == x.cacheOutput) && (this.cacheUnzip == x.cacheUnzip) && (this.appendContentHash == x.appendContentHash) && (this.prependShellScript == x.prependShellScript) && (this.maxHashLength == x.maxHashLength) && (this.shadeRules == x.shadeRules) && (this.scalaVersion == x.scalaVersion) && (this.level == x.level) case _ => false }) override def hashCode: Int = { - 37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbtassembly.AssemblyOption".##) + assemblyDirectory.##) + includeBin.##) + includeScala.##) + includeDependency.##) + excludedJars.##) + excludedFiles.##) + mergeStrategy.##) + cacheOutput.##) + cacheUnzip.##) + appendContentHash.##) + prependShellScript.##) + maxHashLength.##) + shadeRules.##) + scalaVersion.##) + level.##) + 37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbtassembly.AssemblyOption".##) + assemblyDirectory.##) + assemblyUnzipDirectory.##) + includeBin.##) + includeScala.##) + includeDependency.##) + excludedJars.##) + excludedFiles.##) + mergeStrategy.##) + cacheOutput.##) + cacheUnzip.##) + cacheUseHardLinks.##) + appendContentHash.##) + prependShellScript.##) + maxHashLength.##) + shadeRules.##) + scalaVersion.##) + level.##) } override def toString: String = { - "AssemblyOption(" + assemblyDirectory + ", " + includeBin + ", " + includeScala + ", " + includeDependency + ", " + excludedJars + ", " + excludedFiles + ", " + mergeStrategy + ", " + cacheOutput + ", " + cacheUnzip + ", " + appendContentHash + ", " + prependShellScript + ", " + maxHashLength + ", " + shadeRules + ", " + scalaVersion + ", " + level + ")" + "AssemblyOption(" + assemblyDirectory + ", " + assemblyUnzipDirectory + ", " + includeBin + ", " + includeScala + ", " + includeDependency + ", " + excludedJars + ", " + excludedFiles + ", " + mergeStrategy + ", " + cacheOutput + ", " + cacheUnzip + ", " + cacheUseHardLinks + ", " + appendContentHash + ", " + prependShellScript + ", " + maxHashLength + ", " + shadeRules + ", " + scalaVersion + ", " + level + ")" } @deprecated("copy method is deprecated; use withIncludeBin(...) etc", "1.0.0") - def copy(assemblyDirectory: Option[java.io.File] = assemblyDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile = excludedFiles, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[sbtassembly.Assembly.SeqString] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule] = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = { + def copy(assemblyDirectory: Option[java.io.File] = assemblyDirectory, assemblyUnzipDirectory: Option[java.io.File] = assemblyUnzipDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: Seq[java.io.File] => Seq[java.io.File] = excludedFiles, mergeStrategy: String => MergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, cacheUseHardLinks: Boolean = cacheUseHardLinks, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[Seq[String]] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule] = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = { cp(assemblyDirectory = assemblyDirectory, + assemblyUnzipDirectory = assemblyUnzipDirectory, includeBin = includeBin, includeScala = includeScala, includeDependency = includeDependency, @@ -51,16 +54,18 @@ final class AssemblyOption private ( mergeStrategy = mergeStrategy, cacheOutput = cacheOutput, cacheUnzip = cacheUnzip, + cacheUseHardLinks = cacheUseHardLinks, appendContentHash = appendContentHash, prependShellScript = prependShellScript, maxHashLength = maxHashLength, shadeRules = shadeRules, scalaVersion = scalaVersion, - level = level) + level = level + ) } - private def cp(assemblyDirectory: Option[java.io.File] = assemblyDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile = excludedFiles, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[sbtassembly.Assembly.SeqString] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule] = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = { - new AssemblyOption(assemblyDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) + private def cp(assemblyDirectory: Option[java.io.File] = assemblyDirectory, assemblyUnzipDirectory: Option[java.io.File] = assemblyUnzipDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: Seq[java.io.File] => Seq[java.io.File] = excludedFiles, mergeStrategy: String => MergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, cacheUseHardLinks: Boolean = cacheUseHardLinks, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[Seq[String]] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule] = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = { + new AssemblyOption(assemblyDirectory, assemblyUnzipDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) } def withAssemblyDirectory(assemblyDirectory: Option[java.io.File]): AssemblyOption = { @@ -69,6 +74,12 @@ final class AssemblyOption private ( def withAssemblyDirectory(assemblyDirectory: java.io.File): AssemblyOption = { cp(assemblyDirectory = Option(assemblyDirectory)) } + def withAssemblyUnzipDirectory(assemblyUnzipDirectory: Option[java.io.File]): AssemblyOption = { + cp(assemblyUnzipDirectory = assemblyUnzipDirectory) + } + def withAssemblyUnzipDirectory(assemblyUnzipDirectory: java.io.File): AssemblyOption = { + cp(assemblyUnzipDirectory = Option(assemblyUnzipDirectory)) + } def withIncludeBin(includeBin: Boolean): AssemblyOption = { cp(includeBin = includeBin) } @@ -81,10 +92,10 @@ final class AssemblyOption private ( def withExcludedJars(excludedJars: sbt.Keys.Classpath): AssemblyOption = { cp(excludedJars = excludedJars) } - def withExcludedFiles(excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile): AssemblyOption = { + def withExcludedFiles(excludedFiles: Seq[java.io.File] => Seq[java.io.File]): AssemblyOption = { cp(excludedFiles = excludedFiles) } - def withMergeStrategy(mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy): AssemblyOption = { + def withMergeStrategy(mergeStrategy: String => MergeStrategy): AssemblyOption = { cp(mergeStrategy = mergeStrategy) } def withCacheOutput(cacheOutput: Boolean): AssemblyOption = { @@ -93,13 +104,16 @@ final class AssemblyOption private ( def withCacheUnzip(cacheUnzip: Boolean): AssemblyOption = { cp(cacheUnzip = cacheUnzip) } + def withCacheUseHardLinks(cacheUseHardLinks: Boolean): AssemblyOption = { + cp(cacheUseHardLinks = cacheUseHardLinks) + } def withAppendContentHash(appendContentHash: Boolean): AssemblyOption = { cp(appendContentHash = appendContentHash) } - def withPrependShellScript(prependShellScript: Option[sbtassembly.Assembly.SeqString]): AssemblyOption = { + def withPrependShellScript(prependShellScript: Option[Seq[String]]): AssemblyOption = { cp(prependShellScript = prependShellScript) } - def withPrependShellScript(prependShellScript: sbtassembly.Assembly.SeqString): AssemblyOption = { + def withPrependShellScript(prependShellScript: Seq[String]): AssemblyOption = { cp(prependShellScript = Option(prependShellScript)) } def withMaxHashLength(maxHashLength: Option[Int]): AssemblyOption = { @@ -121,6 +135,10 @@ final class AssemblyOption private ( object AssemblyOption { def apply(): AssemblyOption = new AssemblyOption() - def apply(assemblyDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) - def apply(assemblyDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: sbtassembly.Assembly.SeqString, maxHashLength: Int, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level) + // 1.0 and 1.1 Compat + def apply(assemblyDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: Seq[java.io.File] => Seq[java.io.File], mergeStrategy: String => MergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Option[Seq[String]], maxHashLength: Option[Int], shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, None,includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, false, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) + def apply(assemblyDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: Seq[java.io.File] => Seq[java.io.File], mergeStrategy: String => MergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Seq[String], maxHashLength: Int, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), None, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, false, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level) + // Current API + def apply(assemblyDirectory: Option[java.io.File], assemblyUnzipDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: Seq[java.io.File] => Seq[java.io.File], mergeStrategy: String => MergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, cacheUseHardLinks: Boolean, appendContentHash: Boolean, prependShellScript: Option[Seq[String]], maxHashLength: Option[Int], shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, assemblyUnzipDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) + def apply(assemblyDirectory: java.io.File, assemblyUnzipDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: Seq[java.io.File] => Seq[java.io.File], mergeStrategy: String => MergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, cacheUseHardLinks: Boolean, appendContentHash: Boolean, prependShellScript: Seq[String], maxHashLength: Int, shadeRules: Seq[com.eed3si9n.jarjarabrams.ShadeRule], scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), Option(assemblyUnzipDirectory), includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level) } diff --git a/src/main/scala/sbtassembly/AssemblyPlugin.scala b/src/main/scala/sbtassembly/AssemblyPlugin.scala index d52922c4..608ac87d 100644 --- a/src/main/scala/sbtassembly/AssemblyPlugin.scala +++ b/src/main/scala/sbtassembly/AssemblyPlugin.scala @@ -23,6 +23,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { import autoImport.{ Assembly => _, baseAssemblySettings => _, _ } override lazy val globalSettings: Seq[Def.Setting[_]] = Seq( + assemblyUnzipDirectory := None, assemblyMergeStrategy := MergeStrategy.defaultMergeStrategy, assemblyShadeRules := Nil, assemblyExcludedJars := Nil, @@ -32,6 +33,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { assemblyAppendContentHash := false, assemblyCacheUnzip := true, assemblyCacheOutput := true, + assemblyCacheUseHardLinks := false, assemblyPrependShellScript := None ) @@ -50,12 +52,13 @@ object AssemblyPlugin extends sbt.AutoPlugin { ) lazy val baseAssemblySettings: Seq[sbt.Def.Setting[_]] = (Seq( - assembly := Assembly.assemblyTask(assembly).value, - assembledMappings in assembly := Assembly.assembledMappingsTask(assembly).value, - assemblyPackageScala := Assembly.assemblyTask(assemblyPackageScala).value, - assembledMappings in assemblyPackageScala := Assembly.assembledMappingsTask(assemblyPackageScala).value, - assemblyPackageDependency := Assembly.assemblyTask(assemblyPackageDependency).value, - assembledMappings in assemblyPackageDependency := Assembly.assembledMappingsTask(assemblyPackageDependency).value, + assembly := Assembly.assemblyTask(assembly).value, + assembledMappings in assembly := Assembly.assembledMappingsTask(assembly).value, + assemblyPackageScala := Assembly.assemblyTask(assemblyPackageScala).value, + assembledMappings in assemblyPackageScala := Assembly.assembledMappingsTask(assemblyPackageScala).value, + assemblyPackageDependency := Assembly.assemblyTask(assemblyPackageDependency).value, + assembledMappings in assemblyPackageDependency := Assembly.assembledMappingsTask(assemblyPackageDependency).value, + assemblyCacheDependency := Assembly.assemblyCacheDependencyTask(assemblyPackageDependency).value, // test test in assembly := { () }, @@ -94,6 +97,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { ) ++ inTask(assembly)(assemblyOptionSettings) ++ inTask(assemblyPackageScala)(assemblyOptionSettings) ++ inTask(assemblyPackageDependency)(assemblyOptionSettings) + ++ inTask(assemblyCacheDependency)(assemblyOptionSettings) ++ Seq( assemblyOption in assemblyPackageScala ~= { _.withIncludeBin(false) @@ -104,7 +108,12 @@ object AssemblyPlugin extends sbt.AutoPlugin { _.withIncludeBin(false) .withIncludeScala(true) .withIncludeDependency(true) - } + }, + assemblyOption in assemblyCacheDependency ~= { + _.withIncludeBin(false) + .withIncludeScala(true) + .withIncludeDependency(true) + }, )) def assemblyOptionSettings: Seq[Setting[_]] = Seq( @@ -112,6 +121,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { val s = streams.value AssemblyOption() .withAssemblyDirectory(s.cacheDirectory / "assembly") + .withAssemblyUnzipDirectory(assemblyUnzipDirectory.value) .withIncludeBin((assembleArtifact in packageBin).value) .withIncludeScala((assembleArtifact in assemblyPackageScala).value) .withIncludeDependency((assembleArtifact in assemblyPackageDependency).value) @@ -120,6 +130,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { .withExcludedFiles(Assembly.defaultExcludedFiles) .withCacheOutput(assemblyCacheOutput.value) .withCacheUnzip(assemblyCacheUnzip.value) + .withCacheUseHardLinks(assemblyCacheUseHardLinks.value) .withAppendContentHash(assemblyAppendContentHash.value) .withPrependShellScript(assemblyPrependShellScript.value) .withMaxHashLength(assemblyMaxHashLength.?.value) diff --git a/src/main/scala/sbtassembly/AssemblyUtils.scala b/src/main/scala/sbtassembly/AssemblyUtils.scala index 8ffd74b9..7c2719a4 100644 --- a/src/main/scala/sbtassembly/AssemblyUtils.scala +++ b/src/main/scala/sbtassembly/AssemblyUtils.scala @@ -1,6 +1,7 @@ package sbtassembly import sbt._ + import java.io.{File, InputStream} import java.util.zip.ZipInputStream import scala.collection.mutable.HashSet @@ -8,6 +9,9 @@ import ErrorHandling.translate import PluginCompat._ import Using._ +import java.nio.file.{FileSystemException, Files} +import scala.Function.tupled + private[sbtassembly] object AssemblyUtils { private val PathRE = "([^/]+)/(.*)".r @@ -106,4 +110,123 @@ private[sbtassembly] object AssemblyUtils { } loop(rootDir, "", Nil).toVector } + + + def isHardLinkSupported(sourceDir: File, destDir: File): Boolean = { + assert(sourceDir.isDirectory) + assert(destDir.isDirectory) + + withTemporaryFileInDirectory("sbt-assembly", "file", sourceDir) { sourceFile => + try { + val destFile = destDir / sourceFile.getName + Files.createLink(destFile.toPath, sourceFile.toPath) + IO.delete(destFile) + true + } catch { + case ex: FileSystemException if ex.getMessage().contains("Invalid cross-device link") => false + } + } + } + + def withTemporaryFileInDirectory[T](prefix: String, postfix: String, dir: File)( + action: File => T + ): T = { + assert(dir.isDirectory) + val file = File.createTempFile(prefix, postfix, dir) + try { action(file) } finally { file.delete(); () } + } + + // region copyDirectory + + /** This is an experimental port of https://github.com/sbt/io/pull/326 */ + + def copyDirectory( + source: File, + target: File, + overwrite: Boolean = false, + preserveLastModified: Boolean = false, + preserveExecutable: Boolean = true, + hardLink: Boolean = false + ): Unit = { + val sources = PathFinder(source).allPaths pair Path.rebase(source, target) + copy(sources, overwrite, preserveLastModified, preserveExecutable, hardLink) + () + } + + def copy( + sources: Traversable[(File, File)], + overwrite: Boolean, + preserveLastModified: Boolean, + preserveExecutable: Boolean, + hardLink: Boolean + ): Set[File] = + sources + .map(tupled(copyImpl(overwrite, preserveLastModified, preserveExecutable, hardLink))) + .toSet + + private def copyImpl( + overwrite: Boolean, + preserveLastModified: Boolean, + preserveExecutable: Boolean, + hardLink: Boolean + )(from: File, to: File): File = { + if (overwrite || !to.exists || IO.getModifiedTimeOrZero(from) > IO.getModifiedTimeOrZero(to)) { + if (from.isDirectory) { + IO.createDirectory(to) + } else { + IO.createDirectory(to.getParentFile) + copyFile(from, to, preserveLastModified, preserveExecutable, hardLink) + } + } + to + } + + def copyFile( + sourceFile: File, + targetFile: File, + preserveLastModified: Boolean, + preserveExecutable: Boolean, + hardLink: Boolean + ): Unit = { + // NOTE: when modifying this code, test with larger values of CopySpec.MaxFileSizeBits than default + + require(sourceFile.exists, "Source file '" + sourceFile.getAbsolutePath + "' does not exist.") + require( + !sourceFile.isDirectory, + "Source file '" + sourceFile.getAbsolutePath + "' is a directory." + ) + if (hardLink) { + if (targetFile.exists) targetFile.delete() + Files.createLink(targetFile.toPath, sourceFile.toPath) + () + } else { + fileInputChannel(sourceFile) { in => + fileOutputChannel(targetFile) { out => + // maximum bytes per transfer according to from http://dzone.com/snippets/java-filecopy-using-nio + val max = (64L * 1024 * 1024) - (32 * 1024) + val total = in.size + def loop(offset: Long): Long = + if (offset < total) + loop(offset + out.transferFrom(in, offset, max)) + else + offset + val copied = loop(0) + if (copied != in.size) + sys.error( + "Could not copy '" + sourceFile + "' to '" + targetFile + "' (" + copied + "/" + in.size + " bytes copied)" + ) + } + } + if (preserveLastModified) { + IO.copyLastModified(sourceFile, targetFile) + () + } + if (preserveExecutable) { + IO.copyExecutable(sourceFile, targetFile) + () + } + } + } + + // endregion } diff --git a/src/main/scala/sbtassembly/MergeStrategy.scala b/src/main/scala/sbtassembly/MergeStrategy.scala index de8df03f..fd07d7de 100644 --- a/src/main/scala/sbtassembly/MergeStrategy.scala +++ b/src/main/scala/sbtassembly/MergeStrategy.scala @@ -21,7 +21,6 @@ abstract class MergeStrategy extends Function1[(File, String, Seq[File]), Either } object MergeStrategy { - type StringToMergeStrategy = String => MergeStrategy private val FileExtension = """([.]\w+)$""".r private def filenames(tempDir: File, fs: Seq[File]): Seq[String] = diff --git a/src/sbt-test/caching/caching/build.sbt b/src/sbt-test/caching/caching/build.sbt index ea853c2c..a2d47db5 100644 --- a/src/sbt-test/caching/caching/build.sbt +++ b/src/sbt-test/caching/caching/build.sbt @@ -4,6 +4,8 @@ lazy val root = (project in file(".")). scalaVersion := "2.11.12", libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test", libraryDependencies += "ch.qos.logback" % "logback-classic" % "0.9.29" % "runtime", + logLevel := sbt.Level.Info, + logBuffered := false, assembly / assemblyOption ~= { _.withCacheOutput(true) .withCacheUnzip(true) diff --git a/src/sbt-test/caching/caching/project/plugins.sbt b/src/sbt-test/caching/caching/project/plugins.sbt index e50b4e95..7c90d6d0 100644 --- a/src/sbt-test/caching/caching/project/plugins.sbt +++ b/src/sbt-test/caching/caching/project/plugins.sbt @@ -5,3 +5,5 @@ |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) else addSbtPlugin("com.eed3si9n" % "sbt-assembly" % pluginVersion) } + +addSbtPlugin("io.github.er1c" % "sbt-scriptedutils" % "0.1.0") \ No newline at end of file diff --git a/src/sbt-test/caching/caching/test b/src/sbt-test/caching/caching/test index 7ee7b742..b14ddd83 100644 --- a/src/sbt-test/caching/caching/test +++ b/src/sbt-test/caching/caching/test @@ -1,6 +1,12 @@ # check if the file gets created > clean > assembly +# Ensure all warnings have time to be printed +$ sleep 500 +> checkLogContains Unzipping slf4j-api-1.6.1.jar into output cache +> checkLogContains Unzipping logback-classic-0.9.29.jar into output cache +> checkLogContains Unzipping logback-core-0.9.29.jar into output cache +> checkLogContains Unzipping scala-library-2.11.12.jar into output cache $ exists target/scala-2.11/foo.jar # run to cache the hash, then check it's consistent @@ -28,3 +34,13 @@ $ delete src/main/resources/foo.txt > genresource2 > assembly > check + +> clearLog +> assemblyCacheDependency +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains AssemblyOption.assemblyUnzipDirectory must be be supplied. Skipping cache unzip task +> checkLogNotContains Unzipping slf4j-api-1.6.1.jar into unzip cache +> checkLogNotContains Unzipping logback-classic-0.9.29.jar into unzip cache +> checkLogNotContains Unzipping logback-core-0.9.29.jar into unzip cache +> checkLogNotContains Unzipping scala-library-2.11.12.jar into unzip cache \ No newline at end of file diff --git a/src/sbt-test/caching/unzip/build.sbt b/src/sbt-test/caching/unzip/build.sbt new file mode 100644 index 00000000..feee6b9e --- /dev/null +++ b/src/sbt-test/caching/unzip/build.sbt @@ -0,0 +1,67 @@ +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.core.{LogEvent => Log4JLogEvent, _} +import org.apache.logging.log4j.core.Filter.Result +import org.apache.logging.log4j.core.appender.AbstractAppender +import org.apache.logging.log4j.core.filter.LevelRangeFilter +import org.apache.logging.log4j.core.layout.PatternLayout + +lazy val tempUnzipDir = IO.createTemporaryDirectory + +lazy val root = (project in file(".")). + settings( + version := "0.1", + scalaVersion := "2.11.12", + libraryDependencies += "commons-io" % "commons-io" % "2.4", + libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test", + libraryDependencies += "ch.qos.logback" % "logback-classic" % "0.9.29" % "runtime", + assembly / assemblyShadeRules := Seq( + ShadeRule + .rename("org.apache.commons.io.**" -> "shadeio.@1") + .inLibrary("commons-io" % "commons-io" % "2.4") + .inProject + ), + assemblyUnzipDirectory := Some(tempUnzipDir), + assemblyCacheUseHardLinks := true, + logLevel := sbt.Level.Info, + logBuffered := false, + assembly / assemblyJarName := "foo.jar", + TaskKey[Unit]("checkunzip") := { + val opt = (assembly / assemblyOption).value + val assemblyDir = opt.assemblyDirectory.get + val assemblyUnzipDir = opt.assemblyUnzipDirectory.get + val preShadePath = "org.apache.commons.io".replace('.', java.io.File.separatorChar) + val postShadePath = "shadeio" + + val sources = PathFinder(assemblyUnzipDir).allPaths pair Path.rebase(assemblyUnzipDir, assemblyDir) + val ioSources = sources.filter{ case (unzip, _) => unzip.getAbsolutePath.contains(preShadePath) && unzip.isFile } + + assert(ioSources.nonEmpty) + sources.map{ _._1 }.foreach{ f => assert(f.exists) } + + ioSources.foreach { case (unzipFile, origOutFile) => + val outputFile = new java.io.File( + origOutFile + .getAbsolutePath + .toString + .replace(preShadePath, postShadePath) + ) + + assert(unzipFile.exists) + assert(outputFile.exists) + assert(getHashString(unzipFile) != getHashString(outputFile)) + } + () + }, + TaskKey[Unit]("cleanunzip") := { + IO.delete(tempUnzipDir) + } + ) + +def getHashString(file: java.io.File): String = { + import java.security.MessageDigest + MessageDigest + .getInstance("SHA-1") + .digest(IO.readBytes(file)) + .map( b => "%02x".format(b) ) + .mkString +} diff --git a/src/sbt-test/caching/unzip/project/plugins.sbt b/src/sbt-test/caching/unzip/project/plugins.sbt new file mode 100644 index 00000000..7c90d6d0 --- /dev/null +++ b/src/sbt-test/caching/unzip/project/plugins.sbt @@ -0,0 +1,9 @@ +{ + val pluginVersion = System.getProperty("plugin.version") + if(pluginVersion == null) + throw new RuntimeException("""|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) + else addSbtPlugin("com.eed3si9n" % "sbt-assembly" % pluginVersion) +} + +addSbtPlugin("io.github.er1c" % "sbt-scriptedutils" % "0.1.0") \ No newline at end of file diff --git a/src/sbt-test/caching/unzip/src/main/scala/hello.scala b/src/sbt-test/caching/unzip/src/main/scala/hello.scala new file mode 100644 index 00000000..5b2bde64 --- /dev/null +++ b/src/sbt-test/caching/unzip/src/main/scala/hello.scala @@ -0,0 +1,8 @@ +object Main { + def main(args: Array[String]) { + Option(getClass().getResource("foo.txt")) match { + case Some(_) => println("foo.txt") + case _ => println("hello") + } + } +} diff --git a/src/sbt-test/caching/unzip/test b/src/sbt-test/caching/unzip/test new file mode 100644 index 00000000..bfa84a49 --- /dev/null +++ b/src/sbt-test/caching/unzip/test @@ -0,0 +1,63 @@ +# check if the file gets created and unzips and creates hardlinks +> clean +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Unzipping slf4j-api-1.6.1.jar into unzip cache +> checkLogContains Unzipping commons-io-2.4.jar into unzip cache +> checkLogContains Unzipping logback-classic-0.9.29.jar into unzip cache +> checkLogContains Unzipping logback-core-0.9.29.jar into unzip cache +> checkLogContains Unzipping scala-library-2.11.12.jar into unzip cache +> checkLogContains Creating hardlinks of slf4j-api-1.6.1.jar from unzip cache +> checkLogContains Creating hardlinks of commons-io-2.4.jar from unzip cache +> checkLogContains Creating hardlinks of logback-classic-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of logback-core-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of scala-library-2.11.12.jar from unzip cache +$ exists target/scala-2.11/foo.jar + +# check if already cached +> clearLog +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Assembly up to date + +# check if creates from cache files +> clearLog +$ delete target/scala-2.11/foo.jar +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Including slf4j-api-1.6.1.jar from output cache +> checkLogContains Including commons-io-2.4.jar from output cache +> checkLogContains Including logback-classic-0.9.29.jar from output cache +> checkLogContains Including logback-core-0.9.29.jar from output cache +> checkLogContains Including scala-library-2.11.12.jar from output cache + +# check for using unzip cache +> clean +$ absent target/scala-2.11 +> clearLog +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Creating hardlinks of slf4j-api-1.6.1.jar from unzip cache +> checkLogContains Creating hardlinks of commons-io-2.4.jar from unzip cache +> checkLogContains Creating hardlinks of logback-classic-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of logback-core-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of scala-library-2.11.12.jar from unzip cache +> checkunzip + +> cleanunzip +> clean +> clearLog +> assemblyCacheDependency +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Unzipping slf4j-api-1.6.1.jar into unzip cache +> checkLogContains Unzipping commons-io-2.4.jar into unzip cache +> checkLogContains Unzipping logback-classic-0.9.29.jar into unzip cache +> checkLogContains Unzipping logback-core-0.9.29.jar into unzip cache +> checkLogContains Unzipping scala-library-2.11.12.jar into unzip cache + +> cleanunzip \ No newline at end of file