Skip to content

Commit

Permalink
Skip abandoned repos
Browse files Browse the repository at this point in the history
  • Loading branch information
fthomas committed Jan 22, 2025
1 parent f5a5a09 commit fb06636
Show file tree
Hide file tree
Showing 13 changed files with 119 additions and 12 deletions.
2 changes: 2 additions & 0 deletions modules/core/src/main/resources/default.scala-steward.conf
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
// Changes to this file are therefore immediately visible to all
// Scala Steward instances.

lastCommitMaxAge = "540 days"

postUpdateHooks = [
{
groupId = "com.github.liancheng",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import org.scalasteward.core.forge.ForgeType.*
import org.scalasteward.core.git.FileGitAlg.{dotdot, gitCmd}
import org.scalasteward.core.io.process.{ProcessFailedException, SlurpOptions}
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.util.Nel
import org.scalasteward.core.util.{Nel, Timestamp}
import scala.util.Try

final class FileGitAlg[F[_]](config: Config)(implicit
fileAlg: FileAlg[F],
Expand Down Expand Up @@ -102,6 +103,11 @@ final class FileGitAlg[F[_]](config: Config)(implicit
.handleError(_ => List.empty[String])
.map(_.filter(_.nonEmpty))

override def getCommitDate(repo: File, sha1: Sha1): F[Timestamp] =
git("show", "--no-patch", "--format=%ct", sha1.value.value)(repo)
.flatMap(out => F.fromTry(Try(out.mkString.trim.toLong)))
.map(Timestamp.fromEpochSecond)

override def hasConflicts(repo: File, branch: Branch, base: Branch): F[Boolean] = {
val tryMerge = git_("merge", "--no-commit", "--no-ff", branch.name)(repo)
val abortMerge = git_("merge", "--abort")(repo).attempt.void
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import cats.{FlatMap, Monad}
import org.http4s.Uri
import org.scalasteward.core.application.Config
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.util.Timestamp

trait GenGitAlg[F[_], Repo] {
def add(repo: Repo, file: String): F[Unit]
Expand Down Expand Up @@ -57,6 +58,8 @@ trait GenGitAlg[F[_], Repo] {

def findFilesContaining(repo: Repo, string: String): F[List[String]]

def getCommitDate(repo: Repo, sha1: Sha1): F[Timestamp]

/** Returns `true` if merging `branch` into `base` results in merge conflicts. */
def hasConflicts(repo: Repo, branch: Branch, base: Branch): F[Boolean]

Expand Down Expand Up @@ -144,6 +147,9 @@ trait GenGitAlg[F[_], Repo] {
override def findFilesContaining(repo: A, string: String): F[List[String]] =
f(repo).flatMap(self.findFilesContaining(_, string))

override def getCommitDate(repo: A, sha1: Sha1): F[Timestamp] =
f(repo).flatMap(self.getCommitDate(_, sha1))

override def hasConflicts(repo: A, branch: Branch, base: Branch): F[Boolean] =
f(repo).flatMap(self.hasConflicts(_, branch, base))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ import io.circe.generic.semiauto.*
import org.scalasteward.core.data.{ArtifactId, DependencyInfo, GroupId, Scope}
import org.scalasteward.core.git.Sha1
import org.scalasteward.core.repoconfig.RepoConfig
import org.scalasteward.core.util.Timestamp

final case class RepoCache(
sha1: Sha1,
commitDate: Timestamp,
dependencyInfos: List[Scope[List[DependencyInfo]]],
maybeRepoConfig: Option[RepoConfig],
maybeRepoConfigParsingError: Option[String]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,13 @@ import org.scalasteward.core.forge.data.RepoOut
import org.scalasteward.core.forge.{ForgeApiAlg, ForgeRepoAlg}
import org.scalasteward.core.git.GitAlg
import org.scalasteward.core.repoconfig.RepoConfigAlg
import org.scalasteward.core.util.{dateTime, DateTimeAlg}
import org.typelevel.log4cats.Logger
import scala.util.control.NoStackTrace

final class RepoCacheAlg[F[_]](config: Config)(implicit
buildToolDispatcher: BuildToolDispatcher[F],
dateTimeAlg: DateTimeAlg[F],
forgeApiAlg: ForgeApiAlg[F],
forgeRepoAlg: ForgeRepoAlg[F],
gitAlg: GitAlg[F],
Expand All @@ -50,6 +53,7 @@ final class RepoCacheAlg[F[_]](config: Config)(implicit
data <- maybeCache
.filter(_.sha1 === latestSha1)
.fold(cloneAndRefreshCache(repo, repoOut))(supplementCache(repo, _).pure[F])
_ <- throwIfAbandoned(data)
} yield (data, repoOut)

private def supplementCache(repo: Repo, cache: RepoCache): RepoData =
Expand All @@ -68,7 +72,8 @@ final class RepoCacheAlg[F[_]](config: Config)(implicit
private def computeCache(repo: Repo): F[RepoData] =
for {
branch <- gitAlg.currentBranch(repo)
latestSha1 <- gitAlg.latestSha1(repo, branch)
sha1 <- gitAlg.latestSha1(repo, branch)
commitDate <- gitAlg.getCommitDate(repo, sha1)

Check warning on line 76 in modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala

View check run for this annotation

Codecov / codecov/patch

modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala#L75-L76

Added lines #L75 - L76 were not covered by tests
configParsingResult <- repoConfigAlg.readRepoConfig(repo)
maybeConfig = configParsingResult.maybeRepoConfig
maybeConfigParsingError = configParsingResult.maybeParsingError.map(_.getMessage)
Expand All @@ -77,9 +82,21 @@ final class RepoCacheAlg[F[_]](config: Config)(implicit
dependencyInfos <-
dependencies.traverse(_.traverse(_.traverse(gatherDependencyInfo(repo, _))))
_ <- gitAlg.discardChanges(repo)
cache = RepoCache(latestSha1, dependencyInfos, maybeConfig, maybeConfigParsingError)
cache = RepoCache(sha1, commitDate, dependencyInfos, maybeConfig, maybeConfigParsingError)

Check warning on line 85 in modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala

View check run for this annotation

Codecov / codecov/patch

modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala#L85

Added line #L85 was not covered by tests
} yield RepoData(repo, cache, config)

private def gatherDependencyInfo(repo: Repo, dependency: Dependency): F[DependencyInfo] =
gitAlg.findFilesContaining(repo, dependency.version.value).map(DependencyInfo(dependency, _))

private[repocache] def throwIfAbandoned(data: RepoData): F[Unit] =
data.config.lastCommitMaxAge.traverse_ { maxAge =>
dateTimeAlg.currentTimestamp.flatMap { now =>
val sinceLastCommit = data.cache.commitDate.until(now)
val isAbandoned = sinceLastCommit > maxAge
F.raiseWhen(isAbandoned) {
val msg = s"Skipping because last commit is older than ${dateTime.showDuration(maxAge)}"
new Throwable(msg) with NoStackTrace
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ import org.scalasteward.core.buildtool.BuildRoot
import org.scalasteward.core.data.Repo
import org.scalasteward.core.edit.hooks.PostUpdateHook
import org.scalasteward.core.repoconfig.RepoConfig.defaultBuildRoots
import org.scalasteward.core.util.dateTime.*
import org.scalasteward.core.util.{combineOptions, intellijThisImportIsUsed}
import scala.concurrent.duration.FiniteDuration

final case class RepoConfig(
private val commits: Option[CommitsConfig] = None,
Expand All @@ -37,7 +40,8 @@ final case class RepoConfig(
private val assignees: Option[List[String]] = None,
private val reviewers: Option[List[String]] = None,
private val dependencyOverrides: Option[List[GroupRepoConfig]] = None,
signoffCommits: Option[Boolean] = None
signoffCommits: Option[Boolean] = None,
lastCommitMaxAge: Option[FiniteDuration] = None
) {
def commitsOrDefault: CommitsConfig =
commits.getOrElse(CommitsConfig())
Expand Down Expand Up @@ -107,8 +111,11 @@ object RepoConfig {
assignees = x.assignees |+| y.assignees,
reviewers = x.reviewers |+| y.reviewers,
dependencyOverrides = x.dependencyOverrides |+| y.dependencyOverrides,
signoffCommits = x.signoffCommits.orElse(y.signoffCommits)
signoffCommits = x.signoffCommits.orElse(y.signoffCommits),
lastCommitMaxAge = combineOptions(x.lastCommitMaxAge, y.lastCommitMaxAge)(_ max _)
)
}
)

intellijThisImportIsUsed(finiteDurationEncoder)
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ final case class Timestamp(millis: Long) {
}

object Timestamp {
def fromEpochSecond(seconds: Long): Timestamp =
Timestamp(seconds * 1000L)

def fromLocalDateTime(ldt: LocalDateTime): Timestamp =
Timestamp(ldt.toInstant(ZoneOffset.UTC).toEpochMilli)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.scalasteward.core.util

import cats.syntax.all.*
import io.circe.{Decoder, Encoder}
import java.util.concurrent.TimeUnit
import scala.annotation.tailrec
import scala.concurrent.duration.*
Expand All @@ -31,6 +32,12 @@ object dateTime {
def renderFiniteDuration(fd: FiniteDuration): String =
fd.toString.filterNot(_.isSpaceChar)

implicit val finiteDurationDecoder: Decoder[FiniteDuration] =
Decoder[String].emap(parseFiniteDuration(_).leftMap(_.getMessage))

implicit val finiteDurationEncoder: Encoder[FiniteDuration] =
Encoder[String].contramap(renderFiniteDuration)

def showDuration(d: FiniteDuration): String = {
def symbol(unit: TimeUnit): String =
unit match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.scalasteward.core.git.Sha1
import org.scalasteward.core.repocache.RepoCache
import org.scalasteward.core.repoconfig.*
import org.scalasteward.core.repoconfig.PullRequestFrequency.{Asap, Timespan}
import org.scalasteward.core.util.{DateTimeAlg, Timestamp}
import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger
import scala.concurrent.duration.FiniteDuration
Expand All @@ -19,11 +20,14 @@ object TestInstances {
Sha1.unsafeFrom("da39a3ee5e6b4b0d3255bfef95601890afd80709")

val dummyRepoCache: RepoCache =
RepoCache(dummySha1, List.empty, Option.empty, Option.empty)
RepoCache(dummySha1, Timestamp(0L), List.empty, Option.empty, Option.empty)

val dummyRepoCacheWithParsingError: RepoCache =
dummyRepoCache.copy(maybeRepoConfigParsingError = Some("Failed to parse .scala-steward.conf"))

val ioDateTimeAlg: DateTimeAlg[IO] =
DateTimeAlg.create[IO]

implicit val ioLogger: Logger[IO] =
Slf4jLogger.getLogger[IO]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import cats.Monad
import cats.effect.IO
import cats.syntax.all.*
import munit.CatsEffectSuite
import org.scalasteward.core.TestInstances.ioLogger
import org.scalasteward.core.TestInstances.{ioDateTimeAlg, ioLogger}
import org.scalasteward.core.git.FileGitAlgTest.{
conflictsNo,
conflictsYes,
Expand All @@ -18,6 +18,7 @@ import org.scalasteward.core.io.ProcessAlgTest.ioProcessAlg
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.mock.MockConfig.{config, mockRoot}
import org.scalasteward.core.util.Nel
import scala.concurrent.duration.DurationInt

class FileGitAlgTest extends CatsEffectSuite {
private val rootDir = mockRoot / "git-tests"
Expand Down Expand Up @@ -158,6 +159,19 @@ class FileGitAlgTest extends CatsEffectSuite {
} yield ()
}

test("getCommitDate") {
val repo = rootDir / "getCommitDate"
for {
_ <- ioAuxGitAlg.createRepo(repo)
sha1 <- ioGitAlg.latestSha1(repo, master)
commitDate <- ioGitAlg.getCommitDate(repo, sha1)
now <- ioDateTimeAlg.currentTimestamp
diff = commitDate.until(now)
maxDrift = 2.hours
_ = assert(diff > -maxDrift && diff < maxDrift, clue((commitDate, now)))
} yield ()
}

test("hasConflicts") {
val repo = rootDir / "hasConflicts"
for {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ package org.scalasteward.core.io
import cats.effect.IO
import cats.effect.unsafe.implicits.global
import munit.FunSuite
import org.scalasteward.core.TestInstances.ioDateTimeAlg
import org.scalasteward.core.io.process.*
import org.scalasteward.core.util.{DateTimeAlg, Nel}
import org.scalasteward.core.util.Nel
import scala.concurrent.duration.*

class processTest extends FunSuite {
Expand Down Expand Up @@ -66,7 +67,7 @@ class processTest extends FunSuite {
val timeout = 500.milliseconds
val sleep = timeout * 2
val p = slurp2(Nel.of("sleep", sleep.toSeconds.toInt.toString), timeout).attempt
val (Left(t), fd) = DateTimeAlg.create[IO].timed(p).unsafeRunSync(): @unchecked
val (Left(t), fd) = ioDateTimeAlg.timed(p).unsafeRunSync(): @unchecked

assert(clue(t).isInstanceOf[ProcessTimedOutException])
assert(clue(fd) > timeout)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package org.scalasteward.core.repocache

import cats.implicits.toSemigroupKOps
import cats.syntax.all.*
import io.circe.syntax.*
import java.time.LocalDateTime
import munit.CatsEffectSuite
import org.http4s.HttpApp
import org.http4s.circe.*
Expand All @@ -14,7 +15,9 @@ import org.scalasteward.core.forge.github.Repository
import org.scalasteward.core.git.Branch
import org.scalasteward.core.mock.MockContext.context.{repoCacheAlg, repoConfigAlg, workspaceAlg}
import org.scalasteward.core.mock.{GitHubAuth, MockEff, MockEffOps, MockState}
import org.scalasteward.core.util.intellijThisImportIsUsed
import org.scalasteward.core.repoconfig.RepoConfig
import org.scalasteward.core.util.{intellijThisImportIsUsed, Timestamp}
import scala.concurrent.duration.*

class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] {
intellijThisImportIsUsed(encodeUri)
Expand All @@ -36,7 +39,8 @@ class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] {
uri"https://github.com/scala-steward/cats-effect.git",
Branch("main")
)
val repoCache = RepoCache(dummySha1, Nil, None, None)
val now = Timestamp.fromLocalDateTime(LocalDateTime.now())
val repoCache = RepoCache(dummySha1, now, Nil, None, None)
val workspace = workspaceAlg.rootDir.unsafeRunSync()
val httpApp = HttpApp[MockEff] {
case POST -> Root / "repos" / "typelevel" / "cats-effect" / "forks" =>
Expand All @@ -55,4 +59,33 @@ class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] {
val expected = (RepoData(repo, repoCache, repoConfigAlg.mergeWithGlobal(None)), repoOut)
assertIO(obtained, expected)
}

test("throwIfAbandoned: no maxAge") {
val repo = Repo("repo-cache-alg", "test-1")
val cache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None)
val config = RepoConfig.empty
val data = RepoData(repo, cache, config)
val obtained = repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt
assertIO(obtained, Right(()))
}

test("throwIfAbandoned: lastCommit < maxAge") {
val repo = Repo("repo-cache-alg", "test-2")
val commitDate = Timestamp.fromLocalDateTime(LocalDateTime.now())
val cache = RepoCache(dummySha1, commitDate, Nil, None, None)
val config = RepoConfig(lastCommitMaxAge = Some(1.day))
val data = RepoData(repo, cache, config)
val obtained = repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt
assertIO(obtained, Right(()))
}

test("throwIfAbandoned: lastCommit > maxAge") {
val repo = Repo("repo-cache-alg", "test-3")
val cache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None)
val config = RepoConfig(lastCommitMaxAge = Some(1.day))
val data = RepoData(repo, cache, config)
val obtained =
repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt.map(_.leftMap(_.getMessage))
assertIO(obtained, Left("Skipping because last commit is older than 1d"))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos": [],
| "maybeRepoConfig": {
| "pullRequests": {
Expand Down Expand Up @@ -79,6 +80,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down Expand Up @@ -218,6 +220,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down Expand Up @@ -330,6 +333,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down Expand Up @@ -441,6 +445,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down

0 comments on commit fb06636

Please sign in to comment.