Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Skip abandoned repos #3559

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions modules/core/src/main/resources/default.scala-steward.conf
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
// Changes to this file are therefore immediately visible to all
// Scala Steward instances.

lastCommitMaxAge = "540 days"

postUpdateHooks = [
{
groupId = "com.github.liancheng",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import org.scalasteward.core.forge.ForgeType.*
import org.scalasteward.core.git.FileGitAlg.{dotdot, gitCmd}
import org.scalasteward.core.io.process.{ProcessFailedException, SlurpOptions}
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.util.Nel
import org.scalasteward.core.util.{Nel, Timestamp}
import scala.util.Try

final class FileGitAlg[F[_]](config: Config)(implicit
fileAlg: FileAlg[F],
Expand Down Expand Up @@ -102,6 +103,11 @@ final class FileGitAlg[F[_]](config: Config)(implicit
.handleError(_ => List.empty[String])
.map(_.filter(_.nonEmpty))

override def getCommitDate(repo: File, sha1: Sha1): F[Timestamp] =
git("show", "--no-patch", "--format=%ct", sha1.value.value)(repo)
.flatMap(out => F.fromTry(Try(out.mkString.trim.toLong)))
.map(Timestamp.fromEpochSecond)

override def hasConflicts(repo: File, branch: Branch, base: Branch): F[Boolean] = {
val tryMerge = git_("merge", "--no-commit", "--no-ff", branch.name)(repo)
val abortMerge = git_("merge", "--abort")(repo).attempt.void
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import cats.{FlatMap, Monad}
import org.http4s.Uri
import org.scalasteward.core.application.Config
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.util.Timestamp

trait GenGitAlg[F[_], Repo] {
def add(repo: Repo, file: String): F[Unit]
Expand Down Expand Up @@ -57,6 +58,8 @@ trait GenGitAlg[F[_], Repo] {

def findFilesContaining(repo: Repo, string: String): F[List[String]]

def getCommitDate(repo: Repo, sha1: Sha1): F[Timestamp]

/** Returns `true` if merging `branch` into `base` results in merge conflicts. */
def hasConflicts(repo: Repo, branch: Branch, base: Branch): F[Boolean]

Expand Down Expand Up @@ -144,6 +147,9 @@ trait GenGitAlg[F[_], Repo] {
override def findFilesContaining(repo: A, string: String): F[List[String]] =
f(repo).flatMap(self.findFilesContaining(_, string))

override def getCommitDate(repo: A, sha1: Sha1): F[Timestamp] =
f(repo).flatMap(self.getCommitDate(_, sha1))

override def hasConflicts(repo: A, branch: Branch, base: Branch): F[Boolean] =
f(repo).flatMap(self.hasConflicts(_, branch, base))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ import io.circe.generic.semiauto.*
import org.scalasteward.core.data.{ArtifactId, DependencyInfo, GroupId, Scope}
import org.scalasteward.core.git.Sha1
import org.scalasteward.core.repoconfig.RepoConfig
import org.scalasteward.core.util.Timestamp

final case class RepoCache(
sha1: Sha1,
commitDate: Timestamp,
dependencyInfos: List[Scope[List[DependencyInfo]]],
maybeRepoConfig: Option[RepoConfig],
maybeRepoConfigParsingError: Option[String]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,13 @@
import org.scalasteward.core.forge.{ForgeApiAlg, ForgeRepoAlg}
import org.scalasteward.core.git.GitAlg
import org.scalasteward.core.repoconfig.RepoConfigAlg
import org.scalasteward.core.util.{dateTime, DateTimeAlg}
import org.typelevel.log4cats.Logger
import scala.util.control.NoStackTrace

final class RepoCacheAlg[F[_]](config: Config)(implicit
buildToolDispatcher: BuildToolDispatcher[F],
dateTimeAlg: DateTimeAlg[F],
forgeApiAlg: ForgeApiAlg[F],
forgeRepoAlg: ForgeRepoAlg[F],
gitAlg: GitAlg[F],
Expand All @@ -50,6 +53,7 @@
data <- maybeCache
.filter(_.sha1 === latestSha1)
.fold(cloneAndRefreshCache(repo, repoOut))(supplementCache(repo, _).pure[F])
_ <- throwIfAbandoned(data)
} yield (data, repoOut)

private def supplementCache(repo: Repo, cache: RepoCache): RepoData =
Expand All @@ -68,7 +72,8 @@
private def computeCache(repo: Repo): F[RepoData] =
for {
branch <- gitAlg.currentBranch(repo)
latestSha1 <- gitAlg.latestSha1(repo, branch)
sha1 <- gitAlg.latestSha1(repo, branch)
commitDate <- gitAlg.getCommitDate(repo, sha1)

Check warning on line 76 in modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala

View check run for this annotation

Codecov / codecov/patch

modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala#L75-L76

Added lines #L75 - L76 were not covered by tests
configParsingResult <- repoConfigAlg.readRepoConfig(repo)
maybeConfig = configParsingResult.maybeRepoConfig
maybeConfigParsingError = configParsingResult.maybeParsingError.map(_.getMessage)
Expand All @@ -77,9 +82,21 @@
dependencyInfos <-
dependencies.traverse(_.traverse(_.traverse(gatherDependencyInfo(repo, _))))
_ <- gitAlg.discardChanges(repo)
cache = RepoCache(latestSha1, dependencyInfos, maybeConfig, maybeConfigParsingError)
cache = RepoCache(sha1, commitDate, dependencyInfos, maybeConfig, maybeConfigParsingError)

Check warning on line 85 in modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala

View check run for this annotation

Codecov / codecov/patch

modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala#L85

Added line #L85 was not covered by tests
} yield RepoData(repo, cache, config)

private def gatherDependencyInfo(repo: Repo, dependency: Dependency): F[DependencyInfo] =
gitAlg.findFilesContaining(repo, dependency.version.value).map(DependencyInfo(dependency, _))

private[repocache] def throwIfAbandoned(data: RepoData): F[Unit] =
data.config.lastCommitMaxAge.traverse_ { maxAge =>
dateTimeAlg.currentTimestamp.flatMap { now =>
val sinceLastCommit = data.cache.commitDate.until(now)
val isAbandoned = sinceLastCommit > maxAge
F.raiseWhen(isAbandoned) {
val msg = s"Skipping because last commit is older than ${dateTime.showDuration(maxAge)}"
new Throwable(msg) with NoStackTrace
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ import org.scalasteward.core.buildtool.BuildRoot
import org.scalasteward.core.data.Repo
import org.scalasteward.core.edit.hooks.PostUpdateHook
import org.scalasteward.core.repoconfig.RepoConfig.defaultBuildRoots
import org.scalasteward.core.util.dateTime.*
import org.scalasteward.core.util.{combineOptions, intellijThisImportIsUsed}
import scala.concurrent.duration.FiniteDuration

final case class RepoConfig(
private val commits: Option[CommitsConfig] = None,
Expand All @@ -37,7 +40,8 @@ final case class RepoConfig(
private val assignees: Option[List[String]] = None,
private val reviewers: Option[List[String]] = None,
private val dependencyOverrides: Option[List[GroupRepoConfig]] = None,
signoffCommits: Option[Boolean] = None
signoffCommits: Option[Boolean] = None,
lastCommitMaxAge: Option[FiniteDuration] = None
) {
def commitsOrDefault: CommitsConfig =
commits.getOrElse(CommitsConfig())
Expand Down Expand Up @@ -107,8 +111,11 @@ object RepoConfig {
assignees = x.assignees |+| y.assignees,
reviewers = x.reviewers |+| y.reviewers,
dependencyOverrides = x.dependencyOverrides |+| y.dependencyOverrides,
signoffCommits = x.signoffCommits.orElse(y.signoffCommits)
signoffCommits = x.signoffCommits.orElse(y.signoffCommits),
lastCommitMaxAge = combineOptions(x.lastCommitMaxAge, y.lastCommitMaxAge)(_ max _)
)
}
)

intellijThisImportIsUsed(finiteDurationEncoder)
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ final case class Timestamp(millis: Long) {
}

object Timestamp {
def fromEpochSecond(seconds: Long): Timestamp =
Timestamp(seconds * 1000L)

def fromLocalDateTime(ldt: LocalDateTime): Timestamp =
Timestamp(ldt.toInstant(ZoneOffset.UTC).toEpochMilli)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.scalasteward.core.util

import cats.syntax.all.*
import io.circe.{Decoder, Encoder}
import java.util.concurrent.TimeUnit
import scala.annotation.tailrec
import scala.concurrent.duration.*
Expand All @@ -31,6 +32,12 @@ object dateTime {
def renderFiniteDuration(fd: FiniteDuration): String =
fd.toString.filterNot(_.isSpaceChar)

implicit val finiteDurationDecoder: Decoder[FiniteDuration] =
Decoder[String].emap(parseFiniteDuration(_).leftMap(_.getMessage))

implicit val finiteDurationEncoder: Encoder[FiniteDuration] =
Encoder[String].contramap(renderFiniteDuration)

def showDuration(d: FiniteDuration): String = {
def symbol(unit: TimeUnit): String =
unit match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.scalasteward.core.git.Sha1
import org.scalasteward.core.repocache.RepoCache
import org.scalasteward.core.repoconfig.*
import org.scalasteward.core.repoconfig.PullRequestFrequency.{Asap, Timespan}
import org.scalasteward.core.util.{DateTimeAlg, Timestamp}
import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger
import scala.concurrent.duration.FiniteDuration
Expand All @@ -19,11 +20,14 @@ object TestInstances {
Sha1.unsafeFrom("da39a3ee5e6b4b0d3255bfef95601890afd80709")

val dummyRepoCache: RepoCache =
RepoCache(dummySha1, List.empty, Option.empty, Option.empty)
RepoCache(dummySha1, Timestamp(0L), List.empty, Option.empty, Option.empty)

val dummyRepoCacheWithParsingError: RepoCache =
dummyRepoCache.copy(maybeRepoConfigParsingError = Some("Failed to parse .scala-steward.conf"))

val ioDateTimeAlg: DateTimeAlg[IO] =
DateTimeAlg.create[IO]

implicit val ioLogger: Logger[IO] =
Slf4jLogger.getLogger[IO]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import cats.Monad
import cats.effect.IO
import cats.syntax.all.*
import munit.CatsEffectSuite
import org.scalasteward.core.TestInstances.ioLogger
import org.scalasteward.core.TestInstances.{ioDateTimeAlg, ioLogger}
import org.scalasteward.core.git.FileGitAlgTest.{
conflictsNo,
conflictsYes,
Expand All @@ -18,6 +18,7 @@ import org.scalasteward.core.io.ProcessAlgTest.ioProcessAlg
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.mock.MockConfig.{config, mockRoot}
import org.scalasteward.core.util.Nel
import scala.concurrent.duration.DurationInt

class FileGitAlgTest extends CatsEffectSuite {
private val rootDir = mockRoot / "git-tests"
Expand Down Expand Up @@ -158,6 +159,19 @@ class FileGitAlgTest extends CatsEffectSuite {
} yield ()
}

test("getCommitDate") {
val repo = rootDir / "getCommitDate"
for {
_ <- ioAuxGitAlg.createRepo(repo)
sha1 <- ioGitAlg.latestSha1(repo, master)
commitDate <- ioGitAlg.getCommitDate(repo, sha1)
now <- ioDateTimeAlg.currentTimestamp
diff = commitDate.until(now)
maxDrift = 2.hours
_ = assert(diff > -maxDrift && diff < maxDrift, clue((commitDate, now)))
} yield ()
}

test("hasConflicts") {
val repo = rootDir / "hasConflicts"
for {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ package org.scalasteward.core.io
import cats.effect.IO
import cats.effect.unsafe.implicits.global
import munit.FunSuite
import org.scalasteward.core.TestInstances.ioDateTimeAlg
import org.scalasteward.core.io.process.*
import org.scalasteward.core.util.{DateTimeAlg, Nel}
import org.scalasteward.core.util.Nel
import scala.concurrent.duration.*

class processTest extends FunSuite {
Expand Down Expand Up @@ -66,7 +67,7 @@ class processTest extends FunSuite {
val timeout = 500.milliseconds
val sleep = timeout * 2
val p = slurp2(Nel.of("sleep", sleep.toSeconds.toInt.toString), timeout).attempt
val (Left(t), fd) = DateTimeAlg.create[IO].timed(p).unsafeRunSync(): @unchecked
val (Left(t), fd) = ioDateTimeAlg.timed(p).unsafeRunSync(): @unchecked

assert(clue(t).isInstanceOf[ProcessTimedOutException])
assert(clue(fd) > timeout)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package org.scalasteward.core.repocache

import cats.implicits.toSemigroupKOps
import cats.syntax.all.*
import io.circe.syntax.*
import java.time.LocalDateTime
import munit.CatsEffectSuite
import org.http4s.HttpApp
import org.http4s.circe.*
Expand All @@ -14,7 +15,9 @@ import org.scalasteward.core.forge.github.Repository
import org.scalasteward.core.git.Branch
import org.scalasteward.core.mock.MockContext.context.{repoCacheAlg, repoConfigAlg, workspaceAlg}
import org.scalasteward.core.mock.{GitHubAuth, MockEff, MockEffOps, MockState}
import org.scalasteward.core.util.intellijThisImportIsUsed
import org.scalasteward.core.repoconfig.RepoConfig
import org.scalasteward.core.util.{intellijThisImportIsUsed, Timestamp}
import scala.concurrent.duration.*

class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] {
intellijThisImportIsUsed(encodeUri)
Expand All @@ -36,7 +39,8 @@ class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] {
uri"https://github.com/scala-steward/cats-effect.git",
Branch("main")
)
val repoCache = RepoCache(dummySha1, Nil, None, None)
val now = Timestamp.fromLocalDateTime(LocalDateTime.now())
val repoCache = RepoCache(dummySha1, now, Nil, None, None)
val workspace = workspaceAlg.rootDir.unsafeRunSync()
val httpApp = HttpApp[MockEff] {
case POST -> Root / "repos" / "typelevel" / "cats-effect" / "forks" =>
Expand All @@ -55,4 +59,33 @@ class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] {
val expected = (RepoData(repo, repoCache, repoConfigAlg.mergeWithGlobal(None)), repoOut)
assertIO(obtained, expected)
}

test("throwIfAbandoned: no maxAge") {
val repo = Repo("repo-cache-alg", "test-1")
val cache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None)
val config = RepoConfig.empty
val data = RepoData(repo, cache, config)
val obtained = repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt
assertIO(obtained, Right(()))
}

test("throwIfAbandoned: lastCommit < maxAge") {
val repo = Repo("repo-cache-alg", "test-2")
val commitDate = Timestamp.fromLocalDateTime(LocalDateTime.now())
val cache = RepoCache(dummySha1, commitDate, Nil, None, None)
val config = RepoConfig(lastCommitMaxAge = Some(1.day))
val data = RepoData(repo, cache, config)
val obtained = repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt
assertIO(obtained, Right(()))
}

test("throwIfAbandoned: lastCommit > maxAge") {
val repo = Repo("repo-cache-alg", "test-3")
val cache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None)
val config = RepoConfig(lastCommitMaxAge = Some(1.day))
val data = RepoData(repo, cache, config)
val obtained =
repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt.map(_.leftMap(_.getMessage))
assertIO(obtained, Left("Skipping because last commit is older than 1d"))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos": [],
| "maybeRepoConfig": {
| "pullRequests": {
Expand Down Expand Up @@ -79,6 +80,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down Expand Up @@ -218,6 +220,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down Expand Up @@ -330,6 +333,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down Expand Up @@ -441,6 +445,7 @@ class PruningAlgTest extends FunSuite {
val Right(repoCache) = decode[RepoCache](
s"""|{
| "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14",
| "commitDate": 0,
| "dependencyInfos" : [
| {
| "value" : [
Expand Down
Loading