From 2e33740a148ce9398f7ad122fe5d78dad7712516 Mon Sep 17 00:00:00 2001 From: Leonid Dubinsky Date: Fri, 4 Apr 2025 14:35:07 -0400 Subject: [PATCH] - when `Selector`s supplied include only `TestSelector`s and `TestWildcardSelector`s, filter properties to run by matching their names against the `Selector`s; - added a test that actually runs ScalaCheck via the `SBT Test Interface` and demonstrates the (now correct) treatment of the `Selector`s; - test also demonstrates two unfixable infidelities in the treatment of the nested properties; - thankfully, ScalaCheck's implementation of `sbt.testing.Framework` is, unlike in some other test frameworks, shared between the platforms (JVM, Scala.js, Scala Native), so the fixes do not have to be replicated for each platform, but: - the test needs to supply a `testClassLoader: ClassLoader` parameter when calling `sbt.testing.Framework.runner()`; on platforms other than JVM, `getClass.getClassLoader` is not available, so `Platform.getClassLoader: ClassLoader` method was added to every `Platform`; on platforms other than the JVM, it returns `null`, which is fine since on those platforms `sbt.testing.Framework.runner()` ignores the `testClassLoader` parameter anyway. fixes #1105 --- .../main/scala/org/scalacheck/Platform.scala | 2 + .../main/scala/org/scalacheck/Platform.scala | 2 + .../main/scala/org/scalacheck/Platform.scala | 2 + .../org/scalacheck/ScalaCheckFramework.scala | 20 +++- .../org/scalacheck/SbtSpecification.scala | 97 +++++++++++++++++++ 5 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 core/shared/src/test/scala/org/scalacheck/SbtSpecification.scala diff --git a/core/js/src/main/scala/org/scalacheck/Platform.scala b/core/js/src/main/scala/org/scalacheck/Platform.scala index 55ad5aff..37aa985d 100644 --- a/core/js/src/main/scala/org/scalacheck/Platform.scala +++ b/core/js/src/main/scala/org/scalacheck/Platform.scala @@ -46,4 +46,6 @@ private[scalacheck] object Platform { } type EnableReflectiveInstantiation = scala.scalajs.reflect.annotation.EnableReflectiveInstantiation + + def getClassLoader: ClassLoader = null } diff --git a/core/jvm/src/main/scala/org/scalacheck/Platform.scala b/core/jvm/src/main/scala/org/scalacheck/Platform.scala index 009fe587..ce36d3fe 100644 --- a/core/jvm/src/main/scala/org/scalacheck/Platform.scala +++ b/core/jvm/src/main/scala/org/scalacheck/Platform.scala @@ -73,4 +73,6 @@ private[scalacheck] object Platform { Class.forName(name + "$", true, loader).getField("MODULE$").get(null) class EnableReflectiveInstantiation extends scala.annotation.Annotation + + def getClassLoader: ClassLoader = getClass.getClassLoader } diff --git a/core/native/src/main/scala/org/scalacheck/Platform.scala b/core/native/src/main/scala/org/scalacheck/Platform.scala index 7b35c753..7dcd14aa 100644 --- a/core/native/src/main/scala/org/scalacheck/Platform.scala +++ b/core/native/src/main/scala/org/scalacheck/Platform.scala @@ -46,4 +46,6 @@ private[scalacheck] object Platform { } type EnableReflectiveInstantiation = scala.scalanative.reflect.annotation.EnableReflectiveInstantiation + + def getClassLoader: ClassLoader = null } diff --git a/core/shared/src/main/scala/org/scalacheck/ScalaCheckFramework.scala b/core/shared/src/main/scala/org/scalacheck/ScalaCheckFramework.scala index 6ebc028e..3a67ed55 100644 --- a/core/shared/src/main/scala/org/scalacheck/ScalaCheckFramework.scala +++ b/core/shared/src/main/scala/org/scalacheck/ScalaCheckFramework.scala @@ -92,8 +92,23 @@ private abstract class ScalaCheckRunner extends Runner { } def rootTask(td: TaskDef): BaseTask = new BaseTask(td) { - def execute(handler: EventHandler, loggers: Array[Logger]): Array[Task] = - props.map(_._1).toSet.toArray map { name => + def execute(handler: EventHandler, loggers: Array[Logger]): Array[Task] = { + val isTestsOnly: Boolean = td.selectors().forall(selector => + selector.isInstanceOf[TestSelector] || + selector.isInstanceOf[TestWildcardSelector]) + + def isIncluded(name: String): Boolean = !isTestsOnly || td.selectors().exists { + case s: TestWildcardSelector => name.contains(s.testWildcard()) + // For TestSelector, exact comparison with both the full name and the name with + // the suite prefix (properties.name) stripped off works for non-nested suite, + // but results in a false negative for nested suites: test "A.B.test" can not be + // selected by its short name "test"; + // instead, selected test name is matched using `endsWith`. + case s: TestSelector => name.endsWith(s.testName()) + case _ => false + } + + props.map(_._1).toSet.filter(isIncluded).toArray map { name => checkPropTask( new TaskDef( td.fullyQualifiedName(), @@ -102,6 +117,7 @@ private abstract class ScalaCheckRunner extends Runner { Array(new TestSelector(name))), single = true) } + } } def checkPropTask(taskDef: TaskDef, single: Boolean): BaseTask = new BaseTask(taskDef) { self => diff --git a/core/shared/src/test/scala/org/scalacheck/SbtSpecification.scala b/core/shared/src/test/scala/org/scalacheck/SbtSpecification.scala new file mode 100644 index 00000000..1183d3bd --- /dev/null +++ b/core/shared/src/test/scala/org/scalacheck/SbtSpecification.scala @@ -0,0 +1,97 @@ +/* + * ScalaCheck + * Copyright (c) 2007-2021 Rickard Nilsson. All rights reserved. + * http://www.scalacheck.org + * + * This software is released under the terms of the Revised BSD License. + * There is NO WARRANTY. See the file LICENSE for the full text. + */ + +package org.scalacheck + +import sbt.testing.{ + Event, + EventHandler, + Framework, + Selector, + SuiteSelector, + Task, + TaskDef, + TestSelector, + TestWildcardSelector +} + +object SbtFixture extends Properties("SbtFixture") { + property("success") = Prop.passed +} + +object SbtNestingFixture extends Properties("SbtNestingFixture") { + include(SbtFixture) +} + +object SbtSpecification extends Properties("Sbt") { + property("suite") = run(Array(new SuiteSelector)) == List("SbtFixture.success") + + property("exact") = run(Array(new TestSelector("success"))) == List("SbtFixture.success") + property("exactFull") = run(Array(new TestSelector("SbtFixture.success"))) == List("SbtFixture.success") + property("exactMissing") = run(Array(new TestSelector("nonexistent"))) == List.empty + + property("wildcard") = run(Array(new TestWildcardSelector("succ"))) == List("SbtFixture.success") + property("wildcardFull") = run(Array(new TestWildcardSelector("xture.succ"))) == List("SbtFixture.success") + property("wildcardMissing") = run(Array(new TestWildcardSelector("prev"))) == List.empty + + property("nestedFull") = run( + Array(new TestSelector("SbtNestingFixture.SbtFixture.success")), + "org.scalacheck.SbtNestingFixture") == List("SbtNestingFixture.SbtFixture.success") + + property("nestedMedium") = run( + Array(new TestSelector("SbtFixture.success")), + "org.scalacheck.SbtNestingFixture") == List("SbtNestingFixture.SbtFixture.success") + + property("nestedShort") = run( + Array(new TestSelector("success")), + "org.scalacheck.SbtNestingFixture") == List("SbtNestingFixture.SbtFixture.success") + + // Since ScalaCheck does not keep track what class/object a property belongs to, + // the following two issues concerning nested properties can not be fixed: + // + // When `explicitlySpecified = true`, properties from objects other than the one being run + // should *not* run (and the outcome should be `List.empty`) - but they do. + // + // When a property from an object other than the one being run *does* run, + // its status should be reported with a `NestedTestSelector` + // where `suiteId` names the object that the property belongs to - + // but it is reported with a `TestSelector`. + + property("nestedShouldRunAndDoes") = run( + Array(new SuiteSelector), + fullyQualifiedName = "org.scalacheck.SbtNestingFixture", + explicitlySpecified = false + ) == List("SbtNestingFixture.SbtFixture.success") + + property("nestedShouldNotRunButDoes") = run( + Array(new SuiteSelector), + fullyQualifiedName = "org.scalacheck.SbtNestingFixture", + explicitlySpecified = true + ) == List("SbtNestingFixture.SbtFixture.success") // should be List.empty + + // run using SBT Test Interface + def run( + selectors: Array[Selector], + fullyQualifiedName: String = "org.scalacheck.SbtFixture", + explicitlySpecified: Boolean = false + ): List[String] = { + val framework: Framework = new ScalaCheckFramework + var ran: List[String] = List.empty + val eventHandler: EventHandler = + (event: Event) => ran = ran :+ event.selector().asInstanceOf[TestSelector].testName() + def execute(tasks: Array[Task]): Unit = tasks.foreach(task => execute(task.execute(eventHandler, Array.empty))) + execute(framework.runner(Array.empty, Array.empty, Platform.getClassLoader).tasks(Array(new TaskDef( + fullyQualifiedName, + framework.fingerprints()(2), // object ... extends org.scalacheck.Properties + explicitlySpecified, + selectors + )))) + ran + } +}