Introduce a severity level for issues, and a 'warning' severity (#931)

This introduces the concept of severity to the `Issue` type, represented
by a new enum `Issue.Severity` with two cases: `.warning` and `.error`.
Error is the default severity for all issues, matching current behavior,
but warning is provided as a new option which does not cause the test
the issue is associated with to be marked as a failure.

In this PR, these are
[SPI](https://github.com/swiftlang/swift-testing/blob/main/Documentation/SPI.md)
but they could be considered for promotion to public API eventually.
Additional work would be needed to permit test authors to record issues
with severity < `.error`, since APIs like `Issue.record()` are not being
modified at this time to allow customizing the severity.

### Motivation:

There are certain situations where a problem may arise during a test
that doesn't necessarily affect its outcome or signal an important
problem, but is worth calling attention to. A specific example use case
I have in mind is to allow the testing library to record a warning issue
about problems with the arguments passed to a parameterized test, such
as having duplicate arguments.

### Modifications:

- Introduce `Issue.Severity` as an SPI enum.
- Introduce an SPI property `severity` to `Issue` with default value
`.error`.
- Modify entry point logic to exit with `EXIT_SUCCESS` if all issues
recorded had severity < `.error`.
- Modify console output formatting logic and data structures to
represent warning issues sensibly.

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.
- [x] Add new tests

Author: stmontgomery

Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift

@@ -47,7 +47,7 @@ extension ABIv0 {
   /// callback.
   public static var entryPoint: EntryPoint {
     return { configurationJSON, recordHandler in
-      try await Testing.entryPoint(
+      try await _entryPoint(
         configurationJSON: configurationJSON,
         recordHandler: recordHandler
       ) == EXIT_SUCCESS
@@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable (
 @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer {
   let result = UnsafeMutablePointer<ABIEntryPoint_v0>.allocate(capacity: 1)
   result.initialize { configurationJSON, recordHandler in
-    try await entryPoint(
+    try await _entryPoint(
       configurationJSON: configurationJSON,
       eventStreamVersionIfNil: -1,
       recordHandler: recordHandler
@@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable (
 ///
 /// This function will be removed (with its logic incorporated into
 /// ``ABIv0/entryPoint-swift.type.property``) in a future update.
-private func entryPoint(
+private func _entryPoint(
   configurationJSON: UnsafeRawBufferPointer?,
   eventStreamVersionIfNil: Int? = nil,
   recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -20,6 +20,8 @@ private import _TestingInternals
 ///     writes events to the standard error stream in addition to passing them
 ///     to this function.
 ///
+/// - Returns: An exit code representing the result of running tests.
+///
 /// External callers cannot call this function directly. The can use
 /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable
 /// version of this function.
@@ -40,7 +42,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
 
     // Set up the event handler.
     configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-      if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+      if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error {
         exitCode.withLock { exitCode in
           exitCode = EXIT_FAILURE
         }
@@ -270,6 +272,13 @@ public struct __CommandLineArguments_v0: Sendable {
   /// The value(s) of the `--skip` argument.
   public var skip: [String]?
 
+  /// Whether or not to include tests with the `.hidden` trait when constructing
+  /// a test filter based on these arguments.
+  ///
+  /// This property is intended for use in testing the testing library itself.
+  /// It is not parsed as a command-line argument.
+  var includeHiddenTests: Bool?
+
   /// The value of the `--repetitions` argument.
   public var repetitions: Int?
 
@@ -278,6 +287,13 @@ public struct __CommandLineArguments_v0: Sendable {
 
   /// The value of the `--experimental-attachments-path` argument.
   public var experimentalAttachmentsPath: String?
+
+  /// Whether or not the experimental warning issue severity feature should be
+  /// enabled.
+  ///
+  /// This property is intended for use in testing the testing library itself.
+  /// It is not parsed as a command-line argument.
+  var isWarningIssueRecordedEventEnabled: Bool?
 }
 
 extension __CommandLineArguments_v0: Codable {
@@ -517,6 +533,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
   filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding))
 
   configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) }
+  if args.includeHiddenTests == true {
+    configuration.testFilter.includeHiddenTests = true
+  }
 
   // Set up the iteration policy for the test run.
   var repetitionPolicy: Configuration.RepetitionPolicy = .once
@@ -547,6 +566,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
   configuration.exitTestHandler = ExitTest.handlerForEntryPoint()
 #endif
 
+  // Warning issues (experimental).
+  if args.isWarningIssueRecordedEventEnabled == true {
+    configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true
+  } else {
+    switch args.eventStreamVersion {
+    case .some(...0):
+      // If the event stream version was explicitly specified to a value < 1,
+      // disable the warning issue event to maintain legacy behavior.
+      configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false
+    default:
+      // Otherwise the requested event stream version is ≥ 1, so don't change
+      // the warning issue event setting.
+      break
+    }
+  }
+
   return configuration
 }
 

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift

@@ -16,6 +16,19 @@ extension ABIv0 {
   /// assists in converting values to JSON; clients that consume this JSON are
   /// expected to write their own decoders.
   struct EncodedIssue: Sendable {
+    /// An enumeration representing the level of severity of a recorded issue.
+    ///
+    /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``.
+    enum Severity: String, Sendable {
+      case warning
+      case error
+    }
+
+    /// The severity of this issue.
+    ///
+    /// - Warning: Severity is not yet part of the JSON schema.
+    var _severity: Severity
+
     /// Whether or not this issue is known to occur.
     var isKnown: Bool
 
@@ -33,6 +46,11 @@ extension ABIv0 {
     var _error: EncodedError?
 
     init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
+      _severity = switch issue.severity {
+      case .warning: .warning
+      case .error: .error
+      }
+
       isKnown = issue.isKnown
       sourceLocation = issue.sourceLocation
       if let backtrace = issue.sourceContext.backtrace {
@@ -48,3 +66,4 @@ extension ABIv0 {
 // MARK: - Codable
 
 extension ABIv0.EncodedIssue: Codable {}
+extension ABIv0.EncodedIssue.Severity: Codable {}

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift

@@ -25,6 +25,7 @@ extension ABIv0 {
       case `default`
       case skip
       case pass
+      case passWithWarnings = "_passWithWarnings"
       case passWithKnownIssue
       case fail
       case difference
@@ -44,6 +45,8 @@ extension ABIv0 {
           } else {
             .pass
           }
+        case .passWithWarnings:
+          .passWithWarnings
         case .fail:
           .fail
         case .difference:

Sources/Testing/Events/Event.swift

@@ -290,10 +290,7 @@ extension Event {
     if let configuration = configuration ?? Configuration.current {
       // The caller specified a configuration, or the current task has an
       // associated configuration. Post to either configuration's event handler.
-      switch kind {
-      case .expectationChecked where !configuration.deliverExpectationCheckedEvents:
-        break
-      default:
+      if configuration.eventHandlingOptions.shouldHandleEvent(self) {
         configuration.handleEvent(self, in: context)
       }
     } else {

Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift

@@ -162,6 +162,8 @@ extension Event.Symbol {
           return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)"
         }
         return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)"
+      case .passWithWarnings:
+        return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)"
       case .fail:
         return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)"
       case .warning:

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -56,8 +56,9 @@ extension Event {
         /// The instant at which the test started.
         var startInstant: Test.Clock.Instant
 
-        /// The number of issues recorded for the test.
-        var issueCount = 0
+        /// The number of issues recorded for the test, grouped by their
+        /// level of severity.
+        var issueCount: [Issue.Severity: Int] = [:]
 
         /// The number of known issues recorded for the test.
         var knownIssueCount = 0
@@ -114,27 +115,36 @@ extension Event.HumanReadableOutputRecorder {
   ///   - graph: The graph to walk while counting issues.
   ///
   /// - Returns: A tuple containing the number of issues recorded in `graph`.
-  private func _issueCounts(in graph: Graph<String, Event.HumanReadableOutputRecorder._Context.TestData?>?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
+  private func _issueCounts(in graph: Graph<String, Event.HumanReadableOutputRecorder._Context.TestData?>?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
     guard let graph else {
-      return (0, 0, 0, "")
+      return (0, 0, 0, 0, "")
     }
-    let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=)
+    let errorIssueCount = graph.compactMap(\.value?.issueCount[.error]).reduce(into: 0, +=)
+    let warningIssueCount = graph.compactMap(\.value?.issueCount[.warning]).reduce(into: 0, +=)
     let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=)
-    let totalIssueCount = issueCount + knownIssueCount
+    let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount
 
     // Construct a string describing the issue counts.
-    let description = switch (issueCount > 0, knownIssueCount > 0) {
-    case (true, true):
+    let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) {
+    case (true, true, true):
+      " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))"
+    case (true, false, true):
       " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))"
-    case (false, true):
+    case (false, true, true):
+      " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))"
+    case (false, false, true):
       " with \(knownIssueCount.counting("known issue"))"
-    case (true, false):
+    case (true, true, false):
+      " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))"
+    case (true, false, false):
       " with \(totalIssueCount.counting("issue"))"
-    case(false, false):
+    case(false, true, false):
+      " with \(warningIssueCount.counting("warning"))"
+    case(false, false, false):
       ""
     }
 
-    return (issueCount, knownIssueCount, totalIssueCount,  description)
+    return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount,  description)
   }
 }
 
@@ -267,7 +277,8 @@ extension Event.HumanReadableOutputRecorder {
         if issue.isKnown {
           testData.knownIssueCount += 1
         } else {
-          testData.issueCount += 1
+          let issueCount = testData.issueCount[issue.severity] ?? 0
+          testData.issueCount[issue.severity] = issueCount + 1
         }
         context.testData[id] = testData
 
@@ -355,15 +366,15 @@ extension Event.HumanReadableOutputRecorder {
       let testData = testDataGraph?.value ?? .init(startInstant: instant)
       let issues = _issueCounts(in: testDataGraph)
       let duration = testData.startInstant.descriptionOfDuration(to: instant)
-      return if issues.issueCount > 0 {
+      return if issues.errorIssueCount > 0 {
         CollectionOfOne(
           Message(
             symbol: .fail,
             stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)."
           )
         ) + _formattedComments(for: test)
       } else {
-         [
+        [
           Message(
             symbol: .pass(knownIssueCount: issues.knownIssueCount),
             stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)."
@@ -400,13 +411,19 @@ extension Event.HumanReadableOutputRecorder {
         ""
       }
       let symbol: Event.Symbol
-      let known: String
+      let subject: String
       if issue.isKnown {
         symbol = .pass(knownIssueCount: 1)
-        known = " known"
+        subject = "a known issue"
       } else {
-        symbol = .fail
-        known = "n"
+        switch issue.severity {
+        case .warning:
+          symbol = .passWithWarnings
+          subject = "a warning"
+        case .error:
+          symbol = .fail
+          subject = "an issue"
+        }
       }
 
       var additionalMessages = [Message]()
@@ -435,13 +452,13 @@ extension Event.HumanReadableOutputRecorder {
       let primaryMessage: Message = if parameterCount == 0 {
         Message(
           symbol: symbol,
-          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)",
+          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)",
           conciseStringValue: String(describing: issue.kind)
         )
       } else {
         Message(
           symbol: symbol,
-          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
+          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
           conciseStringValue: String(describing: issue.kind)
         )
       }
@@ -498,7 +515,7 @@ extension Event.HumanReadableOutputRecorder {
       let runStartInstant = context.runStartInstant ?? instant
       let duration = runStartInstant.descriptionOfDuration(to: instant)
 
-      return if issues.issueCount > 0 {
+      return if issues.errorIssueCount > 0 {
         [
           Message(
             symbol: .fail,

Sources/Testing/Events/Recorder/Event.Symbol.swift

@@ -22,10 +22,14 @@ extension Event {
     /// The symbol to use when a test passes.
     ///
     /// - Parameters:
-    ///   - knownIssueCount: The number of known issues encountered by the end
-    ///     of the test.
+    ///   - knownIssueCount: The number of known issues recorded for the test.
+    ///     The default value is `0`.
     case pass(knownIssueCount: Int = 0)
 
+    /// The symbol to use when a test passes with one or more warnings.
+    @_spi(Experimental)
+    case passWithWarnings
+
     /// The symbol to use when a test fails.
     case fail
 
@@ -62,6 +66,8 @@ extension Event.Symbol {
       } else {
         ("\u{10105B}", "checkmark.diamond.fill")
       }
+    case .passWithWarnings:
+      ("\u{100123}", "questionmark.diamond.fill")
     case .fail:
       ("\u{100884}", "xmark.diamond.fill")
     case .difference:
@@ -122,6 +128,9 @@ extension Event.Symbol {
         // Unicode: HEAVY CHECK MARK
         return "\u{2714}"
       }
+    case .passWithWarnings:
+      // Unicode: QUESTION MARK
+      return "\u{003F}"
     case .fail:
       // Unicode: HEAVY BALLOT X
       return "\u{2718}"
@@ -157,6 +166,9 @@ extension Event.Symbol {
         // Unicode: SQUARE ROOT
         return "\u{221A}"
       }
+    case .passWithWarnings:
+      // Unicode: QUESTION MARK
+      return "\u{003F}"
     case .fail:
       // Unicode: MULTIPLICATION SIGN
       return "\u{00D7}"