From 1f6778ee51d3e0937dc7ac84c9ed0356d8f14850 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 29 Jun 2024 09:44:11 -0400 Subject: [PATCH] [WIP] Add tools-specific `Issue` kind that can be used by third-party test libraries. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds a new `Issue` kind, `.recordedByTool`, that takes a custom payload provided by a third-party tool or library (e.g. Nimble). This case can then be used to distinguish issues specific to tools while also providing sufficient infrastructural support to allow those tools to distinguish issues they created at later stages of the testing workflow. (If this sounds abstract, it is—the proposed API is meant to be used in a fairly arbitrary fashion by an open set of third-party tools and libraries.) Resolves #490. --- .../ABIv0/Encoded/ABIv0.EncodedIssue.swift | 38 ++++++++++++++++++- Sources/Testing/Issues/Issue+Recording.swift | 26 +++++++++++++ Sources/Testing/Issues/Issue.swift | 37 +++++++++++++++++- Tests/TestingTests/IssueTests.swift | 35 +++++++++++++++++ 4 files changed, 134 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift index 05478645c..3c4a8043e 100644 --- a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift @@ -22,13 +22,49 @@ extension ABIv0 { /// The location in source where this issue occurred, if available. var sourceLocation: SourceLocation? + /// Any tool-specific context about the issue including the name of the tool + /// that recorded it. + /// + /// When decoding using `JSONDecoder`, the value of this property is set to + /// `nil`. Tools that need access to their context values should not use + /// ``ABIv0/EncodedIssue`` to decode issues. + var toolContext: (any Issue.Kind.ToolContext)? + init(encoding issue: borrowing Issue) { isKnown = issue.isKnown sourceLocation = issue.sourceLocation + if case let .recordedByTool(toolContext) = issue.kind { + self.toolContext = toolContext + } } } } // MARK: - Codable -extension ABIv0.EncodedIssue: Codable {} +extension ABIv0.EncodedIssue: Codable { + private enum CodingKeys: String, CodingKey { + case isKnown + case sourceLocation + case toolContext + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(isKnown, forKey: .isKnown) + try container.encode(sourceLocation, forKey: .sourceLocation) + if let toolContext { + func encodeToolContext(_ toolContext: some Issue.Kind.ToolContext) throws { + try container.encode(toolContext, forKey: .toolContext) + } + try encodeToolContext(toolContext) + } + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + isKnown = try container.decode(Bool.self, forKey: .isKnown) + sourceLocation = try container.decode(SourceLocation.self, forKey: .sourceLocation) + toolContext = nil // not decoded + } +} diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index f074f6d7e..78c9780cf 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -115,6 +115,32 @@ extension Issue { let issue = Issue(kind: .unconditional, comments: Array(comment), sourceContext: sourceContext) return issue.record() } + + /// Record an issue on behalf of a tool or library. + /// + /// - Parameters: + /// - comment: A comment describing the expectation. + /// - toolContext: Any tool-specific context about the issue including the + /// name of the tool that recorded it. + /// - sourceLocation: The source location to which the issue should be + /// attributed. + /// + /// - Returns: The issue that was recorded. + /// + /// Test authors do not generally need to use this function. Rather, a tool + /// or library based on the testing library can use it to record a + /// domain-specific issue and to propagatre additional information about that + /// issue to other layers of the testing library's infrastructure. + @_spi(Experimental) + @discardableResult public static func record( + _ comment: Comment? = nil, + context toolContext: some Issue.Kind.ToolContext, + sourceLocation: SourceLocation = #_sourceLocation + ) -> Self { + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + let issue = Issue(kind: .recordedByTool(toolContext), comments: Array(comment), sourceContext: sourceContext) + return issue.record() + } } // MARK: - Recording issues for errors diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 297510335..d3fdcdab6 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -66,6 +66,33 @@ public struct Issue: Sendable { /// An issue due to a failure in the underlying system, not due to a failure /// within the tests being run. case system + + /// A protocol describing additional context provided by an external tool or + /// library that recorded an issue of kind + /// ``Issue/Kind/recordedByTool(_:)``. + /// + /// Test authors do not generally need to use this protocol. Rather, a tool + /// or library based on the testing library can use it to propagate + /// additional information about an issue to other layers of the testing + /// library's infrastructure. + /// + /// A tool or library may conform as many types as it needs to this + /// protocol. Instances of types conforming to this protocol must be + /// encodable as JSON so that they can be included in event streams produced + /// by the testing library. + public protocol ToolContext: Sendable, Encodable { + /// The human-readable name of the tool that recorded the issue. + var toolName: String { get } + } + + /// An issue recorded by an external tool or library that uses the testing + /// library. + /// + /// - Parameters: + /// - toolContext: Any tool-specific context about the issue including the + /// name of the tool that recorded it. + @_spi(Experimental) + indirect case recordedByTool(_ toolContext: any ToolContext) } /// The kind of issue this value represents. @@ -135,7 +162,11 @@ extension Issue: CustomStringConvertible, CustomDebugStringConvertible { let joinedComments = comments.lazy .map(\.rawValue) .joined(separator: "\n") - return "\(kind): \(joinedComments)" + if case let .recordedByTool(toolContext) = kind { + return "\(joinedComments) (from '\(toolContext.toolName)')" + } else { + return "\(kind): \(joinedComments)" + } } public var debugDescription: String { @@ -172,6 +203,8 @@ extension Issue.Kind: CustomStringConvertible { "An API was misused" case .system: "A system failure occurred" + case let .recordedByTool(toolContext): + "'\(toolContext.toolName)' recorded an issue" } } } @@ -310,6 +343,8 @@ extension Issue.Kind { .apiMisused case .system: .system + case .recordedByTool: + .unconditional // TBD } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index ce2022d52..38f396665 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -994,6 +994,41 @@ final class IssueTests: XCTestCase { }.run(configuration: configuration) } + func testFailBecauseOfToolSpecificIssue() async throws { + struct ToolContext: Issue.Kind.ToolContext { + var value: Int + var toolName: String { + "Swift Testing Itself" + } + } + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + XCTAssertFalse(issue.isKnown) + guard case let .recordedByTool(toolContext) = issue.kind else { + XCTFail("Unexpected issue kind \(issue.kind)") + return + } + guard let toolContext = toolContext as? ToolContext else { + XCTFail("Unexpected tool context \(toolContext)") + return + } + XCTAssertEqual(toolContext.toolName, "Swift Testing Itself") + XCTAssertEqual(toolContext.value, 12345) + + XCTAssertEqual(String(describingForTest: issue), "Something went wrong (from 'Swift Testing Itself')") + XCTAssertEqual(String(describingForTest: issue.kind), "'Swift Testing Itself' recorded an issue") + } + + await Test { + let toolContext = ToolContext(value: 12345) + Issue.record("Something went wrong", context: toolContext) + }.run(configuration: configuration) + } + func testErrorPropertyValidForThrownErrors() async throws { var configuration = Configuration() configuration.eventHandler = { event, _ in