Skip to content

Commit 85b59b4

Browse files
committed
[WIP, DNM] Prototype #expect() overloads outside the testing library.
This PR introduces an overload of `#expect()`/`#require()` that's implemented outside the core Swift Testing library. It implements exit tests based on Foundation's `Process` class (AKA `NSTask`.) I do **not** plan to merge this PR. Rather, it's serving as a breadboard to let me prototype overloading `#expect()` outside Swift Testing, which we can then apply to other libraries like `swift-subprocess` and `swift-argument-parser`.
1 parent 664d24f commit 85b59b4

File tree

9 files changed

+267
-7
lines changed

9 files changed

+267
-7
lines changed

Package.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ let package = Package(
265265
name: "_Testing_Foundation",
266266
dependencies: [
267267
"Testing",
268+
"_TestingInternals",
268269
],
269270
path: "Sources/Overlays/_Testing_Foundation",
270271
exclude: ["CMakeLists.txt"],
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
//
2+
// This source file is part of the Swift.org open source project
3+
//
4+
// Copyright (c) 2025 Apple Inc. and the Swift project authors
5+
// Licensed under Apache License v2.0 with Runtime Library Exception
6+
//
7+
// See https://swift.org/LICENSE.txt for license information
8+
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
9+
//
10+
11+
#if canImport(Foundation)
12+
@_spi(ForToolsIntegrationOnly) public import Testing
13+
public import Foundation
14+
private import _TestingInternals
15+
16+
@_spi(Experimental)
17+
@freestanding(expression)
18+
@discardableResult
19+
#if !SWT_NO_EXIT_TESTS
20+
@available(macOS 10.15.4, *)
21+
#else
22+
@_unavailableInEmbedded
23+
@available(*, unavailable, message: "Exit tests are not available on this platform.")
24+
#endif
25+
public macro expect(
26+
_ process: Process,
27+
exitsWith expectedExitCondition: ExitTest.Condition,
28+
observing observedValues: [any PartialKeyPath<ExitTest.Result> & Sendable] = [],
29+
_ comment: @autoclosure () -> Comment? = nil,
30+
sourceLocation: SourceLocation = #_sourceLocation
31+
) -> ExitTest.Result? = #externalMacro(module: "TestingMacros", type: "ExpectNSTaskExitsWithMacro")
32+
33+
@_spi(Experimental)
34+
@freestanding(expression)
35+
@discardableResult
36+
#if !SWT_NO_EXIT_TESTS
37+
@available(macOS 10.15.4, *)
38+
#else
39+
@_unavailableInEmbedded
40+
@available(*, unavailable, message: "Exit tests are not available on this platform.")
41+
#endif
42+
public macro require(
43+
_ process: Process,
44+
exitsWith expectedExitCondition: ExitTest.Condition,
45+
observing observedValues: [any PartialKeyPath<ExitTest.Result> & Sendable] = [],
46+
_ comment: @autoclosure () -> Comment? = nil,
47+
sourceLocation: SourceLocation = #_sourceLocation
48+
) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "RequireNSTaskExitsWithMacro")
49+
50+
// MARK: -
51+
52+
@_spi(Experimental)
53+
@discardableResult
54+
#if !SWT_NO_EXIT_TESTS
55+
@available(macOS 10.15.4, *)
56+
#else
57+
@_unavailableInEmbedded
58+
@available(*, unavailable, message: "Exit tests are not available on this platform.")
59+
#endif
60+
public func __check(
61+
_ process: Process,
62+
exitsWith expectedExitCondition: ExitTest.Condition,
63+
observing observedValues: [any PartialKeyPath<ExitTest.Result> & Sendable] = [],
64+
comments: @autoclosure () -> [Comment],
65+
isRequired: Bool,
66+
isolation: isolated (any Actor)? = #isolation,
67+
sourceLocation: SourceLocation
68+
) async -> Result<ExitTest.Result?, any Error> {
69+
#if !SWT_NO_EXIT_TESTS
70+
// The process may have already started and may already have a termination
71+
// handler set, so it's not possible for us to asynchronously wait for it.
72+
// As such, we'll have to block _some_ thread.
73+
var result: ExitTest.Result
74+
do {
75+
try await withCheckedThrowingContinuation { continuation in
76+
Thread.detachNewThread {
77+
do {
78+
// There's an obvious race condition here, but that's a limitation of
79+
// the Process/NSTask API and we'll just have to accept it.
80+
if !process.isRunning {
81+
try process.run()
82+
}
83+
process.waitUntilExit()
84+
continuation.resume()
85+
} catch {
86+
continuation.resume(throwing: error)
87+
}
88+
}
89+
}
90+
91+
let reason = process.terminationReason
92+
let exitStatus: ExitStatus = switch reason {
93+
case .exit:
94+
.exitCode(process.terminationStatus)
95+
case .uncaughtSignal:
96+
#if os(Windows)
97+
// On Windows, Foundation tries to map exit codes that look like HRESULT
98+
// values to signals, which is not the model Swift Testing uses. The
99+
// conversion is lossy, so there's not much we can do here other than treat
100+
// it as an exit code too.
101+
.exitCode(process.terminationStatus)
102+
#else
103+
.signal(process.terminationStatus)
104+
#endif
105+
@unknown default:
106+
fatalError("Unexpected termination reason '\(reason)' from process \(process). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new")
107+
}
108+
109+
result = ExitTest.Result(exitStatus: exitStatus)
110+
func makeContent(from streamObject: Any?) -> [UInt8] {
111+
if let fileHandle = streamObject as? FileHandle {
112+
if let content = try? fileHandle.readToEnd() {
113+
return Array(content)
114+
}
115+
} else if let pipe = streamObject as? Pipe {
116+
return makeContent(from: pipe.fileHandleForReading)
117+
}
118+
119+
return []
120+
}
121+
if observedValues.contains(\.standardOutputContent) {
122+
result.standardOutputContent = makeContent(from: process.standardOutput)
123+
}
124+
if observedValues.contains(\.standardErrorContent) {
125+
result.standardErrorContent = makeContent(from: process.standardError)
126+
}
127+
} catch {
128+
// As with the main exit test implementation, if an error occurs while
129+
// trying to run the exit test, treat it as a system error and treat the
130+
// condition as a mismatch.
131+
let issue = Issue(
132+
kind: .system,
133+
comments: comments() + CollectionOfOne(Comment(rawValue: String(describingForTest: error))),
134+
sourceContext: SourceContext(backtrace: nil, sourceLocation: sourceLocation)
135+
)
136+
issue.record()
137+
138+
let exitStatus: ExitStatus = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) {
139+
.exitCode(EXIT_SUCCESS)
140+
} else {
141+
.exitCode(EXIT_FAILURE)
142+
}
143+
result = ExitTest.Result(exitStatus: exitStatus)
144+
}
145+
146+
let expression = Expression("expectedExitCondition")
147+
return __checkValue(
148+
expectedExitCondition.isApproximatelyEqual(to: result.exitStatus),
149+
expression: expression,
150+
expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(result.exitStatus),
151+
comments: comments(),
152+
isRequired: isRequired,
153+
sourceLocation: sourceLocation
154+
).map { _ in result }
155+
#else
156+
swt_unreachable()
157+
#endif
158+
}
159+
#endif

Sources/Testing/ExitTests/ExitTest.Condition.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ extension ExitTest.Condition {
216216
///
217217
/// Two exit test conditions can be compared; if either instance is equal to
218218
/// ``failure``, it will compare equal to any instance except ``success``.
219-
func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool {
219+
package func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool {
220220
// Strictly speaking, the C standard treats 0 as a successful exit code and
221221
// potentially distinct from EXIT_SUCCESS. To my knowledge, no modern
222222
// operating system defines EXIT_SUCCESS to any value other than 0, so the

Sources/Testing/Issues/Issue+Recording.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ extension Issue {
1818
///
1919
/// - Returns: The issue that was recorded (`self` or a modified copy of it.)
2020
@discardableResult
21-
func record(configuration: Configuration? = nil) -> Self {
21+
package func record(configuration: Configuration? = nil) -> Self {
2222
// If this issue is a caught error that has a custom issue representation,
2323
// perform that customization now.
2424
if case let .errorCaught(error) = kind {

Sources/Testing/Issues/Issue.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ public struct Issue: Sendable {
172172
/// empty.
173173
/// - sourceContext: A ``SourceContext`` indicating where and how this issue
174174
/// occurred.
175-
init(
175+
package init(
176176
kind: Kind,
177177
severity: Severity = .error,
178178
comments: [Comment],

Sources/Testing/SourceAttribution/Expression.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ public struct __Expression: Sendable {
326326
///
327327
/// - Returns: A copy of `self` with information about the specified runtime
328328
/// value captured for future use.
329-
func capturingRuntimeValue(_ value: (some Any)?) -> Self {
329+
package func capturingRuntimeValue(_ value: (some Any)?) -> Self {
330330
var result = self
331331
result.runtimeValue = value.flatMap(Value.init(reflecting:))
332332
if case let .negation(subexpression, isParenthetical) = kind, let value = value as? Bool {
@@ -348,7 +348,7 @@ public struct __Expression: Sendable {
348348
///
349349
/// If the ``kind`` of `self` is ``Kind/generic`` or ``Kind/stringLiteral``,
350350
/// this function is equivalent to ``capturingRuntimeValue(_:)``.
351-
func capturingRuntimeValues<each T>(_ firstValue: (some Any)?, _ additionalValues: repeat (each T)?) -> Self {
351+
package func capturingRuntimeValues<each T>(_ firstValue: (some Any)?, _ additionalValues: repeat (each T)?) -> Self {
352352
var result = self
353353

354354
// Convert the variadic generic argument list to an array.

Sources/TestingMacros/ConditionMacro.swift

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,15 @@ public import SwiftSyntaxMacros
4343
/// argument (if present) and `sourceLocation` argument are placed at the end of
4444
/// the generated function call's argument list.
4545
public protocol ConditionMacro: ExpressionMacro, Sendable {
46+
/// Whether or not the macro tries to parse/expand its condition argument.
47+
static var parsesConditionArgument: Bool { get }
48+
4649
/// Whether or not the macro's expansion may throw an error.
4750
static var isThrowing: Bool { get }
51+
52+
/// The name of the module containing the `__check()` function this macro
53+
/// expands to call.
54+
static var checkFunctionModuleName: DeclReferenceExprSyntax { get }
4855
}
4956

5057
// MARK: -
@@ -73,6 +80,14 @@ extension ConditionMacro {
7380
.disabled
7481
}
7582

83+
public static var parsesConditionArgument: Bool {
84+
true
85+
}
86+
87+
public static var checkFunctionModuleName: DeclReferenceExprSyntax {
88+
DeclReferenceExprSyntax(baseName: .identifier("Testing"))
89+
}
90+
7691
/// Perform the expansion of this condition macro.
7792
///
7893
/// - Parameters:
@@ -115,7 +130,7 @@ extension ConditionMacro {
115130
// Construct the argument list to __check().
116131
let expandedFunctionName: TokenSyntax
117132
var checkArguments = [Argument]()
118-
do {
133+
if parsesConditionArgument {
119134
if let trailingClosureIndex {
120135
// Include all arguments other than the "comment" and "sourceLocation"
121136
// arguments here.
@@ -156,7 +171,19 @@ extension ConditionMacro {
156171

157172
expandedFunctionName = conditionArgument.expandedFunctionName
158173
}
174+
} else {
175+
// Include all arguments other than the "comment" and "sourceLocation"
176+
// arguments here.
177+
checkArguments += macroArguments.indices.lazy
178+
.filter { $0 != commentIndex }
179+
.filter { $0 != isolationArgumentIndex }
180+
.filter { $0 != sourceLocationArgumentIndex }
181+
.map { macroArguments[$0] }
182+
183+
expandedFunctionName = .identifier("__check")
184+
}
159185

186+
do {
160187
// Capture any comments as well -- either in source, preceding the
161188
// expression macro or one of its lexical context nodes, or as an argument
162189
// to the macro.
@@ -201,7 +228,7 @@ extension ConditionMacro {
201228
}
202229

203230
// Construct and return the call to __check().
204-
let call: ExprSyntax = "Testing.\(expandedFunctionName)(\(LabeledExprListSyntax(checkArguments)))"
231+
let call: ExprSyntax = "\(checkFunctionModuleName).\(expandedFunctionName)(\(LabeledExprListSyntax(checkArguments)))"
205232
if isThrowing {
206233
return "\(call).__required()"
207234
}
@@ -279,6 +306,10 @@ extension RefinedConditionMacro {
279306
public static var isThrowing: Bool {
280307
Base.isThrowing
281308
}
309+
310+
public static var parsesConditionArgument: Bool {
311+
Base.parsesConditionArgument
312+
}
282313
}
283314

284315
// MARK: - Diagnostics-emitting condition macros
@@ -671,3 +702,30 @@ public struct ExitTestExpectMacro: ExitTestConditionMacro {
671702
public struct ExitTestRequireMacro: ExitTestConditionMacro {
672703
public typealias Base = RequireMacro
673704
}
705+
706+
// MARK: - Exit tests using Process/NSTask/Subprocess
707+
708+
public struct ExpectNSTaskExitsWithMacro: RefinedConditionMacro {
709+
public typealias Base = ExpectMacro
710+
711+
public static var parsesConditionArgument: Bool {
712+
false
713+
}
714+
715+
public static var checkFunctionModuleName: DeclReferenceExprSyntax {
716+
DeclReferenceExprSyntax(baseName: .identifier("_Testing_Foundation"))
717+
}
718+
}
719+
720+
public struct RequireNSTaskExitsWithMacro: RefinedConditionMacro {
721+
public typealias Base = RequireMacro
722+
723+
public static var parsesConditionArgument: Bool {
724+
false
725+
}
726+
727+
public static var checkFunctionModuleName: DeclReferenceExprSyntax {
728+
DeclReferenceExprSyntax(baseName: .identifier("_Testing_Foundation"))
729+
}
730+
}
731+

Sources/TestingMacros/TestingMacrosMain.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ struct TestingMacrosMain: CompilerPlugin {
2626
NonOptionalRequireMacro.self,
2727
RequireThrowsMacro.self,
2828
RequireThrowsNeverMacro.self,
29+
ExpectNSTaskExitsWithMacro.self,
30+
RequireNSTaskExitsWithMacro.self,
2931
ExitTestExpectMacro.self,
3032
ExitTestRequireMacro.self,
3133
ExitTestCapturedValueMacro.self,

Tests/TestingTests/ExitTestTests.swift

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@
1111
@testable @_spi(ForToolsIntegrationOnly) import Testing
1212
private import _TestingInternals
1313

14+
#if canImport(Foundation) && canImport(_Testing_Foundation)
15+
import Foundation
16+
@_spi(Experimental) import _Testing_Foundation
17+
#endif
18+
1419
#if !SWT_NO_EXIT_TESTS
1520
@Suite("Exit test tests") struct ExitTestTests {
1621
@Test("Signal names are reported (where supported)") func signalName() {
@@ -627,6 +632,41 @@ private import _TestingInternals
627632
#endif
628633
}
629634

635+
#if canImport(Foundation) && canImport(_Testing_Foundation)
636+
struct `Exit tests using Foundation.Process` {
637+
#if !os(Windows)
638+
@Test func `can consume stdout`() async throws {
639+
let process = Process()
640+
process.executableURL = URL(fileURLWithPath: "/bin/echo", isDirectory: false)
641+
process.arguments = ["Hello world!"]
642+
process.standardOutput = Pipe()
643+
let result = try await #require(process, exitsWith: .success, observing: [\.standardOutputContent])
644+
#expect(result.standardOutputContent.contains("Hello world!".utf8))
645+
}
646+
647+
@Test func `detects exit status`() async throws {
648+
let process = Process()
649+
process.executableURL = URL(fileURLWithPath: "/bin/cat", isDirectory: false)
650+
process.arguments = ["ceci n'est pas un fichier"]
651+
await #expect(process, exitsWith: .failure)
652+
}
653+
654+
@Test func `reports errors back to caller`() async throws {
655+
let process = Process()
656+
process.executableURL = URL(fileURLWithPath: "/bin/this executable does not exist", isDirectory: false)
657+
await withKnownIssue {
658+
await #expect(process, exitsWith: .failure)
659+
} matching: { issue in
660+
if case .system = issue.kind {
661+
return true
662+
}
663+
return false
664+
}
665+
}
666+
#endif
667+
}
668+
#endif
669+
630670
// MARK: - Fixtures
631671

632672
@Suite(.hidden) struct FailingExitTests {

0 commit comments

Comments
 (0)