diff --git a/CMakeLists.txt b/CMakeLists.txt index 1be9a4bed..aa20b9792 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,14 @@ cmake_minimum_required(VERSION 3.19.6...3.29) if(POLICY CMP0157) - cmake_policy(SET CMP0157 NEW) + if(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows AND CMAKE_SYSTEM_NAME STREQUAL Android) + # CMP0157 causes builds to fail when targetting Android with the Windows + # toolchain, because the early swift-driver isn't (yet) available. Disable + # it for now. + cmake_policy(SET CMP0157 OLD) + else() + cmake_policy(SET CMP0157 NEW) + endif() endif() project(SwiftTesting diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md new file mode 100644 index 000000000..f7af8a2f7 --- /dev/null +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -0,0 +1,267 @@ +# Return errors from `#expect(throws:)` + +* Proposal: [SWT-0006](0006-filename.md) +* Authors: [Jonathan Grynspan](https://github.com/grynspan) +* Status: **Awaiting review** +* Bug: rdar://138235250 +* Implementation: [swiftlang/swift-testing#780](https://github.com/swiftlang/swift-testing/pull/780) +* Review: ([pitch](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567)) + +## Introduction + +Swift Testing includes overloads of `#expect()` and `#require()` that can be +used to assert that some code throws an error. They are useful when validating +that your code's failure cases are correctly detected and handled. However, for +more complex validation cases, they aren't particularly ergonomic. This proposal +seeks to resolve that issue by having these overloads return thrown errors for +further inspection. + +## Motivation + +We offer three variants of `#expect(throws:)`: + +- One that takes an error type, and matches any error of the same type; +- One that takes an error _instance_ (conforming to `Equatable`) and matches any + error that compares equal to it; and +- One that takes a trailing closure and allows test authors to write arbitrary + validation logic. + +The third overload has proven to be somewhat problematic. First, it yields the +error to its closure as an instance of `any Error`, which typically forces the +developer to cast it before doing any useful comparisons. Second, the test +author must return `true` to indicate the error matched and `false` to indicate +it didn't, which can be both logically confusing and difficult to express +concisely: + +```swift +try #require { + let potato = try Sack.randomPotato() + try potato.turnIntoFrenchFries() +} throws: { error in + guard let error = error as PotatoError else { + return false + } + guard case .potatoNotPeeled = error else { + return false + } + return error.variety != .russet +} +``` + +The first impulse many test authors have here is to use `#expect()` in the +second closure, but it doesn't return the necessary boolean value _and_ it can +result in multiple issues being recorded in a test when there's really only one. + +## Proposed solution + +I propose deprecating [`#expect(_:sourceLocation:performing:throws:)`](https://developer.apple.com/documentation/testing/expect(_:sourcelocation:performing:throws:)) +and [`#require(_:sourceLocation:performing:throws:)`](https://developer.apple.com/documentation/testing/require(_:sourcelocation:performing:throws:)) +and modifying the other overloads so that, on success, they return the errors +that were thrown. + +## Detailed design + +All overloads of `#expect(throws:)` and `#require(throws:)` will be updated to +return an instance of the error type specified by their arguments, with the +problematic overloads returning `any Error` since more precise type information +is not statically available. The problematic overloads will also be deprecated: + +```diff +--- a/Sources/Testing/Expectations/Expectation+Macro.swift ++++ b/Sources/Testing/Expectations/Expectation+Macro.swift ++@discardableResult + @freestanding(expression) public macro expect( + throws errorType: E.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) ++) -> E? where E: Error + ++@discardableResult + @freestanding(expression) public macro require( + throws errorType: E.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) where E: Error ++) -> E where E: Error + ++@discardableResult + @freestanding(expression) public macro expect( + throws error: E, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) where E: Error & Equatable ++) -> E? where E: Error & Equatable + ++@discardableResult + @freestanding(expression) public macro require( + throws error: E, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) where E: Error & Equatable ++) -> E where E: Error & Equatable + ++@available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") ++@discardableResult + @freestanding(expression) public macro expect( + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R, + throws errorMatcher: (any Error) async throws -> Bool +-) ++) -> (any Error)? + ++@available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") ++@discardableResult + @freestanding(expression) public macro require( + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R, + throws errorMatcher: (any Error) async throws -> Bool +-) ++) -> any Error +``` + +(More detailed information about the deprecations will be provided via DocC.) + +The `#expect(throws:)` overloads return an optional value that is `nil` if the +expectation failed, while the `#require(throws:)` overloads return non-optional +values and throw instances of `ExpectationFailedError` on failure (as before.) + +> [!NOTE] +> Instances of `ExpectationFailedError` thrown by `#require(throws:)` on failure +> are not returned as that would defeat the purpose of using `#require(throws:)` +> instead of `#expect(throws:)`. + +Test authors will be able to use the result of the above functions to verify +that the thrown error is correct: + +```swift +let error = try #require(throws: PotatoError.self) { + let potato = try Sack.randomPotato() + try potato.turnIntoFrenchFries() +} +#expect(error == .potatoNotPeeled) +#expect(error.variety != .russet) +``` + +The new code is more concise than the old code and avoids boilerplate casting +from `any Error`. + +## Source compatibility + +In most cases, this change does not affect source compatibility. Swift does not +allow forming references to macros at runtime, so we don't need to worry about +type mismatches assigning one to some local variable. + +We have identified two scenarios where a new warning will be emitted. + +### Inferred return type from macro invocation + +The return type of the macro may be used by the compiler to infer the return +type of an enclosing closure. If the return value is then discarded, the +compiler may emit a warning: + +```swift +func pokePotato(_ pPotato: UnsafePointer) throws { ... } + +let potato = Potato() +try await Task.sleep(for: .months(3)) +withUnsafePointer(to: potato) { pPotato in + // ^ ^ ^ ⚠️ Result of call to 'withUnsafePointer(to:_:)' is unused + #expect(throws: PotatoError.rotten) { + try pokePotato(pPotato) + } +} +``` + +This warning can be suppressed by assigning the result of the macro invocation +or the result of the function call to `_`: + +```swift +withUnsafePointer(to: potato) { pPotato in + _ = #expect(throws: PotatoError.rotten) { + try pokePotato(pPotato) + } +} +``` + +### Use of `#require(throws:)` in a generic context with `Never.self` + +If `#require(throws:)` (but not `#expect(throws:)`) is used in a generic context +where the type of thrown error is a generic parameter, and the type is resolved +to `Never`, there is no valid value for the invocation to return: + +```swift +func wrapper(throws type: E.Type, _ body: () throws -> Void) throws -> E { + return try #require(throws: type) { + try body() + } +} +let error = try #require(throws: Never.self) { ... } +``` + +We don't think this particular pattern is common (and outside of our own test +target, I'd be surprised if anybody's attempted it yet.) However, we do need to +handle it gracefully. If this pattern is encountered, Swift Testing will record +an "API Misused" issue for the current test and advise the test author to switch +to `#expect(throws:)` or to not pass `Never.self` here. + +## Integration with supporting tools + +N/A + +## Future directions + +- Adopting [typed throws](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0413-typed-throws.md) + to statically require that the error thrown from test code is of the correct + type. + + If we adopted typed throws in the signatures of these macros, it would force + adoption of typed throws in the code under test even when it may not be + appropriate. For example, if we adopted typed throws, the following code would + not compile: + + ```swift + func cook(_ food: consuming some Food) throws { ... } + + let error: PotatoError? = #expect(throws: PotatoError.self) { + var potato = Potato() + potato.fossilize() + try cook(potato) // 🛑 ERROR: Invalid conversion of thrown error type + // 'any Error' to 'PotatoError' + } + ``` + + We believe it may be possible to overload these macros or their expansions so + that the code sample above _does_ compile and behave as intended. We intend to + experiment further with this idea and potentially revisit typed throws support + in a future proposal. + +## Alternatives considered + +- Leaving the existing implementation and signatures in place. We've had + sufficient feedback about the ergonomics of this API that we want to address + the problem. + +- Having the return type of the macros be `any Error` and returning _any_ error + that was thrown even on mismatch. This would make the ergonomics of the + subsequent test code less optimal because the test author would need to cast + the error to the appropriate type before inspecting it. + + There's a philosophical argument to be made here that if a mismatched error is + thrown, then the test has already failed and is in an inconsistent state, so + we should allow the test to fail rather than return what amounts to "bad + output". + + If the test author wants to inspect any arbitrary thrown error, they can + specify `(any Error).self` instead of a concrete error type. + +## Acknowledgments + +Thanks to the team and to [@jakepetroules](https://github.com/jakepetroules) for +starting the discussion that ultimately led to this proposal. diff --git a/Documentation/Proposals/0007-test-scoping-traits.md b/Documentation/Proposals/0007-test-scoping-traits.md new file mode 100644 index 000000000..3b0471550 --- /dev/null +++ b/Documentation/Proposals/0007-test-scoping-traits.md @@ -0,0 +1,510 @@ +# Test Scoping Traits + +* Proposal: [SWT-0007](0007-test-scoping-traits.md) +* Authors: [Stuart Montgomery](https://github.com/stmontgomery) +* Status: **Awaiting review** +* Implementation: [swiftlang/swift-testing#733](https://github.com/swiftlang/swift-testing/pull/733), [swiftlang/swift-testing#86](https://github.com/swiftlang/swift-testing/pull/86) +* Review: ([pitch](https://forums.swift.org/t/pitch-custom-test-execution-traits/75055)) + +### Revision history + +* **v1**: Initial pitch. +* **v2**: Dropped 'Custom' prefix from the proposed API names (although kept the + word in certain documentation passages where it clarified behavior). +* **v3**: Changed the `Trait` requirement from a property to a method which + accepts the test and/or test case, and modify its default implementations such + that custom behavior is either performed per-suite or per-test case by default. +* **v4**: Renamed the APIs to use "scope" as the base verb instead of "execute". + +## Introduction + +This introduces API which enables a `Trait`-conforming type to provide a custom +execution scope for test functions and suites, including running code before or +after them. + +## Motivation + +One of the primary motivations for the trait system in Swift Testing, as +[described in the vision document](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#trait-extensibility), +is to provide a way to customize the behavior of tests which have things in +common. If all the tests in a given suite type need the same custom behavior, +`init` and/or `deinit` (if applicable) can be used today. But if only _some_ of +the tests in a suite need custom behavior, or tests across different levels of +the suite hierarchy need it, traits would be a good place to encapsulate common +logic since they can be applied granularly per-test or per-suite. This aspect of +the vision for traits hasn't been realized yet, though: the `Trait` protocol +does not offer a way for a trait to customize the execution of the tests or +suites it's applied to. + +Customizing a test's behavior typically means running code either before or +after it runs, or both. Consolidating common set-up and tear-down logic allows +each test function to be more succinct with less repetitive boilerplate so it +can focus on what makes it unique. + +## Proposed solution + +At a high level, this proposal entails adding API to the `Trait` protocol +allowing a conforming type to opt-in to providing a custom execution scope for a +test. We discuss how that capability should be exposed to trait types below. + +### Supporting scoped access + +There are different approaches one could take to expose hooks for a trait to +customize test behavior. To illustrate one of them, consider the following +example of a `@Test` function with a custom trait whose purpose is to set mock +API credentials for the duration of each test it's applied to: + +```swift +@Test(.mockAPICredentials) +func example() { + // ... +} + +struct MockAPICredentialsTrait: TestTrait { ... } + +extension Trait where Self == MockAPICredentialsTrait { + static var mockAPICredentials: Self { ... } +} +``` + +In this hypothetical example, the current API credentials are stored via a +static property on an `APICredentials` type which is part of the module being +tested: + +```swift +struct APICredentials { + var apiKey: String + + static var shared: Self? +} +``` + +One way that this custom trait could customize the API credentials during each +test is if the `Trait` protocol were to expose a pair of method requirements +which were then called before and after the test, respectively: + +```swift +public protocol Trait: Sendable { + // ... + func setUp() async throws + func tearDown() async throws +} + +extension Trait { + // ... + public func setUp() async throws { /* No-op */ } + public func tearDown() async throws { /* No-op */ } +} +``` + +The custom trait type could adopt these using code such as the following: + +```swift +extension MockAPICredentialsTrait { + func setUp() { + APICredentials.shared = .init(apiKey: "...") + } + + func tearDown() { + APICredentials.shared = nil + } +} +``` + +Many testing systems use this pattern, including XCTest. However, this approach +encourages the use of global mutable state such as the `APICredentials.shared` +variable, and this limits the testing library's ability to parallelize test +execution, which is +[another part of the Swift Testing vision](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#parallelization-and-concurrency). + +The use of nonisolated static variables is generally discouraged now, and in +Swift 6 the above `APICredentials.shared` property produces an error. One way +to resolve that is to change it to a `@TaskLocal` variable, as this would be +concurrency-safe and still allow tests accessing this state to run in parallel: + +```swift +extension APICredentials { + @TaskLocal static var current: Self? +} +``` + +Binding task local values requires using the scoped access +[`TaskLocal.withValue()`](https://developer.apple.com/documentation/swift/tasklocal/withvalue(_:operation:isolation:file:line:)) +API though, and that would not be possible if `Trait` exposed separate methods +like `setUp()` and `tearDown()`. + +For these reasons, I believe it's important to expose this trait capability +using a single, scoped access-style API which accepts a closure. A simplified +version of that idea might look like this: + +```swift +public protocol Trait: Sendable { + // ... + + // Simplified example, not the actual proposal + func executeTest(_ body: @Sendable () async throws -> Void) async throws +} + +extension MockAPICredentialsTrait { + func executeTest(_ body: @Sendable () async throws -> Void) async throws { + let mockCredentials = APICredentials(apiKey: "...") + try await APICredentials.$current.withValue(mockCredentials) { + try await body() + } + } +} +``` + +### Avoiding unnecessarily lengthy backtraces + +A scoped access-style API has some potential downsides. To apply this approach +to a test function, the scoped call of a trait must wrap the invocation of that +test function, and every _other_ trait applied to that same test which offers +custom behavior _also_ must wrap the other traits' calls in a nesting fashion. +To visualize this, imagine a test function with multiple traits: + +```swift +@Test(.traitA, .traitB, .traitC) +func exampleTest() { + // ... +} +``` + +If all three of those traits provide a custom scope for tests, then each of them +needs to wrap the call to the next one, and the last trait needs to wrap the +invocation of the test, illustrated by the following: + +``` +TraitA.executeTest { + TraitB.executeTest { + TraitC.executeTest { + exampleTest() + } + } +} +``` + +Tests may have an arbitrary number of traits applied to them, including those +inherited from containing suite types. A naïve implementation in which _every_ +trait is given the opportunity to customize test behavior by calling its scoped +access API might cause unnecessarily lengthy backtraces that make debugging the +body of tests more difficult. Or worse: if the number of traits is great enough, +it could cause a stack overflow. + +In practice, most traits probably do _not_ need to provide a custom scope for +the tests they're applied to, so to mitigate these downsides it's important that +there be some way to distinguish traits which customize test behavior. That way, +the testing library can limit these scoped access calls to only traits which +need it. + +### Avoiding unnecessary (re-)execution + +Traits can be applied to either test functions or suites, and traits applied to +suites can optionally support inheritance by implementing the `isRecursive` +property of the `SuiteTrait` protocol. When a trait is directly applied to a +test function, if the trait customizes the behavior of tests it's applied to, it +should be given the opportunity to perform its custom behavior once for every +invocation of that test function. In particular, if the test function is +parameterized and runs multiple times, then the trait applied to it should +perform its custom behavior once for every invocation. This should not be +surprising to users, since it's consistent with the behavior of `init` and +`deinit` for an instance `@Test` method. + +It may be useful for certain kinds of traits to perform custom logic once for +_all_ the invocations of a parameterized test. Although this should be possible, +we believe it shouldn't be the default since it could lead to work being +repeated multiple times needlessly, or unintentional state sharing across tests, +unless the trait is implemented carefully to avoid those problems. + +When a trait conforms to `SuiteTrait` and is applied to a suite, the question of +when its custom scope (if any) should be applied is less obvious. Some suite +traits support inheritance and are recursively applied to all the test functions +they contain (including transitively, via sub-suites). Other suite traits don't +support inheritance, and only affect the specific suite they're applied to. +(It's also worth noting that a sub-suite _can_ have the same non-recursive suite +trait one of its ancestors has, as long as it's applied explicitly.) + +As a general rule of thumb, we believe most traits will either want to perform +custom logic once for _all_ children or once for _each_ child, not both. +Therefore, when it comes to suite traits, the default behavior should depend on +whether it supports inheritance: a recursive suite trait should by default +perform custom logic before each test, and a non-recursive one per-suite. But +the APIs should be flexible enough to support both, for advanced traits which +need it. + +## Detailed design + +I propose the following new APIs: + +- A new protocol `TestScoping` with a single required `provideScope(...)` method. + This will be called to provide scope for a test, and allows the conforming + type to perform custom logic before or after. +- A new method `scopeProvider(for:testCase:)` on the `Trait` protocol whose + result type is an `Optional` value of a type conforming to `TestScoping`. A + `nil` value returned by this method will skip calling the `provideScope(...)` + method. +- A default implementation of `Trait.scopeProvider(...)` which returns `nil`. +- A conditional implementation of `Trait.scopeProvider(...)` which returns `self` + in the common case where the trait type conforms to `TestScoping` itself. + +Since the `scopeProvider(...)` method's return type is optional and returns `nil` +by default, the testing library cannot invoke the `provideScope(...)` method +unless a trait customizes test behavior. This avoids the "unnecessarily lengthy +backtraces" problem above. + +Below are the proposed interfaces: + +```swift +/// A protocol that allows providing a custom execution scope for a test +/// function (and each of its cases) or a test suite by performing custom code +/// before or after it runs. +/// +/// Types conforming to this protocol may be used in conjunction with a +/// ``Trait``-conforming type by implementing the +/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, allowing custom traits +/// to provide custom scope for tests. Consolidating common set-up and tear-down +/// logic for tests which have similar needs allows each test function to be +/// more succinct with less repetitive boilerplate so it can focus on what makes +/// it unique. +public protocol TestScoping: Sendable { + /// Provide custom execution scope for a function call which is related to the + /// specified test and/or test case. + /// + /// - Parameters: + /// - test: The test under which `function` is being performed. + /// - testCase: The test case, if any, under which `function` is being + /// performed. When invoked on a suite, the value of this argument is + /// `nil`. + /// - function: The function to perform. If `test` represents a test suite, + /// this function encapsulates running all the tests in that suite. If + /// `test` represents a test function, this function is the body of that + /// test function (including all cases if it is parameterized.) + /// + /// - Throws: Whatever is thrown by `function`, or an error preventing this + /// type from providing a custom scope correctly. An error thrown from this + /// method is recorded as an issue associated with `test`. If an error is + /// thrown before `function` is called, the corresponding test will not run. + /// + /// When the testing library is preparing to run a test, it starts by finding + /// all traits applied to that test, including those inherited from containing + /// suites. It begins with inherited suite traits, sorting them + /// outermost-to-innermost, and if the test is a function, it then adds all + /// traits applied directly to that functions in the order they were applied + /// (left-to-right). It then asks each trait for its scope provider (if any) + /// by calling ``Trait/scopeProvider(for:testCase:)-cjmg``. Finally, it calls + /// this method on all non-`nil` scope providers, giving each an opportunity + /// to perform arbitrary work before or after invoking `function`. + /// + /// This method should either invoke `function` once before returning or throw + /// an error if it is unable to provide a custom scope. + /// + /// Issues recorded by this method are associated with `test`. + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws +} + +public protocol Trait: Sendable { + // ... + + /// The type of the test scope provider for this trait. + /// + /// The default type is `Never`, which cannot be instantiated. The + /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with this + /// default type must return `nil`, meaning that trait will not provide a + /// custom scope for the tests it's applied to. + associatedtype TestScopeProvider: TestScoping = Never + + /// Get this trait's scope provider for the specified test and/or test case, + /// if any. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which may be + /// used to provide custom scoping for `test` and/or `testCase`, or `nil` if + /// they should not have any custom scope. + /// + /// If this trait's type conforms to ``TestScoping``, the default value + /// returned by this method depends on `test` and/or `testCase`: + /// + /// - If `test` represents a suite, this trait must conform to ``SuiteTrait``. + /// If the value of this suite trait's ``SuiteTrait/isRecursive`` property + /// is `true`, then this method returns `nil`; otherwise, it returns `self`. + /// This means that by default, a suite trait will _either_ provide its + /// custom scope once for the entire suite, or once per-test function it + /// contains. + /// - Otherwise `test` represents a test function. If `testCase` is `nil`, + /// this method returns `nil`; otherwise, it returns `self`. This means that + /// by default, a trait which is applied to or inherited by a test function + /// will provide its custom scope once for each of that function's cases. + /// + /// A trait may explicitly implement this method to further customize the + /// default behaviors above. For example, if a trait should provide custom + /// test scope both once per-suite and once per-test function in that suite, + /// it may implement the method and return a non-`nil` scope provider under + /// those conditions. + /// + /// A trait may also implement this method and return `nil` if it determines + /// that it does not need to provide a custom scope for a particular test at + /// runtime, even if the test has the trait applied. This can improve + /// performance and make diagnostics clearer by avoiding an unnecessary call + /// to ``TestScoping/provideScope(for:testCase:performing:)``. + /// + /// If this trait's type does not conform to ``TestScoping`` and its + /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then + /// this method returns `nil` by default. This means that instances of this + /// trait will not provide a custom scope for tests to which they're applied. + func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? +} + +extension Trait where Self: TestScoping { + // Returns `nil` if `testCase` is `nil`, else `self`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? +} + +extension SuiteTrait where Self: TestScoping { + // If `test` is a suite, returns `nil` if `isRecursive` is `true`, else `self`. + // Otherwise, `test` is a function and this returns `nil` if `testCase` is + // `nil`, else `self`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? +} + +extension Trait where TestScopeProvider == Never { + // Returns `nil`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? +} + +extension Never: TestScoping {} +``` + +Here is a complete example of the usage scenario described earlier, showcasing +the proposed APIs: + +```swift +@Test(.mockAPICredentials) +func example() { + // ...validate API usage, referencing `APICredentials.current`... +} + +struct MockAPICredentialsTrait: TestTrait, TestScoping { + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + let mockCredentials = APICredentials(apiKey: "...") + try await APICredentials.$current.withValue(mockCredentials) { + try await function() + } + } +} + +extension Trait where Self == MockAPICredentialsTrait { + static var mockAPICredentials: Self { + Self() + } +} +``` + +## Source compatibility + +The proposed APIs are purely additive. + +This proposal will replace the existing `CustomExecutionTrait` SPI, and after +further refactoring we anticipate it will obsolete the need for the +`SPIAwareTrait` SPI as well. + +## Integration with supporting tools + +Although some built-in traits are relevant to supporting tools (such as +SourceKit-LSP statically discovering `.tags` traits), custom test behaviors are +only relevant within the test executable process while tests are running. We +don't anticipate any particular need for this feature to integrate with +supporting tools. + +## Future directions + +### Access to suite type instances + +Some test authors have expressed interest in allowing custom traits to access +the instance of a suite type for `@Test` instance methods, so the trait could +inspect or mutate the instance. Currently, only instance-level members of a +suite type (including `init`, `deinit`, and the test function itself) can access +`self`, so this would grant traits applied to an instance test method access to +the instance as well. This is certainly interesting, but poses several technical +challenges that puts it out of scope of this proposal. + +### Convenience trait for setting task locals + +Some reviewers of this proposal pointed out that the hypothetical usage example +shown earlier involving setting a task local value while a test is executing +will likely become a common use of these APIs. To streamline that pattern, it +would be very natural to add a built-in trait type which facilitates this. I +have prototyped this idea and plan to add it once this new trait functionality +lands. + +## Alternatives considered + +### Separate set up & tear down methods on `Trait` + +This idea was discussed in [Supporting scoped access](#supporting-scoped-access) +above, and as mentioned there, the primary problem with this approach is that it +cannot be used with scoped access-style APIs, including (importantly) +`TaskLocal.withValue()`. For that reason, it prevents using that common Swift +concurrency technique and reduces the potential for test parallelization. + +### Add `provideScope(...)` directly to the `Trait` protocol + +The proposed `provideScope(...)` method could be added as a requirement of the +`Trait` protocol instead of being part of a separate `TestScoping` protocol, and +it could have a default implementation which directly invokes the passed-in +closure. But this approach would suffer from the lengthy backtrace problem +described above. + +### Extend the `Trait` protocol + +The original, experimental implementation of this feature included a protocol +named`CustomExecutionTrait` which extended `Trait` and had roughly the same +method requirement as the `TestScoping` protocol proposed above. This design +worked, provided scoped access, and avoided the lengthy backtrace problem. + +After evaluating the design and usage of this SPI though, it seemed unfortunate +to structure it as a sub-protocol of `Trait` because it means that the full +capabilities of the trait system are spread across multiple protocols. In the +proposed design, the ability to return a test scoping provider is exposed via +the main `Trait` protocol, and it relies on an associated type to conditionally +opt-in to custom test behavior. In other words, the proposed design expresses +custom test behavior as just a _capability_ that a trait may have, rather than a +distinct sub-type of trait. + +Also, the implementation of this approach within the testing library was not +ideal as it required a conditional `trait as? CustomExecutionTrait` downcast at +runtime, in contrast to the simpler and more performant Optional property of the +proposed API. + +### API names + +We first considered "execute" as the base verb for the proposed new concept, but +felt this wasn't appropriate since these trait types are not "the executor" of +tests, they merely customize behavior and provide scope(s) for tests to run +within. Also, the term "executor" has prior art in Swift Concurrency, and +although that word is used in other contexts too, it may be helpful to avoid +potential confusion with concurrency executors. + +We also considered "run" as the base verb for the proposed new concept instead +of "execute", which would imply the names `TestRunning`, `TestRunner`, +`runner(for:testCase)`, and `run(_:for:testCase:)`. The word "run" is used in +many other contexts related to testing though, such as the `Runner` SPI type and +more casually to refer to a run which occurred of a test, in the past tense, so +overloading this term again may cause confusion. + +## Acknowledgments + +Thanks to [Dennis Weissmann](https://github.com/dennisweissmann) for originally +implementing this as SPI, and for helping promote its usefulness. + +Thanks to [Jonathan Grynspan](https://github.com/grynspan) for exploring ideas +to refine the API, and considering alternatives to avoid unnecessarily long +backtraces. + +Thanks to [Brandon Williams](https://github.com/mbrandonw) for feedback on the +Forum pitch thread which ultimately led to the refinements described in the +"Avoiding unnecessary (re-)execution" section. diff --git a/Package.swift b/Package.swift index f840a21cb..fea6f6f5e 100644 --- a/Package.swift +++ b/Package.swift @@ -13,6 +13,13 @@ import PackageDescription import CompilerPluginSupport +/// Information about the current state of the package's git repository. +let git = Context.gitInformation + +/// Whether or not this package is being built for development rather than +/// distribution as a package dependency. +let buildingForDevelopment = (git?.currentTag == nil) + let package = Package( name: "swift-testing", @@ -33,7 +40,7 @@ let package = Package( ], dependencies: [ - .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "600.0.0"), + .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "601.0.0-latest"), ], targets: [ @@ -67,10 +74,8 @@ let package = Package( .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), ], exclude: ["CMakeLists.txt"], - swiftSettings: .packageSettings + [ - // When building as a package, the macro plugin always builds as an - // executable rather than a library. - .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), + swiftSettings: .packageSettings + { + var result = [PackageDescription.SwiftSetting]() // The only target which needs the ability to import this macro // implementation target's module is its unit test target. Users of the @@ -78,8 +83,12 @@ let package = Package( // Testing module. This target's module is never distributed to users, // but as an additional guard against accidental misuse, this specifies // the unit test target as the only allowable client. - .unsafeFlags(["-Xfrontend", "-allowable-client", "-Xfrontend", "TestingMacrosTests"]), - ] + if buildingForDevelopment { + result.append(.unsafeFlags(["-Xfrontend", "-allowable-client", "-Xfrontend", "TestingMacrosTests"])) + } + + return result + }() ), // "Support" targets: These contain C family code and are used exclusively @@ -122,14 +131,23 @@ extension Array where Element == PackageDescription.SwiftSetting { /// Settings intended to be applied to every Swift target in this package. /// Analogous to project-level build settings in an Xcode project. static var packageSettings: Self { - availabilityMacroSettings + [ - .unsafeFlags(["-require-explicit-sendable"]), + var result = availabilityMacroSettings + + if buildingForDevelopment { + result.append(.unsafeFlags(["-require-explicit-sendable"])) + } + + result += [ .enableUpcomingFeature("ExistentialAny"), .enableExperimentalFeature("SuppressedAssociatedTypes"), .enableExperimentalFeature("AccessLevelOnImport"), .enableUpcomingFeature("InternalImportsByDefault"), + // When building as a package, the macro plugin always builds as an + // executable rather than a library. + .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), + .define("SWT_TARGET_OS_APPLE", .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS])), .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), @@ -138,6 +156,8 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] + + return result } /// Settings which define commonly-used OS availability macros. @@ -175,7 +195,7 @@ extension Array where Element == PackageDescription.CXXSetting { ] // Capture the testing library's version as a C++ string constant. - if let git = Context.gitInformation { + if let git { let testingLibraryVersion = if let tag = git.currentTag { tag } else if git.hasUncommittedChanges { diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index a0a6a8ee3..b1e90c535 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -304,18 +304,40 @@ extension Event.ConsoleOutputRecorder { /// destination. @discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool { let messages = _humanReadableOutputRecorder.record(event, in: context) - for message in messages { - let symbol = message.symbol?.stringValue(options: options) ?? " " - if case .details = message.symbol, options.useANSIEscapeCodes, options.ansiColorBitDepth > 1 { + // Padding to use in place of a symbol for messages that don't have one. + var padding = " " +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) + if options.useSFSymbols { + padding = " " + } +#endif + + let lines = messages.lazy.map { [test = context.test] message in + let symbol = message.symbol?.stringValue(options: options) ?? padding + + if case .details = message.symbol { // Special-case the detail symbol to apply grey to the entire line of - // text instead of just the symbol. - write("\(_ansiEscapeCodePrefix)90m\(symbol) \(message.stringValue)\(_resetANSIEscapeCode)\n") + // text instead of just the symbol. Details may be multi-line messages, + // so split the message on newlines and indent all lines to align them + // to the indentation provided by the symbol. + var lines = message.stringValue.split(whereSeparator: \.isNewline) + lines = CollectionOfOne(lines[0]) + lines.dropFirst().map { line in + "\(padding) \(line)" + } + let stringValue = lines.joined(separator: "\n") + if options.useANSIEscapeCodes, options.ansiColorBitDepth > 1 { + return "\(_ansiEscapeCodePrefix)90m\(symbol) \(stringValue)\(_resetANSIEscapeCode)\n" + } else { + return "\(symbol) \(stringValue)\n" + } } else { - let colorDots = context.test.map(\.tags).map { self.colorDots(for: $0) } ?? "" - write("\(symbol) \(colorDots)\(message.stringValue)\n") + let colorDots = test.map { self.colorDots(for: $0.tags) } ?? "" + return "\(symbol) \(colorDots)\(message.stringValue)\n" } } + + write(lines.joined()) return !messages.isEmpty } diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index ab1f56702..435f3ac01 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -92,19 +92,7 @@ extension Event.HumanReadableOutputRecorder { /// - Returns: A formatted string representing `comments`, or `nil` if there /// are none. private func _formattedComments(_ comments: [Comment]) -> [Message] { - // Insert an arrow character at the start of each comment, then indent any - // additional lines in the comment to align them with the arrow. - comments.lazy - .flatMap { comment in - let lines = comment.rawValue.split(whereSeparator: \.isNewline) - if let firstLine = lines.first { - let remainingLines = lines.dropFirst() - return CollectionOfOne(Message(symbol: .details, stringValue: String(firstLine))) + remainingLines.lazy - .map(String.init) - .map { Message(stringValue: $0) } - } - return [] - } + comments.map { Message(symbol: .details, stringValue: $0.rawValue) } } /// Get a string representing the comments attached to a test, formatted for diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 5012c93ca..682cb14a9 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -142,6 +142,9 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: If the expectation passes, the instance of `errorType` that was +/// thrown by `expression`. If the expectation fails, the result is `nil`. +/// /// Use this overload of `#expect()` when the expression `expression` _should_ /// throw an error of a given type: /// @@ -157,8 +160,11 @@ public macro require( /// is running in the current task. Any value returned by `expression` is /// discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``expect(throws:_:sourceLocation:performing:)-1xr34`` instead. +/// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. /// /// ## Expressions that should never throw /// @@ -181,12 +187,13 @@ public macro require( /// fail when an error is thrown by `expression`, rather than to explicitly /// check that an error is _not_ thrown by it, do not use this macro. Instead, /// simply call the code in question and allow it to throw an error naturally. +@discardableResult @freestanding(expression) public macro expect( throws errorType: E.Type, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error +) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error /// Check that an expression always throws an error of a given type, and throw /// an error if it does not. @@ -200,6 +207,8 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: The instance of `errorType` that was thrown by `expression`. +/// /// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not /// throw a matching error. The error thrown by `expression` is not rethrown. /// @@ -218,17 +227,21 @@ public macro require( /// is running in the current task and an instance of ``ExpectationFailedError`` /// is thrown. Any value returned by `expression` is discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``require(throws:_:sourceLocation:performing:)-7v83e`` instead. +/// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. +@discardableResult @freestanding(expression) public macro require( throws errorType: E.Type, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error +) -> E = #externalMacro(module: "TestingMacros", type: "RequireThrowsMacro") where E: Error /// Check that an expression never throws an error, and throw an error if it /// does. @@ -261,6 +274,10 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: If the expectation passes, the instance of `E` that was thrown by +/// `expression` and is equal to `error`. If the expectation fails, the result +/// is `nil`. +/// /// Use this overload of `#expect()` when the expression `expression` _should_ /// throw a specific error: /// @@ -275,14 +292,18 @@ public macro require( /// not equal to `error`, an ``Issue`` is recorded for the test that is running /// in the current task. Any value returned by `expression` is discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only be an instance of a particular type, use -/// ``expect(throws:_:sourceLocation:performing:)-79piu`` instead. +/// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. +@discardableResult @freestanding(expression) public macro expect( throws error: E, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error & Equatable +) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error & Equatable /// Check that an expression always throws a specific error, and throw an error /// if it does not. @@ -293,6 +314,9 @@ public macro require( /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. /// - expression: The expression to be evaluated. + +/// - Returns: The instance of `E` that was thrown by `expression` and is equal +/// to `error`. /// /// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not /// throw a matching error. The error thrown by `expression` is not rethrown. @@ -312,14 +336,18 @@ public macro require( /// in the current task and an instance of ``ExpectationFailedError`` is thrown. /// Any value returned by `expression` is discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only be an instance of a particular type, use -/// ``require(throws:_:sourceLocation:performing:)-76bjn`` instead. +/// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. +@discardableResult @freestanding(expression) public macro require( throws error: E, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error & Equatable +) -> E = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error & Equatable // MARK: - Arbitrary error matching @@ -333,6 +361,9 @@ public macro require( /// - errorMatcher: A closure to invoke when `expression` throws an error that /// indicates if it matched or not. /// +/// - Returns: If the expectation passes, the error that was thrown by +/// `expression`. If the expectation fails, the result is `nil`. +/// /// Use this overload of `#expect()` when the expression `expression` _should_ /// throw an error, but the logic to determine if the error matches is complex: /// @@ -353,15 +384,17 @@ public macro require( /// discarded. /// /// If the thrown error need only be an instance of a particular type, use -/// ``expect(throws:_:sourceLocation:performing:)-79piu`` instead. If the thrown +/// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. If the thrown /// error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``expect(throws:_:sourceLocation:performing:)-1xr34`` instead. +/// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. +@available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") +@discardableResult @freestanding(expression) public macro expect( _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R, throws errorMatcher: (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") +) -> (any Error)? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") /// Check that an expression always throws an error matching some condition, and /// throw an error if it does not. @@ -374,6 +407,8 @@ public macro require( /// - errorMatcher: A closure to invoke when `expression` throws an error that /// indicates if it matched or not. /// +/// - Returns: The error that was thrown by `expression`. +/// /// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not /// throw a matching error. The error thrown by `expression` is not rethrown. /// @@ -398,18 +433,20 @@ public macro require( /// discarded. /// /// If the thrown error need only be an instance of a particular type, use -/// ``require(throws:_:sourceLocation:performing:)-76bjn`` instead. If the thrown error need +/// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. If the thrown error need /// only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``require(throws:_:sourceLocation:performing:)-7v83e`` instead. +/// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. +@available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") +@discardableResult @freestanding(expression) public macro require( _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R, throws errorMatcher: (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") +) -> any Error = #externalMacro(module: "TestingMacros", type: "RequireMacro") // MARK: - Exit tests @@ -425,7 +462,7 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// -/// - Returns: If the exit test passed, an instance of ``ExitTestArtifacts`` +/// - Returns: If the exit test passes, an instance of ``ExitTestArtifacts`` /// describing the state of the exit test when it exited. If the exit test /// fails, the result is `nil`. /// diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index eff01e5bf..ca452e2f8 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -824,7 +824,8 @@ public func __checkCast( /// Check that an expression always throws an error. /// /// This overload is used for `#expect(throws:) { }` invocations that take error -/// types. +/// types. It is disfavored so that `#expect(throws: Never.self)` preferentially +/// returns `Void`. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -835,7 +836,7 @@ public func __checkClosureCall( comments: @autoclosure () -> [Comment], isRequired: Bool, sourceLocation: SourceLocation -) -> Result where E: Error { +) -> Result where E: Error { if errorType == Never.self { __checkClosureCall( throws: Never.self, @@ -844,7 +845,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { _ in nil } } else { __checkClosureCall( performing: body, @@ -854,14 +855,15 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } } /// Check that an expression always throws an error. /// /// This overload is used for `await #expect(throws:) { }` invocations that take -/// error types. +/// error types. It is disfavored so that `#expect(throws: Never.self)` +/// preferentially returns `Void`. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -873,7 +875,7 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result where E: Error { +) async -> Result where E: Error { if errorType == Never.self { await __checkClosureCall( throws: Never.self, @@ -883,7 +885,7 @@ public func __checkClosureCall( isRequired: isRequired, isolation: isolation, sourceLocation: sourceLocation - ) + ).map { _ in nil } } else { await __checkClosureCall( performing: body, @@ -894,7 +896,7 @@ public func __checkClosureCall( isRequired: isRequired, isolation: isolation, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } } @@ -932,7 +934,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { _ in } } /// Check that an expression never throws an error. @@ -969,7 +971,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { _ in } } // MARK: - Matching instances of equatable errors @@ -988,7 +990,7 @@ public func __checkClosureCall( comments: @autoclosure () -> [Comment], isRequired: Bool, sourceLocation: SourceLocation -) -> Result where E: Error & Equatable { +) -> Result where E: Error & Equatable { __checkClosureCall( performing: body, throws: { true == (($0 as? E) == error) }, @@ -997,7 +999,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } /// Check that an expression always throws an error. @@ -1015,7 +1017,7 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result where E: Error & Equatable { +) async -> Result where E: Error & Equatable { await __checkClosureCall( performing: body, throws: { true == (($0 as? E) == error) }, @@ -1025,7 +1027,7 @@ public func __checkClosureCall( isRequired: isRequired, isolation: isolation, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } // MARK: - Arbitrary error matching @@ -1044,10 +1046,11 @@ public func __checkClosureCall( comments: @autoclosure () -> [Comment], isRequired: Bool, sourceLocation: SourceLocation -) -> Result { +) -> Result<(any Error)?, any Error> { var errorMatches = false var mismatchExplanationValue: String? = nil var expression = expression + var caughtError: (any Error)? do { let result = try body() @@ -1057,6 +1060,7 @@ public func __checkClosureCall( } mismatchExplanationValue = explanation } catch { + caughtError = error expression = expression.capturingRuntimeValues(error) let secondError = Issue.withErrorRecording(at: sourceLocation) { errorMatches = try errorMatcher(error) @@ -1075,7 +1079,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { caughtError } } /// Check that an expression always throws an error. @@ -1093,10 +1097,11 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result<(any Error)?, any Error> { var errorMatches = false var mismatchExplanationValue: String? = nil var expression = expression + var caughtError: (any Error)? do { let result = try await body() @@ -1106,6 +1111,7 @@ public func __checkClosureCall( } mismatchExplanationValue = explanation } catch { + caughtError = error expression = expression.capturingRuntimeValues(error) let secondError = await Issue.withErrorRecording(at: sourceLocation) { errorMatches = try await errorMatcher(error) @@ -1124,7 +1130,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { caughtError } } // MARK: - Exit tests diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index e13099eaf..8a80e4467 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -29,11 +29,19 @@ extension Issue { func record(configuration: Configuration? = nil) -> Self { // If this issue is a caught error of kind SystemError, reinterpret it as a // testing system issue instead (per the documentation for SystemError.) - if case let .errorCaught(error) = kind, let error = error as? SystemError { - var selfCopy = self - selfCopy.kind = .system - selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) - return selfCopy.record(configuration: configuration) + if case let .errorCaught(error) = kind { + // TODO: consider factoring this logic out into a protocol + if let error = error as? SystemError { + var selfCopy = self + selfCopy.kind = .system + selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) + return selfCopy.record(configuration: configuration) + } else if let error = error as? APIMisuseError { + var selfCopy = self + selfCopy.kind = .apiMisused + selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) + return selfCopy.record(configuration: configuration) + } } // If this issue matches via the known issue matcher, set a copy of it to be diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 954485339..4e20b4b4e 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -56,31 +56,28 @@ public struct Runner: Sendable { // MARK: - Running tests extension Runner { - /// Execute the ``CustomExecutionTrait/execute(_:for:testCase:)`` functions - /// associated with the test in a plan step. + /// Apply the custom scope for any test scope providers of the traits + /// associated with a specified test by calling their + /// ``TestScoping/provideScope(for:testCase:performing:)`` function. /// /// - Parameters: - /// - step: The step being performed. - /// - testCase: The test case, if applicable, for which to execute the - /// custom trait. + /// - test: The test being run, for which to provide custom scope. + /// - testCase: The test case, if applicable, for which to provide custom + /// scope. /// - body: A function to execute from within the - /// ``CustomExecutionTrait/execute(_:for:testCase:)`` functions of each - /// trait applied to `step.test`. + /// ``TestScoping/provideScope(for:testCase:performing:)`` function of + /// each non-`nil` scope provider of the traits applied to `test`. /// /// - Throws: Whatever is thrown by `body` or by any of the - /// ``CustomExecutionTrait/execute(_:for:testCase:)`` functions. - private func _executeTraits( - for step: Plan.Step, + /// ``TestScoping/provideScope(for:testCase:performing:)`` function calls. + private func _applyScopingTraits( + for test: Test, testCase: Test.Case?, _ body: @escaping @Sendable () async throws -> Void ) async throws { // If the test does not have any traits, exit early to avoid unnecessary // heap allocations below. - if step.test.traits.isEmpty { - return try await body() - } - - if case .skip = step.action { + if test.traits.isEmpty { return try await body() } @@ -88,13 +85,13 @@ extension Runner { // function. The order of the sequence is reversed so that the last trait is // the one that invokes body, then the second-to-last invokes the last, etc. // and ultimately the first trait is the first one to be invoked. - let executeAllTraits = step.test.traits.lazy + let executeAllTraits = test.traits.lazy .reversed() - .compactMap { $0 as? any CustomExecutionTrait } - .compactMap { $0.execute(_:for:testCase:) } - .reduce(body) { executeAllTraits, traitExecutor in + .compactMap { $0.scopeProvider(for: test, testCase: testCase) } + .map { $0.provideScope(for:testCase:performing:) } + .reduce(body) { executeAllTraits, provideScope in { - try await traitExecutor(executeAllTraits, step.test, testCase) + try await provideScope(test, testCase, executeAllTraits) } } @@ -200,7 +197,7 @@ extension Runner { if let step = stepGraph.value, case .run = step.action { await Test.withCurrent(step.test) { _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) { - try await _executeTraits(for: step, testCase: nil) { + try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { try await _runTestCases(testCases, within: step) @@ -336,7 +333,7 @@ extension Runner { let sourceLocation = step.test.sourceLocation await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) { try await withTimeLimit(for: step.test, configuration: configuration) { - try await _executeTraits(for: step, testCase: testCase) { + try await _applyScopingTraits(for: step.test, testCase: testCase) { try await testCase.body() } } timeoutHandler: { timeLimit in diff --git a/Sources/Testing/Support/Additions/ResultAdditions.swift b/Sources/Testing/Support/Additions/ResultAdditions.swift index f14f68c85..9a2e6ea5a 100644 --- a/Sources/Testing/Support/Additions/ResultAdditions.swift +++ b/Sources/Testing/Support/Additions/ResultAdditions.swift @@ -31,16 +31,27 @@ extension Result { /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. - @inlinable public func __expected() -> Success where Success == T? { + @discardableResult @inlinable public func __expected() -> Success where Success == T? { try? get() } /// Handle this instance as if it were returned from a call to `#require()`. /// + /// This overload of `__require()` assumes that the result cannot actually be + /// `nil` on success. The optionality is part of our ABI contract for the + /// `__check()` function family so that we can support uninhabited types and + /// "soft" failures. + /// + /// If the value really is `nil` (e.g. we're dealing with `Never`), the + /// testing library throws an error representing an issue of kind + /// ``Issue/Kind-swift.enum/apiMisused``. + /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. - @inlinable public func __required() throws -> T where Success == T? { - // TODO: handle edge case where the value is nil (see #780) - try get()! + @discardableResult public func __required() throws -> T where Success == T? { + guard let result = try get() else { + throw APIMisuseError(description: "Could not unwrap 'nil' value of type Optional<\(T.self)>. Consider using #expect() instead of #require() here.") + } + return result } } diff --git a/Sources/Testing/Support/SystemError.swift b/Sources/Testing/Support/SystemError.swift index d68b9c241..d2d4809e3 100644 --- a/Sources/Testing/Support/SystemError.swift +++ b/Sources/Testing/Support/SystemError.swift @@ -21,3 +21,16 @@ struct SystemError: Error, CustomStringConvertible { var description: String } + +/// A type representing misuse of testing library API. +/// +/// When an error of this type is thrown and caught by the testing library, it +/// is recorded as an issue of kind ``Issue/Kind/apiMisused`` rather than +/// ``Issue/Kind/errorCaught(_:)``. +/// +/// This type is not part of the public interface of the testing library. +/// External callers should generally record issues by throwing their own errors +/// or by calling ``Issue/record(_:sourceLocation:)``. +struct APIMisuseError: Error, CustomStringConvertible { + var description: String +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md new file mode 100644 index 000000000..3114002d2 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md @@ -0,0 +1,28 @@ +# ``expect(_:sourceLocation:performing:throws:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.0) + @Available(Xcode, introduced: 16.0) +} + +@DeprecationSummary { + Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` or + ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: + + ```swift + let error = #expect(throws: FoodTruckError.self) { + ... + } + #expect(error?.napkinCount == 0) + ``` +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md new file mode 100644 index 000000000..291ac0d32 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md @@ -0,0 +1,28 @@ +# ``require(_:sourceLocation:performing:throws:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.0) + @Available(Xcode, introduced: 16.0) +} + +@DeprecationSummary { + Examine the result of ``require(throws:_:sourceLocation:performing:)-7n34r`` + or ``require(throws:_:sourceLocation:performing:)-4djuw`` instead: + + ```swift + let error = try #require(throws: FoodTruckError.self) { + ... + } + #expect(error.napkinCount == 0) + ``` +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md new file mode 100644 index 000000000..6136ee8cb --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)-1z8kh`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md new file mode 100644 index 000000000..25281bfe6 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md @@ -0,0 +1,15 @@ +# ``Trait/TestScopeProvider`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md new file mode 100644 index 000000000..809fb833e --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md @@ -0,0 +1,15 @@ +# ``TestScoping/provideScope(for:testCase:performing:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md new file mode 100644 index 000000000..a0ab00e1f --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md @@ -0,0 +1,15 @@ +# ``TestScoping`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md new file mode 100644 index 000000000..8e903eb3b --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)-9fxg4`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md new file mode 100644 index 000000000..0ff33a204 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)-inmj`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md new file mode 100644 index 000000000..5fcff2667 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/Expectations.md b/Sources/Testing/Testing.docc/Expectations.md index ce92824c1..fd3b0070d 100644 --- a/Sources/Testing/Testing.docc/Expectations.md +++ b/Sources/Testing/Testing.docc/Expectations.md @@ -65,11 +65,11 @@ the test when the code doesn't satisfy a requirement, use ### Checking that errors are thrown - -- ``expect(throws:_:sourceLocation:performing:)-79piu`` -- ``expect(throws:_:sourceLocation:performing:)-1xr34`` +- ``expect(throws:_:sourceLocation:performing:)-1hfms`` +- ``expect(throws:_:sourceLocation:performing:)-7du1h`` - ``expect(_:sourceLocation:performing:throws:)`` -- ``require(throws:_:sourceLocation:performing:)-76bjn`` -- ``require(throws:_:sourceLocation:performing:)-7v83e`` +- ``require(throws:_:sourceLocation:performing:)-7n34r`` +- ``require(throws:_:sourceLocation:performing:)-4djuw`` - ``require(_:sourceLocation:performing:throws:)`` ### Confirming that asynchronous events occur diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index bf0b43e34..44d91b5b0 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -273,7 +273,7 @@ with optional expressions to unwrap them: ### Record issues -Finally, XCTest has a function, [`XCTFail()`](https://developer.apple.com/documentation/xctest/1500970-xctfail), +XCTest has a function, [`XCTFail()`](https://developer.apple.com/documentation/xctest/1500970-xctfail), that causes a test to fail immediately and unconditionally. This function is useful when the syntax of the language prevents the use of an `XCTAssert()` function. To record an unconditional issue using the testing library, use the @@ -326,7 +326,7 @@ their equivalents in the testing library: | `XCTAssertLessThanOrEqual(x, y)` | `#expect(x <= y)` | | `XCTAssertLessThan(x, y)` | `#expect(x < y)` | | `XCTAssertThrowsError(try f())` | `#expect(throws: (any Error).self) { try f() }` | -| `XCTAssertThrowsError(try f()) { error in … }` | `#expect { try f() } throws: { error in return … }` | +| `XCTAssertThrowsError(try f()) { error in … }` | `let error = #expect(throws: (any Error).self) { try f() }` | | `XCTAssertNoThrow(try f())` | `#expect(throws: Never.self) { try f() }` | | `try XCTUnwrap(x)` | `try #require(x)` | | `XCTFail("…")` | `Issue.record("…")` | @@ -692,6 +692,56 @@ of issues: } } +### Run tests sequentially + +By default, the testing library runs all tests in a suite in parallel. The +default behavior of XCTest is to run each test in a suite sequentially. If your +tests use shared state such as global variables, you may see unexpected +behavior including unreliable test outcomes when you run tests in parallel. + +Annotate your test suite with ``Trait/serialized`` to run tests within that +suite serially: + +@Row { + @Column { + ```swift + // Before + class RefrigeratorTests : XCTestCase { + func testLightComesOn() throws { + try FoodTruck.shared.refrigerator.openDoor() + XCTAssertEqual(FoodTruck.shared.refrigerator.lightState, .on) + } + + func testLightGoesOut() throws { + try FoodTruck.shared.refrigerator.openDoor() + try FoodTruck.shared.refrigerator.closeDoor() + XCTAssertEqual(FoodTruck.shared.refrigerator.lightState, .off) + } + } + ``` + } + @Column { + ```swift + // After + @Suite(.serialized) + class RefrigeratorTests { + @Test func lightComesOn() throws { + try FoodTruck.shared.refrigerator.openDoor() + #expect(FoodTruck.shared.refrigerator.lightState == .on) + } + + @Test func lightGoesOut() throws { + try FoodTruck.shared.refrigerator.openDoor() + try FoodTruck.shared.refrigerator.closeDoor() + #expect(FoodTruck.shared.refrigerator.lightState == .off) + } + } + ``` + } +} + +For more information, see . + ## See Also - diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index db8455fc5..b688b4e00 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -10,14 +10,14 @@ See https://swift.org/LICENSE.txt for license information See https://swift.org/CONTRIBUTORS.txt for Swift project authors --> -Add traits to tests to annotate them or customize their behavior. +Annotate test functions and suites, and customize their behavior. ## Overview Pass built-in traits to test functions or suite types to comment, categorize, -classify, and modify runtime behaviors. You can also use the ``Trait``, ``TestTrait``, -and ``SuiteTrait`` protocols to create your own types that customize the -behavior of test functions. +classify, and modify the runtime behavior of test suites and test functions. +Implement the ``TestTrait``, and ``SuiteTrait`` protocols to create your own +types that customize the behavior of your tests. ## Topics @@ -53,6 +53,7 @@ behavior of test functions. - ``Trait`` - ``TestTrait`` - ``SuiteTrait`` +- ``TestScoping`` ### Supporting types diff --git a/Sources/Testing/Testing.docc/Traits/Trait.md b/Sources/Testing/Testing.docc/Traits/Trait.md index 1528ec1b4..d6422167d 100644 --- a/Sources/Testing/Testing.docc/Traits/Trait.md +++ b/Sources/Testing/Testing.docc/Traits/Trait.md @@ -20,17 +20,15 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors - ``Trait/disabled(if:_:sourceLocation:)`` - ``Trait/disabled(_:sourceLocation:_:)`` -### Limiting the running time of tests - -- ``Trait/timeLimit(_:)`` - -### Running tests serially or in parallel +### Controlling how tests are run +- ``Trait/timeLimit(_:)-4kzjp`` - ``Trait/serialized`` - -### Categorizing tests + +### Categorizing tests and adding information - ``Trait/tags(_:)`` +- ``Trait/comments`` ### Associating bugs @@ -38,9 +36,9 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors - ``Trait/bug(_:id:_:)-10yf5`` - ``Trait/bug(_:id:_:)-3vtpl`` -### Adding information to tests -- ``Trait/comments`` - -### Preparing internal state +### Running code before and after a test or suite +- ``TestScoping`` +- ``Trait/scopeProvider(for:testCase:)-cjmg`` +- ``Trait/TestScopeProvider`` - ``Trait/prepare(for:)-3s3zo`` diff --git a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md index 5113202d0..c25a650fe 100644 --- a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md +++ b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md @@ -26,7 +26,7 @@ If the code throws an error, then your test fails. To check that the code under test throws a specific error, or to continue a longer test function after the code throws an error, pass that error as the -first argument of ``expect(throws:_:sourceLocation:performing:)-1xr34``, and +first argument of ``expect(throws:_:sourceLocation:performing:)-7du1h``, and pass a closure that calls the code under test: ```swift @@ -65,4 +65,23 @@ the error to `Never`: If the closure throws _any_ error, the testing library records an issue. If you need the test to stop when the code throws an error, include the code inline in the test function instead of wrapping it in a call to -``expect(throws:_:sourceLocation:performing:)-1xr34``. +``expect(throws:_:sourceLocation:performing:)-7du1h``. + +## Inspect an error thrown by your code + +When you use `#expect(throws:)` or `#require(throws:)` and the error matches the +expectation, it is returned to the caller so that you can perform additional +validation. If the expectation fails because no error was thrown or an error of +a different type was thrown, `#expect(throws:)` returns `nil`: + +```swift +@Test func cannotAddMarshmallowsToPizza() throws { + let error = #expect(throws: PizzaToppings.InvalidToppingError.self) { + try Pizza.current.add(topping: .marshmallows) + } + #expect(error?.topping == .marshmallows) + #expect(error?.reason == .dessertToppingOnly) +} +``` + +If you aren't sure what type of error will be thrown, pass `(any Error).self`. diff --git a/Sources/Testing/Traits/Bug.swift b/Sources/Testing/Traits/Bug.swift index 48a718dfa..14f541557 100644 --- a/Sources/Testing/Traits/Bug.swift +++ b/Sources/Testing/Traits/Bug.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type representing a bug report tracked by a test. +/// A type that represents a bug report tracked by a test. /// /// To add this trait to a test, use one of the following functions: /// @@ -16,7 +16,7 @@ /// - ``Trait/bug(_:id:_:)-10yf5`` /// - ``Trait/bug(_:id:_:)-3vtpl`` public struct Bug { - /// A URL linking to more information about the bug, if available. + /// A URL that links to more information about the bug, if available. /// /// The value of this property represents a URL conforming to /// [RFC 3986](https://www.ietf.org/rfc/rfc3986.txt). @@ -59,42 +59,42 @@ extension Bug: TestTrait, SuiteTrait { } extension Trait where Self == Bug { - /// Construct a bug to track with a test. + /// Constructs a bug to track with a test. /// /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking + /// - url: A URL that refers to this bug in the associated bug-tracking /// system. /// - title: Optionally, the human-readable title of the bug. /// - /// - Returns: An instance of ``Bug`` representing the specified bug. + /// - Returns: An instance of ``Bug`` that represents the specified bug. public static func bug(_ url: _const String, _ title: Comment? = nil) -> Self { Self(url: url, title: title) } - /// Construct a bug to track with a test. + /// Constructs a bug to track with a test. /// /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking + /// - url: A URL that refers to this bug in the associated bug-tracking /// system. /// - id: The unique identifier of this bug in its associated bug-tracking /// system. /// - title: Optionally, the human-readable title of the bug. /// - /// - Returns: An instance of ``Bug`` representing the specified bug. + /// - Returns: An instance of ``Bug`` that represents the specified bug. public static func bug(_ url: _const String? = nil, id: some Numeric, _ title: Comment? = nil) -> Self { Self(url: url, id: String(describing: id), title: title) } - /// Construct a bug to track with a test. + /// Constructs a bug to track with a test. /// /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking + /// - url: A URL that refers to this bug in the associated bug-tracking /// system. /// - id: The unique identifier of this bug in its associated bug-tracking /// system. /// - title: Optionally, the human-readable title of the bug. /// - /// - Returns: An instance of ``Bug`` representing the specified bug. + /// - Returns: An instance of ``Bug`` that represents the specified bug. public static func bug(_ url: _const String? = nil, id: _const String, _ title: Comment? = nil) -> Self { Self(url: url, id: id, title: title) } diff --git a/Sources/Testing/Traits/Comment.swift b/Sources/Testing/Traits/Comment.swift index 3497cf6e9..2a59b2bd5 100644 --- a/Sources/Testing/Traits/Comment.swift +++ b/Sources/Testing/Traits/Comment.swift @@ -8,22 +8,21 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type representing a comment related to a test. +/// A type that represents a comment related to a test. /// -/// This type may be used to provide context or background information about a +/// Use this type to provide context or background information about a /// test's purpose, explain how a complex test operates, or include details /// which may be helpful when diagnosing issues recorded by a test. /// /// To add a comment to a test or suite, add a code comment before its `@Test` /// or `@Suite` attribute. See for more details. /// -/// - Note: This type is not intended to reference bugs related to a test. -/// Instead, use ``Trait/bug(_:_:)``, ``Trait/bug(_:id:_:)-10yf5``, or -/// ``Trait/bug(_:id:_:)-3vtpl``. +/// - Note: To reference bugs related to a test, use ``Trait/bug(_:_:)``, +/// ``Trait/bug(_:id:_:)-10yf5``, or ``Trait/bug(_:id:_:)-3vtpl``. public struct Comment: RawRepresentable, Sendable { - /// The single comment string contained in this instance. + /// The single comment string that this comment contains. /// - /// To obtain the complete set of comments applied to a test, see + /// To get the complete set of comments applied to a test, see /// ``Test/comments``. public var rawValue: String diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index 09c8909dc..245f8e98f 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -8,8 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type that defines a condition which must be satisfied for a test to be -/// enabled. +/// A type that defines a condition which must be satisfied for the testing +/// library to enable a test. /// /// To add this trait to a test, use one of the following functions: /// @@ -19,10 +19,10 @@ /// - ``Trait/disabled(if:_:sourceLocation:)`` /// - ``Trait/disabled(_:sourceLocation:_:)`` public struct ConditionTrait: TestTrait, SuiteTrait { - /// An enumeration describing the kinds of conditions that can be represented - /// by an instance of this type. + /// An enumeration that describes the conditions that an instance of this type + /// can represent. enum Kind: Sendable { - /// The trait is conditional on the result of calling a function. + /// Enabling the test is conditional on the result of calling a function. /// /// - Parameters: /// - body: The function to call. The result of this function determines @@ -39,7 +39,8 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// - body: The function to call. The result of this function determines /// whether or not the condition was met. /// - /// - Returns: An instance of this type. + /// - Returns: A trait that marks a test's enabled status as the result of + /// calling a function. static func conditional(_ body: @escaping @Sendable () async throws -> Bool) -> Self { conditional { () -> (Bool, comment: Comment?) in return (try await body(), nil) @@ -49,14 +50,14 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// The trait is unconditional and always has the same result. /// /// - Parameters: - /// - value: Whether or not the condition was met. + /// - value: Whether or not the test is enabled. case unconditional(_ value: Bool) } - /// The kind of condition represented by this instance. + /// The kind of condition represented by this trait. var kind: Kind - /// Whether or not this trait has a condition that is evaluated at runtime. + /// Whether this trait's condition is constant, or evaluated at runtime. /// /// If this trait was created using a function such as /// ``disabled(_:sourceLocation:)`` that unconditionally enables or disables a @@ -77,7 +78,7 @@ public struct ConditionTrait: TestTrait, SuiteTrait { public var comments: [Comment] - /// The source location where this trait was specified. + /// The source location where this trait is specified. public var sourceLocation: SourceLocation public func prepare(for test: Test) async throws { @@ -110,24 +111,23 @@ public struct ConditionTrait: TestTrait, SuiteTrait { // MARK: - extension Trait where Self == ConditionTrait { - /// Construct a condition trait that causes a test to be disabled if it - /// returns `false`. + /// Constructs a condition trait that disables a test if it returns `false`. /// /// - Parameters: - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `true`, the test is allowed to run. Otherwise, - /// the test is skipped. - /// - comment: An optional, user-specified comment describing this trait. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `true`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the - /// specified closure. - /// - /// @Comment { - /// - Bug: `condition` cannot be `async` without making this function - /// `async` even though `condition` is not evaluated locally. - /// ([103037177](rdar://103037177)) - /// } + /// - Returns: An instance of ``ConditionTrait`` that evaluates the + /// closure you provide. + // + // @Comment { + // - Bug: `condition` cannot be `async` without making this function + // `async` even though `condition` is not evaluated locally. + // ([103037177](rdar://103037177)) + // } public static func enabled( if condition: @autoclosure @escaping @Sendable () throws -> Bool, _ comment: Comment? = nil, @@ -136,18 +136,17 @@ extension Trait where Self == ConditionTrait { Self(kind: .conditional(condition), comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that causes a test to be disabled if it - /// returns `false`. + /// Constructs a condition trait that disables a test if it returns `false`. /// /// - Parameters: - /// - comment: An optional, user-specified comment describing this trait. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `true`, the test is allowed to run. Otherwise, - /// the test is skipped. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `true`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the - /// specified closure. + /// - Returns: An instance of ``ConditionTrait`` that evaluates the + /// closure you provide. public static func enabled( _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, @@ -156,13 +155,13 @@ extension Trait where Self == ConditionTrait { Self(kind: .conditional(condition), comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that disables a test unconditionally. + /// Constructs a condition trait that disables a test unconditionally. /// /// - Parameters: - /// - comment: An optional, user-specified comment describing this trait. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. /// - /// - Returns: An instance of ``ConditionTrait`` that will always disable the + /// - Returns: An instance of ``ConditionTrait`` that always disables the /// test to which it is added. public static func disabled( _ comment: Comment? = nil, @@ -171,24 +170,23 @@ extension Trait where Self == ConditionTrait { Self(kind: .unconditional(false), comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that causes a test to be disabled if it - /// returns `true`. + /// Constructs a condition trait that disables a test if its value is true. /// /// - Parameters: - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `false`, the test is allowed to run. Otherwise, - /// the test is skipped. - /// - comment: An optional, user-specified comment describing this trait. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `false`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the - /// specified closure. - /// - /// @Comment { - /// - Bug: `condition` cannot be `async` without making this function - /// `async` even though `condition` is not evaluated locally. - /// ([103037177](rdar://103037177)) - /// } + /// - Returns: An instance of ``ConditionTrait`` that evaluates the + /// closure you provide. + // + // @Comment { + // - Bug: `condition` cannot be `async` without making this function + // `async` even though `condition` is not evaluated locally. + // ([103037177](rdar://103037177)) + // } public static func disabled( if condition: @autoclosure @escaping @Sendable () throws -> Bool, _ comment: Comment? = nil, @@ -197,17 +195,16 @@ extension Trait where Self == ConditionTrait { Self(kind: .conditional { !(try condition()) }, comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that causes a test to be disabled if it - /// returns `true`. + /// Constructs a condition trait that disables a test if its value is true. /// /// - Parameters: - /// - comment: An optional, user-specified comment describing this trait. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `false`, the test is allowed to run. Otherwise, - /// the test is skipped. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `false`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the + /// - Returns: An instance of ``ConditionTrait`` that evaluates the /// specified closure. public static func disabled( _ comment: Comment? = nil, diff --git a/Sources/Testing/Traits/ParallelizationTrait.swift b/Sources/Testing/Traits/ParallelizationTrait.swift index d41052e25..fec1d22bd 100644 --- a/Sources/Testing/Traits/ParallelizationTrait.swift +++ b/Sources/Testing/Traits/ParallelizationTrait.swift @@ -8,21 +8,21 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type that affects whether or not a test or suite is parallelized. +/// A type that defines whether the testing library runs this test serially +/// or in parallel. /// -/// When added to a parameterized test function, this trait causes that test to -/// run its cases serially instead of in parallel. When applied to a -/// non-parameterized test function, this trait has no effect. When applied to a -/// test suite, this trait causes that suite to run its contained test functions -/// and sub-suites serially instead of in parallel. +/// When you add this trait to a parameterized test function, that test runs its +/// cases serially instead of in parallel. This trait has no effect when you +/// apply it to a non-parameterized test function. /// -/// This trait is recursively applied: if it is applied to a suite, any -/// parameterized tests or test suites contained in that suite are also -/// serialized (as are any tests contained in those suites, and so on.) +/// When you add this trait to a test suite, that suite runs its +/// contained test functions (including their cases, when parameterized) and +/// sub-suites serially instead of in parallel. If the sub-suites have children, +/// they also run serially. /// /// This trait does not affect the execution of a test relative to its peers or -/// to unrelated tests. This trait has no effect if test parallelization is -/// globally disabled (by, for example, passing `--no-parallel` to the +/// to unrelated tests. This trait has no effect if you disable test +/// parallelization globally (for example, by passing `--no-parallel` to the /// `swift test` command.) /// /// To add this trait to a test, use ``Trait/serialized``. diff --git a/Sources/Testing/Traits/TimeLimitTrait.swift b/Sources/Testing/Traits/TimeLimitTrait.swift index fe1d7f787..4e84a1f92 100644 --- a/Sources/Testing/Traits/TimeLimitTrait.swift +++ b/Sources/Testing/Traits/TimeLimitTrait.swift @@ -10,17 +10,16 @@ /// A type that defines a time limit to apply to a test. /// -/// To add this trait to a test, use one of the following functions: -/// -/// - ``Trait/timeLimit(_:)`` +/// To add this trait to a test, use ``Trait/timeLimit(_:)-4kzjp``. @available(_clockAPI, *) public struct TimeLimitTrait: TestTrait, SuiteTrait { /// A type representing the duration of a time limit applied to a test. /// - /// This type is intended for use specifically for specifying test timeouts - /// with ``TimeLimitTrait``. It is used instead of Swift's built-in `Duration` - /// type because test timeouts do not support high-precision, arbitrarily - /// short durations. The smallest allowed unit of time is minutes. + /// Use this type to specify a test timeout with ``TimeLimitTrait``. + /// `TimeLimitTrait` uses this type instead of Swift's built-in `Duration` + /// type because the testing library doesn't support high-precision, + /// arbitrarily short durations for test timeouts. The smallest unit of time + /// you can specify in a `Duration` is minutes. public struct Duration: Sendable { /// The underlying Swift `Duration` which this time limit duration /// represents. @@ -29,8 +28,7 @@ public struct TimeLimitTrait: TestTrait, SuiteTrait { /// Construct a time limit duration given a number of minutes. /// /// - Parameters: - /// - minutes: The number of minutes the resulting duration should - /// represent. + /// - minutes: The length of the duration in minutes. /// /// - Returns: A duration representing the specified number of minutes. public static func minutes(_ minutes: some BinaryInteger) -> Self { @@ -97,26 +95,25 @@ extension Trait where Self == TimeLimitTrait { /// - Returns: An instance of ``TimeLimitTrait``. /// /// Test timeouts do not support high-precision, arbitrarily short durations - /// due to variability in testing environments. The time limit must be at - /// least one minute, and can only be expressed in increments of one minute. + /// due to variability in testing environments. You express the duration in + /// minutes, with a minimum duration of one minute. /// - /// When this trait is associated with a test, that test must complete within - /// a time limit of, at most, `timeLimit`. If the test runs longer, an issue - /// of kind ``Issue/Kind/timeLimitExceeded(timeLimitComponents:)`` is - /// recorded. This timeout is treated as a test failure. + /// When you associate this trait with a test, that test must complete within + /// a time limit of, at most, `timeLimit`. If the test runs longer, the + /// testing library records a + /// ``Issue/Kind/timeLimitExceeded(timeLimitComponents:)`` issue, which it + /// treats as a test failure. /// - /// The time limit amount specified by `timeLimit` may be reduced if the - /// testing library is configured to enforce a maximum per-test limit. When - /// such a maximum is set, the effective time limit of the test this trait is - /// applied to will be the lesser of `timeLimit` and that maximum. This is a - /// policy which may be configured on a global basis by the tool responsible - /// for launching the test process. Refer to that tool's documentation for - /// more details. + /// The testing library can use a shorter time limit than that specified by + /// `timeLimit` if you configure it to enforce a maximum per-test limit. When + /// you configure a maximum per-test limit, the time limit of the test this + /// trait is applied to is the shorter of `timeLimit` and the maximum per-test + /// limit. For information on configuring maximum per-test limits, consult the + /// documentation for the tool you use to run your tests. /// /// If a test is parameterized, this time limit is applied to each of its /// test cases individually. If a test has more than one time limit associated - /// with it, the shortest one is used. A test run may also be configured with - /// a maximum time limit per test case. + /// with it, the testing library uses the shortest time limit. public static func timeLimit(_ timeLimit: Self.Duration) -> Self { return Self(timeLimit: timeLimit.underlyingDuration) } @@ -185,11 +182,9 @@ extension TimeLimitTrait.Duration { @available(_clockAPI, *) extension Test { - /// The maximum amount of time the cases of this test may run for. - /// - /// Time limits are associated with tests using this trait: + /// The maximum amount of time this test's cases may run for. /// - /// - ``Trait/timeLimit(_:)`` + /// Associate a time limit with tests by using ``Trait/timeLimit(_:)-4kzjp``. /// /// If a test has more than one time limit associated with it, the value of /// this property is the shortest one. If a test has no time limits associated diff --git a/Sources/Testing/Traits/Trait.swift b/Sources/Testing/Traits/Trait.swift index e6a42b4d5..7ebbb38d4 100644 --- a/Sources/Testing/Traits/Trait.swift +++ b/Sources/Testing/Traits/Trait.swift @@ -12,53 +12,204 @@ /// test suite. /// /// The testing library defines a number of traits that can be added to test -/// functions and to test suites. Developers can define their own traits by -/// creating types that conform to ``TestTrait`` and/or ``SuiteTrait``. +/// functions and to test suites. Define your own traits by +/// creating types that conform to ``TestTrait`` or ``SuiteTrait``: /// -/// When creating a custom trait type, the type should conform to ``TestTrait`` -/// if it can be added to test functions, ``SuiteTrait`` if it can be added to -/// test suites, and both ``TestTrait`` and ``SuiteTrait`` if it can be added to -/// both test functions _and_ test suites. +/// - term ``TestTrait``: Conform to this type in traits that you add to test +/// functions. +/// - term ``SuiteTrait``: Conform to this type in traits that you add to test +/// suites. +/// +/// You can add a trait that conforms to both ``TestTrait`` and ``SuiteTrait`` +/// to test functions and test suites. public protocol Trait: Sendable { - /// Prepare to run the test to which this trait was added. + /// Prepare to run the test that has this trait. /// /// - Parameters: - /// - test: The test to which this trait was added. + /// - test: The test that has this trait. /// - /// - Throws: Any error that would prevent the test from running. If an error - /// is thrown from this method, the test will be skipped and the error will - /// be recorded as an ``Issue``. + /// - Throws: Any error that prevents the test from running. If an error + /// is thrown from this method, the test is skipped and the error is + /// recorded as an ``Issue``. /// - /// This method is called after all tests and their traits have been - /// discovered by the testing library, but before any test has begun running. - /// It may be used to prepare necessary internal state, or to influence + /// The testing library calls this method after it discovers all tests and + /// their traits, and before it begins to run any tests. + /// Use this method to prepare necessary internal state, or to determine /// whether the test should run. /// /// The default implementation of this method does nothing. func prepare(for test: Test) async throws - /// The user-provided comments for this trait, if any. + /// The user-provided comments for this trait. /// - /// By default, the value of this property is an empty array. + /// The default value of this property is an empty array. var comments: [Comment] { get } + + /// The type of the test scope provider for this trait. + /// + /// The default type is `Never`, which can't be instantiated. The + /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with + /// `Never` as its test scope provider type must return `nil`, meaning that + /// the trait doesn't provide a custom scope for tests it's applied to. + associatedtype TestScopeProvider: TestScoping = Never + + /// Get this trait's scope provider for the specified test and optional test + /// case. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which you + /// use to provide custom scoping for `test` or `testCase`. Returns `nil` if + /// the trait doesn't provide any custom scope for the test or test case. + /// + /// If this trait's type conforms to ``TestScoping``, the default value + /// returned by this method depends on the values of`test` and `testCase`: + /// + /// - If `test` represents a suite, this trait must conform to ``SuiteTrait``. + /// If the value of this suite trait's ``SuiteTrait/isRecursive`` property + /// is `true`, then this method returns `nil`, and the suite trait + /// provides its custom scope once for each test function the test suite + /// contains. If the value of ``SuiteTrait/isRecursive`` is `false`, this + /// method returns `self`, and the suite trait provides its custom scope + /// once for the entire test suite. + /// - If `test` represents a test function, this trait also conforms to + /// ``TestTrait``. If `testCase` is `nil`, this method returns `nil`; + /// otherwise, it returns `self`. This means that by default, a trait which + /// is applied to or inherited by a test function provides its custom scope + /// once for each of that function's cases. + /// + /// A trait may override this method to further customize the + /// default behaviors above. For example, if a trait needs to provide custom + /// test scope both once per-suite and once per-test function in that suite, + /// it implements the method to return a non-`nil` scope provider under + /// those conditions. + /// + /// A trait may also implement this method and return `nil` if it determines + /// that it does not need to provide a custom scope for a particular test at + /// runtime, even if the test has the trait applied. This can improve + /// performance and make diagnostics clearer by avoiding an unnecessary call + /// to ``TestScoping/provideScope(for:testCase:performing:)``. + /// + /// If this trait's type does not conform to ``TestScoping`` and its + /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then + /// this method returns `nil` by default. This means that instances of this + /// trait don't provide a custom scope for tests to which they're applied. + func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? } -/// A protocol describing traits that can be added to a test function. +/// A protocol that tells the test runner to run custom code before or after it +/// runs a test suite or test function. /// -/// The testing library defines a number of traits that can be added to test -/// functions. Developers can also define their own traits by creating types -/// that conform to this protocol and/or to the ``SuiteTrait`` protocol. +/// Provide custom scope for tests by implementing the +/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, returning a type that +/// conforms to this protocol. Create a custom scope to consolidate common +/// set-up and tear-down logic for tests which have similar needs, which allows +/// each test function to focus on the unique aspects of its test. +public protocol TestScoping: Sendable { + /// Provide custom execution scope for a function call which is related to the + /// specified test or test case. + /// + /// - Parameters: + /// - test: The test which `function` encapsulates. + /// - testCase: The test case, if any, which `function` encapsulates. + /// When invoked on a suite, the value of this argument is `nil`. + /// - function: The function to perform. If `test` represents a test suite, + /// this function encapsulates running all the tests in that suite. If + /// `test` represents a test function, this function is the body of that + /// test function (including all cases if the test function is + /// parameterized.) + /// + /// - Throws: Any error that `function` throws, or an error that prevents this + /// type from providing a custom scope correctly. The testing library + /// records an error thrown from this method as an issue associated with + /// `test`. If an error is thrown before this method calls `function`, the + /// corresponding test doesn't run. + /// + /// When the testing library prepares to run a test, it starts by finding + /// all traits applied to that test, including those inherited from containing + /// suites. It begins with inherited suite traits, sorting them + /// outermost-to-innermost, and if the test is a function, it then adds all + /// traits applied directly to that functions in the order they were applied + /// (left-to-right). It then asks each trait for its scope provider (if any) + /// by calling ``Trait/scopeProvider(for:testCase:)-cjmg``. Finally, it calls + /// this method on all non-`nil` scope providers, giving each an opportunity + /// to perform arbitrary work before or after invoking `function`. + /// + /// This method should either invoke `function` once before returning, + /// or throw an error if it's unable to provide a custom scope. + /// + /// Issues recorded by this method are associated with `test`. + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws +} + +extension Trait where Self: TestScoping { + /// Get this trait's scope provider for the specified test or test case. + /// + /// - Parameters: + /// - test: The test for which the testing library requests a + /// scope provider. + /// - testCase: The test case for which the testing library requests a scope + /// provider, if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// The testing library uses this implementation of + /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type conforms + /// to ``TestScoping``. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + testCase == nil ? nil : self + } +} + +extension SuiteTrait where Self: TestScoping { + /// Get this trait's scope provider for the specified test and optional test + /// case. + /// + /// - Parameters: + /// - test: The test for which the testing library requests a scope + /// provider. + /// - testCase: The test case for which the testing library requests a scope + /// provider, if any. When `test` represents a suite, the value of this + /// argument is `nil`. + /// + /// The testing library uses this implementation of + /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type conforms + /// to both ``SuiteTrait`` and ``TestScoping``. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + if test.isSuite { + isRecursive ? nil : self + } else { + testCase == nil ? nil : self + } + } +} + +extension Never: TestScoping { + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws {} +} + +/// A protocol describing a trait that you can add to a test function. +/// +/// The testing library defines a number of traits that you can add to test +/// functions. You can also define your own traits by creating types +/// that conform to this protocol, or to the ``SuiteTrait`` protocol. public protocol TestTrait: Trait {} -/// A protocol describing traits that can be added to a test suite. +/// A protocol describing a trait that you can add to a test suite. /// -/// The testing library defines a number of traits that can be added to test -/// suites. Developers can also define their own traits by creating types that -/// conform to this protocol and/or to the ``TestTrait`` protocol. +/// The testing library defines a number of traits that you can add to test +/// suites. You can also define your own traits by creating types that +/// conform to this protocol, or to the ``TestTrait`` protocol. public protocol SuiteTrait: Trait { /// Whether this instance should be applied recursively to child test suites - /// and test functions or should only be applied to the test suite to which it - /// was directly added. + /// and test functions. + /// + /// If the value is `true`, then the testing library applies this trait + /// recursively to child test suites and test functions. Otherwise, it only + /// applies the trait to the test suite to which you added the trait. /// /// By default, traits are not recursively applied to children. var isRecursive: Bool { get } @@ -72,43 +223,26 @@ extension Trait { } } +extension Trait where TestScopeProvider == Never { + /// Get this trait's scope provider for the specified test or test case. + /// + /// - Parameters: + /// - test: The test for which the testing library requests a + /// scope provider. + /// - testCase: The test case for which the testing library requests a scope + /// provider, if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// The testing library uses this implementation of + /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type's + /// associated ``Trait/TestScopeProvider`` type is `Never`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? { + nil + } +} + extension SuiteTrait { public var isRecursive: Bool { false } } - -/// A protocol extending ``Trait`` that offers an additional customization point -/// for trait authors to execute code before and after each test function (if -/// added to the traits of a test function), or before and after each test suite -/// (if added to the traits of a test suite). -@_spi(Experimental) -public protocol CustomExecutionTrait: Trait { - - /// Execute a function with the effects of this trait applied. - /// - /// - Parameters: - /// - function: The function to perform. If `test` represents a test suite, - /// this function encapsulates running all the tests in that suite. If - /// `test` represents a test function, this function is the body of that - /// test function (including all cases if it is parameterized.) - /// - test: The test under which `function` is being performed. - /// - testCase: The test case, if any, under which `function` is being - /// performed. This is `nil` when invoked on a suite. - /// - /// - Throws: Whatever is thrown by `function`, or an error preventing the - /// trait from running correctly. - /// - /// This function is called for each ``CustomExecutionTrait`` on a test suite - /// or test function and allows additional work to be performed before and - /// after the test runs. - /// - /// This function is invoked once for the test it is applied to, and then once - /// for each test case in that test, if applicable. - /// - /// Issues recorded by this function are recorded against `test`. - /// - /// - Note: If a test function or test suite is skipped, this function does - /// not get invoked by the runner. - func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws -} diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index c91620449..ad58fc35b 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -31,7 +31,7 @@ if(SwiftTesting_BuildMacrosAsExecutables) set(FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_d) FetchContent_Declare(SwiftSyntax GIT_REPOSITORY https://github.com/swiftlang/swift-syntax - GIT_TAG cb53fa1bd3219b0b23ded7dfdd3b2baff266fd25) # 600.0.0 + GIT_TAG 1cd35348b089ff8966588742c69727205d99f8ed) # 601.0.0-prerelease-2024-11-18 FetchContent_MakeAvailable(SwiftSyntax) endif() diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 341b27d7d..a077ab5bf 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -324,7 +324,53 @@ public struct NonOptionalRequireMacro: RefinedConditionMacro { in context: some MacroExpansionContext ) throws -> ExprSyntax { if let argument = macro.arguments.first { +#if !SWT_FIXED_137943258 + // Silence this warning if we see a token (`?`, `nil`, or "Optional") that + // might indicate the test author expects the expression is optional. + let tokenKindsIndicatingOptionality: [TokenKind] = [ + .infixQuestionMark, + .postfixQuestionMark, + .keyword(.nil), + .identifier("Optional") + ] + let looksOptional = argument.tokens(viewMode: .sourceAccurate).lazy + .map(\.tokenKind) + .contains(where: tokenKindsIndicatingOptionality.contains) + if !looksOptional { + context.diagnose(.nonOptionalRequireIsRedundant(argument.expression, in: macro)) + } +#else context.diagnose(.nonOptionalRequireIsRedundant(argument.expression, in: macro)) +#endif + } + + // Perform the normal macro expansion for #require(). + return try RequireMacro.expansion(of: macro, in: context) + } +} + +/// A type describing the expansion of the `#require(throws:)` macro. +/// +/// This macro makes a best effort to check if the type argument is `Never.self` +/// (as we only have the syntax tree here) and diagnoses it as redundant if so. +/// See also ``RequireThrowsNeverMacro`` which is used when full type checking +/// is contextually available. +/// +/// This type is otherwise exactly equivalent to ``RequireMacro``. +public struct RequireThrowsMacro: RefinedConditionMacro { + public typealias Base = RequireMacro + + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) throws -> ExprSyntax { + if let argument = macro.arguments.first { + let argumentTokens: [String] = argument.expression.tokens(viewMode: .fixedUp).lazy + .filter { $0.tokenKind != .period } + .map(\.textWithoutBackticks) + if argumentTokens == ["Swift", "Never", "self"] || argumentTokens == ["Never", "self"] { + context.diagnose(.requireThrowsNeverIsRedundant(argument.expression, in: macro)) + } } // Perform the normal macro expansion for #require(). diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index 3b193bb65..c9fb6bb08 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -19,6 +19,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { public static func expansion( of node: AttributeSyntax, providingMembersOf declaration: some DeclGroupSyntax, + conformingTo protocols: [TypeSyntax], in context: some MacroExpansionContext ) throws -> [DeclSyntax] { guard _diagnoseIssues(with: declaration, suiteAttribute: node, in: context) else { diff --git a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift index 4539ed04d..ca0137b5d 100644 --- a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift @@ -80,6 +80,34 @@ extension MacroExpansionContext { // MARK: - extension MacroExpansionContext { + /// Whether or not our generated warnings are suppressed in the current + /// lexical context. + /// + /// The value of this property is `true` if the current lexical context + /// contains a node with the `@_semantics("testing.macros.nowarnings")` + /// attribute applied to it. + /// + /// - Warning: This functionality is not part of the public interface of the + /// testing library. It may be modified or removed in a future update. + var areWarningsSuppressed: Bool { +#if DEBUG + for lexicalContext in self.lexicalContext { + guard let lexicalContext = lexicalContext.asProtocol((any WithAttributesSyntax).self) else { + continue + } + for attribute in lexicalContext.attributes { + if case let .attribute(attribute) = attribute, + attribute.attributeNameText == "_semantics", + case let .string(argument) = attribute.arguments, + argument.representedLiteralValue == "testing.macros.nowarnings" { + return true + } + } + } +#endif + return false + } + /// Emit a diagnostic message. /// /// - Parameters: @@ -87,23 +115,27 @@ extension MacroExpansionContext { /// arguments to `Diagnostic.init()` are derived from the message's /// `syntax` property. func diagnose(_ message: DiagnosticMessage) { - diagnose( - Diagnostic( - node: message.syntax, - position: message.syntax.positionAfterSkippingLeadingTrivia, - message: message, - fixIts: message.fixIts - ) - ) + diagnose(CollectionOfOne(message)) } /// Emit a sequence of diagnostic messages. /// /// - Parameters: /// - messages: The diagnostic messages to emit. - func diagnose(_ messages: some Sequence) { + func diagnose(_ messages: some Collection) { + lazy var areWarningsSuppressed = areWarningsSuppressed for message in messages { - diagnose(message) + if message.severity == .warning && areWarningsSuppressed { + continue + } + diagnose( + Diagnostic( + node: message.syntax, + position: message.syntax.positionAfterSkippingLeadingTrivia, + message: message, + fixIts: message.fixIts + ) + ) } } diff --git a/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift b/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift index e6e381e94..e1bd346ed 100644 --- a/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift @@ -54,6 +54,7 @@ extension TypeSyntaxProtocol { let nameWithoutGenericParameters = tokens(viewMode: .fixedUp) .prefix { $0.tokenKind != .leftAngle } .filter { $0.tokenKind != .period } + .filter { $0.tokenKind != .leftParen && $0.tokenKind != .rightParen } .map(\.textWithoutBackticks) .joined(separator: ".") diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index 7d8a83c20..420e216ad 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -321,65 +321,57 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { /// generic. /// - attribute: The `@Test` or `@Suite` attribute. /// - decl: The declaration in question (contained in `node`.) + /// - escapableNonConformance: The suppressed conformance to `Escapable` for + /// `decl`, if present. /// /// - Returns: A diagnostic message. - static func containingNodeUnsupported(_ node: some SyntaxProtocol, genericBecauseOf genericClause: Syntax? = nil, whenUsing attribute: AttributeSyntax, on decl: some SyntaxProtocol) -> Self { + static func containingNodeUnsupported(_ node: some SyntaxProtocol, genericBecauseOf genericClause: Syntax? = nil, whenUsing attribute: AttributeSyntax, on decl: some SyntaxProtocol, withSuppressedConformanceToEscapable escapableNonConformance: SuppressedTypeSyntax? = nil) -> Self { // Avoid using a syntax node from a lexical context (it won't have source // location information.) let syntax: Syntax = if let genericClause, attribute.root == genericClause.root { // Prefer the generic clause if available as the root cause. genericClause + } else if let escapableNonConformance, attribute.root == escapableNonConformance.root { + // Then the ~Escapable conformance if present. + Syntax(escapableNonConformance) } else if attribute.root == node.root { - // Second choice is the unsupported containing node. + // Next best choice is the unsupported containing node. Syntax(node) } else { // Finally, fall back to the attribute, which we assume is not detached. Syntax(attribute) } + + // Figure out the message to present. + var message = "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true))" let generic = if genericClause != nil { " generic" } else { "" } if let functionDecl = node.as(FunctionDeclSyntax.self) { - let functionName = functionDecl.completeName - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within\(generic) function '\(functionName)'", - severity: .error - ) + message += " within\(generic) function '\(functionDecl.completeName)'" } else if let namedDecl = node.asProtocol((any NamedDeclSyntax).self) { - let declName = namedDecl.name.textWithoutBackticks - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within\(generic) \(_kindString(for: node)) '\(declName)'", - severity: .error - ) + message += " within\(generic) \(_kindString(for: node)) '\(namedDecl.name.textWithoutBackticks)'" } else if let extensionDecl = node.as(ExtensionDeclSyntax.self) { // Subtly different phrasing from the NamedDeclSyntax case above. - let nodeKind = if genericClause != nil { - "a generic extension to type" + if genericClause != nil { + message += " within a generic extension to type '\(extensionDecl.extendedType.trimmedDescription)'" } else { - "an extension to type" + message += " within an extension to type '\(extensionDecl.extendedType.trimmedDescription)'" } - let declGroupName = extensionDecl.extendedType.trimmedDescription - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within \(nodeKind) '\(declGroupName)'", - severity: .error - ) } else { - let nodeKind = if genericClause != nil { - "a generic \(_kindString(for: node))" + if genericClause != nil { + message += " within a generic \(_kindString(for: node))" } else { - _kindString(for: node, includeA: true) + message += " within \(_kindString(for: node, includeA: true))" } - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within \(nodeKind)", - severity: .error - ) } + if escapableNonConformance != nil { + message += " because its conformance to 'Escapable' has been suppressed" + } + + return Self(syntax: syntax, message: message, severity: .error) } /// Create a diagnostic message stating that the given attribute cannot be diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 0726aa099..5ffb3dc33 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -61,14 +61,15 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } // Check if the lexical context is appropriate for a suite or test. - diagnostics += diagnoseIssuesWithLexicalContext(context.lexicalContext, containing: declaration, attribute: testAttribute) + let lexicalContext = context.lexicalContext + diagnostics += diagnoseIssuesWithLexicalContext(lexicalContext, containing: declaration, attribute: testAttribute) // Suites inheriting from XCTestCase are not supported. We are a bit // conservative here in this check and only check the immediate context. // Presumably, if there's an intermediate lexical context that is *not* a // type declaration, then it must be a function or closure (disallowed // elsewhere) and thus the test function is not a member of any type. - if let containingTypeDecl = context.lexicalContext.first?.asProtocol((any DeclGroupSyntax).self), + if let containingTypeDecl = lexicalContext.first?.asProtocol((any DeclGroupSyntax).self), containingTypeDecl.inherits(fromTypeNamed: "XCTestCase", inModuleNamed: "XCTest") { diagnostics.append(.containingNodeUnsupported(containingTypeDecl, whenUsing: testAttribute, on: declaration)) } @@ -118,6 +119,21 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } } + // Disallow non-escapable types as suites. In order to support them, the + // compiler team needs to finish implementing the lifetime dependency + // feature so that `init()`, ``__requiringTry()`, and `__requiringAwait()` + // can be correctly expressed. + if let containingType = lexicalContext.first?.asProtocol((any DeclGroupSyntax).self), + let inheritedTypes = containingType.inheritanceClause?.inheritedTypes { + let escapableNonConformances = inheritedTypes + .map(\.type) + .compactMap { $0.as(SuppressedTypeSyntax.self) } + .filter { $0.type.isNamed("Escapable", inModuleNamed: "Swift") } + for escapableNonConformance in escapableNonConformances { + diagnostics.append(.containingNodeUnsupported(containingType, whenUsing: testAttribute, on: function, withSuppressedConformanceToEscapable: escapableNonConformance)) + } + } + return !diagnostics.lazy.map(\.severity).contains(.error) } @@ -317,7 +333,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } FunctionParameterSyntax( firstName: .wildcardToken(), - type: "isolated (any Actor)?" as TypeSyntax, + type: "isolated (any _Concurrency.Actor)?" as TypeSyntax, defaultValue: InitializerClauseSyntax(value: "Testing.__defaultSynchronousIsolationContext" as ExprSyntax) ) } diff --git a/Sources/TestingMacros/TestingMacrosMain.swift b/Sources/TestingMacros/TestingMacrosMain.swift index ebc62d660..c6904a6e7 100644 --- a/Sources/TestingMacros/TestingMacrosMain.swift +++ b/Sources/TestingMacros/TestingMacrosMain.swift @@ -24,6 +24,7 @@ struct TestingMacrosMain: CompilerPlugin { RequireMacro.self, AmbiguousRequireMacro.self, NonOptionalRequireMacro.self, + RequireThrowsMacro.self, RequireThrowsNeverMacro.self, ExitTestExpectMacro.self, ExitTestRequireMacro.self, diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index 7ede6233c..e7307476d 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -352,8 +352,26 @@ struct ConditionMacroTests { #expect(diagnostic.message.contains("is redundant")) } +#if !SWT_FIXED_137943258 + @Test( + "#require(optional value mistyped as non-optional) diagnostic is suppressed", + .bug("https://github.com/swiftlang/swift/issues/79202"), + arguments: [ + "#requireNonOptional(expression as? T)", + "#requireNonOptional(expression as Optional)", + "#requireNonOptional(expression ?? nil)", + ] + ) + func requireNonOptionalDiagnosticSuppressed(input: String) throws { + let (_, diagnostics) = try parse(input) + #expect(diagnostics.isEmpty) + } +#endif + @Test("#require(throws: Never.self) produces a diagnostic", arguments: [ + "#requireThrows(throws: Swift.Never.self)", + "#requireThrows(throws: Never.self)", "#requireThrowsNever(throws: Never.self)", ] ) diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index c75166c66..6fbadf2ec 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -140,6 +140,12 @@ struct TestDeclarationMacroTests { "Attribute 'Test' cannot be applied to a function within a generic extension to type 'T!'", "extension T! { @Suite struct S {} }": "Attribute 'Suite' cannot be applied to a structure within a generic extension to type 'T!'", + "struct S: ~Escapable { @Test func f() {} }": + "Attribute 'Test' cannot be applied to a function within structure 'S' because its conformance to 'Escapable' has been suppressed", + "struct S: ~Swift.Escapable { @Test func f() {} }": + "Attribute 'Test' cannot be applied to a function within structure 'S' because its conformance to 'Escapable' has been suppressed", + "struct S: ~(Escapable) { @Test func f() {} }": + "Attribute 'Test' cannot be applied to a function within structure 'S' because its conformance to 'Escapable' has been suppressed", ] ) func apiMisuseErrors(input: String, expectedMessage: String) throws { diff --git a/Tests/TestingMacrosTests/TestSupport/Parse.swift b/Tests/TestingMacrosTests/TestSupport/Parse.swift index fcb0215bc..e6b36e3b2 100644 --- a/Tests/TestingMacrosTests/TestSupport/Parse.swift +++ b/Tests/TestingMacrosTests/TestSupport/Parse.swift @@ -23,6 +23,7 @@ fileprivate let allMacros: [String: any Macro.Type] = [ "require": RequireMacro.self, "requireAmbiguous": AmbiguousRequireMacro.self, // different name needed only for unit testing "requireNonOptional": NonOptionalRequireMacro.self, // different name needed only for unit testing + "requireThrows": RequireThrowsMacro.self, // different name needed only for unit testing "requireThrowsNever": RequireThrowsNeverMacro.self, // different name needed only for unit testing "expectExitTest": ExitTestRequireMacro.self, // different name needed only for unit testing "requireExitTest": ExitTestRequireMacro.self, // different name needed only for unit testing diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 53fe92b84..081317197 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -932,6 +932,77 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } + func testErrorCheckingWithExpect_ResultValue() throws { + let error = #expect(throws: MyDescriptiveError.self) { + throw MyDescriptiveError(description: "abc123") + } + #expect(error?.description == "abc123") + } + + func testErrorCheckingWithRequire_ResultValue() async throws { + let error = try #require(throws: MyDescriptiveError.self) { + throw MyDescriptiveError(description: "abc123") + } + #expect(error.description == "abc123") + } + + func testErrorCheckingWithExpect_ResultValueIsNever() async throws { + let error: Never? = #expect(throws: Never.self) { + throw MyDescriptiveError(description: "abc123") + } + #expect(error == nil) + } + + func testErrorCheckingWithRequire_ResultValueIsNever() async throws { + let errorCaught = expectation(description: "Error caught") + errorCaught.isInverted = true + let apiMisused = expectation(description: "API misused") + let expectationFailed = expectation(description: "Expectation failed") + expectationFailed.isInverted = true + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + if case .errorCaught = issue.kind { + errorCaught.fulfill() + } else if case .apiMisused = issue.kind { + apiMisused.fulfill() + } else { + expectationFailed.fulfill() + } + } + + await Test { + func f(_ type: E.Type) throws -> E where E: Error { + try #require(throws: type) {} + } + try f(Never.self) + }.run(configuration: configuration) + + await fulfillment(of: [errorCaught, apiMisused, expectationFailed], timeout: 0.0) + } + + @_semantics("testing.macros.nowarnings") + func testErrorCheckingWithRequire_ResultValueIsNever_VariousSyntaxes() throws { + // Basic expressions succeed and don't diagnose. + #expect(throws: Never.self) {} + try #require(throws: Never.self) {} + + // Casting to specific types succeeds and doesn't diagnose. + let _: Void = try #require(throws: Never.self) {} + let _: Any = try #require(throws: Never.self) {} + + // Casting to any Error throws an API misuse error because Never cannot be + // instantiated. NOTE: inner function needed for lexical context. + @_semantics("testing.macros.nowarnings") + func castToAnyError() throws { + let _: any Error = try #require(throws: Never.self) {} + } + #expect(throws: APIMisuseError.self, performing: castToAnyError) + } + func testFail() async throws { var configuration = Configuration() configuration.eventHandler = { event, _ in diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 02f2cc768..3177969ee 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -137,6 +137,11 @@ struct SendableTests: Sendable { @Suite("Named Sendable test type", .hidden) struct NamedSendableTests: Sendable {} +// This is meant to help detect unqualified usages of the `Actor` protocol from +// Swift's `_Concurrency` module in macro expansion code, since it's possible +// for another module to declare a type with that name. +private class Actor {} + #if !SWT_NO_GLOBAL_ACTORS @Suite(.hidden) @MainActor diff --git a/Tests/TestingTests/Traits/CustomExecutionTraitTests.swift b/Tests/TestingTests/Traits/CustomExecutionTraitTests.swift deleted file mode 100644 index aedc06de3..000000000 --- a/Tests/TestingTests/Traits/CustomExecutionTraitTests.swift +++ /dev/null @@ -1,104 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing - -@Suite("CustomExecutionTrait Tests") -struct CustomExecutionTraitTests { - @Test("Execute code before and after a non-parameterized test.") - func executeCodeBeforeAndAfterNonParameterizedTest() async { - // `expectedCount` is 2 because we run it both for the test and the test case - await confirmation("Code was run before the test", expectedCount: 2) { before in - await confirmation("Code was run after the test", expectedCount: 2) { after in - await Test(CustomTrait(before: before, after: after)) { - // do nothing - }.run() - } - } - } - - @Test("Execute code before and after a parameterized test.") - func executeCodeBeforeAndAfterParameterizedTest() async { - // `expectedCount` is 3 because we run it both for the test and each test case - await confirmation("Code was run before the test", expectedCount: 3) { before in - await confirmation("Code was run after the test", expectedCount: 3) { after in - await Test(CustomTrait(before: before, after: after), arguments: ["Hello", "World"]) { _ in - // do nothing - }.run() - } - } - } - - @Test("Custom execution trait throws an error") - func customExecutionTraitThrowsAnError() async throws { - var configuration = Configuration() - await confirmation("Error thrown", expectedCount: 1) { errorThrownConfirmation in - configuration.eventHandler = { event, _ in - guard case let .issueRecorded(issue) = event.kind, - case let .errorCaught(error) = issue.kind else { - return - } - - #expect(error is CustomThrowingErrorTrait.CustomTraitError) - errorThrownConfirmation() - } - - await Test(CustomThrowingErrorTrait()) { - // Make sure this does not get reached - Issue.record("Expected trait to fail the test. Should not have reached test body.") - }.run(configuration: configuration) - } - } - - @Test("Teardown occurs after child tests run") - func teardownOccursAtEnd() async throws { - await runTest(for: TestsWithCustomTraitWithStrongOrdering.self, configuration: .init()) - } -} - -// MARK: - Fixtures - -private struct CustomTrait: CustomExecutionTrait, TestTrait { - var before: Confirmation - var after: Confirmation - func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws { - before() - defer { - after() - } - try await function() - } -} - -private struct CustomThrowingErrorTrait: CustomExecutionTrait, TestTrait { - fileprivate struct CustomTraitError: Error {} - - func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws { - throw CustomTraitError() - } -} - -struct DoSomethingBeforeAndAfterTrait: CustomExecutionTrait, SuiteTrait, TestTrait { - static let state = Locked(rawValue: 0) - - func execute(_ function: @Sendable () async throws -> Void, for test: Testing.Test, testCase: Testing.Test.Case?) async throws { - #expect(Self.state.increment() == 1) - - try await function() - #expect(Self.state.increment() == 3) - } -} - -@Suite(.hidden, DoSomethingBeforeAndAfterTrait()) -struct TestsWithCustomTraitWithStrongOrdering { - @Test(.hidden) func f() async { - #expect(DoSomethingBeforeAndAfterTrait.state.increment() == 2) - } -} diff --git a/Tests/TestingTests/Traits/TagListTests.swift b/Tests/TestingTests/Traits/TagListTests.swift index 29b8e3909..1ec8d1248 100644 --- a/Tests/TestingTests/Traits/TagListTests.swift +++ b/Tests/TestingTests/Traits/TagListTests.swift @@ -171,7 +171,7 @@ struct TagListTests { func noTagColorsReadFromBadPath(tagColorJSON: String) throws { var tagColorJSON = tagColorJSON tagColorJSON.withUTF8 { tagColorJSON in - #expect(throws: (any Error).self) { + _ = #expect(throws: (any Error).self) { _ = try JSON.decode(Tag.Color.self, from: .init(tagColorJSON)) } } diff --git a/Tests/TestingTests/Traits/TestScopingTraitTests.swift b/Tests/TestingTests/Traits/TestScopingTraitTests.swift new file mode 100644 index 000000000..af63deb5e --- /dev/null +++ b/Tests/TestingTests/Traits/TestScopingTraitTests.swift @@ -0,0 +1,184 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("TestScoping-conforming Trait Tests") +struct TestScopingTraitTests { + @Test("Execute code before and after a non-parameterized test.") + func executeCodeBeforeAndAfterNonParameterizedTest() async { + await confirmation("Code was run before the test") { before in + await confirmation("Code was run after the test") { after in + await Test(CustomTrait(before: before, after: after)) { + // do nothing + }.run() + } + } + } + + @Test("Execute code before and after a parameterized test.") + func executeCodeBeforeAndAfterParameterizedTest() async { + // `expectedCount` is 2 because we run it for each test case + await confirmation("Code was run before the test", expectedCount: 2) { before in + await confirmation("Code was run after the test", expectedCount: 2) { after in + await Test(CustomTrait(before: before, after: after), arguments: ["Hello", "World"]) { _ in + // do nothing + }.run() + } + } + } + + @Test("Custom execution trait throws an error") + func customExecutionTraitThrowsAnError() async throws { + var configuration = Configuration() + await confirmation("Error thrown", expectedCount: 1) { errorThrownConfirmation in + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind, + case let .errorCaught(error) = issue.kind else { + return + } + + #expect(error is CustomThrowingErrorTrait.CustomTraitError) + errorThrownConfirmation() + } + + await Test(CustomThrowingErrorTrait()) { + // Make sure this does not get reached + Issue.record("Expected trait to fail the test. Should not have reached test body.") + }.run(configuration: configuration) + } + } + + @Test("Teardown occurs after child tests run") + func teardownOccursAtEnd() async throws { + await runTest(for: TestsWithCustomTraitWithStrongOrdering.self, configuration: .init()) + } + + struct ExecutionControl { + @Test("Trait applied directly to function is executed once") + func traitAppliedToFunction() async { + let counter = Locked(rawValue: 0) + await DefaultExecutionTrait.$counter.withValue(counter) { + await Test(DefaultExecutionTrait()) {}.run() + } + #expect(counter.rawValue == 1) + } + + @Test("Non-recursive suite trait with default scope provider implementation") + func nonRecursiveSuiteTrait() async { + let counter = Locked(rawValue: 0) + await DefaultExecutionTrait.$counter.withValue(counter) { + await runTest(for: SuiteWithNonRecursiveDefaultExecutionTrait.self) + } + #expect(counter.rawValue == 1) + } + + @Test("Recursive suite trait with default scope provider implementation") + func recursiveSuiteTrait() async { + let counter = Locked(rawValue: 0) + await DefaultExecutionTrait.$counter.withValue(counter) { + await runTest(for: SuiteWithRecursiveDefaultExecutionTrait.self) + } + #expect(counter.rawValue == 1) + } + + @Test("Recursive, all-inclusive suite trait") + func recursiveAllInclusiveSuiteTrait() async { + let counter = Locked(rawValue: 0) + await AllInclusiveExecutionTrait.$counter.withValue(counter) { + await runTest(for: SuiteWithAllInclusiveExecutionTrait.self) + } + #expect(counter.rawValue == 3) + } + } +} + +// MARK: - Fixtures + +private struct CustomTrait: TestTrait, TestScoping { + var before: Confirmation + var after: Confirmation + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + before() + defer { + after() + } + try await function() + } +} + +private struct CustomThrowingErrorTrait: TestTrait, TestScoping { + fileprivate struct CustomTraitError: Error {} + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + throw CustomTraitError() + } +} + +struct DoSomethingBeforeAndAfterTrait: SuiteTrait, TestTrait, TestScoping { + static let state = Locked(rawValue: 0) + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + #expect(Self.state.increment() == 1) + + try await function() + #expect(Self.state.increment() == 3) + } +} + +@Suite(.hidden, DoSomethingBeforeAndAfterTrait()) +struct TestsWithCustomTraitWithStrongOrdering { + @Test(.hidden) func f() async { + #expect(DoSomethingBeforeAndAfterTrait.state.increment() == 2) + } +} + +private struct DefaultExecutionTrait: SuiteTrait, TestTrait, TestScoping { + @TaskLocal static var counter: Locked? + var isRecursive: Bool = false + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + Self.counter!.increment() + try await function() + } +} + +@Suite(.hidden, DefaultExecutionTrait()) +private struct SuiteWithNonRecursiveDefaultExecutionTrait { + @Test func f() {} +} + +@Suite(.hidden, DefaultExecutionTrait(isRecursive: true)) +private struct SuiteWithRecursiveDefaultExecutionTrait { + @Test func f() {} +} + +private struct AllInclusiveExecutionTrait: SuiteTrait, TestTrait, TestScoping { + @TaskLocal static var counter: Locked? + + var isRecursive: Bool { + true + } + + func scopeProvider(for test: Test, testCase: Test.Case?) -> AllInclusiveExecutionTrait? { + // Unconditionally returning self makes this trait "all inclusive". + self + } + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + Self.counter!.increment() + try await function() + } +} + +@Suite(.hidden, AllInclusiveExecutionTrait()) +private struct SuiteWithAllInclusiveExecutionTrait { + @Test func f() {} +} diff --git a/cmake/modules/LibraryVersion.cmake b/cmake/modules/LibraryVersion.cmake index 720d2c85c..61326fbc0 100644 --- a/cmake/modules/LibraryVersion.cmake +++ b/cmake/modules/LibraryVersion.cmake @@ -8,7 +8,7 @@ # The current version of the Swift Testing release. For release branches, # remember to remove -dev. -set(SWT_TESTING_LIBRARY_VERSION "6.1-dev") +set(SWT_TESTING_LIBRARY_VERSION "6.1.2") find_package(Git QUIET) if(Git_FOUND)