Skip to content

Commit

Permalink
Simplify annotated val file testing (hylo-lang#206)
Browse files Browse the repository at this point in the history
  • Loading branch information
dabrahams authored Jan 6, 2023
1 parent fdc5a57 commit e4980de
Show file tree
Hide file tree
Showing 11 changed files with 188 additions and 226 deletions.
4 changes: 4 additions & 0 deletions Sources/Core/Diagnostic.swift
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,7 @@ extension Diagnostic: CustomStringConvertible {
}

}

// This should perhaps be a first-class type someday.
/// A record of diagnostics produced by processing one or more Val files.
public typealias DiagnosticLog = [Diagnostic]
2 changes: 1 addition & 1 deletion Sources/FrontEnd/AST+Extensions.swift
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ extension AST {
return true
}
} catch let error as DiagnosedError {
fatalError(error.diagnostics.first!.description)
fatalError("\(list: error.diagnostics, joinedBy: "\n")")
} catch let error {
fatalError(error.localizedDescription)
}
Expand Down
35 changes: 27 additions & 8 deletions Tests/ValTests/CXXTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,27 @@ final class CXXTests: XCTestCase {

try checkAnnotatedValFiles(
in: "TestCases/CXX",
{ (source) -> CXXAnnotationHandler in
checkingAnnotationCommands: ["cpp", "h"],

{ (source, cxxAnnotations, diagnostics) in
// Create a module for the input.
var ast = baseAST
let module = try! ast.insert(wellFormed: ModuleDecl(name: source.baseName))

// Parse the input.
let parseResult = Parser.parse(source, into: module, in: &ast)
var diagnostics = parseResult.diagnostics
diagnostics += parseResult.diagnostics
if parseResult.failed {
return .init(module: nil, ranToCompletion: false, diagnostics: diagnostics)
throw DiagnosedError(diagnostics)
}

// Run the type checker.
var checker = TypeChecker(program: ScopedProgram(ast: ast))
diagnostics.append(contentsOf: checker.diagnostics)
if !checker.check(module: module) {
return .init(module: nil, ranToCompletion: false, diagnostics: diagnostics)
diagnostics += checker.diagnostics
let wellTyped = checker.check(module: module)
diagnostics += checker.diagnostics
if !wellTyped {
throw DiagnosedError(diagnostics)
}

let typedProgram = TypedProgram(
Expand All @@ -46,8 +50,23 @@ final class CXXTests: XCTestCase {
var transpiler = CXXTranspiler(program: typedProgram)
let cxxModule = transpiler.emit(module: typedProgram[module])

return .init(module: cxxModule, ranToCompletion: true, diagnostics: diagnostics)
let cxxHeader = cxxModule.emitHeader()
let cxxSource = cxxModule.emitSource()

return cxxAnnotations.compactMap { a in
let expectedCXX = a.argument!.removingTrailingNewlines()
let cxxSourceToSearch = a.command == "cpp" ? cxxSource : cxxHeader

return cxxSourceToSearch.contains(expectedCXX)
? nil
: a.failure(
"""
transpiled code not found:
\(expectedCXX)
--- not found in ---
\(cxxSourceToSearch)
""")
}
})
}

}
21 changes: 10 additions & 11 deletions Tests/ValTests/EmitterTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,26 @@ final class EmitterTests: XCTestCase {
var baseAST = AST()
baseAST.importCoreModule()

try checkAnnotatedValFiles(
try checkAnnotatedValFileDiagnostics(
in: "TestCases/Lowering",
{ (source) -> DefaultTestAnnotationHandler in
{ (source, diagnostics) in
// Create a module for the input.
var ast = baseAST
let module = try! ast.insert(wellFormed: ModuleDecl(name: source.baseName))

// Parse the input.
let parseResult = Parser.parse(source, into: module, in: &ast)
var diagnostics = parseResult.diagnostics
diagnostics += parseResult.diagnostics
if parseResult.failed {
return .init(ranToCompletion: false, diagnostics: diagnostics)
throw DiagnosedError(diagnostics)
}

// Run the type checker.
var checker = TypeChecker(program: ScopedProgram(ast: ast))
diagnostics.append(contentsOf: checker.diagnostics)
if !checker.check(module: module) {
return .init(ranToCompletion: false, diagnostics: diagnostics)
let wellTyped = checker.check(module: module)
diagnostics += checker.diagnostics
if !wellTyped {
throw DiagnosedError(diagnostics)
}

let typedProgram = TypedProgram(
Expand All @@ -54,12 +55,10 @@ final class EmitterTests: XCTestCase {
for i in 0 ..< pipeline.count {
for f in 0 ..< irModule.functions.count {
success = pipeline[i].run(function: f, module: &irModule) && success
diagnostics.append(contentsOf: pipeline[i].diagnostics)
diagnostics += pipeline[i].diagnostics
}
if !success { break }
if !success { throw DiagnosedError(diagnostics) }
}

return .init(ranToCompletion: success, diagnostics: diagnostics)
})
}

Expand Down
7 changes: 4 additions & 3 deletions Tests/ValTests/ParserTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,17 @@ import XCTest
final class ParserTests: XCTestCase {

func testParser() throws {
try checkAnnotatedValFiles(
try checkAnnotatedValFileDiagnostics(
in: "TestCases/Parsing",
{ (source) -> DefaultTestAnnotationHandler in
{ (source, diagnostics) in
// Create a module for the input.
var ast = AST()
let module = try! ast.insert(wellFormed: ModuleDecl(name: source.baseName))

// Parse the input.
let parseResult = Parser.parse(source, into: module, in: &ast)
return .init(ranToCompletion: !parseResult.failed, diagnostics: parseResult.diagnostics)
diagnostics += parseResult.diagnostics
if parseResult.failed { throw DiagnosedError(diagnostics) }
})
}

Expand Down
18 changes: 10 additions & 8 deletions Tests/ValTests/TypeCheckerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,28 @@ final class TypeCheckerTests: XCTestCase {
var baseAST = AST()
baseAST.importCoreModule()

try checkAnnotatedValFiles(
try checkAnnotatedValFileDiagnostics(
in: "TestCases/TypeChecking",
{ (source) -> DefaultTestAnnotationHandler in
{ (source, diagnostics) in
// Create a module for the input.
var ast = baseAST
let module = try! ast.insert(wellFormed: ModuleDecl(name: source.baseName))

// Parse the input.
let parseResult = Parser.parse(source, into: module, in: &ast)
diagnostics += parseResult.diagnostics
if parseResult.failed {
return .init(ranToCompletion: false, diagnostics: parseResult.diagnostics)
throw DiagnosedError(diagnostics)
}

// Run the type checker.
var checker = TypeChecker(program: ScopedProgram(ast: ast))
let success = checker.check(module: module)
return .init(
ranToCompletion: success,
diagnostics: parseResult.diagnostics + Array(checker.diagnostics))
diagnostics += checker.diagnostics
let wellTyped = checker.check(module: module)
diagnostics += checker.diagnostics
if !wellTyped {
throw DiagnosedError(diagnostics)
}
})
}

}
134 changes: 125 additions & 9 deletions Tests/ValTests/Utils/AnnotatedValFileTest.swift
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,64 @@ import Core
import Utils
import XCTest

extension Diagnostic {
/// A test annotation that announces `self` should be expected.
var expectation: TestAnnotation {
return TestAnnotation(
in: location?.source.url ?? URL(string: "nowhere://at/all")!,
atLine: location?.lineAndColumnIndices.line ?? 1,
parsing: "diagnostic " + message
)
}
}

extension XCTestCase {

/// Preprocesses the ".val" files in a directory using `preprocess`, checking results using the
/// returned `Handler` to handle annotations embedded in the file's comments.
func checkAnnotatedValFiles<Handler: TestAnnotationHandler>(
/// The effects of running the `processAndCheck` parameter to `checkAnnotatedValFiles`.
fileprivate typealias ProcessingEffects = (
ranToCompletion: Bool, testFailures: [XCTIssue], diagnostics: DiagnosticLog
)

/// Applies `process` to each ".val" file in `sourceDirectory` and reports XCTest failures where
/// the effects of processing don't match the file's diagnostic annotation commands ("diagnostic",
/// "expect-failure", and "expect-success").
///
/// - Parameter `sourceDirectory`: a path relative to the ValTests/ directory of this project.
/// - Parameter `process`: applies some compilation phases to `source`, updating `diagnostics`
/// with any generated diagnostics. Throws an `Error` if any phases failed.
func checkAnnotatedValFileDiagnostics(
in sourceDirectory: String,
_ process: (_ source: SourceFile, _ diagnostics: inout DiagnosticLog) throws -> Void
) throws {
try checkAnnotatedValFiles(
in: sourceDirectory, checkingAnnotationCommands: [],
{ (source, annotationsToHandle, diagnostics) in
assert(annotationsToHandle.isEmpty)
try process(source, &diagnostics)
return []
}
)
}

/// Applies `processAndCheck` to each ".val" file in `sourceDirectory` along with the subset of
/// that file's annotations whose commands match `checkedCommands`, and reports resulting XCTest
/// failures, along with any additional failures where the effects of processing don't match the
/// file's diagnostic annotation commands ("diagnostic", "expect-failure", and "expect-success").
///
/// - Parameters:
/// - `sourceDirectory`: a path relative to the ValTests/ directory of this project.
/// - `checkedCommands`: the annnotation commands to be validated by `processAndCheck`.
/// - `processAndCheck`: applies some compilation phases to `source`, updating `diagnostics`
/// with any generated diagnostics, then checks `annotationsToCheck` against the results,
/// returning corresponding test failures. Throws an `Error` if any phases failed.
func checkAnnotatedValFiles(
in sourceDirectory: String,
_ preprocess: (_ source: SourceFile) throws -> Handler
checkingAnnotationCommands checkedCommands: Set<String> = [],
_ processAndCheck: (
_ source: SourceFile,
_ annotationsToCheck: ArraySlice<TestAnnotation>,
_ diagnostics: inout DiagnosticLog
) throws -> [XCTIssue]
) throws {
let testCaseDirectory = try XCTUnwrap(
Bundle.module.url(forResource: sourceDirectory, withExtension: nil),
Expand All @@ -21,20 +72,85 @@ extension XCTestCase {
if url.pathExtension != "val" { return true }

let source = try SourceFile(contentsOf: url)
let issues = try XCTContext.runActivity(
var annotations = TestAnnotation.parseAll(from: source)

// Separate the annotations to be checked by default diagnostic annotation checking from
// those to be checked by `processAndCheck`.
let p = annotations.partition(by: { checkedCommands.contains($0.command) })
let (diagnosticAnnotations, processingAnnotations) = (annotations[..<p], annotations[p...])

var diagnostics = DiagnosticLog()
let failures = XCTContext.runActivity(
named: source.baseName,
block: { activity in
var handler = try preprocess(source)
return handler.handles(TestAnnotation.parseAll(from: source))
let completedProcessingTestFailures = try? processAndCheck(
source, processingAnnotations, &diagnostics)

return failuresToReport(
effectsOfProcessing: (
ranToCompletion: completedProcessingTestFailures != nil,
testFailures: completedProcessingTestFailures ?? [],
diagnostics: diagnostics
),
unhandledAnnotations: diagnosticAnnotations)
})

for issue in issues {
record(issue)
for f in failures {
record(f)
}

// Move to the next test case.
return true
})
}

/// Given the effects of processing and the annotations not specifically handled by
/// `processAndCheck` above, returns the final set of test failures to be reported to XCTest.
fileprivate func failuresToReport(
effectsOfProcessing processing: ProcessingEffects,
unhandledAnnotations: ArraySlice<TestAnnotation>
) -> [XCTIssue] {
var testFailures = processing.testFailures

// If this traps due to non-uniqueness of keys, we have two diagnostics that would be matched by
// the same expectation. We can adjust the code to deal with that if it comes up.
var diagnosticsByExpectation = Dictionary(
uniqueKeysWithValues:
zip(processing.diagnostics.lazy.map(\.expectation), processing.diagnostics))

func fail(_ expectation: TestAnnotation, _ message: String) {
testFailures.append(expectation.failure(message))
}

for a in unhandledAnnotations {
switch a.command {
case "diagnostic":
if let i = diagnosticsByExpectation.index(forKey: a) {
diagnosticsByExpectation.remove(at: i)
} else {
fail(a, "missing expected diagnostic\(a.argument.map({": '\($0)'"}) ?? "")")
}
case "expect-failure":
if processing.ranToCompletion {
fail(a, "processing succeeded, but failure was expected")
}
case "expect-success":
if !processing.ranToCompletion {
fail(a, "processing failed, but success was expected")
}
default:
fail(a, "unexpected test command: '\(a.command)'")
}
}

testFailures += diagnosticsByExpectation.values.lazy.map {
XCTIssue(
Diagnostic(
level: .error, message: "unexpected diagnostic: '\($0.message)'",
location: $0.location,
window: $0.window,
children: $0.children))
}
return testFailures
}
}
Loading

0 comments on commit e4980de

Please sign in to comment.