Run a REPL session for nickel #repl snippets (tweag#1615)

* Run a REPL session when checking `nickel #repl` snippets

* Adjust examples in the manual for new checks

* Fix some leftover error messages

* Address code review

core/src/cache.rs

@@ -283,7 +283,7 @@ impl From<SourcePath> for OsString {
             SourcePath::Std(StdlibModule::Std) => "<stdlib/std.ncl>".into(),
             SourcePath::Std(StdlibModule::Internals) => "<stdlib/internals.ncl>".into(),
             SourcePath::Query => "<query>".into(),
-            SourcePath::ReplInput(idx) => format!("<repl-input-{idx}").into(),
+            SourcePath::ReplInput(idx) => format!("<repl-input-{idx}>").into(),
             SourcePath::ReplTypecheck => "<repl-typecheck>".into(),
             SourcePath::ReplQuery => "<repl-query>".into(),
             SourcePath::Override(path) => format!("<override {}>", path.join(".")).into(),

core/src/error.rs

@@ -7,7 +7,7 @@ use crate::cache::Cache;
 pub use codespan::{FileId, Files};
 pub use codespan_reporting::diagnostic::{Diagnostic, Label, LabelStyle};
 
-use codespan_reporting::term::termcolor::{ColorChoice, StandardStream};
+use codespan_reporting::term::termcolor::{ColorChoice, StandardStream, WriteColor};
 use lalrpop_util::ErrorRecovery;
 use malachite::num::conversion::traits::ToSci;
 
@@ -2287,23 +2287,27 @@ impl From<ColorOpt> for ColorChoice {
 /// infrastructure to point at specific locations and print snippets when needed.
 pub fn report<E: IntoDiagnostics<FileId>>(cache: &mut Cache, error: E, color_opt: ColorOpt) {
     let stdlib_ids = cache.get_all_stdlib_modules_file_id();
-    report_with(cache.files_mut(), stdlib_ids.as_ref(), error, color_opt)
+    report_with(
+        &mut StandardStream::stderr(color_opt.into()).lock(),
+        cache.files_mut(),
+        stdlib_ids.as_ref(),
+        error,
+    )
 }
 
 /// Report an error on `stderr`, provided a file database and a list of stdlib file ids.
-fn report_with<E: IntoDiagnostics<FileId>>(
+pub fn report_with<E: IntoDiagnostics<FileId>>(
+    writer: &mut dyn WriteColor,
     files: &mut Files<String>,
     stdlib_ids: Option<&Vec<FileId>>,
     error: E,
-    color_opt: ColorOpt,
 ) {
-    let writer = StandardStream::stderr(color_opt.into());
     let config = codespan_reporting::term::Config::default();
     let diagnostics = error.into_diagnostics(files, stdlib_ids);
 
     let result = diagnostics
         .iter()
-        .try_for_each(|d| codespan_reporting::term::emit(&mut writer.lock(), &config, files, d));
+        .try_for_each(|d| codespan_reporting::term::emit(writer, &config, files, d));
 
     match result {
         Ok(()) => (),

core/src/repl/mod.rs

@@ -43,6 +43,7 @@ pub mod simple_frontend;
 pub mod wasm_frontend;
 
 /// Result of the evaluation of an input.
+#[derive(Debug, Clone)]
 pub enum EvalResult {
     /// The input has been evaluated to a term.
     Evaluated(RichTerm),

core/tests/manual/main.rs

@@ -1,11 +1,18 @@
-use std::{fs::File, io::read_to_string};
+use std::{fs::File, io::read_to_string, iter::once};
 
+use codespan_reporting::term::termcolor::NoColor;
 use comrak::{
     arena_tree::{Node, NodeEdge},
     nodes::{Ast, AstNode, NodeCodeBlock, NodeValue},
     parse_document, ComrakOptions,
 };
+use nickel_lang_core::{
+    error,
+    eval::cache::CacheImpl,
+    repl::{EvalResult, Repl, ReplImpl},
+};
 use nickel_lang_utils::{project_root::project_root, test_program};
+use pretty_assertions::assert_str_eq;
 use test_generator::test_resources;
 use typed_arena::Arena;
 
@@ -28,10 +35,17 @@ enum CodeBlockType {
     /// > std.function.id 5
     /// 5
     /// ```
-    /// We interpret lines starting with `> ` directly after an empty line
-    /// and followed by indented lines (starting with at least one `' '`) as
-    /// a single Nickel program and try to parse them. For now, we don't check
-    /// evaluation. REPL inputs must be separated by an empty line.
+    /// We start a separate REPL session for each such block and interpret lines
+    /// starting with `> ` directly after an empty line and followed by indented
+    /// lines (starting with at least one `' '`) as a single REPL input. In
+    /// particular REPL inputs must be separated by an empty line.
+    ///
+    /// Any text following a REPL input directly, without containing an empty
+    /// line, is interpreted as an expected result. If it starts with `error:`,
+    /// we expect the evaluation to produce an error and match the error report
+    /// with the expected result. A final `[...]` means that the expected
+    /// result is merely a prefix of the error report. If it doesn't start with
+    /// `error:`, the evaluation is expected to succeed.
     Repl,
 }
 
@@ -53,14 +67,103 @@ impl CodeBlockType {
     }
 }
 
-fn check_repl_parts(content: String) {
+#[derive(Debug, Clone)]
+enum MessageExpectation {
+    Full(String),
+    Abridged(String),
+}
+
+#[derive(Debug, Clone)]
+enum ReplResult {
+    Value(String),
+    Error(MessageExpectation),
+    Empty,
+}
+
+fn extract_repl_piece(piece: impl AsRef<str>) -> (String, ReplResult) {
+    let lines: Vec<&str> = piece.as_ref().split_inclusive('\n').collect();
+    let split_index = lines
+        .iter()
+        .position(|l| !l.starts_with(' '))
+        .unwrap_or(lines.len());
+    let (program_lines, result_lines) = lines.split_at(split_index);
+
+    let result_string = result_lines.concat();
+    let result = if result_string.is_empty() {
+        ReplResult::Empty
+    } else if result_string.starts_with("error:") {
+        if let Some((result_string, _)) = result_string.rsplit_once("[...]") {
+            ReplResult::Error(MessageExpectation::Abridged(result_string.to_owned()))
+        } else {
+            ReplResult::Error(MessageExpectation::Full(result_string))
+        }
+    } else {
+        ReplResult::Value(result_string)
+    };
+
+    (program_lines.concat(), result)
+}
+
+/// Assert that two strings are equal except for possible trailing whitespace.
+/// The error reporting by `codespan` sometimes produces trailing whitespace
+/// which will be removed from the documentation markdown files.
+#[track_caller]
+fn assert_str_eq_approx(actual: impl AsRef<str>, expected: impl AsRef<str>) {
+    assert_str_eq!(
+        actual
+            .as_ref()
+            .lines()
+            .flat_map(|l| once(l.trim_end()).chain(once("\n")))
+            .collect::<String>()
+            .trim_end(),
+        expected.as_ref().trim_end()
+    );
+}
+
+#[track_caller]
+fn assert_prefix(actual: impl AsRef<str>, prefix: impl AsRef<str>) {
+    assert!(
+        actual.as_ref().starts_with(prefix.as_ref()),
+        "{} was expected to be a prefix of {}",
+        prefix.as_ref(),
+        actual.as_ref()
+    );
+}
+
+fn check_error_report(actual: impl AsRef<str>, expected: MessageExpectation) {
+    match expected {
+        MessageExpectation::Full(expected) => assert_str_eq_approx(actual, expected),
+        MessageExpectation::Abridged(prefix) => assert_prefix(actual, prefix),
+    }
+}
+
+fn check_repl(content: String) {
+    let mut repl = ReplImpl::<CacheImpl>::new(std::io::sink());
+    repl.load_stdlib().unwrap();
     for piece in content.split("\n\n") {
+        // We only process `piece`s starting with `>`. This way we can make the
+        // testing code ignore unknown REPL statements, e.g. `:query`.
         if let Some(piece) = piece.strip_prefix('>') {
-            let program = piece
-                .split_inclusive('\n')
-                .take_while(|l| l.starts_with(' '))
-                .collect();
-            check_parse_extended(program);
+            let (input, result) = extract_repl_piece(piece);
+
+            eprintln!(">{input}"); // Print the input to stderr to make tracking test failures easier
+            match (repl.eval_full(&input), result) {
+                (Ok(EvalResult::Evaluated(rt)), ReplResult::Value(expected)) => {
+                    assert_str_eq_approx(format!("{rt}"), expected)
+                }
+                (Ok(EvalResult::Bound(_)), ReplResult::Empty) => (),
+                (Err(e), ReplResult::Error(expected)) => {
+                    let mut error = NoColor::new(Vec::<u8>::new());
+                    let stdlib_ids = repl.cache_mut().get_all_stdlib_modules_file_id();
+                    let files = repl.cache_mut().files_mut();
+                    error::report_with(&mut error, files, stdlib_ids.as_ref(), e);
+
+                    check_error_report(String::from_utf8(error.into_inner()).unwrap(), expected);
+                }
+                (actual, expected) => {
+                    panic!("Evaluation produced {actual:?}, expected {expected:?}");
+                }
+            }
         }
     }
 }
@@ -70,11 +173,6 @@ fn check_eval(program: String) {
     test_program::eval(program).unwrap();
 }
 
-fn check_parse_extended(program: String) {
-    eprintln!("{program}"); // Print the program to stderr to make tracking test failures easier
-    test_program::parse_extended(&program).unwrap();
-}
-
 fn check_parse(program: String) {
     eprintln!("{program}"); // Print the program to stderr to make tracking test failures easier
     test_program::parse(&program).unwrap();
@@ -90,7 +188,7 @@ impl CodeBlock {
                 }
             }
             CodeBlockType::Parse => check_parse(self.content),
-            CodeBlockType::Repl => check_repl_parts(self.content),
+            CodeBlockType::Repl => check_repl(self.content),
         }
     }
 }