Commits: 16
Enable proper logging in the interpreter
Now the interpreter will be launched with an environment variable tbb_interpreter_log, containing a lowercase log level (trace, debug, info, warn, error, critical). Interpreters should setup their logging accordingly.
The control program (tbb) will set this variable to either "trace" (if --verbose flag was passed) or "warn" otherwise.
The sample basic interpreter is now using Python logging module to log diagnostic information. Since Python logging doesn't have the trace level, it needs to be coerced to debug.
index 2e9093f..4c67d2d 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -1,12 +1,18 @@
=#!/usr/bin/env python3
=
=import json
-import sys
-import fileinput
+import logging
+import os
+
+
+# Setup logging
+log_level = os.environ.get("tbb_interpreter_log", "warn").upper()
+logging.basicConfig(level=getattr(logging, log_level))
+log = logging.getLogger("basic_interpreter")
=
=def send(message):
= serialized = json.dumps(message)
- print("Interpreter sending:", serialized, file=sys.stderr)
+ log.debug("Sending: %s", serialized)
= print (serialized)
=
=# Send the ready message
@@ -14,11 +20,11 @@ send({ 'ready': True })
=
=# Loop over input
=while True:
- print("Interpreter awaiting message:", file=sys.stderr)
+ log.debug("Awaiting message...")
=
= try:
= received = input()
- print("Interpreter received:", received, file=sys.stderr)
+ log.debug("Received: %s", received)
= payload = json.loads(received)
= send({ 'ok': True })
= except EOFError:index 5ab9676..38737c7 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -22,7 +22,7 @@ struct Cli {
=fn main() -> Result<(), Box<dyn Error>> {
= let cli = Cli::parse();
= let log_env = env_logger::Env::default()
- .filter_or("RUST_LOG", if cli.verbose { "trace" } else { "info" });
+ .filter_or("RUST_LOG", if cli.verbose { "trace" } else { "warn" });
= env_logger::init_from_env(log_env);
=
= log::debug!("Reading the specification from {}", cli.input.display());
@@ -35,22 +35,22 @@ fn main() -> Result<(), Box<dyn Error>> {
= );
= let pattern = format!("{}/**/*.md", input.display());
= for path in glob(&pattern)? {
- process_document(path?)?;
+ process_document(path?, cli.verbose)?;
= }
= Ok(())
= } else if input.is_file() {
= log::debug!("The {} is a file. Reading...", input.display());
- process_document(input)
+ process_document(input, cli.verbose)
= } else {
= Err(format!("The {} is neither a file nor directory", input.display()).into())
= }
=}
=
-fn process_document(input: PathBuf) -> Result<(), Box<dyn Error>> {
+fn process_document(input: PathBuf, verbose: bool) -> Result<(), Box<dyn Error>> {
= log::debug!("Reading {}", input.display());
= let md = std::fs::read_to_string(input)?;
= let suite = spec::Suite::from_markdown(&md)?;
- log::info!("Suite:\n\n{:#?}", suite);
+ log::debug!("Suite:\n\n{:#?}", suite);
=
- suite.evaluate()
+ suite.evaluate(verbose)
=}index a482a02..9d2aaf8 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -33,7 +33,7 @@ pub struct Scenario {
=}
=
=impl Scenario {
- fn run(&self, interpreter: &str) -> Result<(), Box<dyn Error>> {
+ fn run(&self, interpreter: &str, verbose: bool) -> Result<(), Box<dyn Error>> {
= println!("\n {}", self.title);
= // TODO: start the interpreter program and tap into its stdio
= log::debug!("Starting the interpreter process: `{interpreter}`");
@@ -42,6 +42,10 @@ impl Scenario {
= .arg(interpreter)
= .stdin(Stdio::piped())
= .stdout(Stdio::piped())
+ .env(
+ "tbb_interpreter_log",
+ if verbose { "debug" } else { "warn" },
+ )
= .spawn()?;
=
= let input = process
@@ -155,16 +159,16 @@ impl Suite {
= },
= )
= .map_err(|message| message.to_string())?;
- log::info!("Markdown parsed:\n\n{:#?}", mdast);
+ log::debug!("Markdown parsed:\n\n{:#?}", mdast);
=
= Self::try_from(mdast)
= }
=
- pub fn evaluate(&self) -> Result<(), Box<dyn Error>> {
+ pub fn evaluate(&self, verbose: bool) -> Result<(), Box<dyn Error>> {
= println!("\n{}", self.title);
=
= for scenario in self.scenarios.iter() {
- scenario.run(&self.interpreter)?;
+ scenario.run(&self.interpreter, verbose)?;
= }
=
= log::info!("All good!");Implement two steps from the arithmetic scenario
Just to make sure it works before I refactor. Also use a fraction in the division step, to check how it will go.
index 95a7704..c36e714 100644
--- a/samples/basic.md
+++ b/samples/basic.md
@@ -15,8 +15,8 @@ For each scenario the **interpreter** program (see the front-matter above) will
=
=Content outside of bullet points (like this) won't have any effect on the program. You can user it as a humane description of the scenario, or for any other purpose.
=
- * Add `7` and `5` to get `12`.
- * Divide `8` by `4` to get `2`
+ * Add `7` and `5` to get `12`
+ * Divide `10` by `4` to get `2.5`
= * Subtract `7` from `5` to get `-2`
=
=index 4c67d2d..0f49171 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -3,6 +3,7 @@
=import json
=import logging
=import os
+from typing import Any, Callable
=
=
=# Setup logging
@@ -15,20 +16,64 @@ def send(message):
= log.debug("Sending: %s", serialized)
= print (serialized)
=
-# Send the ready message
-send({ 'ready': True })
=
-# Loop over input
-while True:
- log.debug("Awaiting message...")
+steps_implementation = dict()
+def register_step(variant: str, implementation: Callable[..., Any]):
+ log.debug("Registering step implementation for '%s'", variant)
+ steps_implementation[variant] = implementation
=
- try:
- received = input()
- log.debug("Received: %s", received)
- payload = json.loads(received)
- send({ 'ok': True })
- except EOFError:
- # The control program closed the stream.
- # Most likely it indicates the end of scenario.
- break
+def ready():
+ send({ 'ready': True })
=
+ # Loop over input
+ while True:
+ log.debug("Awaiting message...")
+
+ try:
+ received = input()
+ log.debug("Received: %s", received)
+ step = json.loads(received)
+ variant = step.get("variant")
+ arguments = step.get("arguments")
+ log.debug(f"Looking for implementation of '{ variant }'")
+ implementation = steps_implementation.get(variant)
+
+ if implementation:
+ log.debug(f"Found an implementation of '{ variant }'")
+ implementation(*arguments, **step)
+ send({ 'ok': True })
+ else:
+ log.warning(f"Not implemented: {variant}")
+ send({ 'ok': False })
+
+ except EOFError:
+ # The control program closed the stream.
+ # Most likely it indicates the end of scenario.
+ break
+
+
+def add_and_verify(a, b, expected, **kwargs):
+ log.info(f"{ a } + { b } = { expected }?")
+
+ # TODO: Can we do this conversion automatically based on typing information?
+ a = float(a)
+ b = float(b)
+ expected = float(expected)
+
+ assert a + b == expected, f"{ a } + { b } = { a + b }, not { expected }!"
+
+register_step("Add {0} and {1} to get {2}", add_and_verify)
+
+
+def divide_and_verify(a, b, expected, **kwargs):
+ log.info(f"{ a } / { b } = { expected }?")
+
+ # TODO: Can we do this conversion automatically based on typing information?
+ a = float(a)
+ b = float(b)
+ expected = float(expected)
+
+ assert a / b == expected, f"{ a } / { b } = { a / b }, not { expected }!"
+
+register_step("Divide {0} by {1} to get {2}", divide_and_verify)
+ready()Make a decorator for steps in basic interpreter
Also demarcate code that should eventually go to a re-usable Python library and interprete-specific (user) code.
index 0f49171..ecd880a 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -5,6 +5,8 @@ import logging
=import os
=from typing import Any, Callable
=
+# Library code
+# ============
=
=# Setup logging
=log_level = os.environ.get("tbb_interpreter_log", "warn").upper()
@@ -16,12 +18,25 @@ def send(message):
= log.debug("Sending: %s", serialized)
= print (serialized)
=
-
+# This will hold all step implementations
=steps_implementation = dict()
+
+# A helper to register a step implementation
=def register_step(variant: str, implementation: Callable[..., Any]):
= log.debug("Registering step implementation for '%s'", variant)
= steps_implementation[variant] = implementation
=
+# A decorator to register a step implementation
+def step(variant: str) -> Callable[..., Any]:
+ def decorator(implementation: Callable[..., Any]) -> Callable[..., Any]:
+ register_step(variant, implementation)
+
+ # We don't really do anything to the implementation itself
+ return implementation
+
+ return decorator
+
+# Call this when ready, i.e. all implementations are registered
=def ready():
= send({ 'ready': True })
=
@@ -52,6 +67,10 @@ def ready():
= break
=
=
+# User code
+# =========
+
+@step("Add {0} and {1} to get {2}")
=def add_and_verify(a, b, expected, **kwargs):
= log.info(f"{ a } + { b } = { expected }?")
=
@@ -62,9 +81,8 @@ def add_and_verify(a, b, expected, **kwargs):
=
= assert a + b == expected, f"{ a } + { b } = { a + b }, not { expected }!"
=
-register_step("Add {0} and {1} to get {2}", add_and_verify)
-
=
+@step("Divide {0} by {1} to get {2}")
=def divide_and_verify(a, b, expected, **kwargs):
= log.info(f"{ a } / { b } = { expected }?")
=
@@ -75,5 +93,4 @@ def divide_and_verify(a, b, expected, **kwargs):
=
= assert a / b == expected, f"{ a } / { b } = { a / b }, not { expected }!"
=
-register_step("Divide {0} by {1} to get {2}", divide_and_verify)
=ready()Separate reading spec from its evaluation
For this end a new Spec struct was introduced. It's a wrapper for a list of suites, with the evaluate method.
new file mode 100644
index 0000000..e7c23a9
--- /dev/null
+++ b/samples/second.md
@@ -0,0 +1,14 @@
+---
+interpreter: "invalid interpreter"
+---
+
+A second document, just to test if it will be injested correctly.
+
+# Suite 1 from a Second Document
+
+It should be possible to define multiple suites in each document.
+
+## Scenario 1.1
+
+ * Do something `impactful`
+ * Asses the results: `A`, `B` and `C`index 38737c7..686906f 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,6 +4,7 @@ use clap::Parser;
=use env_logger;
=use glob::glob;
=use log;
+use spec::{Spec, Suite};
=use std::error::Error;
=use std::path::PathBuf;
=
@@ -28,29 +29,38 @@ fn main() -> Result<(), Box<dyn Error>> {
= log::debug!("Reading the specification from {}", cli.input.display());
=
= let input = cli.input.canonicalize()?;
+ let mut spec = Spec::default();
+
= if input.is_dir() {
= log::debug!(
= "The {} is a directory. Looking for markdown files...",
= input.display()
= );
= let pattern = format!("{}/**/*.md", input.display());
+
= for path in glob(&pattern)? {
- process_document(path?, cli.verbose)?;
+ spec.suites.append(&mut read_document(path?)?);
= }
- Ok(())
= } else if input.is_file() {
= log::debug!("The {} is a file. Reading...", input.display());
- process_document(input, cli.verbose)
+ spec.suites.append(&mut read_document(input)?);
= } else {
- Err(format!("The {} is neither a file nor directory", input.display()).into())
- }
+ return Err(format!("The {} is neither a file nor directory", input.display()).into());
+ };
+
+ log::info!("Collected {} suites: {:#?}", spec.suites.len(), spec);
+
+ spec.evaluate(cli.verbose);
+
+ Ok(())
=}
=
-fn process_document(input: PathBuf, verbose: bool) -> Result<(), Box<dyn Error>> {
+fn read_document(input: PathBuf) -> Result<Vec<Suite>, Box<dyn Error>> {
= log::debug!("Reading {}", input.display());
= let md = std::fs::read_to_string(input)?;
+
+ // TODO: Return multiple suits from a single document
= let suite = spec::Suite::from_markdown(&md)?;
- log::debug!("Suite:\n\n{:#?}", suite);
=
- suite.evaluate(verbose)
+ Ok(vec![suite])
=}index 9d2aaf8..4ac72e1 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -3,6 +3,23 @@ use std::error::Error;
=use std::io::{BufRead, BufReader, LineWriter, Write};
=use std::process::{Command, Stdio};
=
+#[derive(Debug, Default)]
+pub struct Spec {
+ pub suites: Vec<Suite>,
+}
+
+impl Spec {
+ pub fn evaluate(&self, verbose: bool) -> Result<(), Box<dyn Error>> {
+ for suite in self.suites.iter() {
+ suite.evaluate(verbose)?;
+ }
+
+ // TODO: Return an EvaluationReport
+ log::info!("Done!");
+ Ok(())
+ }
+}
+
=/// Suite is a collection of scenarios that share a common title and interpreter
=///
=/// Other BDD systems often call it "a spec" but in my mind it doesn't makeMove spec evaluation from main to Spec::evaluate
It follows the convention of Suite::evaluate, Scenario::run etc.
Although I'm not sure if it's the final place of this code. It would probably be cleaner to separate evaluation from the spec itself, so it could be easier to port to different platforms. For example ability to run TBB in a web browser would be sweet, but of course the execution model would be very different.
index e7c23a9..8f001a4 100644
--- a/samples/second.md
+++ b/samples/second.md
@@ -4,10 +4,12 @@ interpreter: "invalid interpreter"
=
=A second document, just to test if it will be injested correctly.
=
+
=# Suite 1 from a Second Document
=
=It should be possible to define multiple suites in each document.
=
+
=## Scenario 1.1
=
= * Do something `impactful`index 686906f..362aff8 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,7 +4,7 @@ use clap::Parser;
=use env_logger;
=use glob::glob;
=use log;
-use spec::{Spec, Suite};
+use spec::Spec;
=use std::error::Error;
=use std::path::PathBuf;
=
@@ -39,28 +39,18 @@ fn main() -> Result<(), Box<dyn Error>> {
= let pattern = format!("{}/**/*.md", input.display());
=
= for path in glob(&pattern)? {
- spec.suites.append(&mut read_document(path?)?);
+ let md = std::fs::read_to_string(path?)?;
+ spec.load_document(&md)?;
= }
= } else if input.is_file() {
= log::debug!("The {} is a file. Reading...", input.display());
- spec.suites.append(&mut read_document(input)?);
+ let md = std::fs::read_to_string(input)?;
+ spec.load_document(&md)?;
= } else {
= return Err(format!("The {} is neither a file nor directory", input.display()).into());
= };
=
= log::info!("Collected {} suites: {:#?}", spec.suites.len(), spec);
=
- spec.evaluate(cli.verbose);
-
- Ok(())
-}
-
-fn read_document(input: PathBuf) -> Result<Vec<Suite>, Box<dyn Error>> {
- log::debug!("Reading {}", input.display());
- let md = std::fs::read_to_string(input)?;
-
- // TODO: Return multiple suits from a single document
- let suite = spec::Suite::from_markdown(&md)?;
-
- Ok(vec![suite])
+ spec.evaluate(cli.verbose)
=}index 4ac72e1..ffe218b 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -3,12 +3,26 @@ use std::error::Error;
=use std::io::{BufRead, BufReader, LineWriter, Write};
=use std::process::{Command, Stdio};
=
+/// Spec is a collection of suites that together describe a system
+///
+/// Each system under test has a single specification, whereas each suite
+/// describes a different aspect of it.
=#[derive(Debug, Default)]
=pub struct Spec {
= pub suites: Vec<Suite>,
=}
=
=impl Spec {
+ /// Load suites from a markdown document
+ pub fn load_document(&mut self, md: &str) -> Result<(), Box<dyn Error>> {
+ // TODO: Support loading multiple suits from a single document (demarcated by h1)
+ let suite = Suite::from_markdown(md)?;
+
+ self.suites.push(suite);
+ Ok(())
+ }
+
+ /// Run all the scenarios
= pub fn evaluate(&self, verbose: bool) -> Result<(), Box<dyn Error>> {
= for suite in self.suites.iter() {
= suite.evaluate(verbose)?;Dump some thoughts in a README
new file mode 100644
index 0000000..758bd1a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,44 @@
+# TBB: Tad Better Behavior
+
+A BDD test runner inspired by Gauge, but better.
+
+ * No magic
+ * Flexibility
+ * Cross-platform
+
+## No magic
+
+Test scenarios are evaluated by interpreters, which are normal programs that read JSON on stdin and write JSON to stdout. User can implement them in any language or framework they like, using any tools they like (editors, lanuage servers, libraries).
+
+
+## Flexibility
+
+Different test suites can use different interpreters, which may be implemented using different languages. If your computer can run it, `tbb` can use it as an interpreter.
+
+When executing a step, the interpreter will get more data from the markdown fragment which defined the step
+
+ - tables
+ - lists
+ - code blocks
+
+and even the original markdown fragment itself.
+
+
+## Cross platform
+
+Support for Linux, OS X and Web (WASM).
+
+
+# Roadmap
+
+- [x] Proof of concept
+ - [x] Interpretter in a different language (Python)
+- [ ] Nix package (from Flake)
+- [ ] Use for evaluating Jewiet's Form to Mail specification
+- [ ] Helper libraries
+ - [ ] for Python
+ - [ ] for Clojure
+- [ ] Better reporters
+ - [ ] TUI
+ - [ ] Web
+- [ ] WASM targetImplement pretty printing of a Spec
It implements Display to render as markdown lists. Also, lower default logging level to info, so it can be used for normal, user facing output (like this kind of reporting). It can also be useful while developing interpreters: temporarily log as info, later demote to debug.
index ecd880a..a26ae1f 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -72,7 +72,7 @@ def ready():
=
=@step("Add {0} and {1} to get {2}")
=def add_and_verify(a, b, expected, **kwargs):
- log.info(f"{ a } + { b } = { expected }?")
+ log.debug(f"{ a } + { b } = { expected }?")
=
= # TODO: Can we do this conversion automatically based on typing information?
= a = float(a)
@@ -84,7 +84,7 @@ def add_and_verify(a, b, expected, **kwargs):
=
=@step("Divide {0} by {1} to get {2}")
=def divide_and_verify(a, b, expected, **kwargs):
- log.info(f"{ a } / { b } = { expected }?")
+ log.debug(f"{ a } / { b } = { expected }?")
=
= # TODO: Can we do this conversion automatically based on typing information?
= a = float(a)index 362aff8..7402b26 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -23,7 +23,7 @@ struct Cli {
=fn main() -> Result<(), Box<dyn Error>> {
= let cli = Cli::parse();
= let log_env = env_logger::Env::default()
- .filter_or("RUST_LOG", if cli.verbose { "trace" } else { "warn" });
+ .filter_or("RUST_LOG", if cli.verbose { "trace" } else { "info" });
= env_logger::init_from_env(log_env);
=
= log::debug!("Reading the specification from {}", cli.input.display());
@@ -50,7 +50,7 @@ fn main() -> Result<(), Box<dyn Error>> {
= return Err(format!("The {} is neither a file nor directory", input.display()).into());
= };
=
- log::info!("Collected {} suites: {:#?}", spec.suites.len(), spec);
+ log::info!("Collected {} suites:\n\n{}", spec.suites.len(), spec);
=
= spec.evaluate(cli.verbose)
=}index ffe218b..e25b621 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -1,5 +1,6 @@
=use serde::{Deserialize, Serialize};
=use std::error::Error;
+use std::fmt::Display;
=use std::io::{BufRead, BufReader, LineWriter, Write};
=use std::process::{Command, Stdio};
=
@@ -34,6 +35,35 @@ impl Spec {
= }
=}
=
+impl Display for Spec {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ for suite in self.suites.iter() {
+ writeln!(
+ f,
+ "\n{title} ({interpreter})",
+ title = suite.title,
+ interpreter = suite.interpreter
+ )?;
+
+ for scenario in suite.scenarios.iter() {
+ writeln!(f, "\n * {title}\n", title = scenario.title,)?;
+
+ for (index, step) in scenario.steps.iter().enumerate() {
+ writeln!(
+ f,
+ " {index:02}. {description} {arguments:?}",
+ description = step.description,
+ arguments = step.arguments
+ )?;
+ }
+ }
+ writeln!(f, "")?;
+ }
+
+ Ok(())
+ }
+}
+
=/// Suite is a collection of scenarios that share a common title and interpreter
=///
=/// Other BDD systems often call it "a spec" but in my mind it doesn't make
@@ -75,7 +105,7 @@ impl Scenario {
= .stdout(Stdio::piped())
= .env(
= "tbb_interpreter_log",
- if verbose { "debug" } else { "warn" },
+ if verbose { "debug" } else { "info" },
= )
= .spawn()?;
=Implement EvaluationReport struct and some logic
The idea is that evaluation will be driven by the report. See the doc comment for EvaluationReport::new.
index 7402b26..0a701e8 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,3 +1,4 @@
+mod report;
=mod spec;
=
=use clap::Parser;
@@ -52,5 +53,9 @@ fn main() -> Result<(), Box<dyn Error>> {
=
= log::info!("Collected {} suites:\n\n{}", spec.suites.len(), spec);
=
- spec.evaluate(cli.verbose)
+ let report = spec.evaluate(cli.verbose)?;
+
+ log::info!("Evaluation done. Here's the result:\n\n{report}");
+
+ Ok(())
=}new file mode 100644
index 0000000..54ede7e
--- /dev/null
+++ b/src/report.rs
@@ -0,0 +1,126 @@
+use crate::spec::{Scenario, Spec, Step, Suite};
+use std::fmt::Display;
+
+pub struct EvaluationReport<'a> {
+ spec: &'a Spec,
+ suites: Vec<SuiteReport<'a>>,
+}
+
+pub struct SuiteReport<'a> {
+ suite: &'a Suite,
+ scenarios: Vec<ScenarioReport<'a>>,
+}
+
+pub struct ScenarioReport<'a> {
+ scenario: &'a Scenario,
+ status: ScenarioStatus,
+ steps: Vec<StepReport<'a>>,
+}
+
+pub struct StepReport<'a> {
+ step: &'a Step,
+ status: StepStatus,
+}
+
+pub enum StepStatus {
+ Ok,
+ Failed { reason: String },
+ NotEvaluated,
+}
+
+pub enum ScenarioStatus {
+ Done,
+ Pending,
+ InterpreterFailed { reason: String },
+}
+
+impl<'a> EvaluationReport<'a> {
+ /// Create a blank report to be evaluated
+ ///
+ /// We start with a report pending to be evaluated. It holds references to
+ /// Spec, Suites, Scenarios and Steps (all in pending state). The evaluator
+ /// should walk through the report, and "fill" it. That way, even if it
+ /// fails to finish it's job, the report will be there to display.
+ pub fn new(spec: &'a Spec) -> Self {
+ let suites = spec.suites.iter().map(SuiteReport::from).collect();
+ Self { spec, suites }
+ }
+}
+
+impl Display for EvaluationReport<'_> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ for SuiteReport { suite, scenarios } in self.suites.iter() {
+ writeln!(
+ f,
+ "\n{title} ({interpreter})",
+ title = suite.title,
+ interpreter = suite.interpreter
+ )?;
+
+ for ScenarioReport {
+ scenario,
+ status,
+ steps,
+ } in scenarios.iter()
+ {
+ let sigil = match status {
+ ScenarioStatus::Done => "✔", // TODO: Use different icon depending on steps status
+ ScenarioStatus::Pending => "❔",
+ ScenarioStatus::InterpreterFailed { .. } => "☠️",
+ };
+ writeln!(f, "\n {sigil} {title}\n", title = scenario.title)?;
+ if let ScenarioStatus::InterpreterFailed { reason } = status {
+ writeln!(f, "{reason}")?;
+ }
+
+ for StepReport { step, status } in steps.iter() {
+ let sigil = match status {
+ StepStatus::Ok => "✅",
+ StepStatus::Failed { .. } => "❌",
+ StepStatus::NotEvaluated => "⃞",
+ };
+ writeln!(
+ f,
+ " {sigil} {description} {arguments:?}",
+ description = step.description,
+ arguments = step.arguments
+ )?;
+
+ if let StepStatus::Failed { reason } = status {
+ writeln!(f, "\n {reason}\n")?;
+ }
+ }
+ }
+ writeln!(f, "")?;
+ }
+
+ Ok(())
+ }
+}
+
+impl<'a> From<&'a Suite> for SuiteReport<'a> {
+ fn from(suite: &'a Suite) -> Self {
+ let scenarios = suite.scenarios.iter().map(ScenarioReport::from).collect();
+ Self { suite, scenarios }
+ }
+}
+
+impl<'a> From<&'a Scenario> for ScenarioReport<'a> {
+ fn from(scenario: &'a Scenario) -> Self {
+ let steps = scenario.steps.iter().map(StepReport::from).collect();
+ Self {
+ scenario,
+ status: ScenarioStatus::Pending,
+ steps,
+ }
+ }
+}
+
+impl<'a> From<&'a Step> for StepReport<'a> {
+ fn from(step: &'a Step) -> Self {
+ Self {
+ step,
+ status: StepStatus::NotEvaluated,
+ }
+ }
+}index e25b621..205a3ad 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -1,3 +1,4 @@
+use crate::report::EvaluationReport;
=use serde::{Deserialize, Serialize};
=use std::error::Error;
=use std::fmt::Display;
@@ -24,14 +25,14 @@ impl Spec {
= }
=
= /// Run all the scenarios
- pub fn evaluate(&self, verbose: bool) -> Result<(), Box<dyn Error>> {
+ pub fn evaluate(&'_ self, verbose: bool) -> Result<EvaluationReport<'_>, Box<dyn Error>> {
+ let report = EvaluationReport::new(self);
+
= for suite in self.suites.iter() {
= suite.evaluate(verbose)?;
= }
=
- // TODO: Return an EvaluationReport
- log::info!("Done!");
- Ok(())
+ Ok(report)
= }
=}
=Always print report, even if some scenarios failed
Move evaluation logic to report.
index 0a701e8..97c2c5e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -5,6 +5,7 @@ use clap::Parser;
=use env_logger;
=use glob::glob;
=use log;
+use report::EvaluationReport;
=use spec::Spec;
=use std::error::Error;
=use std::path::PathBuf;
@@ -53,7 +54,8 @@ fn main() -> Result<(), Box<dyn Error>> {
=
= log::info!("Collected {} suites:\n\n{}", spec.suites.len(), spec);
=
- let report = spec.evaluate(cli.verbose)?;
+ let mut report = EvaluationReport::new(&spec);
+ report.evaluate(cli.verbose);
=
= log::info!("Evaluation done. Here's the result:\n\n{report}");
=index 54ede7e..ea61bfa 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,21 +1,160 @@
+use serde::Deserialize;
+
=use crate::spec::{Scenario, Spec, Step, Suite};
+use std::error::Error;
=use std::fmt::Display;
+use std::io::{BufRead, BufReader, LineWriter, Write};
+use std::process::{Command, Stdio};
=
=pub struct EvaluationReport<'a> {
- spec: &'a Spec,
= suites: Vec<SuiteReport<'a>>,
=}
=
+impl<'a> EvaluationReport<'a> {
+ /// Create a blank report to be evaluated
+ ///
+ /// We start with a report pending to be evaluated. It holds references to
+ /// Spec, Suites, Scenarios and Steps (all in pending state). The evaluator
+ /// should walk through the report, and "fill" it. That way, even if it
+ /// fails to finish it's job, the report will be there to display.
+ pub fn new(spec: &'a Spec) -> Self {
+ Self {
+ suites: spec.suites.iter().map(SuiteReport::from).collect(),
+ }
+ }
+
+ /// Run all the scenarios
+ pub fn evaluate(&mut self, verbose: bool) {
+ for suite in self.suites.iter_mut() {
+ suite.evaluate(verbose);
+ }
+ }
+}
+
=pub struct SuiteReport<'a> {
= suite: &'a Suite,
= scenarios: Vec<ScenarioReport<'a>>,
=}
=
+impl<'a> SuiteReport<'a> {
+ fn evaluate(&mut self, verbose: bool) {
+ log::debug!("Evaluating suite {}", self.suite.title);
+
+ for scenario in self.scenarios.iter_mut() {
+ let result = scenario.run(&self.suite.interpreter, verbose);
+ if let Err(error) = result {
+ scenario.status = ScenarioStatus::FailedToRun { error }
+ } else {
+ scenario.status = ScenarioStatus::Done
+ }
+ }
+ }
+}
+
=pub struct ScenarioReport<'a> {
= scenario: &'a Scenario,
= status: ScenarioStatus,
= steps: Vec<StepReport<'a>>,
=}
+impl<'a> ScenarioReport<'a> {
+ // TODO: The ScenarioReport::run method is very effectful. I think it should live in it's own module.
+ fn run(&mut self, interpreter: &str, verbose: bool) -> Result<(), Box<dyn Error>> {
+ log::debug!(
+ "Running scenario '{}' using '{interpreter}' as an interpreter",
+ self.scenario.title
+ );
+
+ let mut process = Command::new("sh")
+ .arg("-c")
+ .arg(interpreter)
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .env(
+ "tbb_interpreter_log",
+ if verbose { "debug" } else { "info" },
+ )
+ .spawn()?;
+
+ let Some(input) = process.stdin.take() else {
+ // TODO: Avoid string errors!
+ return Err("Failed to take the stdin stream of the interpreter process.".into());
+ };
+
+ let Some(output) = process.stdout.take() else {
+ return Err("Failed to take the stdout stream of the interpreter process.".into());
+ };
+
+ let mut reader = BufReader::new(output);
+ let mut writer = LineWriter::new(input);
+
+ loop {
+ let received: String = Self::read_line(&mut reader)?;
+ let interpreter_state: InterpreterState = serde_json::from_str(&received)?;
+ log::debug!("Interpreter state: {interpreter_state:?}");
+ if interpreter_state.ready {
+ break;
+ };
+ }
+
+ for &mut StepReport {
+ step,
+ ref mut status,
+ } in self.steps.iter_mut()
+ {
+ log::debug!("Taking step '{}'", step.description);
+ let json = serde_json::to_string(step)?;
+ Self::write_line(&mut writer, &json)?;
+ let response = Self::read_line(&mut reader)?;
+ let step_result: StepResult = serde_json::from_str(&response)?;
+ log::debug!("Received: {:?}", step_result);
+ if step_result.ok {
+ *status = StepStatus::Ok;
+ } else {
+ *status = StepStatus::Failed {
+ reason: "Unknown".to_string(),
+ };
+ log::debug!("Step {step:?} failed");
+ }
+ }
+
+ drop(writer);
+
+ process
+ .wait()
+ .map_err(|io_error| Box::new(io_error).into()) // No idea why it's needed.
+ .and_then(|exit_status| {
+ log::debug!("Interpreter closed with exit status {exit_status:?}");
+ let exit_code = exit_status.code().unwrap_or(0); // Is that right?
+ if exit_code == 0 {
+ Ok(())
+ } else {
+ let message = format!("Interpreter process {interpreter} exited abnormally. Exit code: {exit_code}");
+ Err(message.into())
+ }
+ })
+ }
+
+ fn write_line(
+ writer: &mut LineWriter<std::process::ChildStdin>,
+ line: &str,
+ ) -> Result<(), Box<dyn Error>> {
+ let buffer = [line, "\n"].concat();
+
+ log::debug!("Sending: {}", buffer);
+ writer.write_all(buffer.as_bytes())?;
+ Ok(())
+ }
+
+ fn read_line(
+ reader: &mut BufReader<std::process::ChildStdout>,
+ ) -> Result<String, Box<dyn Error>> {
+ let mut buffer = String::new();
+ if reader.read_line(&mut buffer)? == 0 {
+ return Err("Can't read from the interpreter process.".into());
+ };
+ Ok(buffer)
+ }
+}
=
=pub struct StepReport<'a> {
= step: &'a Step,
@@ -31,20 +170,7 @@ pub enum StepStatus {
=pub enum ScenarioStatus {
= Done,
= Pending,
- InterpreterFailed { reason: String },
-}
-
-impl<'a> EvaluationReport<'a> {
- /// Create a blank report to be evaluated
- ///
- /// We start with a report pending to be evaluated. It holds references to
- /// Spec, Suites, Scenarios and Steps (all in pending state). The evaluator
- /// should walk through the report, and "fill" it. That way, even if it
- /// fails to finish it's job, the report will be there to display.
- pub fn new(spec: &'a Spec) -> Self {
- let suites = spec.suites.iter().map(SuiteReport::from).collect();
- Self { spec, suites }
- }
+ FailedToRun { error: Box<dyn Error> },
=}
=
=impl Display for EvaluationReport<'_> {
@@ -66,11 +192,11 @@ impl Display for EvaluationReport<'_> {
= let sigil = match status {
= ScenarioStatus::Done => "✔", // TODO: Use different icon depending on steps status
= ScenarioStatus::Pending => "❔",
- ScenarioStatus::InterpreterFailed { .. } => "☠️",
+ ScenarioStatus::FailedToRun { .. } => "☠️",
= };
= writeln!(f, "\n {sigil} {title}\n", title = scenario.title)?;
- if let ScenarioStatus::InterpreterFailed { reason } = status {
- writeln!(f, "{reason}")?;
+ if let ScenarioStatus::FailedToRun { error } = status {
+ writeln!(f, " {error}\n")?;
= }
=
= for StepReport { step, status } in steps.iter() {
@@ -87,7 +213,7 @@ impl Display for EvaluationReport<'_> {
= )?;
=
= if let StepStatus::Failed { reason } = status {
- writeln!(f, "\n {reason}\n")?;
+ writeln!(f, "\n {reason}\n")?;
= }
= }
= }
@@ -124,3 +250,15 @@ impl<'a> From<&'a Step> for StepReport<'a> {
= }
= }
=}
+
+// Messages from interpreter
+
+#[derive(Deserialize, Debug)]
+struct InterpreterState {
+ ready: bool,
+}
+
+#[derive(Deserialize, Debug)]
+struct StepResult {
+ ok: bool,
+}index 205a3ad..2b51f6d 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -1,9 +1,6 @@
-use crate::report::EvaluationReport;
=use serde::{Deserialize, Serialize};
=use std::error::Error;
=use std::fmt::Display;
-use std::io::{BufRead, BufReader, LineWriter, Write};
-use std::process::{Command, Stdio};
=
=/// Spec is a collection of suites that together describe a system
=///
@@ -23,17 +20,6 @@ impl Spec {
= self.suites.push(suite);
= Ok(())
= }
-
- /// Run all the scenarios
- pub fn evaluate(&'_ self, verbose: bool) -> Result<EvaluationReport<'_>, Box<dyn Error>> {
- let report = EvaluationReport::new(self);
-
- for suite in self.suites.iter() {
- suite.evaluate(verbose)?;
- }
-
- Ok(report)
- }
=}
=
=impl Display for Spec {
@@ -94,103 +80,6 @@ pub struct Scenario {
= pub steps: Vec<Step>,
=}
=
-impl Scenario {
- fn run(&self, interpreter: &str, verbose: bool) -> Result<(), Box<dyn Error>> {
- println!("\n {}", self.title);
- // TODO: start the interpreter program and tap into its stdio
- log::debug!("Starting the interpreter process: `{interpreter}`");
- let mut process = Command::new("sh")
- .arg("-c")
- .arg(interpreter)
- .stdin(Stdio::piped())
- .stdout(Stdio::piped())
- .env(
- "tbb_interpreter_log",
- if verbose { "debug" } else { "info" },
- )
- .spawn()?;
-
- let input = process
- .stdin
- .take()
- .ok_or("Failed to take the stdin stream of the interpreter process.")?;
-
- let output = process
- .stdout
- .take()
- .ok_or("Failed to take the stdout stream of the interpreter process.")?;
-
- let mut reader = BufReader::new(output);
- let mut writer = LineWriter::new(input);
-
- loop {
- let received = read_line(&mut reader)?;
- let interpreter_state: InterpreterState = serde_json::from_str(&received)?;
- log::debug!("Interpreter state: {interpreter_state:?}");
- if interpreter_state.ready {
- break;
- };
- }
-
- for step in self.steps.iter() {
- println!(" {}", step.description);
- let json = serde_json::to_string(step)?;
- write_line(&mut writer, &json)?;
- let response = read_line(&mut reader)?;
- let step_result: StepResult = serde_json::from_str(&response)?;
- log::debug!("Received: {:?}", step_result);
- if !step_result.ok {
- return Err(format!("Step {step:?} failed").into());
- }
- }
-
- drop(writer);
-
- process
- .wait()
- .map_err(|io_error| Box::new(io_error).into()) // No idea why it's needed.
- .and_then(|exit_status| {
- log::debug!("Interpreter closed with exit status {exit_status:?}");
- let exit_code = exit_status.code().unwrap_or(0); // Is that right?
- if exit_code == 0 {
- Ok(())
- } else {
- let message = format!("Interpreter process {interpreter} exited abnormally. Exit code: {exit_code}");
- Err(message.into())
- }
- })
- }
-}
-
-#[derive(Deserialize, Debug)]
-struct StepResult {
- ok: bool,
-}
-
-#[derive(Deserialize, Debug)]
-struct InterpreterState {
- ready: bool,
-}
-
-fn write_line(
- writer: &mut LineWriter<std::process::ChildStdin>,
- line: &str,
-) -> Result<(), Box<dyn Error>> {
- let buffer = [line, "\n"].concat();
-
- log::debug!("Sending: {}", buffer);
- writer.write_all(buffer.as_bytes())?;
- Ok(())
-}
-
-fn read_line(reader: &mut BufReader<std::process::ChildStdout>) -> Result<String, Box<dyn Error>> {
- let mut buffer = String::new();
- if reader.read_line(&mut buffer)? == 0 {
- return Err("The interpreter process closed its output stream too early.".into());
- };
- Ok(buffer)
-}
-
=#[derive(Debug, Serialize)]
=pub struct Step {
= /// The headline (without formatting)
@@ -225,17 +114,6 @@ impl Suite {
=
= Self::try_from(mdast)
= }
-
- pub fn evaluate(&self, verbose: bool) -> Result<(), Box<dyn Error>> {
- println!("\n{}", self.title);
-
- for scenario in self.scenarios.iter() {
- scenario.run(&self.interpreter, verbose)?;
- }
-
- log::info!("All good!");
- Ok(())
- }
=}
=
=impl TryFrom<markdown::mdast::Node> for Suite {Update the roadmap
index 758bd1a..27d6dd2 100644
--- a/README.md
+++ b/README.md
@@ -33,11 +33,22 @@ Support for Linux, OS X and Web (WASM).
=
=- [x] Proof of concept
= - [x] Interpretter in a different language (Python)
+ - [ ] Report why steps fail
+- [ ] Pass more step data to interpreters
+ - [ ] Code blocks
+ - [ ] Lists
+ - [ ] Tables
+ - [ ] Definition lists
+ - [ ] Original markdown fragment
=- [ ] Nix package (from Flake)
=- [ ] Use for evaluating Jewiet's Form to Mail specification
=- [ ] Helper libraries
= - [ ] for Python
= - [ ] for Clojure
+- [ ] Capture more data in reports
+ - [ ] Attachments (screenshots, videos, datasets, etc.)
+ - [ ] Performance data (interpreters' startup times, steps' durations)
+ - [ ] Annotations from interpreters
=- [ ] Better reporters
= - [ ] TUI
= - [ ] WebEnumerate control and interpreter messages
For a bit of type safety around message passing. The JSON format of the messages changed. Now each of them need to carry a "type" field indicating which enum variant it implements.
index a26ae1f..e34aae4 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -38,7 +38,10 @@ def step(variant: str) -> Callable[..., Any]:
=
=# Call this when ready, i.e. all implementations are registered
=def ready():
- send({ 'ready': True })
+ send({
+ "type": "InterpreterState",
+ "ready": True
+ })
=
= # Loop over input
= while True:
@@ -47,7 +50,12 @@ def ready():
= try:
= received = input()
= log.debug("Received: %s", received)
- step = json.loads(received)
+ message = json.loads(received)
+ if not message["type"] == "Execute":
+ log.warning(f"Unexpected message received from the control program. Expected Execute type message, got {received}")
+ continue
+
+ step = message["step"] # The only message variant we expect
= variant = step.get("variant")
= arguments = step.get("arguments")
= log.debug(f"Looking for implementation of '{ variant }'")
@@ -56,10 +64,13 @@ def ready():
= if implementation:
= log.debug(f"Found an implementation of '{ variant }'")
= implementation(*arguments, **step)
- send({ 'ok': True })
+ send({ "type": 'Success' })
= else:
= log.warning(f"Not implemented: {variant}")
- send({ 'ok': False })
+ send({
+ "type": 'Failure',
+ 'reason': 'Step not implemented'
+ })
=
= except EOFError:
= # The control program closed the stream.index ea61bfa..862df77 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,4 +1,4 @@
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
=
=use crate::spec::{Scenario, Spec, Step, Suite};
=use std::error::Error;
@@ -88,10 +88,16 @@ impl<'a> ScenarioReport<'a> {
= let mut writer = LineWriter::new(input);
=
= loop {
- let received: String = Self::read_line(&mut reader)?;
- let interpreter_state: InterpreterState = serde_json::from_str(&received)?;
- log::debug!("Interpreter state: {interpreter_state:?}");
- if interpreter_state.ready {
+ let ready = match Self::receive(&mut reader)? {
+ InterpreterMessage::InterpreterState { ready } => ready,
+ message => {
+ let complaint =
+ format!("Unexpected message received from the interpreter: {message:#?}");
+ return Err(complaint.into());
+ }
+ };
+ if ready {
+ log::debug!("Interpreter ready");
= break;
= };
= }
@@ -102,19 +108,29 @@ impl<'a> ScenarioReport<'a> {
= } in self.steps.iter_mut()
= {
= log::debug!("Taking step '{}'", step.description);
- let json = serde_json::to_string(step)?;
- Self::write_line(&mut writer, &json)?;
- let response = Self::read_line(&mut reader)?;
- let step_result: StepResult = serde_json::from_str(&response)?;
- log::debug!("Received: {:?}", step_result);
- if step_result.ok {
- *status = StepStatus::Ok;
- } else {
- *status = StepStatus::Failed {
- reason: "Unknown".to_string(),
- };
- log::debug!("Step {step:?} failed");
- }
+ Self::send(
+ &mut writer,
+ &ControlMessage::Execute {
+ step: step.to_owned(),
+ },
+ )?;
+
+ match Self::receive(&mut reader)? {
+ InterpreterMessage::Success => {
+ log::debug!("Step executed successfully: {step:#?}");
+ *status = StepStatus::Ok;
+ }
+ InterpreterMessage::Failure { reason } => {
+ log::debug!("Step failed:\n\n {step:#?} \n\n {reason:#?}");
+ *status = StepStatus::Failed { reason };
+ }
+ unexpected => {
+ let message = format!(
+ "Unexpected message received from the interpreter: {unexpected:#?}"
+ );
+ return Err(message.into());
+ }
+ };
= }
=
= drop(writer);
@@ -130,10 +146,28 @@ impl<'a> ScenarioReport<'a> {
= } else {
= let message = format!("Interpreter process {interpreter} exited abnormally. Exit code: {exit_code}");
= Err(message.into())
- }
+ }
= })
= }
=
+ /// Send a message to interpreter
+ fn send(
+ writer: &mut LineWriter<std::process::ChildStdin>,
+ message: &ControlMessage,
+ ) -> Result<(), Box<dyn Error>> {
+ let json = serde_json::to_string(&message)?;
+ Self::write_line(writer, &json)
+ }
+
+ fn receive(
+ reader: &mut BufReader<std::process::ChildStdout>,
+ ) -> Result<InterpreterMessage, Box<dyn Error>> {
+ let buffer = Self::read_line(reader)?;
+ let message = serde_json::from_str(&buffer)?;
+
+ Ok(message)
+ }
+
= fn write_line(
= writer: &mut LineWriter<std::process::ChildStdin>,
= line: &str,
@@ -251,14 +285,18 @@ impl<'a> From<&'a Step> for StepReport<'a> {
= }
=}
=
-// Messages from interpreter
-
-#[derive(Deserialize, Debug)]
-struct InterpreterState {
- ready: bool,
+/// Messages from an interpreter to this (control) program
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(tag = "type")]
+pub enum InterpreterMessage {
+ InterpreterState { ready: bool },
+ Success,
+ Failure { reason: String },
=}
=
-#[derive(Deserialize, Debug)]
-struct StepResult {
- ok: bool,
+/// Messages from this (control) program to an interpreter
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(tag = "type")]
+pub enum ControlMessage {
+ Execute { step: Step },
=}index 2b51f6d..4249efd 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -80,7 +80,7 @@ pub struct Scenario {
= pub steps: Vec<Step>,
=}
=
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
=pub struct Step {
= /// The headline (without formatting)
= pub description: String,Fix basic interpreter crashing on assertion errors
and other exceptions when executing a step.
index e34aae4..d2664cb 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -63,8 +63,12 @@ def ready():
=
= if implementation:
= log.debug(f"Found an implementation of '{ variant }'")
- implementation(*arguments, **step)
- send({ "type": 'Success' })
+ try:
+ implementation(*arguments, **step)
+ send({ "type": 'Success' })
+ except Exception as error:
+ send({ "type": "Failure", "reason": str(error) })
+
= else:
= log.warning(f"Not implemented: {variant}")
= send({Make the program skip steps after failure
See the comment for rationale.
index 862df77..b0d5c41 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -123,6 +123,15 @@ impl<'a> ScenarioReport<'a> {
= InterpreterMessage::Failure { reason } => {
= log::debug!("Step failed:\n\n {step:#?} \n\n {reason:#?}");
= *status = StepStatus::Failed { reason };
+
+ // Do not run subsequent steps.
+ //
+ // A scenario is a unit of testing. Later steps are expected
+ // to depend on previous ones. If a step fails, continuing
+ // may lead to a mess.
+ //
+ // Maybe this can be configured via front-matter?
+ break;
= }
= unexpected => {
= let message = format!(Implement one step from the Text scenario
Mainly to make sure that it will be executed, even if the Arithmetic scenario fails.
index d2664cb..f5bbff3 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -108,4 +108,15 @@ def divide_and_verify(a, b, expected, **kwargs):
=
= assert a / b == expected, f"{ a } / { b } = { a / b }, not { expected }!"
=
+
+@step("The word {0} has {1} characters")
+def verify_characters_count(word: str, expected_length, **kwargs):
+ log.info(word)
+ expected_length = int(expected_length)
+ actual_length = len(word)
+
+ assert expected_length == actual_length, f"The word '{word}' is {actual_length} long, not {expected_length}"
+
+
+
=ready()Update the roadmap
index 27d6dd2..cb1c00d 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,11 @@ Support for Linux, OS X and Web (WASM).
=
=- [x] Proof of concept
= - [x] Interpretter in a different language (Python)
- - [ ] Report why steps fail
+ - [x] Report why steps fail
+- [ ] More readable report
+ - [ ] The emojis are misaligned and lack color (at least in my terminal)
+ - [ ] A summary at the bottom (esp. list of errors)
+ - [ ] Use colors
=- [ ] Pass more step data to interpreters
= - [ ] Code blocks
= - [ ] ListsStyle the report
Colors and better sigils! Also, demote logging the spec before evaluation to debug. The output is much more readable now.
index 8f4f3fb..9c1b9d0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -47,7 +47,7 @@ version = "1.1.5"
=source = "registry+https://github.com/rust-lang/crates.io-index"
=checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
=dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
=]
=
=[[package]]
@@ -58,7 +58,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
=dependencies = [
= "anstyle",
= "once_cell_polyfill",
- "windows-sys",
+ "windows-sys 0.61.2",
=]
=
=[[package]]
@@ -107,6 +107,15 @@ version = "1.0.4"
=source = "registry+https://github.com/rust-lang/crates.io-index"
=checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
=
+[[package]]
+name = "colored"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
=[[package]]
=name = "env_filter"
=version = "0.1.4"
@@ -373,6 +382,7 @@ name = "tad-better-behavior"
=version = "0.1.0"
=dependencies = [
= "clap",
+ "colored",
= "env_logger",
= "glob",
= "log",
@@ -412,6 +422,15 @@ version = "0.2.1"
=source = "registry+https://github.com/rust-lang/crates.io-index"
=checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
=
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
=[[package]]
=name = "windows-sys"
=version = "0.61.2"
@@ -420,3 +439,67 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
=dependencies = [
= "windows-link",
=]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"index e75e559..3ffa046 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2024"
=
=[dependencies]
=clap = { version = "4.5.51", features = ["derive"] }
+colored = "3.0.0"
=env_logger = "0.11.8"
=glob = "0.3.3"
=log = "0.4.28"index 97c2c5e..b5da4fa 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -52,7 +52,8 @@ fn main() -> Result<(), Box<dyn Error>> {
= return Err(format!("The {} is neither a file nor directory", input.display()).into());
= };
=
- log::info!("Collected {} suites:\n\n{}", spec.suites.len(), spec);
+ log::info!("Collected {} suites.", spec.suites.len());
+ log::debug!("Evaluating:\n\n{spec}");
=
= let mut report = EvaluationReport::new(&spec);
= report.evaluate(cli.verbose);index b0d5c41..787b9bc 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,6 +1,6 @@
-use serde::{Deserialize, Serialize};
-
=use crate::spec::{Scenario, Spec, Step, Suite};
+use colored::Colorize;
+use serde::{Deserialize, Serialize};
=use std::error::Error;
=use std::fmt::Display;
=use std::io::{BufRead, BufReader, LineWriter, Write};
@@ -222,8 +222,8 @@ impl Display for EvaluationReport<'_> {
= writeln!(
= f,
= "\n{title} ({interpreter})",
- title = suite.title,
- interpreter = suite.interpreter
+ title = suite.title.bold().underline(),
+ interpreter = suite.interpreter.dimmed()
= )?;
=
= for ScenarioReport {
@@ -233,20 +233,20 @@ impl Display for EvaluationReport<'_> {
= } in scenarios.iter()
= {
= let sigil = match status {
- ScenarioStatus::Done => "✔", // TODO: Use different icon depending on steps status
- ScenarioStatus::Pending => "❔",
- ScenarioStatus::FailedToRun { .. } => "☠️",
+ ScenarioStatus::Done => "✓".to_string().bold(), // TODO: Use different icon depending on steps status
+ ScenarioStatus::Pending => "?".to_string().bold().dimmed(),
+ ScenarioStatus::FailedToRun { .. } => "x".to_string().bold().red(),
= };
= writeln!(f, "\n {sigil} {title}\n", title = scenario.title)?;
= if let ScenarioStatus::FailedToRun { error } = status {
- writeln!(f, " {error}\n")?;
+ writeln!(f, " {}\n", error.to_string().red())?;
= }
=
= for StepReport { step, status } in steps.iter() {
= let sigil = match status {
- StepStatus::Ok => "✅",
- StepStatus::Failed { .. } => "❌",
- StepStatus::NotEvaluated => "⃞",
+ StepStatus::Ok => "⊞".to_string().bold().green(),
+ StepStatus::Failed { .. } => "⊠".to_string().bold().red(),
+ StepStatus::NotEvaluated => "□".to_string().bold(),
= };
= writeln!(
= f,
@@ -256,7 +256,7 @@ impl Display for EvaluationReport<'_> {
= )?;
=
= if let StepStatus::Failed { reason } = status {
- writeln!(f, "\n {reason}\n")?;
+ writeln!(f, "\n {}\n", reason.red())?;
= }
= }
= }