Commits: 3
Improve the readme a bit
index 8d5f4a0..969b745 100644
--- a/README.md
+++ b/README.md
@@ -5,32 +5,32 @@
=A BDD test runner inspired by Gauge, but better.
=
= * No magic
- * Flexibility
+ * Simple flexibility
= * Cross-platform
= * Batteries included, but replaceable
=
=
-## No magic
+## No Magic
=
-Test scenarios are evaluated by interpreters, which are normal programs that read JSON on `stdin` and write JSON to `stdout`. User can implement them in any language or framework they like, using any tools they are comfortable with (editors, language servers, libraries, etc).
+Test scenarios are evaluated by interpreters, which are normal programs that read JSON on `stdin` and write JSON to `stdout`. User can implement them in any language or framework they like, using any tools they are comfortable with (editors, language servers, libraries, etc.).
=
=
-## Flexibility
+## Simple Flexibility
=
-Different test suites can use different interpreters, which may be implemented using different languages. If your computer can run it, `tbb` can use it as an interpreter.
=
-When executing a step, the interpreter will get more data from the markdown fragment which defined the step
+You define scenarios and steps to verify them. Steps are executed by programs you write. Different test suites can use different interpreters, which may be implemented using different languages. If your computer can run it, `tbb` can use it as an interpreter.
+
+When executing a step, the interpreter will get a lot of data from the Markdown fragment which defined the step, like:
=
= - tables
= - lists
= - code blocks
-
-and even the original markdown fragment itself.
+ - and even the original Markdown fragment itself.
=
=
-## Cross platform
+## Unix Powered
=
-Support for Linux, BSD, OS X and Web (WASM).
+Support for Linux, BSD, and OS X and uses standard Unix conventions: child processes, input, and output streams, etc.
=
=
=## Batteries included, but replaceableAdd "TBB" to Harper dictionary
Harper is a spell and grammar checker I am using.
new file mode 100644
index 0000000..782251e
--- /dev/null
+++ b/.harper-dictionary.txt
@@ -0,0 +1 @@
+TBBWIP: implement storing snippets in a report
After working with Jewiet on her project it became clear that error messages are often not enough to figure out what's wrong with a system under test. We need more data, especially from preceding steps. Observations like logging, attachments, audio and video clips.
For we designed the observations API. It will allow to attach various pieces of data to step reports.
This commit introduces a first kind of observation - snippet of code.
index 4160666..c30568f 100644
--- a/samples/basic.md
+++ b/samples/basic.md
@@ -58,6 +58,12 @@ This
= * There are `3` `r`s in the word `strawberry`
= * The following table maps words to their lengths
=
+ | word | length |
+ |--------|--------|
+ | cat | 3 |
+ | stork | 5 |
+ | snake | 5 |
+
= | word | length |
= |--------|--------|
= | cat | 3 |index 45dfd6f..01fc945 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -4,7 +4,7 @@ import json
=import unittest
=
=import spec.tbb as tbb
-from spec.tbb import step, log
+from spec.tbb import step, log, send_snippet
=
=
=# Nice assertions with helpful error messages
@@ -12,7 +12,7 @@ tester = unittest.TestCase()
=
=@step("Add {0} and {1} to get {2}")
=def add_and_verify(a: float, b: float, expected: float, **kwargs):
- log.debug(f"{ a } + { b } = { expected }?")
+ send_snippet("text", f"{ a } + { b } = { expected }?")
=
= tester.assertEqual(expected, a + b)
=
@@ -53,8 +53,11 @@ def step_implementation_06(word: str, reverse: str, **kwargs):
=@step("The following table maps words to their lengths")
=def step_implementation_07(**kwargs):
= for table in kwargs["tables"]:
+ send_snippet("text", f"Received a table with {len(table)} x {len(table[0])} cells")
+
= # Skip the first row - it's a heading
= for [word, length] in table[1:]:
+ send_snippet("text", f"Is '{word}' {length} long?")
= actual_length = len(word)
= tester.assertEqual(actual_length, int(length), f"the length of {word=}")
=index b7bf5de..76eb920 100644
--- a/spec/tbb.py
+++ b/spec/tbb.py
@@ -185,6 +185,15 @@ def ready():
= break
=
=
+def send_snippet(language, content, meta = ""):
+ send({
+ "type": "Snippet",
+ "language": language,
+ "content": content,
+ "meta": meta
+ })
+
+
=# TODO: Docstring and test cases for `get_at`. See `self-check` for example use.
=def get_at(collection, path: [str]):
= value = collectionindex d88b6e0..0906f9c 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -154,8 +154,9 @@ impl<'a> ScenarioReport<'a> {
= };
= }
=
- for &mut StepReport {
+ 'steps_loop: for &mut StepReport {
= step,
+ ref mut observations,
= ref mut status,
= } in self.steps.iter_mut()
= {
@@ -167,30 +168,44 @@ impl<'a> ScenarioReport<'a> {
= },
= )?;
=
- match Self::receive(&mut reader)? {
- InterpreterMessage::Success => {
- log::debug!("Step executed successfully: {step:#?}");
- *status = StepStatus::Ok;
- }
- InterpreterMessage::Failure { reason, hint } => {
- log::debug!("Step failed:\n\n {step:#?} \n\n {reason:#?}");
- *status = StepStatus::Failed { reason, hint };
-
- // Do not execute subsequent steps.
- //
- // A scenario is a unit of testing. Later steps are expected
- // to depend on previous ones. If a step fails, continuing
- // may lead to a mess.
- //
- // Maybe this can be configured via front-matter?
- break;
- }
- unexpected => {
- return Err(anyhow!(
- "unexpected message received from the interpreter: {unexpected:#?}"
- ));
- }
- };
+ loop {
+ match Self::receive(&mut reader)? {
+ InterpreterMessage::Snippet {
+ language,
+ meta,
+ content,
+ } => {
+ observations.push(Observation::Snippet {
+ language,
+ meta,
+ content,
+ });
+ }
+ InterpreterMessage::Success => {
+ log::debug!("Step executed successfully: {step:#?}");
+ *status = StepStatus::Ok;
+ break; // Proceed with the next step
+ }
+ InterpreterMessage::Failure { reason, hint } => {
+ log::debug!("Step failed:\n\n {step:#?} \n\n {reason:#?}");
+ *status = StepStatus::Failed { reason, hint };
+
+ // Do not execute subsequent steps.
+ //
+ // A scenario is a unit of testing. Later steps are expected
+ // to depend on previous ones. If a step fails, continuing
+ // may lead to a mess.
+ //
+ // Maybe this can be configured via front-matter?
+ break 'steps_loop;
+ }
+ unexpected => {
+ return Err(anyhow!(
+ "unexpected message received from the interpreter: {unexpected:#?}"
+ ));
+ }
+ };
+ }
= }
=
= drop(writer);
@@ -259,6 +274,32 @@ impl<'a> ScenarioReport<'a> {
=pub struct StepReport<'a> {
= step: &'a Step,
= status: StepStatus,
+ observations: Vec<Observation>,
+}
+
+#[derive(Debug, Serialize)]
+pub enum Observation {
+ Snippet {
+ language: String,
+ meta: String,
+ content: String,
+ },
+}
+
+impl Display for Observation {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Observation::Snippet {
+ language,
+ meta,
+ content,
+ } => {
+ writeln!(f, "``` {language} {meta}")?;
+ writeln!(f, "{content}")?;
+ writeln!(f, "```")
+ }
+ }
+ }
=}
=
=#[derive(Debug, Eq, PartialEq, Serialize)]
@@ -348,7 +389,15 @@ impl Display for EvaluationReport<'_> {
= writeln!(f, "{sigils}")?;
= writeln!(f, "{message}")?;
= } else {
- for (index, StepReport { step, status }) in steps.iter().enumerate() {
+ for (
+ index,
+ StepReport {
+ step,
+ status,
+ observations,
+ },
+ ) in steps.iter().enumerate()
+ {
= let sigil = match status {
= StepStatus::Ok => "⊞".to_string().bold().green(),
= StepStatus::Failed { .. } => "⊠".to_string().bold().red(),
@@ -376,6 +425,10 @@ impl Display for EvaluationReport<'_> {
= indentation = "".indent(4),
= )?;
=
+ for observation in observations {
+ writeln!(f, "\n{}", observation.to_string().indent(6))?;
+ }
+
= if let StepStatus::Failed { reason, hint } = status {
= writeln!(f, "\n{}\n", reason.indent(6).red())?;
= if let Some(hint) = hint {
@@ -409,6 +462,7 @@ impl<'a> From<&'a Step> for StepReport<'a> {
= Self {
= step,
= status: StepStatus::NotEvaluated,
+ observations: Vec::default(),
= }
= }
=}
@@ -420,6 +474,11 @@ pub enum InterpreterMessage {
= InterpreterState {
= ready: bool,
= },
+ Snippet {
+ language: String,
+ meta: String,
+ content: String,
+ },
= Success,
= Failure {
= reason: String,