Commits: 10
On fail expect blank line between reason and hint
index 8a2160d..a69fe72 100644
--- a/spec/basic-usage.md
+++ b/spec/basic-usage.md
@@ -144,8 +144,9 @@ Notice it's similar to the output of `tbb list`, but now contains unicode symbol
= ⊞ The following table maps words to their lengths []
= ⊞ The reverse of abc is cba ["abc", "cba"]
= ⊠ The reverse of CIA is KGB ["CIA", "KGB"]
-
+
= 'KGB' != 'AIC'
+
= - KGB
= + AIC
= ```Block assertion will ignore trailing whitespece
This includes blank lines.
index a69fe72..965b60a 100644
--- a/spec/basic-usage.md
+++ b/spec/basic-usage.md
@@ -126,7 +126,7 @@ Notice it's similar to the output of `tbb list`, but now contains unicode symbol
= ```text
= ✓ Arithmetic
= tagged: math
-
+
= ⊞ Add 7 and 5 to get 12 ["7", "5", "12"]
= ⊞ Divide 10 by 4 to get 2.5 ["10", "4", "2.5"]
= ⊞ Subtract 7 from 5 to get -2 ["7", "5", "-2"]index b4824af..5ce84a8 100644
--- a/spec/self-check.py
+++ b/spec/self-check.py
@@ -93,19 +93,24 @@ def step_implementation_04(label: str, **kwargs):
= block = kwargs['code_blocks'][0]['value']
= output = completed.stdout.decode("utf-8")
=
+ # Trim all blank lines and trailing whitespece.
+ # Without it the assertions are very brittle.
+ needle = re.sub("\s+$", "", block)
+ heystack = re.sub("\s+$", "", output)
= # tester.assertIn gives unreadable output
=
- assert block in output, dedent(f"""
+
+ assert needle in heystack, dedent(f"""
= block not found
=
= ``` text
- {tbb.indent_tail(block, " ")}
+ {indent_tail(block, " ")}
= ```
=
= --- not found in output ---
=
= ``` text
- {tbb.indent_tail(output, " ")}
+ {indent_tail(output, " ")}
= ```
= """)
=Implement the tagging feature
index f655e1d..d2c897c 100644
--- a/src/indentable.rs
+++ b/src/indentable.rs
@@ -118,3 +118,5 @@ mod tests {
= assert_eq!(input.indent(4), expected);
= }
=}
+
+// TODO: Implement Indentable for Formatterindex 178d7f3..74420ca 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -6,6 +6,7 @@ use indoc::formatdoc;
=use serde::{Deserialize, Serialize};
=use std::fmt::Display;
=use std::io::{BufRead, BufReader, LineWriter, Write};
+use std::ops::Not;
=use std::process::{Command, Stdio};
=
=pub struct EvaluationReport<'a> {
@@ -242,6 +243,16 @@ impl Display for EvaluationReport<'_> {
= interpreter = suite.interpreter.dimmed()
= )?;
=
+ if suite.tags.is_empty().not() {
+ let tags = suite
+ .tags
+ .iter()
+ .map(|tag| tag.underline().to_string())
+ .collect::<Vec<String>>()
+ .join(" ");
+ writeln!(f, "tagged: {tags}",)?;
+ }
+
= for ScenarioReport {
= scenario,
= status,
@@ -255,10 +266,21 @@ impl Display for EvaluationReport<'_> {
= };
= writeln!(
= f,
- "\n{indentation}{sigil} {title}\n",
+ "\n{indentation}{sigil} {title}",
= indentation = "".indent(0),
- title = scenario.title
+ title = scenario.title.bold()
= )?;
+ if scenario.tags.is_empty().not() {
+ let tags = scenario
+ .tags
+ .iter()
+ .map(|tag| tag.underline().to_string())
+ .collect::<Vec<String>>()
+ .join(" ");
+ writeln!(f, "{indentation}tagged: {tags}", indentation = "".indent(2))?;
+ };
+ writeln!(f, "")?;
+
= if let ScenarioStatus::FailedToRun { error } = status {
= writeln!(
= f,index 22546c8..4e87ba4 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -1,7 +1,9 @@
+use crate::indentable::Indentable;
=use anyhow::{Context, anyhow, bail};
+use colored::Colorize;
=use indoc::indoc;
=use serde::{Deserialize, Serialize};
-use std::fmt::Display;
+use std::{fmt::Display, ops::Not};
=
=/// Spec is a collection of suites that together describe a system
=///
@@ -25,6 +27,7 @@ pub struct Spec {
=#[derive(Debug, Clone)]
=pub struct Suite {
= pub title: String,
+ pub tags: Vec<String>,
= pub interpreter: String,
= pub scenarios: Vec<Scenario>,
=}
@@ -38,6 +41,7 @@ pub struct Suite {
=#[derive(Debug, Clone)]
=pub struct Scenario {
= pub title: String,
+ pub tags: Vec<String>,
= pub steps: Vec<Step>,
=}
=
@@ -89,8 +93,27 @@ impl Display for Spec {
= interpreter = suite.interpreter
= )?;
=
+ if suite.tags.is_empty().not() {
+ writeln!(f, "tagged: {tags}", tags = suite.tags.join(" "))?;
+ }
+
= for scenario in suite.scenarios.iter() {
- writeln!(f, "\n * {title}\n", title = scenario.title,)?;
+ writeln!(
+ f,
+ "\n{indentation}* {title}",
+ title = scenario.title,
+ indentation = "".indent(2)
+ )?;
+ if scenario.tags.is_empty().not() {
+ let tags = scenario
+ .tags
+ .iter()
+ .map(|tag| tag.underline().to_string())
+ .collect::<Vec<String>>()
+ .join(" ");
+ writeln!(f, "{indentation}tagged: {tags}", indentation = "".indent(4))?;
+ }
+ writeln!(f, "")?;
=
= for (index, step) in scenario.steps.iter().enumerate() {
= writeln!(
@@ -151,6 +174,14 @@ impl From<&markdown::mdast::Table> for Table {
=#[derive(Deserialize)]
=pub struct FrontMatter {
= pub interpreter: String,
+ #[serde(default)]
+ pub tags: Vec<String>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct Metadata {
+ #[serde(default)]
+ pub tags: Vec<String>,
=}
=
=impl Suite {
@@ -224,6 +255,10 @@ impl TryFrom<markdown::mdast::Node> for Suite {
= "#})?
= .to_string();
=
+ // Find any yaml meta code block after the h1 but before any h2
+
+ let mut suite_tags = frontmatter.tags.to_owned();
+
= // Extract scenarios and steps
= // Split into sections, each starting at h2
= // Convert each section into a scenario
@@ -234,6 +269,7 @@ impl TryFrom<markdown::mdast::Node> for Suite {
= if heading.depth == 2 {
= scenarios.push(Scenario {
= title: node.to_string(),
+ tags: vec![],
= steps: [].into(),
= });
= }
@@ -258,12 +294,34 @@ impl TryFrom<markdown::mdast::Node> for Suite {
= // A list before any scenario. Ignoring.
= }
= }
+ markdown::mdast::Node::Code(code) => {
+ if code.meta == Some("tbb".to_string()) {
+ if code.lang != Some("yaml".to_string()) {
+ bail!(
+ "Found a metadata code block with {lang} syntax at {position:?}. Currently metadata can only be provided as YAML.",
+ lang = code.lang.as_deref().unwrap_or("unspecified"),
+ position = code.position.clone().context(
+ "No position information for the metadata code block"
+ )?
+ )
+ }
+ let mut metadata: Metadata = serde_yaml::from_str(&code.value)?;
+ log::debug!("Found metadata block: {metadata:?}");
+
+ if let Some(scenario) = scenarios.last_mut() {
+ scenario.tags.append(&mut metadata.tags);
+ } else {
+ suite_tags.append(&mut metadata.tags);
+ }
+ }
+ }
= _ => continue,
= }
= }
=
= Ok(Self {
= title,
+ tags: suite_tags,
= scenarios,
= interpreter: frontmatter.interpreter,
= })Colorize output from list sub-command
index 4e87ba4..c144dff 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -89,19 +89,25 @@ impl Display for Spec {
= writeln!(
= f,
= "\n{title} ({interpreter})",
- title = suite.title,
- interpreter = suite.interpreter
+ title = suite.title.bold().underline(),
+ interpreter = suite.interpreter.dimmed()
= )?;
=
= if suite.tags.is_empty().not() {
- writeln!(f, "tagged: {tags}", tags = suite.tags.join(" "))?;
+ let tags = suite
+ .tags
+ .iter()
+ .map(|tag| tag.underline().to_string())
+ .collect::<Vec<String>>()
+ .join(" ");
+ writeln!(f, "tagged: {tags}")?;
= }
=
= for scenario in suite.scenarios.iter() {
= writeln!(
= f,
= "\n{indentation}* {title}",
- title = scenario.title,
+ title = scenario.title.bold(),
= indentation = "".indent(2)
= )?;
= if scenario.tags.is_empty().not() {
@@ -118,9 +124,9 @@ impl Display for Spec {
= for (index, step) in scenario.steps.iter().enumerate() {
= writeln!(
= f,
- " {index:02}. {description} {arguments:?}",
+ " {index:02}. {description} {arguments}",
= description = step.description,
- arguments = step.arguments
+ arguments = format!("{:?}", step.arguments).dimmed()
= )?;
= }
= }Define vocabulary
Previously the word "run" was used ambiguously. Now it's clarified that "a specs is evaluated", "a scenario is run" and "a step is executed".
index e6c1b16..cab0847 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,29 @@ and even the original markdown fragment itself.
=
=Support for Linux, BSD, OS X and Web (WASM).
=
+# Vocabulary
+
+<!-- TODO: Reformat vocabulary as a definition list? -->
+
+When developing a system, the totality of requirements is called "**specification**" or "**spec**" for short. Making sure that **system under test** conforms to a _spec_ is called "**evaluating a spec**". The goal of _evaluating a spec_ is to produce a **report**.
+
+A _spec_ consists of multiple **suites**. A _suite_ should concern a broad feature of a system, like authentication, processing orders, etc.
+
+_Suites_ consist of multiple **scenarios**. To _evaluate a spec_ TBB will **run** every _scenario_ from every _suite_. Scenarios should be isolated from each other, i.e. they should not depend on other scenarios. Even if some scenarios fail, all should be evaluated.
+
+Scenarios involve **steps** that are being **executed**. Typically steps will implement one of "arrange", "act" and "assert" operations, but there is no strict distinction between them. Each step can **succeed** or **fail** with a **reason** and an optional **hint**. A scenario is only considered successful when all it's steps are successfully executed.
+
+_Suites_ and _scenarios_ can be tagged. **Tags** can be used for filtering which part of the spec should be evaluated. This is useful during development, to focus on a narrow set of features.
+
+A _spec_ can be define across one or more **documents**. These are markdown files from which TBB reads a spec to evaluate.
+
+_Scenarios_ are _run_ using **interpreters**. These are programs responsible for running scenarios. The **control program** (`tbb evaluate`) will start an _interpreter_ process for each _scenario_ and sequentially pass _steps_ to it.
+
+In summary:
+
+ - A spec is evaluated to produce a report
+ - A scenario is run
+ - A step is executed
=
=# Roadmap
=
@@ -77,7 +100,7 @@ Support for Linux, BSD, OS X and Web (WASM).
= - [ ] HTTP client
= - [ ] Web Driver client
= - [ ] E-Mail client
- - [ ] Recursive calls (call `tbb run` and such)
+ - [ ] Recursive calls (call `tbb evaluate` and such)
=- [ ] Capture more data in reports
= - [ ] Attachments (screenshots, videos, datasets, etc.)
= - [ ] Performance data (interpreters' startup times, steps' durations)index 965b60a..3ba4249 100644
--- a/spec/basic-usage.md
+++ b/spec/basic-usage.md
@@ -25,9 +25,9 @@ interpreter: "python -m spec.self-check"
=
= ``` text
= Commands:
- list Print the suites, scenarios and steps of the specification
- run Evaluate the specification
- help Print this message or the help of the given subcommand(s)
+ list Print the suites, scenarios and steps of the specification
+ evaluate Evaluate the specification
+ help Print this message or the help of the given subcommand(s)
= ```
=
=## Getting a version
@@ -75,7 +75,7 @@ interpreter: "python -m spec.self-check"
= ```
=
=
-## Running a spec from a single document
+## Evaluating a spec from a single document
=
=A complete sample output is like this:
=
@@ -109,7 +109,7 @@ A complete sample output is like this:
=Notice it's similar to the output of `tbb list`, but now contains unicode symbols to indicate the results of each step. Each completed scenario has a check mark `✓`. Successful steps have a squared plus `⊞` . The failing step is markd with a squared times symbol `⊠`. Once a step fails, the subsequent steps in a scenario are not exercised. In the report they are marked with a white square symbol `□`.
=
=
- * Run the program with `run samples/basic.md` command line arguments
+ * Run the program with `evaluate samples/basic.md` command line arguments
= * The exit code should be `1`
=
= The `basic.md` suit is intentionally wrong. It should be reflected in the status code.index 67f9fb3..5a5fb59 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -32,7 +32,7 @@ enum Command {
= },
=
= /// Evaluate the specification
- Run {
+ Evaluate {
= /// A directory or a markdown file with the spec to evaluate
= #[arg(value_name = "SPEC PATH", default_value = "./spec/")]
= input: PathBuf,
@@ -46,7 +46,7 @@ fn main() -> Result<(), Error> {
=
= match cli.command {
= Command::List { input } => list(input),
- Command::Run { input } => run(input, cli.verbose),
+ Command::Evaluate { input } => evaluate(input, cli.verbose),
= }
=}
=
@@ -85,7 +85,7 @@ fn list(input: PathBuf) -> Result<(), Error> {
= Ok(())
=}
=
-fn run(input: PathBuf, verbose: bool) -> Result<(), Error> {
+fn evaluate(input: PathBuf, verbose: bool) -> Result<(), Error> {
= log::debug!("Reading the specification from {}", input.display());
=
= let input = input.canonicalize()?;index 74420ca..e328949 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -134,7 +134,7 @@ impl<'a> ScenarioReport<'a> {
= log::debug!("Step failed:\n\n {step:#?} \n\n {reason:#?}");
= *status = StepStatus::Failed { reason, hint };
=
- // Do not run subsequent steps.
+ // Do not execute subsequent steps.
= //
= // A scenario is a unit of testing. Later steps are expected
= // to depend on previous ones. If a step fails, continuingSpecify the filtering feature (prose)
new file mode 100644
index 0000000..c0f9263
--- /dev/null
+++ b/spec/filtering.md
@@ -0,0 +1,57 @@
+---
+interpreter: "python -m spec.self-check"
+---
+
+# Filtering suites and scenarios
+
+This suite specifies the behavior of `--only` and `--exclude` options for the `evaluate` sub-command. The values given to this options will match against tags. They _can_ be prefixed with a `suite:` or `scenario:` specifier. If the prefix is given, they will only have effect on the appropriate level.
+
+
+## Only some suites included
+
+With the `--only suite:<tag>` only suites that have the given tag should be included in the report. What's not in the report, should not be evaluated (laziness).
+
+
+## Only some scenarios included
+
+With the `--exclude suite:<tag>` any suites that have the given tag should be excluded from the report. What's not in the report, should not be evaluated (laziness).
+
+
+## Some suites excluded
+
+With the `--only scenario:<tag>` only scenarios that have the given tag should be included in the report. What's not in the report, should not be evaluated (laziness).
+
+
+## Some scenarios excluded
+
+With the `--exclude scenario:<tag>` any scenarios that have the given tag should be excluded from the report. What's not in the report, should not be evaluated (laziness).
+
+
+## Only some suites included and then some scenarios excluded
+
+With the `--only suite:<tag>` only suites that have the given tag should be included in the report. What's not in the report, should not be evaluated (laziness).
+
+
+## Excluding by a universal tag
+
+The `--exclude <tag>` (i.e. without a prefix) option should have effect on both suites and scenarios tagged with `<tag>`.
+
+
+## Filtering by a universal tag
+
+Same for `--only <tag>`.
+
+
+## Nothing to evaluate in a suite
+
+If filtering results in some suites having no scenarios to evaluate, they should still be mentioned in the report.
+
+
+## No scenarios to evaluate at all
+
+If as a result of filtering there are no scenarios to evaluate, the program should print a warning and exit with an error code. This is to prevent false positives when running unsupervised (e.g. in a CI/CD system).
+
+
+## No suites to evaluate at all
+
+Same as above.Specify recursive behavior (prose)
new file mode 100644
index 0000000..6c9e67e
--- /dev/null
+++ b/spec/recursion.md
@@ -0,0 +1,23 @@
+---
+interpreter: "python -m spec.self-check"
+---
+
+# Running the program recursively
+
+It is already possible to run it recursively (by having an interpreter that as part of its job starts `tbb`), but it's a footgun. We need to provide some safeguards around it.
+
+
+## By default max recursion is 5
+
+The program should keep track of every scenario it runs, accumulating it into a list. If a scenario to be executed is present in the list more than 5 times, refuse to run it.
+
+
+## Modify max recursion via a document front-matter
+
+## Modify max recursion via a suite properties
+
+Takes precedence over front-matter.
+
+## Modify max recursion via a scenario properties
+
+Takes precedence over suite property.Specify loading all documents from directories
This is the default behavior, and currently it is not being evaluated.
index 3ba4249..22ac0e2 100644
--- a/spec/basic-usage.md
+++ b/spec/basic-usage.md
@@ -37,6 +37,29 @@ interpreter: "python -m spec.self-check"
= * The output will contain `tad-better-behavior \d+\.\d+\.\d+`
=
=
+## Listing a spec from the `./spec/` directory
+
+This is the default behavior when a path is not given. We should prepare the samples to replicate this directory structure like this
+
+```
+samples/python/
+ flake.nix
+ my-program.py
+ ...rest of the python project files
+ spec/
+ arithmentic.md
+ text.md
+ interpreters/basic.py
+```
+
+We would also need a step like `Change working directory to ...` to execute before running the program.
+
+
+## Listing a spec from a different directory
+
+Whe a directory is given as the last argument, load all documents inside (recursively).
+
+
=## Listing suites and scenarios from a single document
=
= * Run the program with `list samples/basic.md` command line argumentsSpecify that tags are a set and make it so
index 22ac0e2..5f5380b 100644
--- a/spec/basic-usage.md
+++ b/spec/basic-usage.md
@@ -189,3 +189,12 @@ Sometimes the failur is not in any particular step, but in a whole scenario, e.g
=## Multiple scenarios failure
=
=When several different scenarios fail, each one should be mentioned in the summary.
+
+
+## Tags are always in alphabetical order
+
+Consider moving it to a new Tags suite.
+
+## Tagging is idempotent.
+
+Tags are a set. Specifying the same tag more than once doesn't have any effect.index e328949..e62f399 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -4,6 +4,7 @@ use anyhow::{Context, anyhow};
=use colored::Colorize;
=use indoc::formatdoc;
=use serde::{Deserialize, Serialize};
+use std::collections::BTreeSet;
=use std::fmt::Display;
=use std::io::{BufRead, BufReader, LineWriter, Write};
=use std::ops::Not;index c144dff..e023441 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -3,7 +3,9 @@ use anyhow::{Context, anyhow, bail};
=use colored::Colorize;
=use indoc::indoc;
=use serde::{Deserialize, Serialize};
-use std::{fmt::Display, ops::Not};
+use std::collections::BTreeSet;
+use std::fmt::Display;
+use std::ops::Not;
=
=/// Spec is a collection of suites that together describe a system
=///
@@ -27,7 +29,7 @@ pub struct Spec {
=#[derive(Debug, Clone)]
=pub struct Suite {
= pub title: String,
- pub tags: Vec<String>,
+ pub tags: BTreeSet<String>,
= pub interpreter: String,
= pub scenarios: Vec<Scenario>,
=}
@@ -41,7 +43,7 @@ pub struct Suite {
=#[derive(Debug, Clone)]
=pub struct Scenario {
= pub title: String,
- pub tags: Vec<String>,
+ pub tags: BTreeSet<String>,
= pub steps: Vec<Step>,
=}
=
@@ -181,13 +183,13 @@ impl From<&markdown::mdast::Table> for Table {
=pub struct FrontMatter {
= pub interpreter: String,
= #[serde(default)]
- pub tags: Vec<String>,
+ pub tags: BTreeSet<String>,
=}
=
=#[derive(Deserialize, Debug)]
=pub struct Metadata {
= #[serde(default)]
- pub tags: Vec<String>,
+ pub tags: BTreeSet<String>,
=}
=
=impl Suite {
@@ -275,7 +277,7 @@ impl TryFrom<markdown::mdast::Node> for Suite {
= if heading.depth == 2 {
= scenarios.push(Scenario {
= title: node.to_string(),
- tags: vec![],
+ tags: BTreeSet::default(),
= steps: [].into(),
= });
= }
@@ -311,13 +313,13 @@ impl TryFrom<markdown::mdast::Node> for Suite {
= )?
= )
= }
- let mut metadata: Metadata = serde_yaml::from_str(&code.value)?;
+ let metadata: Metadata = serde_yaml::from_str(&code.value)?;
= log::debug!("Found metadata block: {metadata:?}");
=
= if let Some(scenario) = scenarios.last_mut() {
- scenario.tags.append(&mut metadata.tags);
+ scenario.tags.extend(metadata.tags);
= } else {
- suite_tags.append(&mut metadata.tags);
+ suite_tags.extend(metadata.tags);
= }
= }
= }Specify steps to verify --only suite:... option
Prepare two sample spec documents: one passing and one invalid. Only the
passing one (tagged passing) should be evaluated.
new file mode 100644
index 0000000..1e99cc7
--- /dev/null
+++ b/samples/invalid.md
@@ -0,0 +1,28 @@
+---
+interpreter: "invalid interpreter"
+tags: [ not-implemented ]
+---
+
+A second document. None of the suites from this document can pass, because the interpreter is intentionally wrong.
+
+
+# Suite 1 from the invalid document
+
+It should be possible to define multiple suites in each document.
+
+``` yaml tbb
+tags: [ seriously-underbaked ]
+```
+
+## Scenario 1.1
+
+``` yaml tbb
+tags: [ work-in-progress, very-important ]
+```
+
+ * Do something `impactful`
+ * Asses the results: `A`, `B` and `C`
+
+``` yaml tbb
+tags: [ even, more tags ]
+```new file mode 100644
index 0000000..45ce718
--- /dev/null
+++ b/samples/passing.md
@@ -0,0 +1,12 @@
+---
+interpreter: "python -m samples.basic"
+tags: [passing, basic]
+---
+
+# A little suite that could
+
+This suite should always pass.
+
+## Easy scenario
+
+ * Add `2` and `2` to get `4`deleted file mode 100644
index 8f001a4..0000000
--- a/samples/second.md
+++ /dev/null
@@ -1,16 +0,0 @@
----
-interpreter: "invalid interpreter"
----
-
-A second document, just to test if it will be injested correctly.
-
-
-# Suite 1 from a Second Document
-
-It should be possible to define multiple suites in each document.
-
-
-## Scenario 1.1
-
- * Do something `impactful`
- * Asses the results: `A`, `B` and `C`index c0f9263..70433c0 100644
--- a/spec/filtering.md
+++ b/spec/filtering.md
@@ -11,6 +11,17 @@ This suite specifies the behavior of `--only` and `--exclude` options for the `e
=
=With the `--only suite:<tag>` only suites that have the given tag should be included in the report. What's not in the report, should not be evaluated (laziness).
=
+ * Run the program with `evaluate --only suite:passing samples/` command line arguments
+ * The exit code should be `0`
+ * The output will contain `the expected suite header` block
+
+ ```text
+ A little suite that could (python -m samples.basic)
+ tagged: basic passing
+ ```
+
+ * The output will not contain `Suite 1 from the invalid document`
+
=
=## Only some scenarios included
=