Week 50 of 2025
Development log of Tad Better Behavior
6 items
Selfcheck: call ready() only when running as main
On by
So the module can be imported without triggering any IO.
index ce81686..5b69a9e 100644
--- a/spec/self-check.py
+++ b/spec/self-check.py
@@ -191,4 +191,5 @@ def step_implementation_14(first_line: int, last_line: int, document_path: str,
= )
=
=
-tbb.ready()
+if __name__ == "__main__":
+ tbb.ready()Self-check: improve white-space logic in blocks
On by
It was only trimming the whitespace at the end of the output and the block. Now it trims on every line.
index 5b69a9e..d59ea92 100644
--- a/spec/self-check.py
+++ b/spec/self-check.py
@@ -90,12 +90,11 @@ def step_implementation_04(label: str, **kwargs):
=
= # Trim all blank lines and trailing whitespece.
= # Without it the assertions are very brittle.
- needle = re.sub("\\s+$", "", block)
- heystack = re.sub("\\s+$", "", output)
+ needle = re.sub("\\s+\n", "\n", block)
+ haystack = re.sub("\\s+\n", "\n", output)
= # tester.assertIn gives unreadable output
=
-
- assert needle in heystack, dedent(f"""
+ assert needle in haystack, dedent(f"""
= block not found
=
= ``` textSpecify and implement the JSON output format
On by
index a22c28e..d579577 100644
--- a/spec/basic-usage.md
+++ b/spec/basic-usage.md
@@ -15,9 +15,23 @@ interpreter: "python -m spec.self-check"
=
= ``` text
= Options:
- -v, --verbose Enable verbose logging
- -h, --help Print help
- -V, --version Print version
+ -v, --verbose
+ Enable verbose logging
+
+ -f, --format <FORMAT>
+ Set output format
+
+ Possible values:
+ - text: Formatted text output
+ - json: JSON output
+
+ [default: text]
+
+ -h, --help
+ Print help (see a summary with '-h')
+
+ -V, --version
+ Print version
= ```
=
= * The output will contain `the sub-commands` blockindex d59ea92..aebf88c 100644
--- a/spec/self-check.py
+++ b/spec/self-check.py
@@ -189,6 +189,56 @@ def step_implementation_14(first_line: int, last_line: int, document_path: str,
= kwargs["source"]
= )
=
+@step("Parse the output as JSON and store it as {0}")
+def step_implementation_15(name: str, **kwargs):
+ import json
+ global completed
+ global state
+ state[name] = json.loads(completed.stdout)
+
+@step("From {0} get the property {1} as {2}")
+def step_implementation_16(from_name: str, property_name: str, name: str, **kwargs):
+ global state
+ state[name] = get_at(state, [from_name, property_name])
+
+@step("From {0} get the element where {1} equals {2} as {3}")
+def step_implementation_17(from_name: str, property_name: str, json_value: str, name: str, **kwargs):
+ import json
+ collection = state[from_name]
+ expected = json.loads(json_value)
+ matching = list(filter(lambda element: element[property_name] == expected, collection))
+ if len(matching) == 0:
+ raise Exception("No matching element found")
+
+ state[name] = matching[0]
+
+@step("The property {0} of {1} equals to {2}")
+def step_implementation_18(property_name: str, name: str, json_value: str, **kwargs):
+ import json
+ global state
+ expected = json.loads(json_value)
+ actual = get_at(state, [name, property_name])
+ tester.assertEqual(expected, actual)
+
+@step("The property {0} of {1} is an array that includes the value {2}")
+def step_implementation_19(property_name: str, name: str, json_value: str, **kwargs):
+ import json
+ global state
+ expected = json.loads(json_value)
+ actual = get_at(state, [name, property_name])
+ tester.assertIn(expected, actual)
+
+@step("From {0} get the element with the value at {1} equal to {2} as {3}")
+def step_implementation_20(from_name: str, json_path: str, json_value: str, name: str, **kwargs):
+ import json
+ global state
+ path = json.loads(json_path)
+ expected = json.loads(json_value)
+ matching = list(filter(lambda element: get_at(element, path) == expected, state[from_name]))
+ if len(matching) == 0:
+ raise Exception("No matching element found")
+
+ state[name] = matching[0]
=
=if __name__ == "__main__":
= tbb.ready()index d674ae7..7da77aa 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,13 +3,14 @@ mod report;
=mod spec;
=
=use anyhow::{Context, Error, bail};
-use clap::{Parser, Subcommand};
+use clap::{Parser, Subcommand, ValueEnum};
=use env_logger;
=use glob::glob;
=use log;
=use report::{EvaluationReport, EvaluationSummary, ReportOptions};
=use spec::Spec;
-use std::{path::PathBuf, process};
+use std::path::PathBuf;
+use std::process;
=
=#[derive(Parser)]
=#[command(version, about, long_about=None)]
@@ -18,10 +19,23 @@ struct Cli {
= #[arg(short, long)]
= verbose: bool,
=
+ /// Set output format
+ #[arg(short, long, default_value = "text")]
+ format: OutputFormat,
+
= #[command(subcommand)]
= command: Command,
=}
=
+#[derive(Clone, Copy, ValueEnum, Eq, PartialEq, PartialOrd, Ord, Debug)]
+enum OutputFormat {
+ /// Formatted text output
+ Text,
+
+ /// JSON output
+ JSON,
+}
+
=#[derive(Subcommand)]
=enum Command {
= /// Print the suites, scenarios and steps of the specification
@@ -52,17 +66,19 @@ fn main() -> Result<(), Error> {
= .filter_or("RUST_LOG", if cli.verbose { "trace" } else { "info" });
= env_logger::init_from_env(log_env);
=
+ log::debug!("The output format is {:?}", cli.format);
+
= match cli.command {
- Command::Show { input } => show(input),
+ Command::Show { input } => show(input, cli.format),
= Command::Evaluate {
= input,
= only,
= exclude,
- } => evaluate(input, only, exclude, cli.verbose),
+ } => evaluate(input, only, exclude, cli.verbose, cli.format),
= }
=}
=
-fn show(input: PathBuf) -> Result<(), Error> {
+fn show(input: PathBuf, format: OutputFormat) -> Result<(), Error> {
= log::debug!("Reading the specification from {}", input.display());
=
= let input = input.canonicalize()?;
@@ -93,7 +109,10 @@ fn show(input: PathBuf) -> Result<(), Error> {
= };
=
= log::debug!("Collected {} suites.", spec.suites.len());
- println!("{spec}");
+ match format {
+ OutputFormat::Text => println!("{spec}"),
+ OutputFormat::JSON => println!("{json}", json = serde_json::to_string(&spec)?),
+ }
= Ok(())
=}
=
@@ -102,6 +121,7 @@ fn evaluate(
= only: Vec<String>,
= exclude: Vec<String>,
= verbose: bool,
+ format: OutputFormat,
=) -> Result<(), Error> {
= log::debug!("Reading the specification from {}", input.display());
=
@@ -172,7 +192,10 @@ fn evaluate(
= report.evaluate(verbose);
=
= // Print the report on STDOUT
- println!("{report}");
+ match format {
+ OutputFormat::Text => println!("{report}"),
+ OutputFormat::JSON => println!("{json}", json = serde_json::to_string(&report)?),
+ }
=
= match EvaluationSummary::from(report) {
= EvaluationSummary::AllOk => Ok(()),
@@ -199,7 +222,6 @@ fn evaluate(
= )
= }
= }
- // TODO: Print errors from failing steps
= process::exit(1)
= }
= }index 4cd1f31..e89089b 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -3,7 +3,7 @@ use crate::spec::{Scenario, Spec, Step, Suite};
=use anyhow::{Context, anyhow};
=use colored::Colorize;
=use indoc::formatdoc;
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize, Serializer};
=use std::collections::BTreeSet;
=use std::fmt::Display;
=use std::io::{BufRead, BufReader, LineWriter, Write};
@@ -25,6 +25,7 @@ pub struct ReportOptions {
= pub exclude_scenarios: BTreeSet<String>,
=}
=
+#[derive(Debug, Serialize)]
=pub struct EvaluationReport<'a> {
= suites: Vec<SuiteReport<'a>>,
=}
@@ -60,6 +61,7 @@ impl<'a> EvaluationReport<'a> {
= }
=}
=
+#[derive(Debug, Serialize)]
=pub struct SuiteReport<'a> {
= suite: &'a Suite,
= scenarios: Vec<ScenarioReport<'a>>,
@@ -95,6 +97,7 @@ impl<'a> SuiteReport<'a> {
= }
=}
=
+#[derive(Debug, Serialize)]
=pub struct ScenarioReport<'a> {
= scenario: &'a Scenario,
= status: ScenarioStatus,
@@ -252,12 +255,13 @@ impl<'a> ScenarioReport<'a> {
= }
=}
=
+#[derive(Debug, Serialize)]
=pub struct StepReport<'a> {
= step: &'a Step,
= status: StepStatus,
=}
=
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Eq, PartialEq, Serialize)]
=pub enum StepStatus {
= Ok,
= Failed {
@@ -267,10 +271,21 @@ pub enum StepStatus {
= NotEvaluated,
=}
=
+#[derive(Debug, Serialize)]
=pub enum ScenarioStatus {
= Done,
= Pending,
- FailedToRun { error: anyhow::Error },
+ FailedToRun {
+ #[serde(serialize_with = "serialize_error")]
+ error: anyhow::Error,
+ },
+}
+
+fn serialize_error<S>(error: &anyhow::Error, serializer: S) -> Result<S::Ok, S::Error>
+where
+ S: Serializer,
+{
+ serializer.serialize_str(&format!("{error:?}"))
=}
=
=impl Display for EvaluationReport<'_> {index 955e225..49432c3 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -12,7 +12,7 @@ use std::path::PathBuf;
=///
=/// Each system under test has a single specification, whereas each suite
=/// describes a different aspect of it.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Serialize)]
=pub struct Spec {
= pub suites: Vec<Suite>,
=}
@@ -27,7 +27,7 @@ pub struct Spec {
=/// From a markdown perspective, a single document can contain multiple suites,
=/// demarcated by an h1 heading. In such a case all those suites use the same
=/// interpreter, since there can be only one front-matter per document.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
=pub struct Suite {
= pub title: String,
= pub tags: BTreeSet<String>,
@@ -42,7 +42,7 @@ pub struct Suite {
=/// spawned and fed steps. Scenarios can be stateful, i.e. running a step can
=/// affect subsequent steps. It's up to the interpreter to implement state
=/// management.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
=pub struct Scenario {
= pub title: String,
= pub tags: BTreeSet<String>,Fix extra blank lines in report (text format)
On by
index e89089b..d88b6e0 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -312,7 +312,6 @@ impl Display for EvaluationReport<'_> {
= ScenarioStatus::FailedToRun { .. } => "x".to_string().bold().red(),
= };
= writeln!(f, "\n {sigil} {}", scenario.to_string().indent(4).trim())?;
- writeln!(f, "")?;
=
= if let ScenarioStatus::FailedToRun { error } = status {
= writeln!(
@@ -328,7 +327,6 @@ impl Display for EvaluationReport<'_> {
= indentation = "".indent(6)
= )?
= }
- writeln!(f, "\n")?;
= }
=
= if scenario.steps.is_empty() {Add specification for JSON output
On by
It was not checked in by accident.
new file mode 100644
index 0000000..6189733
--- /dev/null
+++ b/spec/json-output.md
@@ -0,0 +1,43 @@
+---
+interpreter: "python -m spec.self-check"
+---
+
+# JSON Output
+
+## Printing suites and scenarios in JSON format
+
+ * Run the program with `--format=json show samples/some-invalid/` command line arguments
+ * Parse the output as JSON and store it as `the output`
+ * From `the output` get the property `suites` as `the suites`
+ * From `the suites` get the element where `title` equals `"Suite 1 from the invalid document"` as `the invalid suite`
+ * The property `interpreter` of `the invalid suite` equals to `"invalid interpreter"`
+ * The property `tags` of `the invalid suite` is an array that includes the value `"not-implemented"`
+ * The exit code should be `0`
+
+
+## Printing a report in JSON format
+
+ * Run the program with `--format=json evaluate samples/basic.md` command line arguments
+
+ * Parse the output as JSON and store it as `the output`
+ * From `the output` get the property `suites` as `the suites`
+ * From `the suites` get the element with the value at `["suite", "title"]` equal to `"Basic BDD suite"` as `the suite report`
+
+ Notice that in the structure of a suite report is a bit different than a spec - each suite contains the `suite` and `scenarios` keys. The `suite` is the same as in the spec JSON, while `scenarios` contains report of each scenario.
+
+ TODO: Consider if we could get rid of this difference. Basically only have a `Report` struct. Upon loading, it would have all steps in `Pending` state. This way a lot of code could be removed and the structure would become flatter.
+
+ * From `the suite report` get the property `suite` as `the suite spec`
+ * The property `interpreter` of `the suite spec` equals to `"python -m samples.basic"`
+ * From `the suite report` get the property `scenarios` as `the scenarios`
+ * From `the scenarios` get the element with the value at `["scenario", "title"]` equal to `"Text"` as `the text scenario report`
+ * The property `status` of `the text scenario report` equals to `"Done"`
+
+ The execution is done, although some steps failed!
+
+ * From `the text scenario report` get the property `steps` as `the text scenario steps`
+ * From `the text scenario steps` get the element with the value at `["step", "description"]` equal to `"The reverse of CIA is KGB"` as `the failing step`
+ * From `the failing step` get the property `status` as `the failed status`
+ * From `the failed status` get the property `Failed` as `the details of failure`
+ * The property `reason` of `the details of failure` equals to `"'KGB' != 'AIC'"`
+ * The exit code should be `1`Bump minor version to 0.10.0
On by
To celebrate JSON output format.
index 0297ef9..c7f81fa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -400,7 +400,7 @@ dependencies = [
=
=[[package]]
=name = "tad-better-behavior"
-version = "0.9.0"
+version = "0.10.0"
=dependencies = [
= "anyhow",
= "clap",index e4b9e37..782f21d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
=[package]
=name = "tad-better-behavior"
-version = "0.9.0"
+version = "0.10.0"
=edition = "2024"
=
=[dependencies]