Commits: 9
Use Python unittest assertion for better messages
It has specialized functions for different assertions, and includes compared values in assertion errors.
index f5bbff3..52c10db 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -1,5 +1,6 @@
=#!/usr/bin/env python3
=
+import unittest
=import json
=import logging
=import os
@@ -84,7 +85,9 @@ def ready():
=
=# User code
=# =========
-
+
+tester = unittest.TestCase()
+
=@step("Add {0} and {1} to get {2}")
=def add_and_verify(a, b, expected, **kwargs):
= log.debug(f"{ a } + { b } = { expected }?")
@@ -94,7 +97,7 @@ def add_and_verify(a, b, expected, **kwargs):
= b = float(b)
= expected = float(expected)
=
- assert a + b == expected, f"{ a } + { b } = { a + b }, not { expected }!"
+ tester.assertEqual(expected, a + b)
=
=
=@step("Divide {0} by {1} to get {2}")
@@ -106,7 +109,7 @@ def divide_and_verify(a, b, expected, **kwargs):
= b = float(b)
= expected = float(expected)
=
- assert a / b == expected, f"{ a } / { b } = { a / b }, not { expected }!"
+ tester.assertAlmostEqual (expected, a / b)
=
=
=@step("The word {0} has {1} characters")
@@ -115,7 +118,7 @@ def verify_characters_count(word: str, expected_length, **kwargs):
= expected_length = int(expected_length)
= actual_length = len(word)
=
- assert expected_length == actual_length, f"The word '{word}' is {actual_length} long, not {expected_length}"
+ tester.assertEqual(expected_length, actual_length)
=
=
=Implement automatic argument parsing in python
The python interpreter now can parse arguments into types indicated by step function annotation, i.e. if a function looks like this:
def fun(a: float, b):
# ...Then the first argument coming from tbb will be converted to float. Second parameter is not annotated, so the argument will be passed as string.
This will only work for a type with a constructor that accepts a string.
index aed3aea..e7d619c 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,11 @@ Support for Linux, OS X and Web (WASM).
=- [ ] Use for evaluating Jewiet's Form to Mail specification
=- [ ] Helper libraries
= - [ ] for Python
+ - [x] Proof-of-concept
+ - [x] `@step` decorator
+ - [x] Automatic arguments conversion (casting)
+ - [x] Insightful assertion errors
+ - [ ] Split library code from the basic interpreter
= - [ ] for Clojure
=- [ ] Capture more data in reports
= - [ ] Attachments (screenshots, videos, datasets, etc.)index 52c10db..b472e3c 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -1,6 +1,7 @@
=#!/usr/bin/env python3
=
=import unittest
+import inspect
=import json
=import logging
=import os
@@ -65,6 +66,18 @@ def ready():
= if implementation:
= log.debug(f"Found an implementation of '{ variant }'")
= try:
+ signature = inspect.signature(implementation)
+ log.info(f"Signature: {signature}")
+ for index, key in enumerate(signature.parameters):
+ parameter = signature.parameters[key]
+ log.info(f"Processing parameter {index} ({parameter.annotation})")
+ if parameter.annotation == inspect._empty:
+ log.info (f"No annotation for parameter {index} ({parameter.name}). Skipping.")
+ continue
+
+ log.info (f"Converting argument {index} to {parameter.annotation.__name__}")
+ arguments[index] = parameter.annotation(arguments[index])
+
= implementation(*arguments, **step)
= send({ "type": 'Success' })
= except Exception as error:
@@ -89,37 +102,25 @@ def ready():
=tester = unittest.TestCase()
=
=@step("Add {0} and {1} to get {2}")
-def add_and_verify(a, b, expected, **kwargs):
+def add_and_verify(a: float, b: float, expected: float, **kwargs):
= log.debug(f"{ a } + { b } = { expected }?")
=
- # TODO: Can we do this conversion automatically based on typing information?
- a = float(a)
- b = float(b)
- expected = float(expected)
-
= tester.assertEqual(expected, a + b)
=
=
=@step("Divide {0} by {1} to get {2}")
-def divide_and_verify(a, b, expected, **kwargs):
+def divide_and_verify(a: float, b: float, expected: float, **kwargs):
= log.debug(f"{ a } / { b } = { expected }?")
-
- # TODO: Can we do this conversion automatically based on typing information?
- a = float(a)
- b = float(b)
- expected = float(expected)
=
= tester.assertAlmostEqual (expected, a / b)
=
=
=@step("The word {0} has {1} characters")
-def verify_characters_count(word: str, expected_length, **kwargs):
- log.info(word)
- expected_length = int(expected_length)
+def verify_characters_count(word, expected_length: int, **kwargs):
+ log.info(f"Is '{word}' of length {expected_length}?")
= actual_length = len(word)
=
= tester.assertEqual(expected_length, actual_length)
=
-
=
=ready()Update the roadmap
index e7d619c..142bd8a 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,7 @@ Support for Linux, OS X and Web (WASM).
= - [ ] A summary at the bottom (esp. list of errors)
= - [x] Use colors
= - [ ] More instructive error messages
+ - [ ] Indent multiline error messages
=- [ ] Pass more step data to interpreters
= - [ ] Code blocks
= - [ ] ListsLet basic.py chill with the logging
index b472e3c..08a839a 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -67,15 +67,15 @@ def ready():
= log.debug(f"Found an implementation of '{ variant }'")
= try:
= signature = inspect.signature(implementation)
- log.info(f"Signature: {signature}")
+ log.debug(f"Signature: {signature}")
= for index, key in enumerate(signature.parameters):
= parameter = signature.parameters[key]
- log.info(f"Processing parameter {index} ({parameter.annotation})")
+ log.debug(f"Processing parameter {index} ({parameter.annotation})")
= if parameter.annotation == inspect._empty:
- log.info (f"No annotation for parameter {index} ({parameter.name}). Skipping.")
+ log.debug (f"No annotation for parameter {index} ({parameter.name}). Skipping.")
= continue
=
- log.info (f"Converting argument {index} to {parameter.annotation.__name__}")
+ log.debug (f"Converting argument {index} to {parameter.annotation.__name__}")
= arguments[index] = parameter.annotation(arguments[index])
=
= implementation(*arguments, **step)
@@ -117,7 +117,7 @@ def divide_and_verify(a: float, b: float, expected: float, **kwargs):
=
=@step("The word {0} has {1} characters")
=def verify_characters_count(word, expected_length: int, **kwargs):
- log.info(f"Is '{word}' of length {expected_length}?")
+ log.debug(f"Is '{word}' of length {expected_length}?")
= actual_length = len(word)
=
= tester.assertEqual(expected_length, actual_length)Implement code block passing
Now interpreters can access code blocks at .step.code_blocks.
index 101d5af..c347baa 100644
--- a/samples/basic.md
+++ b/samples/basic.md
@@ -23,6 +23,18 @@ Content outside of bullet points (like this) won't have any effect on the progra
=## Text
=
= * The word `blocks` has `6` characters
+ * There are `3` properties in the following JSON
+
+ This step includes a code block. All code blocks, together with their (optional) language and metadata (text following the language on the same line) will be passed to interpreters as `step.code_blocks`. It's an array, so you can have multiple blocks.
+
+ ``` json
+ {
+ "one": "Hello",
+ "two": [ 1, 2, 1, 2 ],
+ "three": null
+ }
+ ```
+
= * There are `3` `r`s in the word `strawberry`
= * The reverse of `abc` is `cba`
= * The reverse of `CIA` is `KGB`index 08a839a..f5a28bd 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -122,5 +122,13 @@ def verify_characters_count(word, expected_length: int, **kwargs):
=
= tester.assertEqual(expected_length, actual_length)
=
-
+
+@step("There are {0} properties in the following JSON")
+def count_json_properties(expected_count: int, **kwargs):
+ code = kwargs['code_blocks'][0]['value']
+ log.debug(f"Are there {expected_count} properties in the following?\n{code}")
+ data = json.loads(code)
+ tester.assertEqual(expected_count, len(data))
+
+
=ready()index 9da0529..8788fab 100644
--- a/src/spec.rs
+++ b/src/spec.rs
@@ -91,6 +91,26 @@ pub struct Step {
=
= /// Values of the arguments
= pub arguments: Vec<String>,
+
+ /// List of code blocks
+ pub code_blocks: Vec<CodeBlock>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CodeBlock {
+ value: String,
+ language: Option<String>,
+ meta: Option<String>,
+}
+
+impl From<&markdown::mdast::Code> for CodeBlock {
+ fn from(code: &markdown::mdast::Code) -> Self {
+ Self {
+ language: code.lang.to_owned(),
+ value: code.value.to_owned(),
+ meta: code.meta.to_owned(),
+ }
+ }
=}
=
=#[derive(Deserialize)]
@@ -242,11 +262,29 @@ impl TryFrom<&markdown::mdast::ListItem> for Step {
= }
=
= // TODO: Extract list, tables, code blocks etc. from subsequent children of the list item
+ let mut code_blocks = vec![];
+
+ for child in item.children.iter() {
+ match child {
+ markdown::mdast::Node::Code(code) => {
+ log::info!(
+ "Found a code block: {language} ({meta})",
+ language = code.lang.as_deref().unwrap_or("unspecified"),
+ meta = code.meta.as_deref().unwrap_or_default(),
+ );
+
+ code_blocks.push(CodeBlock::from(code))
+ }
+
+ _ => continue,
+ }
+ }
=
= Ok(Self {
= description: headline.to_string(),
= variant: variant_headline.to_string(),
= arguments,
+ code_blocks,
= })
= }
=}Allow interpreters to hint about failed steps
The hint is an optional field of the Failure message. It is supposed to provide advice to user about a potential solution to a problem. If present, it will be printed below the reason.
An example of hint is in the basic.py interpreter.
index f5a28bd..5b9f78a 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -1,12 +1,15 @@
=#!/usr/bin/env python3
=
-import unittest
=import inspect
=import json
=import logging
=import os
+import re
+import unittest
+from textwrap import dedent
=from typing import Any, Callable
=
+
=# Library code
=# ============
=
@@ -84,10 +87,26 @@ def ready():
= send({ "type": "Failure", "reason": str(error) })
=
= else:
+ # Generate helpful hint
+ indices = re.findall(r"\{(\d)\}", variant)
+ params = map(lambda index: f"arg_{index}: str", indices)
+ name = f"step_implementation_{len(steps_implementation):02}"
+
+ hint = dedent(f"""
+ To get started, place the following in {__file__}:
+
+ ``` python
+ @step("{variant}")
+ def {name}({str.join(", ", params)}, **kwargs):
+ assert "cat" == "dog", "Obviously you need to replace this!"
+ ```
+ """)
+
= log.warning(f"Not implemented: {variant}")
= send({
= "type": 'Failure',
- 'reason': 'Step not implemented'
+ "reason": "step not implemented",
+ "hint": hint
= })
=
= except EOFError:index 4eed1ef..99d105a 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -128,9 +128,9 @@ impl<'a> ScenarioReport<'a> {
= log::debug!("Step executed successfully: {step:#?}");
= *status = StepStatus::Ok;
= }
- InterpreterMessage::Failure { reason } => {
+ InterpreterMessage::Failure { reason, hint } => {
= log::debug!("Step failed:\n\n {step:#?} \n\n {reason:#?}");
- *status = StepStatus::Failed { reason };
+ *status = StepStatus::Failed { reason, hint };
=
= // Do not run subsequent steps.
= //
@@ -218,7 +218,10 @@ pub struct StepReport<'a> {
=
=pub enum StepStatus {
= Ok,
- Failed { reason: String },
+ Failed {
+ reason: String,
+ hint: Option<String>,
+ },
= NotEvaluated,
=}
=
@@ -271,8 +274,12 @@ impl Display for EvaluationReport<'_> {
= arguments = step.arguments
= )?;
=
- if let StepStatus::Failed { reason } = status {
+ if let StepStatus::Failed { reason, hint } = status {
= writeln!(f, "\n {}\n", reason.red())?;
+ if let Some(hint) = hint {
+ // TODO: Indent the hint properly
+ writeln!(f, "\n\n{}\n", hint)?;
+ }
= }
= }
= }
@@ -314,9 +321,14 @@ impl<'a> From<&'a Step> for StepReport<'a> {
=#[derive(Deserialize, Serialize, Debug)]
=#[serde(tag = "type")]
=pub enum InterpreterMessage {
- InterpreterState { ready: bool },
+ InterpreterState {
+ ready: bool,
+ },
= Success,
- Failure { reason: String },
+ Failure {
+ reason: String,
+ hint: Option<String>,
+ },
=}
=
=/// Messages from this (control) program to an interpreterImplement the remaining step variants in basic.py
Only the CIA <-> KGB step fails now, but that's intentional.
index 5b9f78a..484bdc3 100644
--- a/samples/basic.py
+++ b/samples/basic.py
@@ -149,5 +149,19 @@ def count_json_properties(expected_count: int, **kwargs):
= data = json.loads(code)
= tester.assertEqual(expected_count, len(data))
=
+@step("Subtract {0} from {1} to get {2}")
+def step_implementation_04(a: float, b: float, expected: float, **kwargs):
+ tester.assertAlmostEqual(expected, b - a)
+
+
+@step("There are {0} {1}s in the word {2}")
+def step_implementation_05(count: int, letter: str, word: str, **kwargs):
+ actual_count = word.count(letter)
+ tester.assertEqual(count, actual_count)
+
+@step("The reverse of {0} is {1}")
+def step_implementation_06(word: str, reverse: str, **kwargs):
+ actual_reverse = word[::-1]
+ tester.assertEqual(reverse, actual_reverse)
=
=ready()Improve report indentation
Make sure that multiline strings are indented correctly.
For this, I made the Indentable trait and implemented it for &str.
Also, the arguments are now printed in a dimmed style.
new file mode 100644
index 0000000..f655e1d
--- /dev/null
+++ b/src/indentable.rs
@@ -0,0 +1,120 @@
+pub trait Indentable {
+ fn indent(self, width: usize) -> String;
+}
+
+impl Indentable for &str {
+ fn indent(self, width: usize) -> String {
+ let spaces = " ".repeat(width);
+ self.replace("\r\n", "\n")
+ .split("\n")
+ .map(|line| spaces.clone() + line)
+ .reduce(|accu, line| accu + "\n" + &line)
+ .unwrap_or(spaces)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_single_line() {
+ let input = "hello";
+ let expected = " hello";
+ // 4 spaces
+
+ assert_eq!(input.indent(4), expected);
+ }
+
+ #[test]
+ fn test_single_line_2() {
+ let input = "hello";
+ let expected = " hello";
+ // 2 spaces
+
+ assert_eq!(input.indent(2), expected);
+ }
+
+ #[test]
+ fn test_multiple_lines() {
+ let input = "hello\nworld";
+ let expected = " hello\n world";
+
+ assert_eq!(input.indent(2), expected);
+ }
+
+ #[test]
+ fn test_multiple_lines_with_cariege_return() {
+ let input = "hello\r\nworld\r\nand\r\nmoon";
+ let expected = " hello\n world\n and\n moon";
+ // \r\n will be replaced with \n
+
+ assert_eq!(input.indent(2), expected);
+ }
+
+ #[test]
+ fn test_empty_string() {
+ let input = "";
+ let expected = " ";
+ // 4 spaces
+
+ assert_eq!(input.indent(4), expected);
+ }
+
+ #[test]
+ fn test_leading_line() {
+ let input = "\nhello";
+ let expected = " \n hello";
+ // 4 spaces
+
+ assert_eq!(input.indent(4), expected);
+ }
+
+ #[test]
+ fn test_trailing_newline() {
+ let input = "hello\n";
+ let expected = " hello\n ";
+
+ assert_eq!(input.indent(4), expected);
+ }
+
+ #[test]
+ fn test_multiple_empty_lines() {
+ let input = "\n\n\n";
+ let expected = " \n \n \n ";
+ // four lines each containing four spaces
+
+ assert_eq!(input.indent(4), expected);
+ }
+
+ #[test]
+ fn test_whitespace_in_content() {
+ let input = " leading spaces \n trailing spaces ";
+ let expected = " leading spaces \n trailing spaces ";
+ // leading and trailing spaces will be preserved, so:
+ // line 1: 4 (indentation) + 2 (original) leading spaces
+ // 2 trailing spaces untouched
+ // line 2: 4 + 3 leading spaces
+ // 1 trailing space
+ //
+
+ assert_eq!(input.indent(4), expected);
+ }
+
+ #[test]
+ fn test_large_indent() {
+ let input = "a";
+ let expected = " a";
+ // 40 spaces
+
+ assert_eq!(input.indent(40), expected);
+ }
+
+ #[test]
+ fn test_no_newline_in_string() {
+ let input = "abc";
+ let expected = " abc";
+
+ assert_eq!(input.indent(4), expected);
+ }
+}index 1e3dab7..b539e3c 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,3 +1,4 @@
+mod indentable;
=mod report;
=mod spec;
=index 99d105a..b3df963 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,3 +1,4 @@
+use crate::indentable::Indentable;
=use crate::spec::{Scenario, Spec, Step, Suite};
=use anyhow::{Context, anyhow};
=use colored::Colorize;
@@ -252,11 +253,20 @@ impl Display for EvaluationReport<'_> {
= ScenarioStatus::Pending => "?".to_string().bold().dimmed(),
= ScenarioStatus::FailedToRun { .. } => "x".to_string().bold().red(),
= };
- writeln!(f, "\n {sigil} {title}\n", title = scenario.title)?;
+ writeln!(
+ f,
+ "\n{indentation}{sigil} {title}\n",
+ indentation = "".indent(0),
+ title = scenario.title
+ )?;
= if let ScenarioStatus::FailedToRun { error } = status {
- writeln!(f, " {}\n", error.root_cause().to_string().red())?;
+ writeln!(
+ f,
+ "{cause}\n",
+ cause = error.root_cause().to_string().red().indent(2)
+ )?;
= for cause in error.chain() {
- writeln!(f, " * {}", cause)?;
+ writeln!(f, " * {}", cause.to_string().indent(2))?
= }
= writeln!(f, "\n")?;
= }
@@ -269,16 +279,16 @@ impl Display for EvaluationReport<'_> {
= };
= writeln!(
= f,
- " {sigil} {description} {arguments:?}",
+ "{indentation}{sigil} {description} {arguments}",
+ indentation = "".indent(2),
= description = step.description,
- arguments = step.arguments
+ arguments = format!("{:?}", step.arguments).dimmed()
= )?;
=
= if let StepStatus::Failed { reason, hint } = status {
- writeln!(f, "\n {}\n", reason.red())?;
+ writeln!(f, "\n{}\n", reason.indent(4).red())?;
= if let Some(hint) = hint {
- // TODO: Indent the hint properly
- writeln!(f, "\n\n{}\n", hint)?;
+ writeln!(f, "{}\n", hint.as_str().indent(4))?;
= }
= }
= }Bump minor version
To celebrate the code blocks feature.
index 922de52..c2e992f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -400,7 +400,7 @@ dependencies = [
=
=[[package]]
=name = "tad-better-behavior"
-version = "0.1.2"
+version = "0.2.0"
=dependencies = [
= "anyhow",
= "clap",index b7c5dcd..9297cf6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
=[package]
=name = "tad-better-behavior"
-version = "0.1.2"
+version = "0.2.0"
=edition = "2024"
=
=[dependencies]