Commits: 3

Implement state management for TBB interpreter

It's implemented the functional / immutable way. Essentially each step is an iterator in a fold operation. If the step returns a record, it's going to be merged with current state and passed to the next steps as data.state.

Currently there is a lot of noise on stderr. I decided not to clean it up, so later I can transform it to TBB observations.

index 15b6d3f..d426ca8 100644
--- a/spec/BASIC.md
+++ b/spec/BASIC.md
@@ -95,10 +95,12 @@ Any project that is not listed, but had activity within the period for which a d
=
=This is not real spec, just a quick test of the WIP interpreter.
=
+  * Do the twist `2` times
+
=  * Do something `nice` and then `sleep` for `2` hours
=  
=    ``` text
=    Zzzzz.....
=    ```
=
-  * Do the twist `7` times
+  * Do the twist `4` times
index 77cd3e2..3b4cbac 100644
--- a/spec/interpreters/basic.nu
+++ b/spec/interpreters/basic.nu
@@ -7,13 +7,28 @@ def main [] {
=            $data.code_blocks | first | get value | print --stderr 
=        },
=        "Do the twist {0} times": { |how_many, data|
+            # All arguments (except data) are passed as strings
+            # TODO: Can we cast them based on closure parameter types?
=            $how_many | into int | let how_many
-            if $how_many > 5 {
-            error make "That's too many!"
-            }
+
+            # This step demonstrates use of state. We count the total number of twists.
+            $data.state
+            | get --optional "twists"   # On first run it won't be there
+            | default 0                 # ...so we need an initial value
+            | let twists_so_far
+            
=            for n in 1..$how_many {
-                $"Doing the twist ($n)" | print --stderr 
+                let done_twists = $n + $twists_so_far
+
+                $"Doing the twist ($done_twists) 🕺" | print --stderr 
+
+                if $done_twists > 5 {
+                    error make "That's too many! My head is spinning 🤣"
+                }
=            }
+
+            # Update the state for other steps. This will be merged.
+            { twists: ($twists_so_far + $how_many) }
=        }
=    }
=}
index afb7882..e8512cd 100644
--- a/spec/interpreters/tbb.nu
+++ b/spec/interpreters/tbb.nu
@@ -6,23 +6,44 @@ export def run [steps: record]: string -> nothing {
=        | print
=    }
=    | lines
-    | each { handle incoming $steps }
+    | reduce --fold {} { |line, state|
+        print --stderr  "State before the step"
+        $state | print --stderr 
+        
+        $line
+        | handle incoming $state $steps
+        | let state_update
+
+        print --stderr "Record update"
+        $state_update | describe --detailed | print --stderr 
+
+        if ($state_update | describe | str starts-with "record") {
+            $state | merge $state_update
+        } else {
+            print --stderr $"Not a record: ($state_update)"
+            $state
+        }
+    }
=
=    print --stderr "Interpreter done."
=}
=
-def "handle incoming" [steps: record] {
+def "handle incoming" [
+    state: record,
+    steps: record,
+] {
=    let incoming = $in | from json
=
=    print --stderr --raw  $incoming
=    match $incoming.type {
=        Execute => {
=            try {
-                let step = $incoming.step
+                let step = $incoming.step | insert state $state
=                print --stderr $"Executing step ($step.description)"
=                let closure = $steps | get $step.variant
-                do $closure ...$step.arguments $step
+                let state_update = do $closure ...$step.arguments $step
=                { type: Success } | to json --raw | print
+                $state_update
=            } catch { |err|
=                {
=                    type: Failure,

Convert the stderr logging to TBB observations

Recently I've started working on observations API for TBB. It allows to collect additional information during steps execution. Currently only two types of observation are implemented:

Accordingly TBB exports two commands:

Both take content as input, allow a custom transformation before sending the observation (as demonstrated in basic.nu interpreter) and output the original value. This is important, as it allows to make observations from the middle of a pipeline.

The observe snippet by default sends a formatted Nushell table, with language set to "table". To send code, one can use --raw <language> flag. If the language is anything else than Nuon (Nushell Object Notation), then the code should be a string (use transform if the input is not a string already). As a special (but likely common) case, structure data can be piped with --raw nuon and will be handled correctly (although not formatter, i.e. the code will be all on a single line). This is thanks to the fact that Nushell will serialize any data structure to a Nuon string.

Finally the --caption flag is useful to explain what the snippet represents. The text command doesn't have such a flag, as a text observaion should be self-explanatory.

The tbb module itself is making a lot of observations. I'm still trying to find out what is the best practice here, but I think it's good to make generous observations, as they will all be hidden in a successful scenario report. Should the scenario fail, the observations can aid debugging.

In general, after the initial trouble with I/O, I think Nushell might be a great platform for TBB interpreters.

index 671416e..745410f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -849,11 +849,11 @@
=        "systems": "systems_2"
=      },
=      "locked": {
-        "lastModified": 1775638737,
-        "narHash": "sha256-h2QcCWhV7hA+v8Tmyh5zmiZmeDLj23Rh5ciJwWiuhJI=",
+        "lastModified": 1776280929,
+        "narHash": "sha256-Z6wXuMFjoRzeCgMSEhb1H93w5SoqtfC539ZRdemaDEU=",
=        "ref": "refs/heads/main",
-        "rev": "ed447ca6ee1a377bf4afcde6a785fdc0870e4f13",
-        "revCount": 162,
+        "rev": "b70531b7ce4b77625cd8824842d5b0cdb62a2b88",
+        "revCount": 163,
=        "type": "git",
=        "url": "http://codeberg.org/tad-lispy/tad-better-behavior"
=      },
index 3b4cbac..801b3f1 100644
--- a/spec/interpreters/basic.nu
+++ b/spec/interpreters/basic.nu
@@ -3,8 +3,12 @@ use tbb.nu
=def main [] {
=    tbb run {
=        "Do something {0} and then {1} for {2} hours": { |how, what, how_long, data|
-            print --stderr $"OK boss. I will do something ($how) for ($how_long) hours and then I'll ($what)"
-            $data.code_blocks | first | get value | print --stderr 
+            $"OK boss. I will do something ($how) for ($how_long) hours and then I'll ($what)"
+            | tbb observe text --transform { str upcase }
+            $data.code_blocks
+            | first
+            | tbb observe snippet --caption "Received code block"
+            | ignore # Do not send any state update.
=        },
=        "Do the twist {0} times": { |how_many, data|
=            # All arguments (except data) are passed as strings
@@ -20,14 +24,15 @@ def main [] {
=            for n in 1..$how_many {
=                let done_twists = $n + $twists_so_far
=
-                $"Doing the twist ($done_twists) 🕺" | print --stderr 
+                $"Doing the twist ($done_twists) 🕺"
+                | tbb observe text 
=
=                if $done_twists > 5 {
=                    error make "That's too many! My head is spinning 🤣"
=                }
=            }
=
-            # Update the state for other steps. This will be merged.
+            # Update the state for other steps. This record will be merged into the previous state.
=            { twists: ($twists_so_far + $how_many) }
=        }
=    }
index e8512cd..45dc7f5 100644
--- a/spec/interpreters/tbb.nu
+++ b/spec/interpreters/tbb.nu
@@ -7,20 +7,20 @@ export def run [steps: record]: string -> nothing {
=    }
=    | lines
=    | reduce --fold {} { |line, state|
-        print --stderr  "State before the step"
-        $state | print --stderr 
+        $state
+        | observe snippet --caption "State before the step" 
=        
=        $line
=        | handle incoming $state $steps
=        | let state_update
=
-        print --stderr "Record update"
-        $state_update | describe --detailed | print --stderr 
-
-        if ($state_update | describe | str starts-with "record") {
+        if ($state_update | describe --detailed | $in.type == "record") {
+            $state_update
+            | observe snippet --caption "State update to be merged"
=            $state | merge $state_update
=        } else {
-            print --stderr $"Not a record: ($state_update)"
+            $state_update
+            | observe snippet --caption "The value returned by the step is not a record; Not updating"
=            $state
=        }
=    }
@@ -32,14 +32,19 @@ def "handle incoming" [
=    state: record,
=    steps: record,
=] {
-    let incoming = $in | from json
+    $in
+    | observe snippet --raw "json" --caption  "Received input"
+    | from json
+    | observe snippet --caption  "Decoded control message"
+    | let incoming
=
-    print --stderr --raw  $incoming
=    match $incoming.type {
=        Execute => {
=            try {
-                let step = $incoming.step | insert state $state
-                print --stderr $"Executing step ($step.description)"
+                $incoming.step
+                | insert state $state
+                | let step
+
=                let closure = $steps | get $step.variant
=                let state_update = do $closure ...$step.arguments $step
=                { type: Success } | to json --raw | print
@@ -57,3 +62,56 @@ def "handle incoming" [
=        }
=    }
=}
+
+export def "observe snippet" [
+    --raw: string,
+    --caption: string,
+    --meta: string = "",
+    --transform: closure
+]: any -> any {
+    # By default transform is identity
+    $transform
+    | default { {||} }
+    | let transform
+
+    $in
+    | do $transform
+    | let transformed
+    | if ($raw | is-empty) { table --expand } else { $in }
+    | let formatted
+
+    {
+        type: Snippet
+        caption: $caption,
+        language: ($raw | default "table"),
+        meta: $meta,
+        content: ($formatted | default "nothing")
+    }
+    | to json --raw
+    | print
+
+    $in # Pass the value on
+}
+
+export def "observe text" [
+    --transform: closure
+] {
+    # By default transform is identity
+    $transform
+    | default { {||} }
+    | let transform
+
+    $in
+    | do $transform
+    | let formatted
+
+
+    {
+        type: Text
+        content: $formatted
+    }
+    | to json --raw
+    | print 
+
+    $in # Pass the value on
+}

Allow setting initial state of a TBB interpreter

Now with the --initial-state flag to tbb run command, an interpreter can set an initial state. This simplifies steps implementation, as they don't have to worry about expected state missing - as evidenced by the "Do the twist" step.

The only remaining plumbing in the steps is type conversion from string. It would be very nice to have it done implicitly and in a generic fashion (i.e. in the tbb module).

index 801b3f1..517943f 100644
--- a/spec/interpreters/basic.nu
+++ b/spec/interpreters/basic.nu
@@ -1,7 +1,7 @@
=use tbb.nu
=
=def main [] {
-    tbb run {
+    tbb run --initial-state { twists: 0 } {
=        "Do something {0} and then {1} for {2} hours": { |how, what, how_long, data|
=            $"OK boss. I will do something ($how) for ($how_long) hours and then I'll ($what)"
=            | tbb observe text --transform { str upcase }
@@ -16,13 +16,8 @@ def main [] {
=            $how_many | into int | let how_many
=
=            # This step demonstrates use of state. We count the total number of twists.
-            $data.state
-            | get --optional "twists"   # On first run it won't be there
-            | default 0                 # ...so we need an initial value
-            | let twists_so_far
-            
=            for n in 1..$how_many {
-                let done_twists = $n + $twists_so_far
+                let done_twists = $n + $data.state.twists
=
=                $"Doing the twist ($done_twists) 🕺"
=                | tbb observe text 
@@ -33,7 +28,7 @@ def main [] {
=            }
=
=            # Update the state for other steps. This record will be merged into the previous state.
-            { twists: ($twists_so_far + $how_many) }
+            { twists: ($data.state.twists + $how_many) }
=        }
=    }
=}
index 45dc7f5..b2d1284 100644
--- a/spec/interpreters/tbb.nu
+++ b/spec/interpreters/tbb.nu
@@ -1,4 +1,7 @@
-export def run [steps: record]: string -> nothing {
+export def run [
+    steps: record
+    --initial-state: record = {}
+]: string -> nothing {
=    tee { # This preserves the input stream for the lines command
=        print --stderr "Interpreter listening..."
=        { type: InterpreterState, ready: true }
@@ -6,7 +9,7 @@ export def run [steps: record]: string -> nothing {
=        | print
=    }
=    | lines
-    | reduce --fold {} { |line, state|
+    | reduce --fold $initial_state { |line, state|
=        $state
=        | observe snippet --caption "State before the step" 
=