From 3f724336f451e3012ade654a9d8f28719486e1bc Mon Sep 17 00:00:00 2001 From: gastown/crew/jack Date: Sat, 17 Jan 2026 04:45:24 -0800 Subject: [PATCH] feat(patrol): add backoff test formula and fix await-signal Add mol-backoff-test formula for integration testing exponential backoff with short intervals (2s base, 10s max) to observe multiple cycles quickly. Fix await-signal to use --since 1s when subscribing to activity feed. Without this, historical events would immediately wake the signal, preventing proper timeout and backoff behavior. Co-Authored-By: Claude Opus 4.5 --- .beads/formulas/mol-backoff-test.formula.toml | 95 +++++++++++++++++++ internal/cmd/molecule_await_signal.go | 6 +- .../formulas/mol-backoff-test.formula.toml | 95 +++++++++++++++++++ 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 .beads/formulas/mol-backoff-test.formula.toml create mode 100644 internal/formula/formulas/mol-backoff-test.formula.toml diff --git a/.beads/formulas/mol-backoff-test.formula.toml b/.beads/formulas/mol-backoff-test.formula.toml new file mode 100644 index 00000000..a8245adf --- /dev/null +++ b/.beads/formulas/mol-backoff-test.formula.toml @@ -0,0 +1,95 @@ +description = """ +Test molecule for verifying exponential backoff behavior. + +Uses SHORT intervals (2s base, 10s max) so you can observe multiple +backoff cycles in under a minute. + +Expected backoff progression: +- Cycle 0: 2s +- Cycle 1: 4s +- Cycle 2: 8s +- Cycle 3+: 10s (capped) + +Full backoff cycle observable in ~24 seconds. + +## Running This Test + +1. Create a test agent bead (if not exists): + ```bash + bd create --type=agent --title="Backoff Test Agent" --id=hq-backoff-test + ``` + +2. Run the deacon with this formula, or execute steps manually: + ```bash + gt mol step await-signal --agent-bead hq-backoff-test \ + --backoff-base 2s --backoff-mult 2 --backoff-max 10s + ``` + +3. Watch the idle counter increment on each timeout: + ```bash + bd show hq-backoff-test --json | jq '.[] | .labels' + ``` + +4. Trigger activity to test signal wake: + ```bash + bd create --title="Wake signal" --silent # Any bd command triggers activity + ``` + +5. Reset idle counter after signal (caller responsibility): + ```bash + bd update hq-backoff-test --set-labels=idle:0 + ``` +""" +formula = "mol-backoff-test" +version = 1 + +[[steps]] +id = "heartbeat" +title = "Touch heartbeat" +description = """ +Touch heartbeat to show we're alive. + +```bash +gt deacon heartbeat "backoff test cycle" +``` + +This updates the deacon's heartbeat file to signal liveness. +""" + +[[steps]] +id = "await-signal" +title = "Wait with exponential backoff (SHORT intervals)" +needs = ["heartbeat"] +description = """ +Wait for activity with exponential backoff using TEST intervals. + +**IMPORTANT**: This uses SHORT intervals for testing: +- Base: 2s (production: 60s) +- Multiplier: 2 +- Max: 10s (production: 10m) + +```bash +gt mol step await-signal --agent-bead hq-backoff-test \ + --backoff-base 2s --backoff-mult 2 --backoff-max 10s +``` + +**Expected behavior:** + +| Idle Cycles | Timeout | +|-------------|---------| +| 0 | 2s | +| 1 | 4s | +| 2 | 8s | +| 3+ | 10s | + +**On timeout**: The command auto-increments the `idle:N` label on the agent bead. +Continue to the next patrol cycle (loop back to heartbeat). + +**On signal**: Activity was detected. Reset the idle counter: +```bash +bd update hq-backoff-test --set-labels=idle:0 +``` +Then loop back to heartbeat for the next cycle. + +**To exit**: When context is high or testing is complete, exit cleanly. +""" diff --git a/internal/cmd/molecule_await_signal.go b/internal/cmd/molecule_await_signal.go index 696a1c33..b26bd2ac 100644 --- a/internal/cmd/molecule_await_signal.go +++ b/internal/cmd/molecule_await_signal.go @@ -237,8 +237,10 @@ func calculateEffectiveTimeout(idleCycles int) (time.Duration, error) { // waitForActivitySignal starts bd activity --follow and waits for any output. // Returns immediately when a line is received, or when context is canceled. func waitForActivitySignal(ctx context.Context, workDir string) (*AwaitSignalResult, error) { - // Start bd activity --follow - cmd := exec.CommandContext(ctx, "bd", "activity", "--follow") + // Start bd activity --follow --since 1s + // The --since flag ensures we only wait for NEW events, not historical ones. + // Without this, the feed would immediately return old activity and never timeout. + cmd := exec.CommandContext(ctx, "bd", "activity", "--follow", "--since", "1s") cmd.Dir = workDir stdout, err := cmd.StdoutPipe() diff --git a/internal/formula/formulas/mol-backoff-test.formula.toml b/internal/formula/formulas/mol-backoff-test.formula.toml new file mode 100644 index 00000000..a8245adf --- /dev/null +++ b/internal/formula/formulas/mol-backoff-test.formula.toml @@ -0,0 +1,95 @@ +description = """ +Test molecule for verifying exponential backoff behavior. + +Uses SHORT intervals (2s base, 10s max) so you can observe multiple +backoff cycles in under a minute. + +Expected backoff progression: +- Cycle 0: 2s +- Cycle 1: 4s +- Cycle 2: 8s +- Cycle 3+: 10s (capped) + +Full backoff cycle observable in ~24 seconds. + +## Running This Test + +1. Create a test agent bead (if not exists): + ```bash + bd create --type=agent --title="Backoff Test Agent" --id=hq-backoff-test + ``` + +2. Run the deacon with this formula, or execute steps manually: + ```bash + gt mol step await-signal --agent-bead hq-backoff-test \ + --backoff-base 2s --backoff-mult 2 --backoff-max 10s + ``` + +3. Watch the idle counter increment on each timeout: + ```bash + bd show hq-backoff-test --json | jq '.[] | .labels' + ``` + +4. Trigger activity to test signal wake: + ```bash + bd create --title="Wake signal" --silent # Any bd command triggers activity + ``` + +5. Reset idle counter after signal (caller responsibility): + ```bash + bd update hq-backoff-test --set-labels=idle:0 + ``` +""" +formula = "mol-backoff-test" +version = 1 + +[[steps]] +id = "heartbeat" +title = "Touch heartbeat" +description = """ +Touch heartbeat to show we're alive. + +```bash +gt deacon heartbeat "backoff test cycle" +``` + +This updates the deacon's heartbeat file to signal liveness. +""" + +[[steps]] +id = "await-signal" +title = "Wait with exponential backoff (SHORT intervals)" +needs = ["heartbeat"] +description = """ +Wait for activity with exponential backoff using TEST intervals. + +**IMPORTANT**: This uses SHORT intervals for testing: +- Base: 2s (production: 60s) +- Multiplier: 2 +- Max: 10s (production: 10m) + +```bash +gt mol step await-signal --agent-bead hq-backoff-test \ + --backoff-base 2s --backoff-mult 2 --backoff-max 10s +``` + +**Expected behavior:** + +| Idle Cycles | Timeout | +|-------------|---------| +| 0 | 2s | +| 1 | 4s | +| 2 | 8s | +| 3+ | 10s | + +**On timeout**: The command auto-increments the `idle:N` label on the agent bead. +Continue to the next patrol cycle (loop back to heartbeat). + +**On signal**: Activity was detected. Reset the idle counter: +```bash +bd update hq-backoff-test --set-labels=idle:0 +``` +Then loop back to heartbeat for the next cycle. + +**To exit**: When context is high or testing is complete, exit cleanly. +"""