feat(deacon): add heartbeat mechanism for daemon coordination (gt-5af.6)
Add deacon package with heartbeat read/write helpers: - Heartbeat struct with Timestamp, Cycle, LastAction, health counts - WriteHeartbeat/ReadHeartbeat for persistence to deacon/heartbeat.json - IsFresh/IsStale/IsVeryStale for age checks - ShouldPoke to determine if daemon should wake the Deacon - Touch/TouchWithAction convenience functions The Deacon writes heartbeat on each wake cycle. The Go daemon reads it to decide whether to poke the Deacon (only if very stale >5 minutes). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Steve Yegge
149
internal/deacon/heartbeat.go
Normal file
149
internal/deacon/heartbeat.go
Normal file
@@ -0,0 +1,149 @@
|
||||
// Package deacon provides the Deacon agent infrastructure.
|
||||
// The Deacon is a Claude agent that monitors Mayor and Witnesses,
|
||||
// handles lifecycle requests, and keeps Gas Town running.
|
||||
package deacon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Heartbeat represents the Deacon's heartbeat file contents.
|
||||
// Written by the Deacon on each wake cycle.
|
||||
// Read by the Go daemon to decide whether to poke.
|
||||
type Heartbeat struct {
|
||||
// Timestamp is when the heartbeat was written.
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
|
||||
// Cycle is the current wake cycle number.
|
||||
Cycle int64 `json:"cycle"`
|
||||
|
||||
// LastAction describes what the Deacon did in this cycle.
|
||||
LastAction string `json:"last_action,omitempty"`
|
||||
|
||||
// HealthyAgents is the count of healthy agents observed.
|
||||
HealthyAgents int `json:"healthy_agents"`
|
||||
|
||||
// UnhealthyAgents is the count of unhealthy agents observed.
|
||||
UnhealthyAgents int `json:"unhealthy_agents"`
|
||||
}
|
||||
|
||||
// HeartbeatFile returns the path to the Deacon heartbeat file.
|
||||
func HeartbeatFile(townRoot string) string {
|
||||
return filepath.Join(townRoot, "deacon", "heartbeat.json")
|
||||
}
|
||||
|
||||
// WriteHeartbeat writes a new heartbeat to disk.
|
||||
// Called by the Deacon at the start of each wake cycle.
|
||||
func WriteHeartbeat(townRoot string, hb *Heartbeat) error {
|
||||
hbFile := HeartbeatFile(townRoot)
|
||||
|
||||
// Ensure deacon directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(hbFile), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set timestamp if not already set
|
||||
if hb.Timestamp.IsZero() {
|
||||
hb.Timestamp = time.Now().UTC()
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(hb, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(hbFile, data, 0644)
|
||||
}
|
||||
|
||||
// ReadHeartbeat reads the Deacon heartbeat from disk.
|
||||
// Returns nil if the file doesn't exist or can't be read.
|
||||
func ReadHeartbeat(townRoot string) *Heartbeat {
|
||||
hbFile := HeartbeatFile(townRoot)
|
||||
|
||||
data, err := os.ReadFile(hbFile)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var hb Heartbeat
|
||||
if err := json.Unmarshal(data, &hb); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &hb
|
||||
}
|
||||
|
||||
// Age returns how old the heartbeat is.
|
||||
// Returns a very large duration if the heartbeat is nil.
|
||||
func (hb *Heartbeat) Age() time.Duration {
|
||||
if hb == nil {
|
||||
return 24 * time.Hour * 365 // Very stale
|
||||
}
|
||||
return time.Since(hb.Timestamp)
|
||||
}
|
||||
|
||||
// IsFresh returns true if the heartbeat is less than 2 minutes old.
|
||||
// A fresh heartbeat means the Deacon is actively working.
|
||||
func (hb *Heartbeat) IsFresh() bool {
|
||||
return hb != nil && hb.Age() < 2*time.Minute
|
||||
}
|
||||
|
||||
// IsStale returns true if the heartbeat is 2-5 minutes old.
|
||||
// A stale heartbeat may indicate the Deacon is slow or stuck.
|
||||
func (hb *Heartbeat) IsStale() bool {
|
||||
if hb == nil {
|
||||
return false
|
||||
}
|
||||
age := hb.Age()
|
||||
return age >= 2*time.Minute && age < 5*time.Minute
|
||||
}
|
||||
|
||||
// IsVeryStale returns true if the heartbeat is more than 5 minutes old.
|
||||
// A very stale heartbeat means the Deacon should be poked.
|
||||
func (hb *Heartbeat) IsVeryStale() bool {
|
||||
return hb == nil || hb.Age() >= 5*time.Minute
|
||||
}
|
||||
|
||||
// ShouldPoke returns true if the daemon should poke the Deacon.
|
||||
// The Deacon should be poked if:
|
||||
// - No heartbeat exists
|
||||
// - Heartbeat is very stale (>5 minutes)
|
||||
func (hb *Heartbeat) ShouldPoke() bool {
|
||||
return hb.IsVeryStale()
|
||||
}
|
||||
|
||||
// Touch writes a minimal heartbeat with just the timestamp.
|
||||
// This is a convenience function for simple heartbeat updates.
|
||||
func Touch(townRoot string) error {
|
||||
// Read existing heartbeat to increment cycle
|
||||
existing := ReadHeartbeat(townRoot)
|
||||
cycle := int64(1)
|
||||
if existing != nil {
|
||||
cycle = existing.Cycle + 1
|
||||
}
|
||||
|
||||
return WriteHeartbeat(townRoot, &Heartbeat{
|
||||
Timestamp: time.Now().UTC(),
|
||||
Cycle: cycle,
|
||||
})
|
||||
}
|
||||
|
||||
// TouchWithAction writes a heartbeat with an action description.
|
||||
func TouchWithAction(townRoot, action string, healthy, unhealthy int) error {
|
||||
existing := ReadHeartbeat(townRoot)
|
||||
cycle := int64(1)
|
||||
if existing != nil {
|
||||
cycle = existing.Cycle + 1
|
||||
}
|
||||
|
||||
return WriteHeartbeat(townRoot, &Heartbeat{
|
||||
Timestamp: time.Now().UTC(),
|
||||
Cycle: cycle,
|
||||
LastAction: action,
|
||||
HealthyAgents: healthy,
|
||||
UnhealthyAgents: unhealthy,
|
||||
})
|
||||
}
|
||||
381
internal/deacon/heartbeat_test.go
Normal file
381
internal/deacon/heartbeat_test.go
Normal file
@@ -0,0 +1,381 @@
|
||||
package deacon
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHeartbeatFile(t *testing.T) {
|
||||
townRoot := "/tmp/test-town"
|
||||
expected := filepath.Join(townRoot, "deacon", "heartbeat.json")
|
||||
|
||||
result := HeartbeatFile(townRoot)
|
||||
if result != expected {
|
||||
t.Errorf("HeartbeatFile() = %q, want %q", result, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReadHeartbeat(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "deacon-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
hb := &Heartbeat{
|
||||
Timestamp: time.Now().UTC(),
|
||||
Cycle: 42,
|
||||
LastAction: "health check",
|
||||
HealthyAgents: 3,
|
||||
UnhealthyAgents: 1,
|
||||
}
|
||||
|
||||
// Write heartbeat
|
||||
if err := WriteHeartbeat(tmpDir, hb); err != nil {
|
||||
t.Fatalf("WriteHeartbeat error: %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists
|
||||
hbFile := HeartbeatFile(tmpDir)
|
||||
if _, err := os.Stat(hbFile); err != nil {
|
||||
t.Errorf("heartbeat file not created: %v", err)
|
||||
}
|
||||
|
||||
// Read heartbeat
|
||||
loaded := ReadHeartbeat(tmpDir)
|
||||
if loaded == nil {
|
||||
t.Fatal("ReadHeartbeat returned nil")
|
||||
}
|
||||
|
||||
if loaded.Cycle != 42 {
|
||||
t.Errorf("Cycle = %d, want 42", loaded.Cycle)
|
||||
}
|
||||
if loaded.LastAction != "health check" {
|
||||
t.Errorf("LastAction = %q, want 'health check'", loaded.LastAction)
|
||||
}
|
||||
if loaded.HealthyAgents != 3 {
|
||||
t.Errorf("HealthyAgents = %d, want 3", loaded.HealthyAgents)
|
||||
}
|
||||
if loaded.UnhealthyAgents != 1 {
|
||||
t.Errorf("UnhealthyAgents = %d, want 1", loaded.UnhealthyAgents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHeartbeat_NonExistent(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "deacon-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
// Read from non-existent file
|
||||
hb := ReadHeartbeat(tmpDir)
|
||||
if hb != nil {
|
||||
t.Error("expected nil for non-existent heartbeat")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeat_Age(t *testing.T) {
|
||||
// Test nil heartbeat
|
||||
var nilHb *Heartbeat
|
||||
if nilHb.Age() < 24*time.Hour {
|
||||
t.Error("nil heartbeat should have very large age")
|
||||
}
|
||||
|
||||
// Test recent heartbeat
|
||||
hb := &Heartbeat{
|
||||
Timestamp: time.Now().Add(-30 * time.Second),
|
||||
}
|
||||
if hb.Age() > time.Minute {
|
||||
t.Errorf("Age() = %v, expected < 1 minute", hb.Age())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeat_IsFresh(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hb *Heartbeat
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "nil heartbeat",
|
||||
hb: nil,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "just now",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now(),
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "1 minute old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-1 * time.Minute),
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "3 minutes old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-3 * time.Minute),
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := tc.hb.IsFresh()
|
||||
if result != tc.expected {
|
||||
t.Errorf("IsFresh() = %v, want %v", result, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeat_IsStale(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hb *Heartbeat
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "nil heartbeat",
|
||||
hb: nil,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "1 minute old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-1 * time.Minute),
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "3 minutes old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-3 * time.Minute),
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "6 minutes old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-6 * time.Minute),
|
||||
},
|
||||
expected: false, // Very stale, not stale
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := tc.hb.IsStale()
|
||||
if result != tc.expected {
|
||||
t.Errorf("IsStale() = %v, want %v", result, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeat_IsVeryStale(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hb *Heartbeat
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "nil heartbeat",
|
||||
hb: nil,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "1 minute old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-1 * time.Minute),
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "4 minutes old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-4 * time.Minute),
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "6 minutes old",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-6 * time.Minute),
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := tc.hb.IsVeryStale()
|
||||
if result != tc.expected {
|
||||
t.Errorf("IsVeryStale() = %v, want %v", result, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeat_ShouldPoke(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hb *Heartbeat
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "nil heartbeat - should poke",
|
||||
hb: nil,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "fresh - no poke",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now(),
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "stale - no poke",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-3 * time.Minute),
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "very stale - should poke",
|
||||
hb: &Heartbeat{
|
||||
Timestamp: time.Now().Add(-6 * time.Minute),
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := tc.hb.ShouldPoke()
|
||||
if result != tc.expected {
|
||||
t.Errorf("ShouldPoke() = %v, want %v", result, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouch(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "deacon-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
// First touch
|
||||
if err := Touch(tmpDir); err != nil {
|
||||
t.Fatalf("Touch error: %v", err)
|
||||
}
|
||||
|
||||
hb := ReadHeartbeat(tmpDir)
|
||||
if hb == nil {
|
||||
t.Fatal("expected heartbeat after Touch")
|
||||
}
|
||||
if hb.Cycle != 1 {
|
||||
t.Errorf("first Touch: Cycle = %d, want 1", hb.Cycle)
|
||||
}
|
||||
|
||||
// Second touch should increment cycle
|
||||
if err := Touch(tmpDir); err != nil {
|
||||
t.Fatalf("Touch error: %v", err)
|
||||
}
|
||||
|
||||
hb = ReadHeartbeat(tmpDir)
|
||||
if hb.Cycle != 2 {
|
||||
t.Errorf("second Touch: Cycle = %d, want 2", hb.Cycle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouchWithAction(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "deacon-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
if err := TouchWithAction(tmpDir, "health scan", 5, 2); err != nil {
|
||||
t.Fatalf("TouchWithAction error: %v", err)
|
||||
}
|
||||
|
||||
hb := ReadHeartbeat(tmpDir)
|
||||
if hb == nil {
|
||||
t.Fatal("expected heartbeat after TouchWithAction")
|
||||
}
|
||||
if hb.Cycle != 1 {
|
||||
t.Errorf("Cycle = %d, want 1", hb.Cycle)
|
||||
}
|
||||
if hb.LastAction != "health scan" {
|
||||
t.Errorf("LastAction = %q, want 'health scan'", hb.LastAction)
|
||||
}
|
||||
if hb.HealthyAgents != 5 {
|
||||
t.Errorf("HealthyAgents = %d, want 5", hb.HealthyAgents)
|
||||
}
|
||||
if hb.UnhealthyAgents != 2 {
|
||||
t.Errorf("UnhealthyAgents = %d, want 2", hb.UnhealthyAgents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteHeartbeat_CreatesDirectory(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "deacon-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
// Ensure deacon directory doesn't exist
|
||||
deaconDir := filepath.Join(tmpDir, "deacon")
|
||||
if _, err := os.Stat(deaconDir); !os.IsNotExist(err) {
|
||||
t.Fatal("deacon directory should not exist initially")
|
||||
}
|
||||
|
||||
// Write heartbeat should create directory
|
||||
hb := &Heartbeat{Cycle: 1}
|
||||
if err := WriteHeartbeat(tmpDir, hb); err != nil {
|
||||
t.Fatalf("WriteHeartbeat error: %v", err)
|
||||
}
|
||||
|
||||
// Verify directory was created
|
||||
if _, err := os.Stat(deaconDir); err != nil {
|
||||
t.Errorf("deacon directory should exist: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteHeartbeat_SetsTimestamp(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "deacon-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
// Write heartbeat without timestamp
|
||||
hb := &Heartbeat{Cycle: 1}
|
||||
if err := WriteHeartbeat(tmpDir, hb); err != nil {
|
||||
t.Fatalf("WriteHeartbeat error: %v", err)
|
||||
}
|
||||
|
||||
// Read back and verify timestamp was set
|
||||
loaded := ReadHeartbeat(tmpDir)
|
||||
if loaded == nil {
|
||||
t.Fatal("expected heartbeat")
|
||||
}
|
||||
if loaded.Timestamp.IsZero() {
|
||||
t.Error("expected Timestamp to be set")
|
||||
}
|
||||
if time.Since(loaded.Timestamp) > time.Minute {
|
||||
t.Error("Timestamp should be recent")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user