Files
gastown/internal/doctor/lifecycle_check.go
Steve Yegge 9ae23a2bca Add crew-state and lifecycle-hygiene doctor checks
New checks:
- crew-state: Validates crew worker state.json files for completeness
  Can regenerate missing/invalid state files with --fix

- lifecycle-hygiene: Detects stale lifecycle state that can wedge the deacon
  - Stale lifecycle messages in deacon inbox
  - Stuck requesting_* flags in state.json when session is healthy
  Can clean up with --fix (external intervention when deacon is stuck)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-23 00:48:10 -08:00

303 lines
7.6 KiB
Go

package doctor
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
// LifecycleHygieneCheck detects and cleans up stale lifecycle state.
// This can happen when:
// - Lifecycle messages weren't properly deleted after processing
// - Agent state.json has stuck requesting_* flags
// - Session was manually killed without clearing state
type LifecycleHygieneCheck struct {
FixableCheck
staleMessages []staleMessage
stuckStateFiles []stuckState
}
type staleMessage struct {
ID string
Subject string
From string
}
type stuckState struct {
stateFile string
identity string
flag string
}
// NewLifecycleHygieneCheck creates a new lifecycle hygiene check.
func NewLifecycleHygieneCheck() *LifecycleHygieneCheck {
return &LifecycleHygieneCheck{
FixableCheck: FixableCheck{
BaseCheck: BaseCheck{
CheckName: "lifecycle-hygiene",
CheckDescription: "Check for stale lifecycle messages and stuck state flags",
},
},
}
}
// Run checks for stale lifecycle state.
func (c *LifecycleHygieneCheck) Run(ctx *CheckContext) *CheckResult {
c.staleMessages = nil
c.stuckStateFiles = nil
var details []string
// Check for stale lifecycle messages in deacon inbox
staleCount := c.checkDeaconInbox(ctx)
if staleCount > 0 {
details = append(details, fmt.Sprintf("%d stale lifecycle message(s) in deacon inbox", staleCount))
}
// Check for stuck requesting_* flags in state files
stuckCount := c.checkStateFiles(ctx)
if stuckCount > 0 {
details = append(details, fmt.Sprintf("%d agent(s) with stuck requesting_* flags", stuckCount))
}
total := staleCount + stuckCount
if total == 0 {
return &CheckResult{
Name: c.Name(),
Status: StatusOK,
Message: "No stale lifecycle state found",
}
}
return &CheckResult{
Name: c.Name(),
Status: StatusWarning,
Message: fmt.Sprintf("Found %d lifecycle hygiene issue(s)", total),
Details: details,
FixHint: "Run 'gt doctor --fix' to clean up",
}
}
// checkDeaconInbox looks for stale lifecycle messages.
func (c *LifecycleHygieneCheck) checkDeaconInbox(ctx *CheckContext) int {
// Get deacon inbox via gt mail
cmd := exec.Command("gt", "mail", "inbox", "--identity", "deacon/", "--json")
cmd.Dir = ctx.TownRoot
output, err := cmd.Output()
if err != nil {
return 0 // Can't check, assume OK
}
if len(output) == 0 || string(output) == "[]" || string(output) == "[]\n" {
return 0
}
var messages []struct {
ID string `json:"id"`
From string `json:"from"`
Subject string `json:"subject"`
}
if err := json.Unmarshal(output, &messages); err != nil {
return 0
}
// Look for lifecycle messages
for _, msg := range messages {
if strings.HasPrefix(strings.ToLower(msg.Subject), "lifecycle:") {
c.staleMessages = append(c.staleMessages, staleMessage{
ID: msg.ID,
Subject: msg.Subject,
From: msg.From,
})
}
}
return len(c.staleMessages)
}
// checkStateFiles looks for stuck requesting_* flags in state.json files.
func (c *LifecycleHygieneCheck) checkStateFiles(ctx *CheckContext) int {
stateFiles := c.findStateFiles(ctx.TownRoot)
for _, sf := range stateFiles {
data, err := os.ReadFile(sf.path)
if err != nil {
continue
}
var state map[string]interface{}
if err := json.Unmarshal(data, &state); err != nil {
continue
}
// Check for any requesting_* flags
for key, val := range state {
if strings.HasPrefix(key, "requesting_") {
if boolVal, ok := val.(bool); ok && boolVal {
// Found a stuck flag - verify session is actually healthy
if c.isSessionHealthy(sf.identity) {
c.stuckStateFiles = append(c.stuckStateFiles, stuckState{
stateFile: sf.path,
identity: sf.identity,
flag: key,
})
}
}
}
}
}
return len(c.stuckStateFiles)
}
type stateFileInfo struct {
path string
identity string
}
// findStateFiles locates all state.json files for agents.
func (c *LifecycleHygieneCheck) findStateFiles(townRoot string) []stateFileInfo {
var files []stateFileInfo
// Mayor state
mayorState := filepath.Join(townRoot, "mayor", "state.json")
if _, err := os.Stat(mayorState); err == nil {
files = append(files, stateFileInfo{path: mayorState, identity: "mayor"})
}
// Scan rigs for witness, refinery, and crew state files
entries, err := os.ReadDir(townRoot)
if err != nil {
return files
}
for _, entry := range entries {
if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") || entry.Name() == "mayor" {
continue
}
rigName := entry.Name()
rigPath := filepath.Join(townRoot, rigName)
// Witness state
witnessState := filepath.Join(rigPath, "witness", "state.json")
if _, err := os.Stat(witnessState); err == nil {
files = append(files, stateFileInfo{
path: witnessState,
identity: rigName + "-witness",
})
}
// Refinery state
refineryState := filepath.Join(rigPath, "refinery", "state.json")
if _, err := os.Stat(refineryState); err == nil {
files = append(files, stateFileInfo{
path: refineryState,
identity: rigName + "-refinery",
})
}
// Crew state files
crewPath := filepath.Join(rigPath, "crew")
crewEntries, err := os.ReadDir(crewPath)
if err != nil {
continue
}
for _, crew := range crewEntries {
if !crew.IsDir() || strings.HasPrefix(crew.Name(), ".") {
continue
}
crewState := filepath.Join(crewPath, crew.Name(), "state.json")
if _, err := os.Stat(crewState); err == nil {
files = append(files, stateFileInfo{
path: crewState,
identity: rigName + "-crew-" + crew.Name(),
})
}
}
}
return files
}
// isSessionHealthy checks if the tmux session for this identity exists and is running.
func (c *LifecycleHygieneCheck) isSessionHealthy(identity string) bool {
sessionName := identityToSessionName(identity)
if sessionName == "" {
return false
}
// Check if session exists
cmd := exec.Command("tmux", "has-session", "-t", sessionName)
return cmd.Run() == nil
}
// identityToSessionName converts an identity to its tmux session name.
func identityToSessionName(identity string) string {
switch identity {
case "mayor":
return "gt-mayor"
default:
if strings.HasSuffix(identity, "-witness") ||
strings.HasSuffix(identity, "-refinery") ||
strings.Contains(identity, "-crew-") {
return "gt-" + identity
}
return ""
}
}
// Fix cleans up stale lifecycle state.
func (c *LifecycleHygieneCheck) Fix(ctx *CheckContext) error {
var errors []string
// Delete stale lifecycle messages
for _, msg := range c.staleMessages {
cmd := exec.Command("gt", "mail", "delete", msg.ID)
cmd.Dir = ctx.TownRoot
if err := cmd.Run(); err != nil {
errors = append(errors, fmt.Sprintf("failed to delete message %s: %v", msg.ID, err))
}
}
// Clear stuck requesting_* flags
for _, stuck := range c.stuckStateFiles {
if err := c.clearRequestingFlag(stuck); err != nil {
errors = append(errors, fmt.Sprintf("failed to clear %s in %s: %v", stuck.flag, stuck.identity, err))
}
}
if len(errors) > 0 {
return fmt.Errorf("%s", strings.Join(errors, "; "))
}
return nil
}
// clearRequestingFlag removes the stuck requesting_* flag from a state file.
func (c *LifecycleHygieneCheck) clearRequestingFlag(stuck stuckState) error {
data, err := os.ReadFile(stuck.stateFile)
if err != nil {
return err
}
var state map[string]interface{}
if err := json.Unmarshal(data, &state); err != nil {
return err
}
// Remove the requesting flag and any associated timestamp
delete(state, stuck.flag)
delete(state, "requesting_time")
newData, err := json.MarshalIndent(state, "", " ")
if err != nil {
return err
}
return os.WriteFile(stuck.stateFile, newData, 0644)
}