feat(sync): prevent zombie resurrection from stale clones

Add JSONL sanitization after git pull to remove deleted issues that
git's 3-way merge may resurrect. Also add bd doctor check to hydrate
deletions.jsonl from git history for pre-v0.25.0 deletions.

Changes:
- Add sanitizeJSONLWithDeletions() in sync.go (Step 3.6)
- Add checkDeletionsManifest() in doctor.go (Check 18)
- Add HydrateDeletionsManifest() fix in doctor/fix/deletions.go
- Add looksLikeIssueID() validation to prevent false positives
- Add comprehensive tests for sanitization logic

🤖 Generated with Claude Code

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Steve Yegge
2025-11-25 18:11:45 -08:00
parent 8051cc911b
commit e4f9c3556f
4 changed files with 658 additions and 0 deletions

View File

@@ -200,6 +200,8 @@ func applyFixes(result doctorResult) {
err = fix.SyncBranchConfig(result.Path)
case "Database Config":
err = fix.DatabaseConfig(result.Path)
case "Deletions Manifest":
err = fix.HydrateDeletionsManifest(result.Path)
default:
fmt.Printf(" ⚠ No automatic fix available for %s\n", check.Name)
fmt.Printf(" Manual fix: %s\n", check.Fix)
@@ -367,6 +369,11 @@ func runDiagnostics(path string) doctorResult {
result.Checks = append(result.Checks, syncBranchCheck)
// Don't fail overall check for missing sync.branch, just warn
// Check 18: Deletions manifest (prevents zombie resurrection)
deletionsCheck := checkDeletionsManifest(path)
result.Checks = append(result.Checks, deletionsCheck)
// Don't fail overall check for missing deletions manifest, just warn
return result
}
@@ -1840,6 +1847,89 @@ func checkSyncBranchConfig(path string) doctorCheck {
}
}
func checkDeletionsManifest(path string) doctorCheck {
beadsDir := filepath.Join(path, ".beads")
// Skip if .beads doesn't exist
if _, err := os.Stat(beadsDir); os.IsNotExist(err) {
return doctorCheck{
Name: "Deletions Manifest",
Status: statusOK,
Message: "N/A (no .beads directory)",
}
}
// Check if we're in a git repository
gitDir := filepath.Join(path, ".git")
if _, err := os.Stat(gitDir); os.IsNotExist(err) {
return doctorCheck{
Name: "Deletions Manifest",
Status: statusOK,
Message: "N/A (not a git repository)",
}
}
deletionsPath := filepath.Join(beadsDir, "deletions.jsonl")
// Check if deletions.jsonl exists and has content
info, err := os.Stat(deletionsPath)
if err == nil && info.Size() > 0 {
// Count entries
file, err := os.Open(deletionsPath) // #nosec G304 - controlled path
if err == nil {
defer file.Close()
count := 0
scanner := bufio.NewScanner(file)
for scanner.Scan() {
if len(scanner.Bytes()) > 0 {
count++
}
}
return doctorCheck{
Name: "Deletions Manifest",
Status: statusOK,
Message: fmt.Sprintf("Present (%d entries)", count),
}
}
}
// deletions.jsonl doesn't exist or is empty
// Check if there's git history that might have deletions
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
if _, err := os.Stat(jsonlPath); os.IsNotExist(err) {
jsonlPath = filepath.Join(beadsDir, "issues.jsonl")
if _, err := os.Stat(jsonlPath); os.IsNotExist(err) {
return doctorCheck{
Name: "Deletions Manifest",
Status: statusOK,
Message: "N/A (no JSONL file)",
}
}
}
// Check if JSONL has any git history
relPath, _ := filepath.Rel(path, jsonlPath)
cmd := exec.Command("git", "log", "--oneline", "-1", "--", relPath)
cmd.Dir = path
if output, err := cmd.Output(); err != nil || len(output) == 0 {
// No git history for JSONL
return doctorCheck{
Name: "Deletions Manifest",
Status: statusOK,
Message: "Not yet created (no deletions recorded)",
}
}
// There's git history but no deletions manifest - recommend hydration
return doctorCheck{
Name: "Deletions Manifest",
Status: statusWarning,
Message: "Missing or empty (may have pre-v0.25.0 deletions)",
Detail: "Deleted issues from before v0.25.0 are not tracked and may resurrect on sync",
Fix: "Run 'bd doctor --fix' to hydrate deletions manifest from git history",
}
}
func init() {
rootCmd.AddCommand(doctorCmd)
doctorCmd.Flags().BoolVar(&perfMode, "perf", false, "Run performance diagnostics and generate CPU profile")

View File

@@ -0,0 +1,223 @@
package fix
import (
"bufio"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/steveyegge/beads/internal/deletions"
)
// HydrateDeletionsManifest populates deletions.jsonl from git history.
// It finds all issue IDs that were ever in the JSONL but are no longer present,
// and adds them to the deletions manifest.
func HydrateDeletionsManifest(path string) error {
if err := validateBeadsWorkspace(path); err != nil {
return err
}
beadsDir := filepath.Join(path, ".beads")
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
// Also check for legacy issues.jsonl
if _, err := os.Stat(jsonlPath); os.IsNotExist(err) {
legacyPath := filepath.Join(beadsDir, "issues.jsonl")
if _, err := os.Stat(legacyPath); err == nil {
jsonlPath = legacyPath
} else {
return fmt.Errorf("no JSONL file found in .beads/")
}
}
// Load existing deletions manifest to avoid duplicates
deletionsPath := deletions.DefaultPath(beadsDir)
existingDeletions, err := deletions.LoadDeletions(deletionsPath)
if err != nil {
return fmt.Errorf("failed to load existing deletions: %w", err)
}
// Get current IDs from JSONL
currentIDs, err := getCurrentJSONLIDs(jsonlPath)
if err != nil {
return fmt.Errorf("failed to read current JSONL: %w", err)
}
// Get historical IDs from git
historicalIDs, err := getHistoricalJSONLIDs(path, jsonlPath)
if err != nil {
return fmt.Errorf("failed to get historical IDs from git: %w", err)
}
// Find deleted IDs (in history but not in current, and not already in manifest)
var deletedIDs []string
for id := range historicalIDs {
if !currentIDs[id] {
// Skip if already in deletions manifest
if _, exists := existingDeletions.Records[id]; exists {
continue
}
deletedIDs = append(deletedIDs, id)
}
}
if len(deletedIDs) == 0 {
fmt.Println(" No new deleted issues found in git history")
return nil
}
// Add to deletions manifest
now := time.Now()
for _, id := range deletedIDs {
record := deletions.DeletionRecord{
ID: id,
Timestamp: now,
Actor: "bd-doctor-hydrate",
Reason: "Hydrated from git history",
}
if err := deletions.AppendDeletion(deletionsPath, record); err != nil {
return fmt.Errorf("failed to append deletion record for %s: %w", id, err)
}
}
fmt.Printf(" Added %d deletion records to manifest\n", len(deletedIDs))
return nil
}
// getCurrentJSONLIDs reads the current JSONL file and returns a set of IDs.
func getCurrentJSONLIDs(jsonlPath string) (map[string]bool, error) {
ids := make(map[string]bool)
file, err := os.Open(jsonlPath) // #nosec G304 - path validated by caller
if err != nil {
if os.IsNotExist(err) {
return ids, nil
}
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 0, 64*1024), 10*1024*1024)
for scanner.Scan() {
line := scanner.Bytes()
if len(line) == 0 {
continue
}
var issue struct {
ID string `json:"id"`
}
if err := json.Unmarshal(line, &issue); err != nil {
continue
}
if issue.ID != "" {
ids[issue.ID] = true
}
}
return ids, scanner.Err()
}
// getHistoricalJSONLIDs uses git log to find all IDs that were ever in the JSONL.
func getHistoricalJSONLIDs(repoPath, jsonlPath string) (map[string]bool, error) {
// Get the relative path for the JSONL file
relPath, err := filepath.Rel(repoPath, jsonlPath)
if err != nil {
relPath = jsonlPath
}
// Use the commit-by-commit approach which is more memory efficient
// and allows us to properly parse JSON rather than regex matching
return getHistoricalIDsViaDiff(repoPath, relPath)
}
// looksLikeIssueID validates that a string looks like a beads issue ID.
// Issue IDs have the format: prefix-hash or prefix-number (e.g., bd-abc123, myproject-42)
func looksLikeIssueID(id string) bool {
if id == "" {
return false
}
// Must contain at least one dash
dashIdx := strings.Index(id, "-")
if dashIdx <= 0 || dashIdx >= len(id)-1 {
return false
}
// Prefix should be alphanumeric (letters/numbers/underscores)
prefix := id[:dashIdx]
for _, c := range prefix {
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_') {
return false
}
}
// Suffix should be alphanumeric (base36 hash or number), may contain dots for children
suffix := id[dashIdx+1:]
for _, c := range suffix {
if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.') {
return false
}
}
return true
}
// getHistoricalIDsViaDiff walks through git history commit-by-commit to find all IDs.
// This is more memory efficient than git log -p and allows proper JSON parsing.
func getHistoricalIDsViaDiff(repoPath, relPath string) (map[string]bool, error) {
ids := make(map[string]bool)
// Get list of all commits that touched the file
cmd := exec.Command("git", "log", "--all", "--format=%H", "--", relPath)
cmd.Dir = repoPath
output, err := cmd.Output()
if err != nil {
return ids, fmt.Errorf("git log failed: %w", err)
}
commits := strings.Split(strings.TrimSpace(string(output)), "\n")
if len(commits) == 0 || (len(commits) == 1 && commits[0] == "") {
return ids, nil
}
// For each commit, get the file content and extract IDs
for _, commit := range commits {
if commit == "" {
continue
}
// Get file content at this commit
showCmd := exec.Command("git", "show", commit+":"+relPath)
showCmd.Dir = repoPath
content, err := showCmd.Output()
if err != nil {
// File might not exist at this commit
continue
}
// Parse each line for IDs
scanner := bufio.NewScanner(strings.NewReader(string(content)))
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, `"id"`) {
var issue struct {
ID string `json:"id"`
}
if err := json.Unmarshal([]byte(line), &issue); err == nil && issue.ID != "" {
// Validate the ID looks like an issue ID to avoid false positives
if looksLikeIssueID(issue.ID) {
ids[issue.ID] = true
}
}
}
}
}
return ids, nil
}

View File

@@ -1,6 +1,8 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
@@ -301,6 +303,20 @@ Use --merge to merge the sync branch back to main branch.`,
}
}
// Step 3.6: Sanitize JSONL - remove any resurrected zombies
// Git's 3-way merge may re-add deleted issues to JSONL.
// We must remove them before import to prevent resurrection.
sanitizeResult, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to sanitize JSONL: %v\n", err)
// Non-fatal - continue with import
} else if sanitizeResult.RemovedCount > 0 {
fmt.Printf("→ Sanitized JSONL: removed %d deleted issue(s) that were resurrected by git merge\n", sanitizeResult.RemovedCount)
for _, id := range sanitizeResult.RemovedIDs {
fmt.Printf(" - %s\n", id)
}
}
// Step 4: Import updated JSONL after pull
fmt.Println("→ Importing updated JSONL...")
if err := importFromJSONL(ctx, jsonlPath, renameOnImport); err != nil {
@@ -1237,3 +1253,120 @@ func maybeAutoCompactDeletions(ctx context.Context, jsonlPath string) error {
return nil
}
// SanitizeResult contains statistics about the JSONL sanitization operation.
type SanitizeResult struct {
RemovedCount int // Number of issues removed from JSONL
RemovedIDs []string // IDs that were removed
}
// sanitizeJSONLWithDeletions removes any issues from the JSONL file that are
// in the deletions manifest. This prevents zombie resurrection when git's
// 3-way merge re-adds deleted issues to the JSONL during pull.
//
// This should be called after git pull but before import.
func sanitizeJSONLWithDeletions(jsonlPath string) (*SanitizeResult, error) {
result := &SanitizeResult{
RemovedIDs: []string{},
}
// Get deletions manifest path
beadsDir := filepath.Dir(jsonlPath)
deletionsPath := deletions.DefaultPath(beadsDir)
// Load deletions manifest
loadResult, err := deletions.LoadDeletions(deletionsPath)
if err != nil {
return nil, fmt.Errorf("failed to load deletions manifest: %w", err)
}
// If no deletions, nothing to sanitize
if len(loadResult.Records) == 0 {
return result, nil
}
// Read current JSONL
f, err := os.Open(jsonlPath) // #nosec G304 - controlled path
if err != nil {
if os.IsNotExist(err) {
return result, nil // No JSONL file yet
}
return nil, fmt.Errorf("failed to open JSONL: %w", err)
}
var keptLines [][]byte
scanner := bufio.NewScanner(f)
// Allow large lines (up to 10MB for issues with large descriptions)
scanner.Buffer(make([]byte, 0, 64*1024), 10*1024*1024)
for scanner.Scan() {
line := scanner.Bytes()
if len(bytes.TrimSpace(line)) == 0 {
continue
}
// Quick extraction of ID without full unmarshal
// Look for "id":"..." pattern
var issue struct {
ID string `json:"id"`
}
if err := json.Unmarshal(line, &issue); err != nil {
// Keep malformed lines (let import handle them)
keptLines = append(keptLines, append([]byte{}, line...))
continue
}
// Check if this ID is in deletions manifest
if _, deleted := loadResult.Records[issue.ID]; deleted {
result.RemovedCount++
result.RemovedIDs = append(result.RemovedIDs, issue.ID)
} else {
keptLines = append(keptLines, append([]byte{}, line...))
}
}
if err := scanner.Err(); err != nil {
_ = f.Close()
return nil, fmt.Errorf("failed to read JSONL: %w", err)
}
_ = f.Close()
// If nothing was removed, we're done
if result.RemovedCount == 0 {
return result, nil
}
// Write sanitized JSONL atomically
dir := filepath.Dir(jsonlPath)
base := filepath.Base(jsonlPath)
tempFile, err := os.CreateTemp(dir, base+".sanitize.*")
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
tempPath := tempFile.Name()
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempPath) // Clean up on error
}()
for _, line := range keptLines {
if _, err := tempFile.Write(line); err != nil {
return nil, fmt.Errorf("failed to write line: %w", err)
}
if _, err := tempFile.Write([]byte("\n")); err != nil {
return nil, fmt.Errorf("failed to write newline: %w", err)
}
}
if err := tempFile.Close(); err != nil {
return nil, fmt.Errorf("failed to close temp file: %w", err)
}
// Atomic replace
if err := os.Rename(tempPath, jsonlPath); err != nil {
return nil, fmt.Errorf("failed to replace JSONL: %w", err)
}
return result, nil
}

View File

@@ -788,3 +788,215 @@ func TestMaybeAutoCompactDeletions_BelowThreshold(t *testing.T) {
t.Error("deletions file should not be modified when below threshold")
}
}
func TestSanitizeJSONLWithDeletions_NoDeletions(t *testing.T) {
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
os.MkdirAll(beadsDir, 0755)
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
jsonlContent := `{"id":"bd-1","title":"Issue 1"}
{"id":"bd-2","title":"Issue 2"}
{"id":"bd-3","title":"Issue 3"}
`
os.WriteFile(jsonlPath, []byte(jsonlContent), 0644)
// No deletions.jsonl file - should return without changes
result, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.RemovedCount != 0 {
t.Errorf("expected 0 removed, got %d", result.RemovedCount)
}
// Verify JSONL unchanged
afterContent, _ := os.ReadFile(jsonlPath)
if string(afterContent) != jsonlContent {
t.Error("JSONL should not be modified when no deletions")
}
}
func TestSanitizeJSONLWithDeletions_EmptyDeletions(t *testing.T) {
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
os.MkdirAll(beadsDir, 0755)
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
deletionsPath := filepath.Join(beadsDir, "deletions.jsonl")
jsonlContent := `{"id":"bd-1","title":"Issue 1"}
{"id":"bd-2","title":"Issue 2"}
`
os.WriteFile(jsonlPath, []byte(jsonlContent), 0644)
os.WriteFile(deletionsPath, []byte(""), 0644)
result, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.RemovedCount != 0 {
t.Errorf("expected 0 removed, got %d", result.RemovedCount)
}
}
func TestSanitizeJSONLWithDeletions_RemovesDeletedIssues(t *testing.T) {
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
os.MkdirAll(beadsDir, 0755)
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
deletionsPath := filepath.Join(beadsDir, "deletions.jsonl")
// JSONL with 4 issues
jsonlContent := `{"id":"bd-1","title":"Issue 1"}
{"id":"bd-2","title":"Issue 2"}
{"id":"bd-3","title":"Issue 3"}
{"id":"bd-4","title":"Issue 4"}
`
os.WriteFile(jsonlPath, []byte(jsonlContent), 0644)
// Deletions manifest marks bd-2 and bd-4 as deleted
now := time.Now().Format(time.RFC3339)
deletionsContent := fmt.Sprintf(`{"id":"bd-2","ts":"%s","by":"user","reason":"cleanup"}
{"id":"bd-4","ts":"%s","by":"user","reason":"duplicate"}
`, now, now)
os.WriteFile(deletionsPath, []byte(deletionsContent), 0644)
result, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.RemovedCount != 2 {
t.Errorf("expected 2 removed, got %d", result.RemovedCount)
}
if len(result.RemovedIDs) != 2 {
t.Errorf("expected 2 RemovedIDs, got %d", len(result.RemovedIDs))
}
// Verify correct IDs were removed
removedMap := make(map[string]bool)
for _, id := range result.RemovedIDs {
removedMap[id] = true
}
if !removedMap["bd-2"] || !removedMap["bd-4"] {
t.Errorf("expected bd-2 and bd-4 to be removed, got %v", result.RemovedIDs)
}
// Verify JSONL now only has bd-1 and bd-3
afterContent, _ := os.ReadFile(jsonlPath)
afterCount, _ := countIssuesInJSONL(jsonlPath)
if afterCount != 2 {
t.Errorf("expected 2 issues in JSONL after sanitize, got %d", afterCount)
}
if !strings.Contains(string(afterContent), `"id":"bd-1"`) {
t.Error("JSONL should still contain bd-1")
}
if !strings.Contains(string(afterContent), `"id":"bd-3"`) {
t.Error("JSONL should still contain bd-3")
}
if strings.Contains(string(afterContent), `"id":"bd-2"`) {
t.Error("JSONL should NOT contain deleted bd-2")
}
if strings.Contains(string(afterContent), `"id":"bd-4"`) {
t.Error("JSONL should NOT contain deleted bd-4")
}
}
func TestSanitizeJSONLWithDeletions_NoMatchingDeletions(t *testing.T) {
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
os.MkdirAll(beadsDir, 0755)
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
deletionsPath := filepath.Join(beadsDir, "deletions.jsonl")
// JSONL with issues
jsonlContent := `{"id":"bd-1","title":"Issue 1"}
{"id":"bd-2","title":"Issue 2"}
`
os.WriteFile(jsonlPath, []byte(jsonlContent), 0644)
// Deletions for different IDs
now := time.Now().Format(time.RFC3339)
deletionsContent := fmt.Sprintf(`{"id":"bd-99","ts":"%s","by":"user"}
{"id":"bd-100","ts":"%s","by":"user"}
`, now, now)
os.WriteFile(deletionsPath, []byte(deletionsContent), 0644)
result, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.RemovedCount != 0 {
t.Errorf("expected 0 removed (no matching IDs), got %d", result.RemovedCount)
}
// Verify JSONL unchanged
afterContent, _ := os.ReadFile(jsonlPath)
if string(afterContent) != jsonlContent {
t.Error("JSONL should not be modified when no matching deletions")
}
}
func TestSanitizeJSONLWithDeletions_PreservesMalformedLines(t *testing.T) {
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
os.MkdirAll(beadsDir, 0755)
jsonlPath := filepath.Join(beadsDir, "beads.jsonl")
deletionsPath := filepath.Join(beadsDir, "deletions.jsonl")
// JSONL with a malformed line
jsonlContent := `{"id":"bd-1","title":"Issue 1"}
this is not valid json
{"id":"bd-2","title":"Issue 2"}
`
os.WriteFile(jsonlPath, []byte(jsonlContent), 0644)
// Delete bd-2
now := time.Now().Format(time.RFC3339)
os.WriteFile(deletionsPath, []byte(fmt.Sprintf(`{"id":"bd-2","ts":"%s","by":"user"}`, now)+"\n"), 0644)
result, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.RemovedCount != 1 {
t.Errorf("expected 1 removed, got %d", result.RemovedCount)
}
// Verify malformed line is preserved (let import handle it)
afterContent, _ := os.ReadFile(jsonlPath)
if !strings.Contains(string(afterContent), "this is not valid json") {
t.Error("malformed line should be preserved")
}
if !strings.Contains(string(afterContent), `"id":"bd-1"`) {
t.Error("bd-1 should be preserved")
}
if strings.Contains(string(afterContent), `"id":"bd-2"`) {
t.Error("bd-2 should be removed")
}
}
func TestSanitizeJSONLWithDeletions_NonexistentJSONL(t *testing.T) {
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
os.MkdirAll(beadsDir, 0755)
jsonlPath := filepath.Join(beadsDir, "nonexistent.jsonl")
deletionsPath := filepath.Join(beadsDir, "deletions.jsonl")
// Create deletions file
now := time.Now().Format(time.RFC3339)
os.WriteFile(deletionsPath, []byte(fmt.Sprintf(`{"id":"bd-1","ts":"%s","by":"user"}`, now)+"\n"), 0644)
// Should handle missing JSONL gracefully
result, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
t.Fatalf("unexpected error for missing JSONL: %v", err)
}
if result.RemovedCount != 0 {
t.Errorf("expected 0 removed for missing file, got %d", result.RemovedCount)
}
}