johno/beads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove all deletions.jsonl code (bd-fom)

Browse Source 
Complete removal of the legacy deletions.jsonl manifest system.
Tombstones are now the sole deletion mechanism.

Removed:
- internal/deletions/ - entire package
- cmd/bd/deleted.go - deleted command
- cmd/bd/doctor/fix/deletions.go - HydrateDeletionsManifest
- Tests for all removed functionality

Cleaned:
- cmd/bd/sync.go - removed sanitize, auto-compact
- cmd/bd/delete.go - removed dual-writes
- cmd/bd/doctor.go - removed checkDeletionsManifest
- internal/importer/importer.go - removed deletions checks
- internal/syncbranch/worktree.go - removed deletions merge
- cmd/bd/integrity.go - updated validation (warn-only on decrease)

Files removed: 12
Lines removed: ~7500

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Steve Yegge
2025-12-16 14:20:32 -08:00
parent e0528de590
commit 9f76cfda01
32 changed files with 298 additions and 7534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

267
cmd/bd/sync.go
View File

@@ -10,15 +10,12 @@ import (
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/debug"
"github.com/steveyegge/beads/internal/deletions"
"github.com/steveyegge/beads/internal/git"
"github.com/steveyegge/beads/internal/rpc"
"github.com/steveyegge/beads/internal/syncbranch"
@@ -609,29 +606,6 @@ Use --merge to merge the sync branch back to main branch.`,
}
}
// Step 3.6: Sanitize JSONL - remove any resurrected zombies
// Git's 3-way merge may re-add deleted issues to JSONL.
// We must remove them before import to prevent resurrection.
sanitizeResult, err := sanitizeJSONLWithDeletions(jsonlPath)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to sanitize JSONL: %v\n", err)
// Non-fatal - continue with import
} else {
// bd-3ee1 fix: Log protected issues (local work that would have been incorrectly removed)
if sanitizeResult.ProtectedCount > 0 {
fmt.Printf("→ Protected %d locally exported issue(s) from incorrect sanitization (bd-3ee1)\n", sanitizeResult.ProtectedCount)
for _, id := range sanitizeResult.ProtectedIDs {
fmt.Printf(" - %s (in left snapshot)\n", id)
}
}
if sanitizeResult.RemovedCount > 0 {
fmt.Printf("→ Sanitized JSONL: removed %d deleted issue(s) that were resurrected by git merge\n", sanitizeResult.RemovedCount)
for _, id := range sanitizeResult.RemovedIDs {
fmt.Printf(" - %s\n", id)
}
}
}
// Step 4: Import updated JSONL after pull
// Enable --protect-left-snapshot to prevent git-history-backfill from
// tombstoning issues that were in our local export but got lost during merge (bd-sync-deletion fix)
@@ -648,12 +622,7 @@ Use --merge to merge the sync branch back to main branch.`,
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to count issues after import: %v\n", err)
} else {
// Account for expected deletions from sanitize step (bd-tt0 fix)
expectedDeletions := 0
if sanitizeResult != nil {
expectedDeletions = sanitizeResult.RemovedCount
}
if err := validatePostImportWithExpectedDeletions(beforeCount, afterCount, expectedDeletions, jsonlPath); err != nil {
if err := validatePostImportWithExpectedDeletions(beforeCount, afterCount, 0, jsonlPath); err != nil {
fmt.Fprintf(os.Stderr, "Post-import validation failed: %v\n", err)
os.Exit(1)
}
@@ -765,12 +734,6 @@ Use --merge to merge the sync branch back to main branch.`,
fmt.Fprintf(os.Stderr, "Warning: failed to clean up snapshots: %v\n", err)
}
// Auto-compact deletions manifest if enabled and threshold exceeded
if err := maybeAutoCompactDeletions(ctx, jsonlPath); err != nil {
// Non-fatal - just log warning
fmt.Fprintf(os.Stderr, "Warning: auto-compact deletions failed: %v\n", err)
}
// When using sync.branch, restore .beads/ from current branch to keep
// working directory clean. The actual beads data lives on the sync branch,
// and the main branch's .beads/ should match what's committed there.
@@ -1688,234 +1651,6 @@ func importFromJSONL(ctx context.Context, jsonlPath string, renameOnImport bool,
return nil
}
// Default configuration values for auto-compact
const (
defaultAutoCompact = false
defaultAutoCompactThreshold = 1000
)
// maybeAutoCompactDeletions checks if auto-compact is enabled and threshold exceeded,
// and if so, prunes the deletions manifest.
func maybeAutoCompactDeletions(ctx context.Context, jsonlPath string) error {
// Ensure store is initialized for config access
if err := ensureStoreActive(); err != nil {
return nil // Can't access config, skip silently
}
// Check if auto-compact is enabled (disabled by default)
autoCompactStr, err := store.GetConfig(ctx, "deletions.auto_compact")
if err != nil || autoCompactStr == "" {
return nil // Not configured, skip
}
autoCompact := autoCompactStr == "true" || autoCompactStr == "1" || autoCompactStr == "yes"
if !autoCompact {
return nil // Disabled, skip
}
// Get threshold (default 1000)
threshold := defaultAutoCompactThreshold
if thresholdStr, err := store.GetConfig(ctx, "deletions.auto_compact_threshold"); err == nil && thresholdStr != "" {
if parsed, err := strconv.Atoi(thresholdStr); err == nil && parsed > 0 {
threshold = parsed
}
}
// Get deletions path
beadsDir := filepath.Dir(jsonlPath)
deletionsPath := deletions.DefaultPath(beadsDir)
// Count current deletions
count, err := deletions.Count(deletionsPath)
if err != nil {
return fmt.Errorf("failed to count deletions: %w", err)
}
// Check if threshold exceeded
if count <= threshold {
return nil // Below threshold, skip
}
// Get retention days (default 7)
retentionDays := configfile.DefaultDeletionsRetentionDays
if retentionStr, err := store.GetConfig(ctx, "deletions.retention_days"); err == nil && retentionStr != "" {
if parsed, err := strconv.Atoi(retentionStr); err == nil && parsed > 0 {
retentionDays = parsed
}
}
// Prune deletions
fmt.Printf("→ Auto-compacting deletions manifest (%d entries > %d threshold)...\n", count, threshold)
result, err := deletions.PruneDeletions(deletionsPath, retentionDays)
if err != nil {
return fmt.Errorf("failed to prune deletions: %w", err)
}
if result.PrunedCount > 0 {
fmt.Printf(" Pruned %d entries older than %d days, kept %d entries\n",
result.PrunedCount, retentionDays, result.KeptCount)
} else {
fmt.Printf(" No entries older than %d days to prune\n", retentionDays)
}
return nil
}
// SanitizeResult contains statistics about the JSONL sanitization operation.
type SanitizeResult struct {
RemovedCount int // Number of issues removed from JSONL
RemovedIDs []string // IDs that were removed
ProtectedCount int // Number of issues protected from removal (bd-3ee1)
ProtectedIDs []string // IDs that were protected
}
// sanitizeJSONLWithDeletions removes non-tombstone issues from the JSONL file
// if they are in the deletions manifest. This prevents zombie resurrection when
// git's 3-way merge re-adds deleted issues to the JSONL during pull.
//
// IMPORTANT (bd-kzxd fix): Tombstones are NOT removed. Tombstones are the proper
// representation of deletions in the JSONL format. Removing them would cause
// the importer to re-create tombstones from deletions.jsonl, leading to
// UNIQUE constraint errors when the tombstone already exists in the database.
//
// IMPORTANT (bd-3ee1 fix): Issues that were in the left snapshot (local export
// before pull) are protected from removal. This prevents newly created issues
// from being incorrectly removed when they happen to have an ID that matches
// an entry in the deletions manifest (possible with hash-based IDs if content
// is similar to a previously deleted issue).
//
// This should be called after git pull but before import.
func sanitizeJSONLWithDeletions(jsonlPath string) (*SanitizeResult, error) {
result := &SanitizeResult{
RemovedIDs: []string{},
ProtectedIDs: []string{},
}
// Get deletions manifest path
beadsDir := filepath.Dir(jsonlPath)
deletionsPath := deletions.DefaultPath(beadsDir)
// Load deletions manifest
loadResult, err := deletions.LoadDeletions(deletionsPath)
if err != nil {
return nil, fmt.Errorf("failed to load deletions manifest: %w", err)
}
// If no deletions, nothing to sanitize
if len(loadResult.Records) == 0 {
return result, nil
}
// bd-3ee1 fix: Load left snapshot to protect locally exported issues
// Issues in the left snapshot were exported before pull and represent
// local work that should not be removed by sanitize
sm := NewSnapshotManager(jsonlPath)
_, leftPath := sm.getSnapshotPaths()
protectedIDs := make(map[string]bool)
if leftIDs, err := sm.buildIDSet(leftPath); err == nil && len(leftIDs) > 0 {
protectedIDs = leftIDs
}
// Read current JSONL
f, err := os.Open(jsonlPath) // #nosec G304 - controlled path
if err != nil {
if os.IsNotExist(err) {
return result, nil // No JSONL file yet
}
return nil, fmt.Errorf("failed to open JSONL: %w", err)
}
var keptLines [][]byte
scanner := bufio.NewScanner(f)
// Allow large lines (up to 10MB for issues with large descriptions)
scanner.Buffer(make([]byte, 0, 64*1024), 10*1024*1024)
for scanner.Scan() {
line := scanner.Bytes()
if len(bytes.TrimSpace(line)) == 0 {
continue
}
// Extract ID and status to check for tombstones
var issue struct {
ID string `json:"id"`
Status string `json:"status"`
}
if err := json.Unmarshal(line, &issue); err != nil {
// Keep malformed lines (let import handle them)
keptLines = append(keptLines, append([]byte{}, line...))
continue
}
// Check if this ID is in deletions manifest
if _, deleted := loadResult.Records[issue.ID]; deleted {
// bd-kzxd fix: Keep tombstones! They are the proper representation of deletions.
// Only remove non-tombstone issues that were resurrected by git merge.
if issue.Status == string(types.StatusTombstone) {
// Keep the tombstone - it's the authoritative deletion record
keptLines = append(keptLines, append([]byte{}, line...))
} else if protectedIDs[issue.ID] {
// bd-3ee1 fix: Issue was in left snapshot (local export before pull)
// This is local work, not a resurrected zombie - protect it!
keptLines = append(keptLines, append([]byte{}, line...))
result.ProtectedCount++
result.ProtectedIDs = append(result.ProtectedIDs, issue.ID)
} else {
// Remove non-tombstone issue that was resurrected
result.RemovedCount++
result.RemovedIDs = append(result.RemovedIDs, issue.ID)
}
} else {
keptLines = append(keptLines, append([]byte{}, line...))
}
}
if err := scanner.Err(); err != nil {
_ = f.Close()
return nil, fmt.Errorf("failed to read JSONL: %w", err)
}
_ = f.Close()
// If nothing was removed, we're done
if result.RemovedCount == 0 {
return result, nil
}
// Write sanitized JSONL atomically
dir := filepath.Dir(jsonlPath)
base := filepath.Base(jsonlPath)
tempFile, err := os.CreateTemp(dir, base+".sanitize.*")
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
tempPath := tempFile.Name()
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempPath) // Clean up on error
}()
for _, line := range keptLines {
if _, err := tempFile.Write(line); err != nil {
return nil, fmt.Errorf("failed to write line: %w", err)
}
if _, err := tempFile.Write([]byte("\n")); err != nil {
return nil, fmt.Errorf("failed to write newline: %w", err)
}
}
if err := tempFile.Close(); err != nil {
return nil, fmt.Errorf("failed to close temp file: %w", err)
}
// Atomic replace
if err := os.Rename(tempPath, jsonlPath); err != nil {
return nil, fmt.Errorf("failed to replace JSONL: %w", err)
}
return result, nil
}
// resolveNoGitHistoryForFromMain returns the resolved noGitHistory value for sync operations.
// When syncing from main (--from-main), noGitHistory is forced to true to prevent creating
// incorrect deletion records for locally-created beads that don't exist on main.