johno/beads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(import,sync): add --no-git-history flag to prevent spurious deletions

Browse Source 
Fixes bd-0b2: The git history backfill mechanism was causing data loss
during JSONL filename migrations (beads.jsonl → issues.jsonl). When issues
existed in the old filename's git history, the backfill incorrectly treated
them as "deleted" and purged them from the database.

Changes:
- Add NoGitHistory field to importer.Options and ImportOptions structs
- Modify purgeDeletedIssues() to skip git history check when flag is set
- Add --no-git-history flag to bd import command
- Add --no-git-history flag to bd sync command
- Update purge_test.go to pass Options argument

Usage:
  bd import -i .beads/issues.jsonl --no-git-history
  bd sync --no-git-history

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Steve Yegge
2025-11-26 23:10:43 -08:00
parent 6294ef0cc6
commit 5506486dc5
6 changed files with 33 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.beads/issues.jsonl
View File

@@ -1,6 +1,6 @@
{"id":"bd-03r","title":"Document deletions manifest in AGENTS.md and README","description":"Parent: bd-imj\n\n## Task\nAdd documentation about the deletions manifest feature.\n\n## Locations to Update\n\n### AGENTS.md\n- Explain that deletions.jsonl is tracked in git\n- Document that `bd delete` records to the manifest\n- Explain cross-clone propagation mechanism\n\n### README.md \n- Brief mention in .beads directory structure section\n- Link to detailed docs if needed\n\n### docs/deletions.md (new file)\n- Full technical documentation\n- Format specification\n- Pruning policy\n- Git history fallback\n- Troubleshooting\n\n## Acceptance Criteria\n- AGENTS.md updated with deletion workflow\n- README.md mentions deletions.jsonl purpose\n- New docs/deletions.md with complete reference","status":"closed","priority":2,"issue_type":"task","created_at":"2025-11-25T14:56:49.13027-08:00","updated_at":"2025-11-25T15:17:23.145944-08:00","closed_at":"2025-11-25T15:17:23.145944-08:00"}
{"id":"bd-055","title":"Fix hyphenated prefix parsing in ExtractIssuePrefix (GH #395)","description":"","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-11-26T15:22:22.395177-08:00","updated_at":"2025-11-26T16:46:57.8927-08:00","closed_at":"2025-11-26T16:46:57.8927-08:00"}
{"id":"bd-0b2","title":"Need --no-git-history flag to disable git history backfill during import","description":"During JSONL migration (beads.jsonl → issues.jsonl), the git history backfill mechanism causes data loss by finding issues in the old beads.jsonl git history and incorrectly treating them as deleted.\n\nA --no-git-history flag for 'bd import' and 'bd sync' would allow users to disable the git history fallback when it's causing problems.\n\nUse cases:\n- JSONL filename migrations\n- Repos with complex git history\n- Debugging import issues\n- Performance (skip slow git scans)\n\nRelated: bd-0gh (migration causes spurious deletions)","status":"open","priority":2,"issue_type":"feature","created_at":"2025-11-26T22:28:22.5286-08:00","updated_at":"2025-11-26T22:28:22.5286-08:00"}
{"id":"bd-0b2","title":"Need --no-git-history flag to disable git history backfill during import","description":"During JSONL migration (beads.jsonl → issues.jsonl), the git history backfill mechanism causes data loss by finding issues in the old beads.jsonl git history and incorrectly treating them as deleted.\n\nA --no-git-history flag for 'bd import' and 'bd sync' would allow users to disable the git history fallback when it's causing problems.\n\nUse cases:\n- JSONL filename migrations\n- Repos with complex git history\n- Debugging import issues\n- Performance (skip slow git scans)\n\nRelated: bd-0gh (migration causes spurious deletions)","status":"closed","priority":2,"issue_type":"feature","created_at":"2025-11-26T22:28:22.5286-08:00","updated_at":"2025-11-26T23:09:29.398357-08:00","closed_at":"2025-11-26T23:09:29.398357-08:00"}
{"id":"bd-1pj6","title":"Proposal: Custom status states via config","description":"Proposal to add 'custom status states' via `bd config`.\nUsers could define an optional issue status enum (e.g., awaiting_review, review_in_progress) in the config.\nThis would enable multi-step pipelines to process issues where each step correlates to a specific status.\n\nExamples:\n- awaiting_verification\n- awaiting_docs\n- awaiting_testing\n","status":"open","priority":3,"issue_type":"feature","created_at":"2025-11-20T18:55:48.670499-05:00","updated_at":"2025-11-20T18:55:48.670499-05:00"}
{"id":"bd-2em","title":"Expand checkHooksQuick to verify all hook versions","description":"Currently checkHooksQuick only checks post-merge hook version. Should also check pre-commit, pre-push, and post-checkout for completeness. Keep it lightweight but catch more outdated hooks.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-11-25T19:27:47.432243-08:00","updated_at":"2025-11-25T19:50:21.378464-08:00","closed_at":"2025-11-25T19:50:21.378464-08:00"}
{"id":"bd-39o","title":"Rename last_import_hash metadata key to jsonl_content_hash","description":"The metadata key 'last_import_hash' is misleading because it's updated on both import AND export (sync.go:614, import.go:320).\n\nBetter names:\n- jsonl_content_hash (more accurate)\n- last_sync_hash (clearer intent)\n\nThis is a breaking change requiring migration of existing metadata values.","status":"open","priority":2,"issue_type":"task","created_at":"2025-11-20T21:31:07.568739-05:00","updated_at":"2025-11-20T21:31:07.568739-05:00"}

3
cmd/bd/import.go
View File

@@ -84,6 +84,7 @@ NOTE: Import requires direct database access and does not work with daemon mode.
clearDuplicateExternalRefs, _ := cmd.Flags().GetBool("clear-duplicate-external-refs")
orphanHandling, _ := cmd.Flags().GetString("orphan-handling")
force, _ := cmd.Flags().GetBool("force")
noGitHistory, _ := cmd.Flags().GetBool("no-git-history")
// Check if stdin is being used interactively (not piped)
if input == "" && term.IsTerminal(int(os.Stdin.Fd())) {
@@ -242,6 +243,7 @@ NOTE: Import requires direct database access and does not work with daemon mode.
RenameOnImport: renameOnImport,
ClearDuplicateExternalRefs: clearDuplicateExternalRefs,
OrphanHandling: orphanHandling,
NoGitHistory: noGitHistory,
}
result, err := importIssuesCore(ctx, dbPath, store, allIssues, opts)
@@ -743,6 +745,7 @@ func init() {
importCmd.Flags().Bool("clear-duplicate-external-refs", false, "Clear duplicate external_ref values (keeps first occurrence)")
importCmd.Flags().String("orphan-handling", "", "How to handle missing parent issues: strict/resurrect/skip/allow (default: use config or 'allow')")
importCmd.Flags().Bool("force", false, "Force metadata update even when database is already in sync with JSONL")
importCmd.Flags().Bool("no-git-history", false, "Skip git history backfill for deletions (use during JSONL filename migrations)")
importCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output import statistics in JSON format")
rootCmd.AddCommand(importCmd)
}

2
cmd/bd/import_shared.go
View File

@@ -165,6 +165,7 @@ type ImportOptions struct {
SkipPrefixValidation bool // Skip prefix validation (for auto-import)
ClearDuplicateExternalRefs bool // Clear duplicate external_ref values instead of erroring
OrphanHandling string // Orphan handling mode: strict/resurrect/skip/allow (empty = use config)
NoGitHistory bool // Skip git history backfill for deletions (prevents spurious deletion during JSONL migrations)
}
// ImportResult contains statistics about the import operation
@@ -221,6 +222,7 @@ func importIssuesCore(ctx context.Context, dbPath string, store storage.Storage,
SkipPrefixValidation: opts.SkipPrefixValidation,
ClearDuplicateExternalRefs: opts.ClearDuplicateExternalRefs,
OrphanHandling: importer.OrphanHandling(orphanHandling),
NoGitHistory: opts.NoGitHistory,
}
// Delegate to the importer package

22
cmd/bd/sync.go
View File

@@ -51,6 +51,7 @@ Use --merge to merge the sync branch back to main branch.`,
status, _ := cmd.Flags().GetBool("status")
merge, _ := cmd.Flags().GetBool("merge")
fromMain, _ := cmd.Flags().GetBool("from-main")
noGitHistory, _ := cmd.Flags().GetBool("no-git-history")
// Find JSONL path
jsonlPath := findJSONLPath()
@@ -79,7 +80,7 @@ Use --merge to merge the sync branch back to main branch.`,
// If from-main mode, one-way sync from main branch (gt-ick9: ephemeral branch support)
if fromMain {
if err := doSyncFromMain(ctx, jsonlPath, renameOnImport, dryRun); err != nil {
if err := doSyncFromMain(ctx, jsonlPath, renameOnImport, dryRun, noGitHistory); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
@@ -92,7 +93,7 @@ Use --merge to merge the sync branch back to main branch.`,
fmt.Println("→ [DRY RUN] Would import from JSONL")
} else {
fmt.Println("→ Importing from JSONL...")
if err := importFromJSONL(ctx, jsonlPath, renameOnImport); err != nil {
if err := importFromJSONL(ctx, jsonlPath, renameOnImport, noGitHistory); err != nil {
fmt.Fprintf(os.Stderr, "Error importing: %v\n", err)
os.Exit(1)
}
@@ -137,7 +138,7 @@ Use --merge to merge the sync branch back to main branch.`,
if hasGitRemote(ctx) {
// Remote exists but no upstream - use from-main mode
fmt.Println("→ No upstream configured, using --from-main mode")
if err := doSyncFromMain(ctx, jsonlPath, renameOnImport, dryRun); err != nil {
if err := doSyncFromMain(ctx, jsonlPath, renameOnImport, dryRun, noGitHistory); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
@@ -163,7 +164,7 @@ Use --merge to merge the sync branch back to main branch.`,
if divergence > 0.5 { // >50% more issues in DB than JSONL
fmt.Printf("→ DB has %d issues but JSONL has %d (stale DB detected)\n", dbCount, jsonlCount)
fmt.Println("→ Importing JSONL first (ZFC)...")
if err := importFromJSONL(ctx, jsonlPath, renameOnImport); err != nil {
if err := importFromJSONL(ctx, jsonlPath, renameOnImport, noGitHistory); err != nil {
fmt.Fprintf(os.Stderr, "Error importing (ZFC): %v\n", err)
os.Exit(1)
}
@@ -319,7 +320,7 @@ Use --merge to merge the sync branch back to main branch.`,
// Step 4: Import updated JSONL after pull
fmt.Println("→ Importing updated JSONL...")
if err := importFromJSONL(ctx, jsonlPath, renameOnImport); err != nil {
if err := importFromJSONL(ctx, jsonlPath, renameOnImport, noGitHistory); err != nil {
fmt.Fprintf(os.Stderr, "Error importing: %v\n", err)
os.Exit(1)
}
@@ -450,6 +451,7 @@ func init() {
syncCmd.Flags().Bool("status", false, "Show diff between sync branch and main branch")
syncCmd.Flags().Bool("merge", false, "Merge sync branch back to main branch")
syncCmd.Flags().Bool("from-main", false, "One-way sync from main branch (for ephemeral branches without upstream)")
syncCmd.Flags().Bool("no-git-history", false, "Skip git history backfill for deletions (use during JSONL filename migrations)")
syncCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output sync statistics in JSON format")
rootCmd.AddCommand(syncCmd)
}
@@ -758,7 +760,7 @@ func getDefaultBranch(ctx context.Context) string {
// doSyncFromMain performs a one-way sync from the default branch (main/master)
// Used for ephemeral branches without upstream tracking (gt-ick9)
// This fetches beads from main and imports them, discarding local beads changes.
func doSyncFromMain(ctx context.Context, jsonlPath string, renameOnImport bool, dryRun bool) error {
func doSyncFromMain(ctx context.Context, jsonlPath string, renameOnImport bool, dryRun bool, noGitHistory bool) error {
if dryRun {
fmt.Println("→ [DRY RUN] Would sync beads from main branch")
fmt.Println(" 1. Fetch origin main")
@@ -796,7 +798,7 @@ func doSyncFromMain(ctx context.Context, jsonlPath string, renameOnImport bool,
// Step 3: Import JSONL
fmt.Println("→ Importing JSONL...")
if err := importFromJSONL(ctx, jsonlPath, renameOnImport); err != nil {
if err := importFromJSONL(ctx, jsonlPath, renameOnImport, noGitHistory); err != nil {
return fmt.Errorf("import failed: %w", err)
}
@@ -1156,7 +1158,7 @@ func mergeSyncBranch(ctx context.Context, dryRun bool) error {
}
// importFromJSONL imports the JSONL file by running the import command
func importFromJSONL(ctx context.Context, jsonlPath string, renameOnImport bool) error {
func importFromJSONL(ctx context.Context, jsonlPath string, renameOnImport bool, noGitHistory ...bool) error {
// Get current executable path to avoid "./bd" path issues
exe, err := os.Executable()
if err != nil {
@@ -1168,6 +1170,10 @@ func importFromJSONL(ctx context.Context, jsonlPath string, renameOnImport bool)
if renameOnImport {
args = append(args, "--rename-on-import")
}
// Handle optional noGitHistory parameter
if len(noGitHistory) > 0 && noGitHistory[0] {
args = append(args, "--no-git-history")
}
// Run import command
cmd := exec.CommandContext(ctx, exe, args...) // #nosec G204 - bd import command from trusted binary

14
internal/importer/importer.go
View File

@@ -42,6 +42,7 @@ type Options struct {
SkipPrefixValidation bool // Skip prefix validation (for auto-import)
OrphanHandling OrphanHandling // How to handle missing parent issues (default: allow)
ClearDuplicateExternalRefs bool // Clear duplicate external_ref values instead of erroring
NoGitHistory bool // Skip git history backfill for deletions (prevents spurious deletion during JSONL migrations)
}
// Result contains statistics about the import operation
@@ -155,7 +156,7 @@ func ImportIssues(ctx context.Context, dbPath string, store storage.Storage, iss
// Purge deleted issues from DB based on deletions manifest
// Issues that are in the manifest but not in JSONL should be deleted from DB
if !opts.DryRun {
if err := purgeDeletedIssues(ctx, sqliteStore, dbPath, issues, result); err != nil {
if err := purgeDeletedIssues(ctx, sqliteStore, dbPath, issues, opts, result); err != nil {
// Non-fatal - just log warning
fmt.Fprintf(os.Stderr, "Warning: failed to purge deleted issues: %v\n", err)
}
@@ -757,8 +758,9 @@ func importComments(ctx context.Context, sqliteStore *sqlite.SQLiteStorage, issu
// purgeDeletedIssues removes issues from the DB that are in the deletions manifest
// but not in the incoming JSONL. This enables deletion propagation across clones.
// Also uses git history fallback for deletions that were pruned from the manifest.
func purgeDeletedIssues(ctx context.Context, sqliteStore *sqlite.SQLiteStorage, dbPath string, jsonlIssues []*types.Issue, result *Result) error {
// Also uses git history fallback for deletions that were pruned from the manifest,
// unless opts.NoGitHistory is set (useful during JSONL filename migrations).
func purgeDeletedIssues(ctx context.Context, sqliteStore *sqlite.SQLiteStorage, dbPath string, jsonlIssues []*types.Issue, opts Options, result *Result) error {
// Get deletions manifest path (same directory as database)
beadsDir := filepath.Dir(dbPath)
deletionsPath := deletions.DefaultPath(beadsDir)
@@ -824,7 +826,8 @@ func purgeDeletedIssues(ctx context.Context, sqliteStore *sqlite.SQLiteStorage,
}
// Git history fallback for potential pruned deletions
if len(needGitCheck) > 0 {
// Skip if --no-git-history flag is set (prevents spurious deletions during JSONL migrations)
if len(needGitCheck) > 0 && !opts.NoGitHistory {
deletedViaGit := checkGitHistoryForDeletions(beadsDir, needGitCheck)
for _, id := range deletedViaGit {
// Backfill the deletions manifest (self-healing)
@@ -848,6 +851,9 @@ func purgeDeletedIssues(ctx context.Context, sqliteStore *sqlite.SQLiteStorage,
result.Purged++
result.PurgedIDs = append(result.PurgedIDs, id)
}
} else if len(needGitCheck) > 0 && opts.NoGitHistory {
// Log that we skipped git history check due to flag
fmt.Fprintf(os.Stderr, "Skipped git history check for %d issue(s) (--no-git-history flag set)\n", len(needGitCheck))
}
return nil

6
internal/importer/purge_test.go
View File

@@ -80,7 +80,7 @@ func TestPurgeDeletedIssues(t *testing.T) {
}
// Call purgeDeletedIssues
if err := purgeDeletedIssues(ctx, store, dbPath, jsonlIssues, result); err != nil {
if err := purgeDeletedIssues(ctx, store, dbPath, jsonlIssues, Options{}, result); err != nil {
t.Fatalf("purgeDeletedIssues failed: %v", err)
}
@@ -159,7 +159,7 @@ func TestPurgeDeletedIssues_NoDeletionsManifest(t *testing.T) {
}
// Call purgeDeletedIssues - should succeed with no errors
if err := purgeDeletedIssues(ctx, store, dbPath, jsonlIssues, result); err != nil {
if err := purgeDeletedIssues(ctx, store, dbPath, jsonlIssues, Options{}, result); err != nil {
t.Fatalf("purgeDeletedIssues failed: %v", err)
}
@@ -222,7 +222,7 @@ func TestPurgeDeletedIssues_EmptyDeletionsManifest(t *testing.T) {
}
// Call purgeDeletedIssues - should succeed with no errors
if err := purgeDeletedIssues(ctx, store, dbPath, jsonlIssues, result); err != nil {
if err := purgeDeletedIssues(ctx, store, dbPath, jsonlIssues, Options{}, result); err != nil {
t.Fatalf("purgeDeletedIssues failed: %v", err)
}