johno/beads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CRITICAL: Disable export deduplication (bd-160)

Browse Source 
The timestamp-only deduplication feature causes data loss when
export_hashes table gets out of sync with JSONL file (after git
operations, imports, etc). This leads to exports skipping issues
that aren't actually in the file.

Symptoms we saw:
- Export reports 'Skipped 128 issues with timestamp-only changes'
- JSONL only has 38 lines but DB has 149 issues
- Two repos on same commit show different issue counts
- Auto-import doesn't trigger (hash matches despite missing data)

Fix: Disable the feature entirely until we can implement proper
JSONL integrity validation (see bd-160 for proposed solutions).
This commit is contained in:
Steve Yegge
2025-10-29 21:21:04 -07:00
parent c9704f8bd4
commit bafa8374f9
2 changed files with 45 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
cmd/bd/autoflush.go
View File

@@ -454,43 +454,40 @@ func writeJSONLAtomic(jsonlPath string, issues []*types.Issue) ([]string, error)
} }
}() }()
// Write all issues as JSONL (with timestamp-only deduplication for bd-159) // Write all issues as JSONL (timestamp-only deduplication DISABLED - bd-160)
ctx := context.Background()
encoder := json.NewEncoder(f) encoder := json.NewEncoder(f)
skippedCount := 0 skippedCount := 0
exportedIDs := make([]string, 0, len(issues)) exportedIDs := make([]string, 0, len(issues))
for _, issue := range issues { for _, issue := range issues {
// Check if this is only a timestamp change (bd-159) // DISABLED: timestamp-only deduplication causes data loss (bd-160)
skip, err := shouldSkipExport(ctx, issue) // skip, err := shouldSkipExport(ctx, issue)
if err != nil { // if err != nil {
// Log warning but continue - don't fail export on hash check errors // if os.Getenv("BD_DEBUG") != "" {
if os.Getenv("BD_DEBUG") != "" { // fmt.Fprintf(os.Stderr, "Debug: failed to check if %s should skip: %v\n", issue.ID, err)
fmt.Fprintf(os.Stderr, "Debug: failed to check if %s should skip: %v\n", issue.ID, err) // }
} // skip = false
skip = false // }
} // if skip {
// skippedCount++
if skip { // continue
skippedCount++ // }
continue
}
if err := encoder.Encode(issue); err != nil { if err := encoder.Encode(issue); err != nil {
return nil, fmt.Errorf("failed to encode issue %s: %w", issue.ID, err) return nil, fmt.Errorf("failed to encode issue %s: %w", issue.ID, err)
} }
// Save content hash after successful export (bd-159) // DISABLED: export hash tracking (bd-160)
contentHash, err := computeIssueContentHash(issue) // contentHash, err := computeIssueContentHash(issue)
if err != nil { // if err != nil {
if os.Getenv("BD_DEBUG") != "" { // if os.Getenv("BD_DEBUG") != "" {
fmt.Fprintf(os.Stderr, "Debug: failed to compute hash for %s: %v\n", issue.ID, err) // fmt.Fprintf(os.Stderr, "Debug: failed to compute hash for %s: %v\n", issue.ID, err)
} // }
} else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil { // } else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
if os.Getenv("BD_DEBUG") != "" { // if os.Getenv("BD_DEBUG") != "" {
fmt.Fprintf(os.Stderr, "Debug: failed to save export hash for %s: %v\n", issue.ID, err) // fmt.Fprintf(os.Stderr, "Debug: failed to save export hash for %s: %v\n", issue.ID, err)
} // }
} // }
exportedIDs = append(exportedIDs, issue.ID) exportedIDs = append(exportedIDs, issue.ID)
} }

41
cmd/bd/export.go
View File

@@ -222,36 +222,37 @@ Output to stdout by default, or use -o flag for file output.`,
out = tempFile out = tempFile
} }
// Write JSONL (with timestamp-only deduplication for bd-164) // Write JSONL (timestamp-only deduplication DISABLED due to bd-160)
encoder := json.NewEncoder(out) encoder := json.NewEncoder(out)
exportedIDs := make([]string, 0, len(issues)) exportedIDs := make([]string, 0, len(issues))
skippedCount := 0 skippedCount := 0
for _, issue := range issues { for _, issue := range issues {
// Check if this is only a timestamp change (bd-164) // DISABLED: timestamp-only deduplication causes data loss (bd-160)
skip, err := shouldSkipExport(ctx, issue) // The export_hashes table gets out of sync with JSONL after git operations,
if err != nil { // causing exports to skip issues that aren't actually in the file.
// Log warning but continue - don't fail export on hash check errors //
fmt.Fprintf(os.Stderr, "Warning: failed to check if %s should skip: %v\n", issue.ID, err) // skip, err := shouldSkipExport(ctx, issue)
skip = false // if err != nil {
} // fmt.Fprintf(os.Stderr, "Warning: failed to check if %s should skip: %v\n", issue.ID, err)
// skip = false
if skip { // }
skippedCount++ // if skip {
continue // skippedCount++
} // continue
// }
if err := encoder.Encode(issue); err != nil { if err := encoder.Encode(issue); err != nil {
fmt.Fprintf(os.Stderr, "Error encoding issue %s: %v\n", issue.ID, err) fmt.Fprintf(os.Stderr, "Error encoding issue %s: %v\n", issue.ID, err)
os.Exit(1) os.Exit(1)
} }
// Save content hash after successful export (bd-164) // DISABLED: export hash tracking (bd-160)
contentHash, err := computeIssueContentHash(issue) // contentHash, err := computeIssueContentHash(issue)
if err != nil { // if err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to compute hash for %s: %v\n", issue.ID, err) // fmt.Fprintf(os.Stderr, "Warning: failed to compute hash for %s: %v\n", issue.ID, err)
} else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil { // } else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to save export hash for %s: %v\n", issue.ID, err) // fmt.Fprintf(os.Stderr, "Warning: failed to save export hash for %s: %v\n", issue.ID, err)
} // }
exportedIDs = append(exportedIDs, issue.ID) exportedIDs = append(exportedIDs, issue.ID)
} }