johno/beads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CRITICAL: Disable export deduplication (bd-160)

Browse Source 
The timestamp-only deduplication feature causes data loss when
export_hashes table gets out of sync with JSONL file (after git
operations, imports, etc). This leads to exports skipping issues
that aren't actually in the file.

Symptoms we saw:
- Export reports 'Skipped 128 issues with timestamp-only changes'
- JSONL only has 38 lines but DB has 149 issues
- Two repos on same commit show different issue counts
- Auto-import doesn't trigger (hash matches despite missing data)

Fix: Disable the feature entirely until we can implement proper
JSONL integrity validation (see bd-160 for proposed solutions).
This commit is contained in:
Steve Yegge
2025-10-29 21:21:04 -07:00
parent c9704f8bd4
commit bafa8374f9
2 changed files with 45 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
cmd/bd/autoflush.go
View File

@@ -454,43 +454,40 @@ func writeJSONLAtomic(jsonlPath string, issues []*types.Issue) ([]string, error)
}
}()
// Write all issues as JSONL (with timestamp-only deduplication for bd-159)
ctx := context.Background()
// Write all issues as JSONL (timestamp-only deduplication DISABLED - bd-160)
encoder := json.NewEncoder(f)
skippedCount := 0
exportedIDs := make([]string, 0, len(issues))
for _, issue := range issues {
// Check if this is only a timestamp change (bd-159)
skip, err := shouldSkipExport(ctx, issue)
if err != nil {
// Log warning but continue - don't fail export on hash check errors
if os.Getenv("BD_DEBUG") != "" {
fmt.Fprintf(os.Stderr, "Debug: failed to check if %s should skip: %v\n", issue.ID, err)
}
skip = false
}
if skip {
skippedCount++
continue
}
// DISABLED: timestamp-only deduplication causes data loss (bd-160)
// skip, err := shouldSkipExport(ctx, issue)
// if err != nil {
// if os.Getenv("BD_DEBUG") != "" {
// fmt.Fprintf(os.Stderr, "Debug: failed to check if %s should skip: %v\n", issue.ID, err)
// }
// skip = false
// }
// if skip {
// skippedCount++
// continue
// }
if err := encoder.Encode(issue); err != nil {
return nil, fmt.Errorf("failed to encode issue %s: %w", issue.ID, err)
}
// Save content hash after successful export (bd-159)
contentHash, err := computeIssueContentHash(issue)
if err != nil {
if os.Getenv("BD_DEBUG") != "" {
fmt.Fprintf(os.Stderr, "Debug: failed to compute hash for %s: %v\n", issue.ID, err)
}
} else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
if os.Getenv("BD_DEBUG") != "" {
fmt.Fprintf(os.Stderr, "Debug: failed to save export hash for %s: %v\n", issue.ID, err)
}
}
// DISABLED: export hash tracking (bd-160)
// contentHash, err := computeIssueContentHash(issue)
// if err != nil {
// if os.Getenv("BD_DEBUG") != "" {
// fmt.Fprintf(os.Stderr, "Debug: failed to compute hash for %s: %v\n", issue.ID, err)
// }
// } else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
// if os.Getenv("BD_DEBUG") != "" {
// fmt.Fprintf(os.Stderr, "Debug: failed to save export hash for %s: %v\n", issue.ID, err)
// }
// }
exportedIDs = append(exportedIDs, issue.ID)
}

41
cmd/bd/export.go
View File

@@ -222,36 +222,37 @@ Output to stdout by default, or use -o flag for file output.`,
out = tempFile
}
// Write JSONL (with timestamp-only deduplication for bd-164)
// Write JSONL (timestamp-only deduplication DISABLED due to bd-160)
encoder := json.NewEncoder(out)
exportedIDs := make([]string, 0, len(issues))
skippedCount := 0
for _, issue := range issues {
// Check if this is only a timestamp change (bd-164)
skip, err := shouldSkipExport(ctx, issue)
if err != nil {
// Log warning but continue - don't fail export on hash check errors
fmt.Fprintf(os.Stderr, "Warning: failed to check if %s should skip: %v\n", issue.ID, err)
skip = false
}
if skip {
skippedCount++
continue
}
// DISABLED: timestamp-only deduplication causes data loss (bd-160)
// The export_hashes table gets out of sync with JSONL after git operations,
// causing exports to skip issues that aren't actually in the file.
//
// skip, err := shouldSkipExport(ctx, issue)
// if err != nil {
// fmt.Fprintf(os.Stderr, "Warning: failed to check if %s should skip: %v\n", issue.ID, err)
// skip = false
// }
// if skip {
// skippedCount++
// continue
// }
if err := encoder.Encode(issue); err != nil {
fmt.Fprintf(os.Stderr, "Error encoding issue %s: %v\n", issue.ID, err)
os.Exit(1)
}
// Save content hash after successful export (bd-164)
contentHash, err := computeIssueContentHash(issue)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to compute hash for %s: %v\n", issue.ID, err)
} else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to save export hash for %s: %v\n", issue.ID, err)
}
// DISABLED: export hash tracking (bd-160)
// contentHash, err := computeIssueContentHash(issue)
// if err != nil {
// fmt.Fprintf(os.Stderr, "Warning: failed to compute hash for %s: %v\n", issue.ID, err)
// } else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
// fmt.Fprintf(os.Stderr, "Warning: failed to save export hash for %s: %v\n", issue.ID, err)
// }
exportedIDs = append(exportedIDs, issue.ID)
}