diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 262bf62c..14ed14b1 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -477,6 +477,7 @@ func upsertIssues(ctx context.Context, sqliteStore *sqlite.SQLiteStorage, issues // Track what we need to create var newIssues []*types.Issue seenHashes := make(map[string]bool) + seenIDs := make(map[string]bool) // Track IDs to prevent UNIQUE constraint errors for _, incoming := range issues { hash := incoming.ContentHash @@ -486,13 +487,21 @@ func upsertIssues(ctx context.Context, sqliteStore *sqlite.SQLiteStorage, issues incoming.ContentHash = hash } - // Skip duplicates within incoming batch + // Skip duplicates within incoming batch (by content hash) if seenHashes[hash] { result.Skipped++ continue } seenHashes[hash] = true + // Skip duplicates by ID to prevent UNIQUE constraint violations + // This handles JSONL files with multiple versions of the same issue + if seenIDs[incoming.ID] { + result.Skipped++ + continue + } + seenIDs[incoming.ID] = true + // CRITICAL: Check for tombstone FIRST, before any other matching (bd-4q8 fix) // This prevents ghost resurrection regardless of which phase would normally match. // If this ID has a tombstone in the DB, skip importing it entirely. @@ -701,6 +710,20 @@ if len(newIssues) > 0 { return newIssues[i].ID < newIssues[j].ID // Stable sort }) +// Deduplicate by ID to prevent UNIQUE constraint errors during batch insert +// This handles cases where JSONL contains multiple versions of the same issue +seenNewIDs := make(map[string]bool) +var dedupedNewIssues []*types.Issue +for _, issue := range newIssues { + if !seenNewIDs[issue.ID] { + seenNewIDs[issue.ID] = true + dedupedNewIssues = append(dedupedNewIssues, issue) + } else { + result.Skipped++ // Count duplicates that were skipped + } +} +newIssues = dedupedNewIssues + // Create in batches by depth level (max depth 3) for depth := 0; depth <= 3; depth++ { var batchForDepth []*types.Issue diff --git a/internal/storage/sqlite/issues.go b/internal/storage/sqlite/issues.go index 91c17b57..23ec183a 100644 --- a/internal/storage/sqlite/issues.go +++ b/internal/storage/sqlite/issues.go @@ -4,10 +4,22 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/steveyegge/beads/internal/types" ) +// isUniqueConstraintError checks if error is a UNIQUE constraint violation +// Used to detect and handle duplicate IDs in JSONL imports gracefully +func isUniqueConstraintError(err error) bool { + if err == nil { + return false + } + errMsg := err.Error() + return strings.Contains(errMsg, "UNIQUE constraint failed") || + strings.Contains(errMsg, "constraint failed: UNIQUE") +} + // insertIssue inserts a single issue into the database func insertIssue(ctx context.Context, conn *sql.Conn, issue *types.Issue) error { sourceRepo := issue.SourceRepo @@ -21,7 +33,7 @@ func insertIssue(ctx context.Context, conn *sql.Conn, issue *types.Issue) error } _, err := conn.ExecContext(ctx, ` - INSERT INTO issues ( + INSERT OR IGNORE INTO issues ( id, content_hash, title, description, design, acceptance_criteria, notes, status, priority, issue_type, assignee, estimated_minutes, created_at, updated_at, closed_at, external_ref, source_repo, close_reason, @@ -38,7 +50,12 @@ func insertIssue(ctx context.Context, conn *sql.Conn, issue *types.Issue) error issue.Sender, ephemeral, ) if err != nil { - return fmt.Errorf("failed to insert issue: %w", err) + // INSERT OR IGNORE should handle duplicates, but driver may still return error + // Explicitly ignore UNIQUE constraint errors (expected for duplicate IDs in JSONL) + if !isUniqueConstraintError(err) { + return fmt.Errorf("failed to insert issue: %w", err) + } + // Duplicate ID detected and ignored (INSERT OR IGNORE succeeded) } return nil } @@ -46,7 +63,7 @@ func insertIssue(ctx context.Context, conn *sql.Conn, issue *types.Issue) error // insertIssues bulk inserts multiple issues using a prepared statement func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) error { stmt, err := conn.PrepareContext(ctx, ` - INSERT INTO issues ( + INSERT OR IGNORE INTO issues ( id, content_hash, title, description, design, acceptance_criteria, notes, status, priority, issue_type, assignee, estimated_minutes, created_at, updated_at, closed_at, external_ref, source_repo, close_reason, @@ -80,7 +97,12 @@ func insertIssues(ctx context.Context, conn *sql.Conn, issues []*types.Issue) er issue.Sender, ephemeral, ) if err != nil { - return fmt.Errorf("failed to insert issue %s: %w", issue.ID, err) + // INSERT OR IGNORE should handle duplicates, but driver may still return error + // Explicitly ignore UNIQUE constraint errors (expected for duplicate IDs in JSONL) + if !isUniqueConstraintError(err) { + return fmt.Errorf("failed to insert issue %s: %w", issue.ID, err) + } + // Duplicate ID detected and ignored (INSERT OR IGNORE succeeded) } } return nil