Implement incremental JSONL export with dirty issue tracking

Optimize auto-flush by tracking which issues have changed instead of
exporting the entire database on every flush. For large projects with
1000+ issues, this provides significant performance improvements.

Changes:
- Add dirty_issues table to schema with issue_id and marked_at columns
- Implement dirty tracking functions in new dirty.go file:
  * MarkIssueDirty() - Mark single issue as needing export
  * MarkIssuesDirty() - Batch mark multiple issues efficiently
  * GetDirtyIssues() - Query which issues need export
  * ClearDirtyIssues() - Clear tracking after successful export
  * GetDirtyIssueCount() - Monitor dirty issue count
- Update all CRUD operations to mark affected issues as dirty:
  * CreateIssue, UpdateIssue, DeleteIssue
  * AddDependency, RemoveDependency (marks both issues)
  * AddLabel, RemoveLabel, AddEvent
- Modify export to support incremental mode:
  * Add --incremental flag to export only dirty issues
  * Used by auto-flush for performance
  * Full export still available without flag
- Add Storage interface methods for dirty tracking

Performance impact: With incremental export, large databases only write
changed issues instead of regenerating entire JSONL file on every
auto-flush.

Closes bd-39

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

cmd/bd/export.go

@@ -83,9 +83,18 @@ Output to stdout by default, or use -o flag for file output.`,
 			}
 		}
 
-		// Clear auto-flush state since we just manually exported
-		// This cancels any pending auto-flush timer and marks DB as clean
-		clearAutoFlushState()
+		// Only clear dirty issues and auto-flush state if exporting to the default JSONL path
+		// This prevents clearing dirty flags when exporting to custom paths (e.g., bd export -o backup.jsonl)
+		if output == "" || output == findJSONLPath() {
+			// Clear dirty issues since we just exported to the canonical JSONL file
+			if err := store.ClearDirtyIssues(ctx); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: failed to clear dirty issues: %v\n", err)
+			}
+
+			// Clear auto-flush state since we just manually exported
+			// This cancels any pending auto-flush timer and marks DB as clean
+			clearAutoFlushState()
+		}
 	},
 }
 

cmd/bd/main.go

@@ -118,39 +118,8 @@ var rootCmd = &cobra.Command{
 		flushMutex.Unlock()
 
 		if needsFlush {
-			// Flush without checking isDirty again (we already cleared it)
-			jsonlPath := findJSONLPath()
-			ctx := context.Background()
-			issues, err := store.SearchIssues(ctx, "", types.IssueFilter{})
-			if err == nil {
-				sort.Slice(issues, func(i, j int) bool {
-					return issues[i].ID < issues[j].ID
-				})
-				allDeps, err := store.GetAllDependencyRecords(ctx)
-				if err == nil {
-					for _, issue := range issues {
-						issue.Dependencies = allDeps[issue.ID]
-					}
-					tempPath := jsonlPath + ".tmp"
-					f, err := os.Create(tempPath)
-					if err == nil {
-						encoder := json.NewEncoder(f)
-						hasError := false
-						for _, issue := range issues {
-							if err := encoder.Encode(issue); err != nil {
-								hasError = true
-								break
-							}
-						}
-						f.Close()
-						if !hasError {
-							os.Rename(tempPath, jsonlPath)
-						} else {
-							os.Remove(tempPath)
-						}
-					}
-				}
-			}
+			// Call the shared flush function (no code duplication)
+			flushToJSONL()
 		}
 
 		if store != nil {
@@ -233,6 +202,9 @@ func autoImportIfNewer() {
 	jsonlInfo, err := os.Stat(jsonlPath)
 	if err != nil {
 		// JSONL doesn't exist or can't be accessed, skip import
+		if os.Getenv("BD_DEBUG") != "" {
+			fmt.Fprintf(os.Stderr, "Debug: auto-import skipped, JSONL not found: %v\n", err)
+		}
 		return
 	}
 
@@ -240,6 +212,9 @@ func autoImportIfNewer() {
 	dbInfo, err := os.Stat(dbPath)
 	if err != nil {
 		// DB doesn't exist (new init?), skip import
+		if os.Getenv("BD_DEBUG") != "" {
+			fmt.Fprintf(os.Stderr, "Debug: auto-import skipped, DB not found: %v\n", err)
+		}
 		return
 	}
 
@@ -383,7 +358,7 @@ func clearAutoFlushState() {
 	lastFlushError = nil
 }
 
-// flushToJSONL exports all issues to JSONL if dirty
+// flushToJSONL exports dirty issues to JSONL using incremental updates
 func flushToJSONL() {
 	// Check if store is still active (not closed)
 	storeMutex.Lock()
@@ -439,29 +414,77 @@ func flushToJSONL() {
 		flushMutex.Unlock()
 	}
 
-	// Get all issues
 	ctx := context.Background()
-	issues, err := store.SearchIssues(ctx, "", types.IssueFilter{})
+
+	// Get dirty issue IDs (bd-39: incremental export optimization)
+	dirtyIDs, err := store.GetDirtyIssues(ctx)
 	if err != nil {
-		recordFailure(fmt.Errorf("failed to get issues: %w", err))
+		recordFailure(fmt.Errorf("failed to get dirty issues: %w", err))
 		return
 	}
 
-	// Sort by ID for consistent output
+	// No dirty issues? Nothing to do!
+	if len(dirtyIDs) == 0 {
+		recordSuccess()
+		return
+	}
+
+	// Read existing JSONL into a map
+	issueMap := make(map[string]*types.Issue)
+	if existingFile, err := os.Open(jsonlPath); err == nil {
+		scanner := bufio.NewScanner(existingFile)
+		lineNum := 0
+		for scanner.Scan() {
+			lineNum++
+			line := scanner.Text()
+			if line == "" {
+				continue
+			}
+			var issue types.Issue
+			if err := json.Unmarshal([]byte(line), &issue); err == nil {
+				issueMap[issue.ID] = &issue
+			} else {
+				// Warn about malformed JSONL lines
+				fmt.Fprintf(os.Stderr, "Warning: skipping malformed JSONL line %d: %v\n", lineNum, err)
+			}
+		}
+		existingFile.Close()
+	}
+
+	// Fetch only dirty issues from DB
+	for _, issueID := range dirtyIDs {
+		issue, err := store.GetIssue(ctx, issueID)
+		if err != nil {
+			recordFailure(fmt.Errorf("failed to get issue %s: %w", issueID, err))
+			return
+		}
+		if issue == nil {
+			// Issue was deleted, remove from map
+			delete(issueMap, issueID)
+			continue
+		}
+
+		// Get dependencies for this issue
+		deps, err := store.GetDependencyRecords(ctx, issueID)
+		if err != nil {
+			recordFailure(fmt.Errorf("failed to get dependencies for %s: %w", issueID, err))
+			return
+		}
+		issue.Dependencies = deps
+
+		// Update map
+		issueMap[issueID] = issue
+	}
+
+	// Convert map to sorted slice
+	issues := make([]*types.Issue, 0, len(issueMap))
+	for _, issue := range issueMap {
+		issues = append(issues, issue)
+	}
 	sort.Slice(issues, func(i, j int) bool {
 		return issues[i].ID < issues[j].ID
 	})
 
-	// Populate dependencies for all issues
-	allDeps, err := store.GetAllDependencyRecords(ctx)
-	if err != nil {
-		recordFailure(fmt.Errorf("failed to get dependencies: %w", err))
-		return
-	}
-	for _, issue := range issues {
-		issue.Dependencies = allDeps[issue.ID]
-	}
-
 	// Write to temp file first, then rename (atomic)
 	tempPath := jsonlPath + ".tmp"
 	f, err := os.Create(tempPath)
@@ -493,6 +516,12 @@ func flushToJSONL() {
 		return
 	}
 
+	// Clear dirty issues after successful export
+	if err := store.ClearDirtyIssues(ctx); err != nil {
+		// Don't fail the whole flush for this, but warn
+		fmt.Fprintf(os.Stderr, "Warning: failed to clear dirty issues: %v\n", err)
+	}
+
 	// Success!
 	recordSuccess()
 }