Implement incremental JSONL export with dirty issue tracking

Optimize auto-flush by tracking which issues have changed instead of exporting the entire database on every flush. For large projects with 1000+ issues, this provides significant performance improvements. Changes: - Add dirty_issues table to schema with issue_id and marked_at columns - Implement dirty tracking functions in new dirty.go file: * MarkIssueDirty() - Mark single issue as needing export * MarkIssuesDirty() - Batch mark multiple issues efficiently * GetDirtyIssues() - Query which issues need export * ClearDirtyIssues() - Clear tracking after successful export * GetDirtyIssueCount() - Monitor dirty issue count - Update all CRUD operations to mark affected issues as dirty: * CreateIssue, UpdateIssue, DeleteIssue * AddDependency, RemoveDependency (marks both issues) * AddLabel, RemoveLabel, AddEvent - Modify export to support incremental mode: * Add --incremental flag to export only dirty issues * Used by auto-flush for performance * Full export still available without flag - Add Storage interface methods for dirty tracking Performance impact: With incremental export, large databases only write changed issues instead of regenerating entire JSONL file on every auto-flush. Closes bd-39 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-14 00:17:23 -07:00
parent 25644d9717
commit bafb2801c5
11 changed files with 372 additions and 99 deletions
--- a/internal/storage/sqlite/dependencies.go
+++ b/internal/storage/sqlite/dependencies.go
@@ -110,6 +110,26 @@ func (s *SQLiteStorage) AddDependency(ctx context.Context, dep *types.Dependency
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark both issues as dirty for incremental export
+	// (dependencies are exported with each issue, so both need updating)
+	now := time.Now()
+	stmt, err := tx.PrepareContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to prepare dirty statement: %w", err)
+	}
+	defer stmt.Close()
+
+	if _, err := stmt.ExecContext(ctx, dep.IssueID, now); err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+	if _, err := stmt.ExecContext(ctx, dep.DependsOnID, now); err != nil {
+		return fmt.Errorf("failed to mark dependency target dirty: %w", err)
+	}
+
 	return tx.Commit()
 }

@@ -137,6 +157,25 @@ func (s *SQLiteStorage) RemoveDependency(ctx context.Context, issueID, dependsOn
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark both issues as dirty for incremental export
+	now := time.Now()
+	stmt, err := tx.PrepareContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to prepare dirty statement: %w", err)
+	}
+	defer stmt.Close()
+
+	if _, err := stmt.ExecContext(ctx, issueID, now); err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+	if _, err := stmt.ExecContext(ctx, dependsOnID, now); err != nil {
+		return fmt.Errorf("failed to mark dependency target dirty: %w", err)
+	}
+
 	return tx.Commit()
 }

--- a/internal/storage/sqlite/dirty.go
+++ b/internal/storage/sqlite/dirty.go
@@ -0,0 +1,96 @@
+// Package sqlite implements dirty issue tracking for incremental JSONL export.
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+// MarkIssueDirty marks an issue as dirty (needs to be exported to JSONL)
+// This should be called whenever an issue is created, updated, or has dependencies changed
+func (s *SQLiteStorage) MarkIssueDirty(ctx context.Context, issueID string) error {
+	_, err := s.db.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, issueID, time.Now())
+	return err
+}
+
+// MarkIssuesDirty marks multiple issues as dirty in a single transaction
+// More efficient when marking multiple issues (e.g., both sides of a dependency)
+func (s *SQLiteStorage) MarkIssuesDirty(ctx context.Context, issueIDs []string) error {
+	if len(issueIDs) == 0 {
+		return nil
+	}
+
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("failed to begin transaction: %w", err)
+	}
+	defer tx.Rollback()
+
+	now := time.Now()
+	stmt, err := tx.PrepareContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to prepare statement: %w", err)
+	}
+	defer stmt.Close()
+
+	for _, issueID := range issueIDs {
+		if _, err := stmt.ExecContext(ctx, issueID, now); err != nil {
+			return fmt.Errorf("failed to mark issue %s dirty: %w", issueID, err)
+		}
+	}
+
+	return tx.Commit()
+}
+
+// GetDirtyIssues returns the list of issue IDs that need to be exported
+func (s *SQLiteStorage) GetDirtyIssues(ctx context.Context) ([]string, error) {
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT issue_id FROM dirty_issues
+		ORDER BY marked_at ASC
+	`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get dirty issues: %w", err)
+	}
+	defer rows.Close()
+
+	var issueIDs []string
+	for rows.Next() {
+		var issueID string
+		if err := rows.Scan(&issueID); err != nil {
+			return nil, fmt.Errorf("failed to scan issue ID: %w", err)
+		}
+		issueIDs = append(issueIDs, issueID)
+	}
+
+	return issueIDs, rows.Err()
+}
+
+// ClearDirtyIssues removes all entries from the dirty_issues table
+// This should be called after a successful JSONL export
+func (s *SQLiteStorage) ClearDirtyIssues(ctx context.Context) error {
+	_, err := s.db.ExecContext(ctx, `DELETE FROM dirty_issues`)
+	if err != nil {
+		return fmt.Errorf("failed to clear dirty issues: %w", err)
+	}
+	return nil
+}
+
+// GetDirtyIssueCount returns the count of dirty issues (for monitoring/debugging)
+func (s *SQLiteStorage) GetDirtyIssueCount(ctx context.Context) (int, error) {
+	var count int
+	err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dirty_issues`).Scan(&count)
+	if err != nil && err != sql.ErrNoRows {
+		return 0, fmt.Errorf("failed to count dirty issues: %w", err)
+	}
+	return count, nil
+}
--- a/internal/storage/sqlite/events.go
+++ b/internal/storage/sqlite/events.go
@@ -4,13 +4,20 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"time"

 	"github.com/steveyegge/beads/internal/types"
 )

 // AddComment adds a comment to an issue
 func (s *SQLiteStorage) AddComment(ctx context.Context, issueID, actor, comment string) error {
-	_, err := s.db.ExecContext(ctx, `
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("failed to begin transaction: %w", err)
+	}
+	defer tx.Rollback()
+
+	_, err = tx.ExecContext(ctx, `
 		INSERT INTO events (issue_id, event_type, actor, comment)
 		VALUES (?, ?, ?, ?)
 	`, issueID, types.EventCommented, actor, comment)
@@ -19,14 +26,25 @@ func (s *SQLiteStorage) AddComment(ctx context.Context, issueID, actor, comment
 	}

 	// Update issue updated_at timestamp
-	_, err = s.db.ExecContext(ctx, `
-		UPDATE issues SET updated_at = CURRENT_TIMESTAMP WHERE id = ?
-	`, issueID)
+	now := time.Now()
+	_, err = tx.ExecContext(ctx, `
+		UPDATE issues SET updated_at = ? WHERE id = ?
+	`, now, issueID)
 	if err != nil {
 		return fmt.Errorf("failed to update timestamp: %w", err)
 	}

-	return nil
+	// Mark issue as dirty for incremental export
+	_, err = tx.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, issueID, now)
+	if err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+
+	return tx.Commit()
 }

 // GetEvents returns the event history for an issue
--- a/internal/storage/sqlite/labels.go
+++ b/internal/storage/sqlite/labels.go
@@ -3,6 +3,7 @@ package sqlite
 import (
 	"context"
 	"fmt"
+	"time"

 	"github.com/steveyegge/beads/internal/types"
 )
@@ -31,6 +32,16 @@ func (s *SQLiteStorage) AddLabel(ctx context.Context, issueID, label, actor stri
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark issue as dirty for incremental export
+	_, err = tx.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, issueID, time.Now())
+	if err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+
 	return tx.Commit()
 }

@@ -57,6 +68,16 @@ func (s *SQLiteStorage) RemoveLabel(ctx context.Context, issueID, label, actor s
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark issue as dirty for incremental export
+	_, err = tx.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, issueID, time.Now())
+	if err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+
 	return tx.Commit()
 }

--- a/internal/storage/sqlite/schema.go
+++ b/internal/storage/sqlite/schema.go
@@ -71,6 +71,16 @@ CREATE TABLE IF NOT EXISTS config (
    value TEXT NOT NULL
 );

+-- Dirty issues table (for incremental JSONL export)
+-- Tracks which issues have changed since last export
+CREATE TABLE IF NOT EXISTS dirty_issues (
+    issue_id TEXT PRIMARY KEY,
+    marked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_dirty_issues_marked_at ON dirty_issues(marked_at);
+
 -- Ready work view
 CREATE VIEW IF NOT EXISTS ready_issues AS
 SELECT i.*
--- a/internal/storage/sqlite/sqlite.go
+++ b/internal/storage/sqlite/sqlite.go
@@ -173,6 +173,16 @@ func (s *SQLiteStorage) CreateIssue(ctx context.Context, issue *types.Issue, act
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark issue as dirty for incremental export
+	_, err = tx.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, issue.ID, time.Now())
+	if err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+
 	return tx.Commit()
 }

@@ -336,6 +346,16 @@ func (s *SQLiteStorage) UpdateIssue(ctx context.Context, id string, updates map[
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark issue as dirty for incremental export
+	_, err = tx.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, id, time.Now())
+	if err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+
 	return tx.Commit()
 }

@@ -366,6 +386,16 @@ func (s *SQLiteStorage) CloseIssue(ctx context.Context, id string, reason string
 		return fmt.Errorf("failed to record event: %w", err)
 	}

+	// Mark issue as dirty for incremental export
+	_, err = tx.ExecContext(ctx, `
+		INSERT INTO dirty_issues (issue_id, marked_at)
+		VALUES (?, ?)
+		ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
+	`, id, time.Now())
+	if err != nil {
+		return fmt.Errorf("failed to mark issue dirty: %w", err)
+	}
+
 	return tx.Commit()
 }