Implement incremental JSONL export with dirty issue tracking
Optimize auto-flush by tracking which issues have changed instead of exporting the entire database on every flush. For large projects with 1000+ issues, this provides significant performance improvements. Changes: - Add dirty_issues table to schema with issue_id and marked_at columns - Implement dirty tracking functions in new dirty.go file: * MarkIssueDirty() - Mark single issue as needing export * MarkIssuesDirty() - Batch mark multiple issues efficiently * GetDirtyIssues() - Query which issues need export * ClearDirtyIssues() - Clear tracking after successful export * GetDirtyIssueCount() - Monitor dirty issue count - Update all CRUD operations to mark affected issues as dirty: * CreateIssue, UpdateIssue, DeleteIssue * AddDependency, RemoveDependency (marks both issues) * AddLabel, RemoveLabel, AddEvent - Modify export to support incremental mode: * Add --incremental flag to export only dirty issues * Used by auto-flush for performance * Full export still available without flag - Add Storage interface methods for dirty tracking Performance impact: With incremental export, large databases only write changed issues instead of regenerating entire JSONL file on every auto-flush. Closes bd-39 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
96
internal/storage/sqlite/dirty.go
Normal file
96
internal/storage/sqlite/dirty.go
Normal file
@@ -0,0 +1,96 @@
|
||||
// Package sqlite implements dirty issue tracking for incremental JSONL export.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MarkIssueDirty marks an issue as dirty (needs to be exported to JSONL)
|
||||
// This should be called whenever an issue is created, updated, or has dependencies changed
|
||||
func (s *SQLiteStorage) MarkIssueDirty(ctx context.Context, issueID string) error {
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO dirty_issues (issue_id, marked_at)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
|
||||
`, issueID, time.Now())
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkIssuesDirty marks multiple issues as dirty in a single transaction
|
||||
// More efficient when marking multiple issues (e.g., both sides of a dependency)
|
||||
func (s *SQLiteStorage) MarkIssuesDirty(ctx context.Context, issueIDs []string) error {
|
||||
if len(issueIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
now := time.Now()
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
INSERT INTO dirty_issues (issue_id, marked_at)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare statement: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, issueID := range issueIDs {
|
||||
if _, err := stmt.ExecContext(ctx, issueID, now); err != nil {
|
||||
return fmt.Errorf("failed to mark issue %s dirty: %w", issueID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetDirtyIssues returns the list of issue IDs that need to be exported
|
||||
func (s *SQLiteStorage) GetDirtyIssues(ctx context.Context) ([]string, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT issue_id FROM dirty_issues
|
||||
ORDER BY marked_at ASC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get dirty issues: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var issueIDs []string
|
||||
for rows.Next() {
|
||||
var issueID string
|
||||
if err := rows.Scan(&issueID); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan issue ID: %w", err)
|
||||
}
|
||||
issueIDs = append(issueIDs, issueID)
|
||||
}
|
||||
|
||||
return issueIDs, rows.Err()
|
||||
}
|
||||
|
||||
// ClearDirtyIssues removes all entries from the dirty_issues table
|
||||
// This should be called after a successful JSONL export
|
||||
func (s *SQLiteStorage) ClearDirtyIssues(ctx context.Context) error {
|
||||
_, err := s.db.ExecContext(ctx, `DELETE FROM dirty_issues`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to clear dirty issues: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetDirtyIssueCount returns the count of dirty issues (for monitoring/debugging)
|
||||
func (s *SQLiteStorage) GetDirtyIssueCount(ctx context.Context) (int, error) {
|
||||
var count int
|
||||
err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dirty_issues`).Scan(&count)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
return 0, fmt.Errorf("failed to count dirty issues: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
Reference in New Issue
Block a user