Extract SQLite migrations into separate files (bd-fb95094c.7)

- Created migrations/ subdirectory with 14 individual migration files - Reduced migrations.go from 680 to 98 lines (orchestration only) - Updated test imports to use migrations package - Updated MULTI_REPO_HYDRATION.md documentation - All tests passing
2025-11-06 20:06:38 -08:00
parent 0095464e83
commit b655b29ad9
18 changed files with 642 additions and 624 deletions
--- a/internal/storage/sqlite/migrations/001_dirty_issues_table.go
+++ b/internal/storage/sqlite/migrations/001_dirty_issues_table.go
@@ -0,0 +1,52 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateDirtyIssuesTable(db *sql.DB) error {
+	var tableName string
+	err := db.QueryRow(`
+		SELECT name FROM sqlite_master
+		WHERE type='table' AND name='dirty_issues'
+	`).Scan(&tableName)
+
+	if err == sql.ErrNoRows {
+		_, err := db.Exec(`
+			CREATE TABLE dirty_issues (
+				issue_id TEXT PRIMARY KEY,
+				marked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+				FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
+			);
+			CREATE INDEX idx_dirty_issues_marked_at ON dirty_issues(marked_at);
+		`)
+		if err != nil {
+			return fmt.Errorf("failed to create dirty_issues table: %w", err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to check for dirty_issues table: %w", err)
+	}
+
+	var hasContentHash bool
+	err = db.QueryRow(`
+		SELECT COUNT(*) > 0 FROM pragma_table_info('dirty_issues')
+		WHERE name = 'content_hash'
+	`).Scan(&hasContentHash)
+	
+	if err != nil {
+		return fmt.Errorf("failed to check for content_hash column: %w", err)
+	}
+	
+	if !hasContentHash {
+		_, err = db.Exec(`ALTER TABLE dirty_issues ADD COLUMN content_hash TEXT`)
+		if err != nil {
+			return fmt.Errorf("failed to add content_hash column: %w", err)
+		}
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/002_external_ref_column.go
+++ b/internal/storage/sqlite/migrations/002_external_ref_column.go
@@ -0,0 +1,43 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateExternalRefColumn(db *sql.DB) error {
+	var columnExists bool
+	rows, err := db.Query("PRAGMA table_info(issues)")
+	if err != nil {
+		return fmt.Errorf("failed to check schema: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var cid int
+		var name, typ string
+		var notnull, pk int
+		var dflt *string
+		err := rows.Scan(&cid, &name, &typ, &notnull, &dflt, &pk)
+		if err != nil {
+			return fmt.Errorf("failed to scan column info: %w", err)
+		}
+		if name == "external_ref" {
+			columnExists = true
+			break
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("error reading column info: %w", err)
+	}
+
+	if !columnExists {
+		_, err := db.Exec(`ALTER TABLE issues ADD COLUMN external_ref TEXT`)
+		if err != nil {
+			return fmt.Errorf("failed to add external_ref column: %w", err)
+		}
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/003_composite_indexes.go
+++ b/internal/storage/sqlite/migrations/003_composite_indexes.go
@@ -0,0 +1,30 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateCompositeIndexes(db *sql.DB) error {
+	var indexName string
+	err := db.QueryRow(`
+		SELECT name FROM sqlite_master
+		WHERE type='index' AND name='idx_dependencies_depends_on_type'
+	`).Scan(&indexName)
+
+	if err == sql.ErrNoRows {
+		_, err := db.Exec(`
+			CREATE INDEX idx_dependencies_depends_on_type ON dependencies(depends_on_id, type)
+		`)
+		if err != nil {
+			return fmt.Errorf("failed to create composite index idx_dependencies_depends_on_type: %w", err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to check for composite index: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/004_closed_at_constraint.go
+++ b/internal/storage/sqlite/migrations/004_closed_at_constraint.go
@@ -0,0 +1,43 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateClosedAtConstraint(db *sql.DB) error {
+	var count int
+	err := db.QueryRow(`
+		SELECT COUNT(*)
+		FROM issues
+		WHERE (CASE WHEN status = 'closed' THEN 1 ELSE 0 END) <>
+		      (CASE WHEN closed_at IS NOT NULL THEN 1 ELSE 0 END)
+	`).Scan(&count)
+	if err != nil {
+		return fmt.Errorf("failed to count inconsistent issues: %w", err)
+	}
+
+	if count == 0 {
+		return nil
+	}
+
+	_, err = db.Exec(`
+		UPDATE issues
+		SET closed_at = NULL
+		WHERE status != 'closed' AND closed_at IS NOT NULL
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to clear closed_at for non-closed issues: %w", err)
+	}
+
+	_, err = db.Exec(`
+		UPDATE issues
+		SET closed_at = COALESCE(updated_at, CURRENT_TIMESTAMP)
+		WHERE status = 'closed' AND closed_at IS NULL
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to set closed_at for closed issues: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/005_compaction_columns.go
+++ b/internal/storage/sqlite/migrations/005_compaction_columns.go
@@ -0,0 +1,33 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateCompactionColumns(db *sql.DB) error {
+	var columnExists bool
+	err := db.QueryRow(`
+		SELECT COUNT(*) > 0
+		FROM pragma_table_info('issues')
+		WHERE name = 'compaction_level'
+	`).Scan(&columnExists)
+	if err != nil {
+		return fmt.Errorf("failed to check compaction_level column: %w", err)
+	}
+
+	if columnExists {
+		return nil
+	}
+
+	_, err = db.Exec(`
+		ALTER TABLE issues ADD COLUMN compaction_level INTEGER DEFAULT 0;
+		ALTER TABLE issues ADD COLUMN compacted_at DATETIME;
+		ALTER TABLE issues ADD COLUMN original_size INTEGER;
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to add compaction columns: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/006_snapshots_table.go
+++ b/internal/storage/sqlite/migrations/006_snapshots_table.go
@@ -0,0 +1,43 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateSnapshotsTable(db *sql.DB) error {
+	var tableExists bool
+	err := db.QueryRow(`
+		SELECT COUNT(*) > 0
+		FROM sqlite_master
+		WHERE type='table' AND name='issue_snapshots'
+	`).Scan(&tableExists)
+	if err != nil {
+		return fmt.Errorf("failed to check issue_snapshots table: %w", err)
+	}
+
+	if tableExists {
+		return nil
+	}
+
+	_, err = db.Exec(`
+		CREATE TABLE issue_snapshots (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			issue_id TEXT NOT NULL,
+			snapshot_time DATETIME NOT NULL,
+			compaction_level INTEGER NOT NULL,
+			original_size INTEGER NOT NULL,
+			compressed_size INTEGER NOT NULL,
+			original_content TEXT NOT NULL,
+			archived_events TEXT,
+			FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
+		);
+		CREATE INDEX idx_snapshots_issue ON issue_snapshots(issue_id);
+		CREATE INDEX idx_snapshots_level ON issue_snapshots(compaction_level);
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to create issue_snapshots table: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/007_compaction_config.go
+++ b/internal/storage/sqlite/migrations/007_compaction_config.go
@@ -0,0 +1,26 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateCompactionConfig(db *sql.DB) error {
+	_, err := db.Exec(`
+		INSERT OR IGNORE INTO config (key, value) VALUES
+			('compaction_enabled', 'false'),
+			('compact_tier1_days', '30'),
+			('compact_tier1_dep_levels', '2'),
+			('compact_tier2_days', '90'),
+			('compact_tier2_dep_levels', '5'),
+			('compact_tier2_commits', '100'),
+			('compact_model', 'claude-3-5-haiku-20241022'),
+			('compact_batch_size', '50'),
+			('compact_parallel_workers', '5'),
+			('auto_compact_enabled', 'false')
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to add compaction config defaults: %w", err)
+	}
+	return nil
+}
--- a/internal/storage/sqlite/migrations/008_compacted_at_commit_column.go
+++ b/internal/storage/sqlite/migrations/008_compacted_at_commit_column.go
@@ -0,0 +1,29 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateCompactedAtCommitColumn(db *sql.DB) error {
+	var columnExists bool
+	err := db.QueryRow(`
+		SELECT COUNT(*) > 0
+		FROM pragma_table_info('issues')
+		WHERE name = 'compacted_at_commit'
+	`).Scan(&columnExists)
+	if err != nil {
+		return fmt.Errorf("failed to check compacted_at_commit column: %w", err)
+	}
+
+	if columnExists {
+		return nil
+	}
+
+	_, err = db.Exec(`ALTER TABLE issues ADD COLUMN compacted_at_commit TEXT`)
+	if err != nil {
+		return fmt.Errorf("failed to add compacted_at_commit column: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/009_export_hashes_table.go
+++ b/internal/storage/sqlite/migrations/009_export_hashes_table.go
@@ -0,0 +1,35 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateExportHashesTable(db *sql.DB) error {
+	var tableName string
+	err := db.QueryRow(`
+		SELECT name FROM sqlite_master
+		WHERE type='table' AND name='export_hashes'
+	`).Scan(&tableName)
+
+	if err == sql.ErrNoRows {
+		_, err := db.Exec(`
+			CREATE TABLE export_hashes (
+				issue_id TEXT PRIMARY KEY,
+				content_hash TEXT NOT NULL,
+				exported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+				FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
+			)
+		`)
+		if err != nil {
+			return fmt.Errorf("failed to create export_hashes table: %w", err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to check export_hashes table: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/010_content_hash_column.go
+++ b/internal/storage/sqlite/migrations/010_content_hash_column.go
@@ -0,0 +1,94 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/steveyegge/beads/internal/types"
+)
+
+func MigrateContentHashColumn(db *sql.DB) error {
+	var colName string
+	err := db.QueryRow(`
+		SELECT name FROM pragma_table_info('issues')
+		WHERE name = 'content_hash'
+	`).Scan(&colName)
+
+	if err == sql.ErrNoRows {
+		_, err := db.Exec(`ALTER TABLE issues ADD COLUMN content_hash TEXT`)
+		if err != nil {
+			return fmt.Errorf("failed to add content_hash column: %w", err)
+		}
+
+		_, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_issues_content_hash ON issues(content_hash)`)
+		if err != nil {
+			return fmt.Errorf("failed to create content_hash index: %w", err)
+		}
+
+		rows, err := db.Query(`
+			SELECT id, title, description, design, acceptance_criteria, notes,
+			       status, priority, issue_type, assignee, external_ref
+			FROM issues
+		`)
+		if err != nil {
+			return fmt.Errorf("failed to query existing issues: %w", err)
+		}
+		defer rows.Close()
+
+		updates := make(map[string]string)
+		for rows.Next() {
+			var issue types.Issue
+			var assignee sql.NullString
+			var externalRef sql.NullString
+			err := rows.Scan(
+				&issue.ID, &issue.Title, &issue.Description, &issue.Design,
+				&issue.AcceptanceCriteria, &issue.Notes, &issue.Status,
+				&issue.Priority, &issue.IssueType, &assignee, &externalRef,
+			)
+			if err != nil {
+				return fmt.Errorf("failed to scan issue: %w", err)
+			}
+			if assignee.Valid {
+				issue.Assignee = assignee.String
+			}
+			if externalRef.Valid {
+				issue.ExternalRef = &externalRef.String
+			}
+
+			updates[issue.ID] = issue.ComputeContentHash()
+		}
+		if err := rows.Err(); err != nil {
+			return fmt.Errorf("error iterating issues: %w", err)
+		}
+
+		tx, err := db.Begin()
+		if err != nil {
+			return fmt.Errorf("failed to begin transaction: %w", err)
+		}
+		defer tx.Rollback()
+
+		stmt, err := tx.Prepare(`UPDATE issues SET content_hash = ? WHERE id = ?`)
+		if err != nil {
+			return fmt.Errorf("failed to prepare update statement: %w", err)
+		}
+		defer stmt.Close()
+
+		for id, hash := range updates {
+			if _, err := stmt.Exec(hash, id); err != nil {
+				return fmt.Errorf("failed to update content_hash for issue %s: %w", id, err)
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			return fmt.Errorf("failed to commit transaction: %w", err)
+		}
+
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to check content_hash column: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/011_external_ref_unique.go
+++ b/internal/storage/sqlite/migrations/011_external_ref_unique.go
@@ -0,0 +1,66 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+func MigrateExternalRefUnique(db *sql.DB) error {
+	var hasConstraint bool
+	err := db.QueryRow(`
+		SELECT COUNT(*) > 0
+		FROM sqlite_master
+		WHERE type = 'index'
+		  AND name = 'idx_issues_external_ref_unique'
+	`).Scan(&hasConstraint)
+	if err != nil {
+		return fmt.Errorf("failed to check for UNIQUE constraint: %w", err)
+	}
+
+	if hasConstraint {
+		return nil
+	}
+
+	existingDuplicates, err := findExternalRefDuplicates(db)
+	if err != nil {
+		return fmt.Errorf("failed to check for duplicate external_ref values: %w", err)
+	}
+
+	if len(existingDuplicates) > 0 {
+		return fmt.Errorf("cannot add UNIQUE constraint: found %d duplicate external_ref values (resolve with 'bd duplicates' or manually)", len(existingDuplicates))
+	}
+
+	_, err = db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_issues_external_ref_unique ON issues(external_ref) WHERE external_ref IS NOT NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to create UNIQUE index on external_ref: %w", err)
+	}
+
+	return nil
+}
+
+func findExternalRefDuplicates(db *sql.DB) (map[string][]string, error) {
+	rows, err := db.Query(`
+		SELECT external_ref, GROUP_CONCAT(id, ',') as ids
+		FROM issues
+		WHERE external_ref IS NOT NULL
+		GROUP BY external_ref
+		HAVING COUNT(*) > 1
+	`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	duplicates := make(map[string][]string)
+	for rows.Next() {
+		var externalRef, idsCSV string
+		if err := rows.Scan(&externalRef, &idsCSV); err != nil {
+			return nil, err
+		}
+		ids := strings.Split(idsCSV, ",")
+		duplicates[externalRef] = ids
+	}
+
+	return duplicates, rows.Err()
+}
--- a/internal/storage/sqlite/migrations/012_source_repo_column.go
+++ b/internal/storage/sqlite/migrations/012_source_repo_column.go
@@ -0,0 +1,34 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateSourceRepoColumn(db *sql.DB) error {
+	var columnExists bool
+	err := db.QueryRow(`
+		SELECT COUNT(*) > 0
+		FROM pragma_table_info('issues')
+		WHERE name = 'source_repo'
+	`).Scan(&columnExists)
+	if err != nil {
+		return fmt.Errorf("failed to check source_repo column: %w", err)
+	}
+
+	if columnExists {
+		return nil
+	}
+
+	_, err = db.Exec(`ALTER TABLE issues ADD COLUMN source_repo TEXT DEFAULT '.'`)
+	if err != nil {
+		return fmt.Errorf("failed to add source_repo column: %w", err)
+	}
+
+	_, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_issues_source_repo ON issues(source_repo)`)
+	if err != nil {
+		return fmt.Errorf("failed to create source_repo index: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/013_repo_mtimes_table.go
+++ b/internal/storage/sqlite/migrations/013_repo_mtimes_table.go
@@ -0,0 +1,36 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateRepoMtimesTable(db *sql.DB) error {
+	var tableName string
+	err := db.QueryRow(`
+		SELECT name FROM sqlite_master
+		WHERE type='table' AND name='repo_mtimes'
+	`).Scan(&tableName)
+
+	if err == sql.ErrNoRows {
+		_, err := db.Exec(`
+			CREATE TABLE repo_mtimes (
+				repo_path TEXT PRIMARY KEY,
+				jsonl_path TEXT NOT NULL,
+				mtime_ns INTEGER NOT NULL,
+				last_checked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+			);
+			CREATE INDEX idx_repo_mtimes_checked ON repo_mtimes(last_checked);
+		`)
+		if err != nil {
+			return fmt.Errorf("failed to create repo_mtimes table: %w", err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to check for repo_mtimes table: %w", err)
+	}
+
+	return nil
+}
--- a/internal/storage/sqlite/migrations/014_child_counters_table.go
+++ b/internal/storage/sqlite/migrations/014_child_counters_table.go
@@ -0,0 +1,34 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func MigrateChildCountersTable(db *sql.DB) error {
+	var tableName string
+	err := db.QueryRow(`
+		SELECT name FROM sqlite_master
+		WHERE type='table' AND name='child_counters'
+	`).Scan(&tableName)
+
+	if err == sql.ErrNoRows {
+		_, err := db.Exec(`
+			CREATE TABLE child_counters (
+				parent_id TEXT PRIMARY KEY,
+				last_child INTEGER NOT NULL DEFAULT 0,
+				FOREIGN KEY (parent_id) REFERENCES issues(id) ON DELETE CASCADE
+			)
+		`)
+		if err != nil {
+			return fmt.Errorf("failed to create child_counters table: %w", err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to check for child_counters table: %w", err)
+	}
+
+	return nil
+}