Fix CI regressions and stabilize tests
This commit is contained in:
@@ -150,7 +150,7 @@ func Merge3Way(outputPath, basePath, leftPath, rightPath string, debug bool) err
|
||||
if err := outFile.Sync(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to sync output file: %v\n", err)
|
||||
}
|
||||
if content, err := os.ReadFile(outputPath); err == nil {
|
||||
if content, err := os.ReadFile(outputPath); err == nil { // #nosec G304 -- debug output reads file created earlier in same function
|
||||
lines := 0
|
||||
fmt.Fprintf(os.Stderr, "Output file preview (first 10 lines):\n")
|
||||
for _, line := range splitLines(string(content)) {
|
||||
@@ -195,7 +195,7 @@ func splitLines(s string) []string {
|
||||
}
|
||||
|
||||
func readIssues(path string) ([]Issue, error) {
|
||||
file, err := os.Open(path)
|
||||
file, err := os.Open(path) // #nosec G304 -- path supplied by CLI flag and validated upstream
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func (s *SQLiteStorage) GetLabelsForIssues(ctx context.Context, issueIDs []strin
|
||||
FROM labels
|
||||
WHERE issue_id IN (%s)
|
||||
ORDER BY issue_id, label
|
||||
`, buildPlaceholders(len(issueIDs)))
|
||||
`, buildPlaceholders(len(issueIDs))) // #nosec G201 -- placeholders are generated internally
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, placeholders...)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,24 +2,30 @@ package migrations
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func MigrateExternalRefColumn(db *sql.DB) error {
|
||||
func MigrateExternalRefColumn(db *sql.DB) (retErr error) {
|
||||
var columnExists bool
|
||||
rows, err := db.Query("PRAGMA table_info(issues)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check schema: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if rows != nil {
|
||||
if closeErr := rows.Close(); closeErr != nil {
|
||||
retErr = errors.Join(retErr, fmt.Errorf("failed to close schema rows: %w", closeErr))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, typ string
|
||||
var notnull, pk int
|
||||
var dflt *string
|
||||
err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk)
|
||||
if err != nil {
|
||||
rows.Close()
|
||||
if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil {
|
||||
return fmt.Errorf("failed to scan column info: %w", err)
|
||||
}
|
||||
if name == "external_ref" {
|
||||
@@ -29,12 +35,14 @@ func MigrateExternalRefColumn(db *sql.DB) error {
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("error reading column info: %w", err)
|
||||
}
|
||||
|
||||
// Close rows before executing any statements to avoid deadlock with MaxOpenConns(1)
|
||||
rows.Close()
|
||||
// Close rows before executing any statements to avoid deadlock with MaxOpenConns(1).
|
||||
if err := rows.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close schema rows: %w", err)
|
||||
}
|
||||
rows = nil
|
||||
|
||||
if !columnExists {
|
||||
_, err := db.Exec(`ALTER TABLE issues ADD COLUMN external_ref TEXT`)
|
||||
|
||||
@@ -19,26 +19,26 @@ var expectedSchema = map[string][]string{
|
||||
"created_at", "updated_at", "closed_at", "content_hash", "external_ref",
|
||||
"compaction_level", "compacted_at", "compacted_at_commit", "original_size",
|
||||
},
|
||||
"dependencies": {"issue_id", "depends_on_id", "type", "created_at", "created_by"},
|
||||
"labels": {"issue_id", "label"},
|
||||
"comments": {"id", "issue_id", "author", "text", "created_at"},
|
||||
"events": {"id", "issue_id", "event_type", "actor", "old_value", "new_value", "comment", "created_at"},
|
||||
"config": {"key", "value"},
|
||||
"metadata": {"key", "value"},
|
||||
"dirty_issues": {"issue_id", "marked_at"},
|
||||
"export_hashes": {"issue_id", "content_hash", "exported_at"},
|
||||
"child_counters": {"parent_id", "last_child"},
|
||||
"issue_snapshots": {"id", "issue_id", "snapshot_time", "compaction_level", "original_size", "compressed_size", "original_content", "archived_events"},
|
||||
"dependencies": {"issue_id", "depends_on_id", "type", "created_at", "created_by"},
|
||||
"labels": {"issue_id", "label"},
|
||||
"comments": {"id", "issue_id", "author", "text", "created_at"},
|
||||
"events": {"id", "issue_id", "event_type", "actor", "old_value", "new_value", "comment", "created_at"},
|
||||
"config": {"key", "value"},
|
||||
"metadata": {"key", "value"},
|
||||
"dirty_issues": {"issue_id", "marked_at"},
|
||||
"export_hashes": {"issue_id", "content_hash", "exported_at"},
|
||||
"child_counters": {"parent_id", "last_child"},
|
||||
"issue_snapshots": {"id", "issue_id", "snapshot_time", "compaction_level", "original_size", "compressed_size", "original_content", "archived_events"},
|
||||
"compaction_snapshots": {"id", "issue_id", "compaction_level", "snapshot_json", "created_at"},
|
||||
"repo_mtimes": {"repo_path", "jsonl_path", "mtime_ns", "last_checked"},
|
||||
"repo_mtimes": {"repo_path", "jsonl_path", "mtime_ns", "last_checked"},
|
||||
}
|
||||
|
||||
// SchemaProbeResult contains the results of a schema compatibility check
|
||||
type SchemaProbeResult struct {
|
||||
Compatible bool
|
||||
MissingTables []string
|
||||
MissingColumns map[string][]string // table -> missing columns
|
||||
ErrorMessage string
|
||||
Compatible bool
|
||||
MissingTables []string
|
||||
MissingColumns map[string][]string // table -> missing columns
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// probeSchema verifies all expected tables and columns exist
|
||||
@@ -52,19 +52,19 @@ func probeSchema(db *sql.DB) SchemaProbeResult {
|
||||
|
||||
for table, expectedCols := range expectedSchema {
|
||||
// Try to query the table with all expected columns
|
||||
query := fmt.Sprintf("SELECT %s FROM %s LIMIT 0", strings.Join(expectedCols, ", "), table)
|
||||
query := fmt.Sprintf("SELECT %s FROM %s LIMIT 0", strings.Join(expectedCols, ", "), table) // #nosec G201 -- table/column names sourced from hardcoded schema
|
||||
_, err := db.Exec(query)
|
||||
|
||||
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
|
||||
|
||||
// Check if table doesn't exist
|
||||
if strings.Contains(errMsg, "no such table") {
|
||||
result.Compatible = false
|
||||
result.MissingTables = append(result.MissingTables, table)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
// Check if column doesn't exist
|
||||
if strings.Contains(errMsg, "no such column") {
|
||||
result.Compatible = false
|
||||
@@ -97,25 +97,25 @@ func probeSchema(db *sql.DB) SchemaProbeResult {
|
||||
// findMissingColumns determines which columns are missing from a table
|
||||
func findMissingColumns(db *sql.DB, table string, expectedCols []string) []string {
|
||||
missing := []string{}
|
||||
|
||||
|
||||
for _, col := range expectedCols {
|
||||
query := fmt.Sprintf("SELECT %s FROM %s LIMIT 0", col, table)
|
||||
query := fmt.Sprintf("SELECT %s FROM %s LIMIT 0", col, table) // #nosec G201 -- table/column names sourced from hardcoded schema
|
||||
_, err := db.Exec(query)
|
||||
if err != nil && strings.Contains(err.Error(), "no such column") {
|
||||
missing = append(missing, col)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return missing
|
||||
}
|
||||
|
||||
// verifySchemaCompatibility runs schema probe and returns detailed error on failure
|
||||
func verifySchemaCompatibility(db *sql.DB) error {
|
||||
result := probeSchema(db)
|
||||
|
||||
|
||||
if !result.Compatible {
|
||||
return fmt.Errorf("%w: %s", ErrSchemaIncompatible, result.ErrorMessage)
|
||||
}
|
||||
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -19,24 +19,30 @@ import (
|
||||
// - For shared memory (not recommended): ":memory:"
|
||||
func newTestStore(t *testing.T, dbPath string) *SQLiteStorage {
|
||||
t.Helper()
|
||||
|
||||
|
||||
// Default to temp file for test isolation
|
||||
// File-based databases are more reliable than in-memory for connection pool scenarios
|
||||
if dbPath == "" {
|
||||
dbPath = t.TempDir() + "/test.db"
|
||||
}
|
||||
|
||||
|
||||
store, err := New(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test database: %v", err)
|
||||
}
|
||||
|
||||
|
||||
t.Cleanup(func() {
|
||||
if cerr := store.Close(); cerr != nil {
|
||||
t.Fatalf("Failed to close test database: %v", cerr)
|
||||
}
|
||||
})
|
||||
|
||||
// CRITICAL (bd-166): Set issue_prefix to prevent "database not initialized" errors
|
||||
ctx := context.Background()
|
||||
if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil {
|
||||
_ = store.Close()
|
||||
t.Fatalf("Failed to set issue_prefix: %v", err)
|
||||
}
|
||||
|
||||
|
||||
return store
|
||||
}
|
||||
|
||||
@@ -90,47 +90,47 @@ var taskTitles = []string{
|
||||
|
||||
// DataConfig controls the distribution and characteristics of generated test data
|
||||
type DataConfig struct {
|
||||
TotalIssues int // total number of issues to generate
|
||||
EpicRatio float64 // percentage of issues that are epics (e.g., 0.1 for 10%)
|
||||
FeatureRatio float64 // percentage of issues that are features (e.g., 0.3 for 30%)
|
||||
OpenRatio float64 // percentage of issues that are open (e.g., 0.5 for 50%)
|
||||
CrossLinkRatio float64 // percentage of tasks with cross-epic blocking dependencies (e.g., 0.2 for 20%)
|
||||
MaxEpicAgeDays int // maximum age in days for epics (e.g., 180)
|
||||
MaxFeatureAgeDays int // maximum age in days for features (e.g., 150)
|
||||
MaxTaskAgeDays int // maximum age in days for tasks (e.g., 120)
|
||||
MaxClosedAgeDays int // maximum days since closure (e.g., 30)
|
||||
RandSeed int64 // random seed for reproducibility
|
||||
TotalIssues int // total number of issues to generate
|
||||
EpicRatio float64 // percentage of issues that are epics (e.g., 0.1 for 10%)
|
||||
FeatureRatio float64 // percentage of issues that are features (e.g., 0.3 for 30%)
|
||||
OpenRatio float64 // percentage of issues that are open (e.g., 0.5 for 50%)
|
||||
CrossLinkRatio float64 // percentage of tasks with cross-epic blocking dependencies (e.g., 0.2 for 20%)
|
||||
MaxEpicAgeDays int // maximum age in days for epics (e.g., 180)
|
||||
MaxFeatureAgeDays int // maximum age in days for features (e.g., 150)
|
||||
MaxTaskAgeDays int // maximum age in days for tasks (e.g., 120)
|
||||
MaxClosedAgeDays int // maximum days since closure (e.g., 30)
|
||||
RandSeed int64 // random seed for reproducibility
|
||||
}
|
||||
|
||||
// DefaultLargeConfig returns configuration for 10K issue dataset
|
||||
func DefaultLargeConfig() DataConfig {
|
||||
return DataConfig{
|
||||
TotalIssues: 10000,
|
||||
EpicRatio: 0.1,
|
||||
FeatureRatio: 0.3,
|
||||
OpenRatio: 0.5,
|
||||
CrossLinkRatio: 0.2,
|
||||
MaxEpicAgeDays: 180,
|
||||
TotalIssues: 10000,
|
||||
EpicRatio: 0.1,
|
||||
FeatureRatio: 0.3,
|
||||
OpenRatio: 0.5,
|
||||
CrossLinkRatio: 0.2,
|
||||
MaxEpicAgeDays: 180,
|
||||
MaxFeatureAgeDays: 150,
|
||||
MaxTaskAgeDays: 120,
|
||||
MaxClosedAgeDays: 30,
|
||||
RandSeed: 42,
|
||||
MaxTaskAgeDays: 120,
|
||||
MaxClosedAgeDays: 30,
|
||||
RandSeed: 42,
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultXLargeConfig returns configuration for 20K issue dataset
|
||||
func DefaultXLargeConfig() DataConfig {
|
||||
return DataConfig{
|
||||
TotalIssues: 20000,
|
||||
EpicRatio: 0.1,
|
||||
FeatureRatio: 0.3,
|
||||
OpenRatio: 0.5,
|
||||
CrossLinkRatio: 0.2,
|
||||
MaxEpicAgeDays: 180,
|
||||
TotalIssues: 20000,
|
||||
EpicRatio: 0.1,
|
||||
FeatureRatio: 0.3,
|
||||
OpenRatio: 0.5,
|
||||
CrossLinkRatio: 0.2,
|
||||
MaxEpicAgeDays: 180,
|
||||
MaxFeatureAgeDays: 150,
|
||||
MaxTaskAgeDays: 120,
|
||||
MaxClosedAgeDays: 30,
|
||||
RandSeed: 43,
|
||||
MaxTaskAgeDays: 120,
|
||||
MaxClosedAgeDays: 30,
|
||||
RandSeed: 43,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ func XLargeFromJSONL(ctx context.Context, store storage.Storage, tempDir string)
|
||||
|
||||
// generateIssuesWithConfig creates issues with realistic epic hierarchies and cross-links using provided configuration
|
||||
func generateIssuesWithConfig(ctx context.Context, store storage.Storage, cfg DataConfig) error {
|
||||
rng := rand.New(rand.NewSource(cfg.RandSeed))
|
||||
rng := rand.New(rand.NewSource(cfg.RandSeed)) // #nosec G404 -- deterministic math/rand used for repeatable fixture data
|
||||
|
||||
// Calculate breakdown using configuration ratios
|
||||
numEpics := int(float64(cfg.TotalIssues) * cfg.EpicRatio)
|
||||
|
||||
Reference in New Issue
Block a user