johno/beads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(multirepo): handle out-of-order dependencies during JSONL import (#414)

Browse Source 
* bd sync: 2025-11-29 00:08:58

* fix(multirepo): handle out-of-order dependencies during JSONL import

Fixes #413. When importing issues from multi-repo JSONL files, if issue A
(line 1) has a dependency on issue B (line 5), the import would fail with
FK constraint error because B doesn't exist yet.

Solution:
- Disable FK checks at start of importJSONLFile()
- Re-enable FK checks before commit
- Run PRAGMA foreign_key_check to validate data integrity
- Fail with clear error if orphaned dependencies are detected

This allows out-of-order dependencies while still catching corrupted data.

---------

Co-authored-by: Shaun Cutts <shauncutts@factfiber.com>
This commit is contained in:
Shaun Cutts
2025-11-30 01:07:52 -05:00
committed by GitHub
parent ba6f06e98b
commit 3a2e9d5852
3 changed files with 182 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
internal/storage/sqlite/multirepo.go
View File

@@ -117,6 +117,7 @@ func (s *SQLiteStorage) hydrateFromRepo(ctx context.Context, repoPath, sourceRep
}
// importJSONLFile imports issues from a JSONL file, setting the source_repo field.
// Disables FK checks during import to handle out-of-order dependencies.
func (s *SQLiteStorage) importJSONLFile(ctx context.Context, jsonlPath, sourceRepo string) (int, error) {
file, err := os.Open(jsonlPath) // #nosec G304 -- jsonlPath is from trusted source
if err != nil {
@@ -138,8 +139,22 @@ func (s *SQLiteStorage) importJSONLFile(ctx context.Context, jsonlPath, sourceRe
count := 0
lineNum := 0
// Get exclusive connection to ensure PRAGMA applies
conn, err := s.db.Conn(ctx)
if err != nil {
return 0, fmt.Errorf("failed to get connection: %w", err)
}
defer func() { _ = conn.Close() }()
// Disable foreign keys on this connection to handle out-of-order deps
// (issue A may depend on issue B that appears later in the file)
_, err = conn.ExecContext(ctx, `PRAGMA foreign_keys = OFF`)
if err != nil {
return 0, fmt.Errorf("failed to disable foreign keys: %w", err)
}
// Begin transaction for bulk import
tx, err := s.db.BeginTx(ctx, nil)
tx, err := conn.BeginTx(ctx, nil)
if err != nil {
return 0, fmt.Errorf("failed to begin transaction: %w", err)
}
@@ -179,6 +194,28 @@ func (s *SQLiteStorage) importJSONLFile(ctx context.Context, jsonlPath, sourceRe
return 0, fmt.Errorf("failed to read JSONL file: %w", err)
}
// Re-enable foreign keys before commit to validate data integrity
_, err = conn.ExecContext(ctx, `PRAGMA foreign_keys = ON`)
if err != nil {
return 0, fmt.Errorf("failed to re-enable foreign keys: %w", err)
}
// Validate FK constraints on imported data
rows, err := conn.QueryContext(ctx, `PRAGMA foreign_key_check`)
if err != nil {
return 0, fmt.Errorf("failed to check foreign keys: %w", err)
}
defer rows.Close()
if rows.Next() {
var table, rowid, parent, fkid string
_ = rows.Scan(&table, &rowid, &parent, &fkid)
return 0, fmt.Errorf(
"foreign key violation in imported data: table=%s rowid=%s parent=%s",
table, rowid, parent,
)
}
if err := tx.Commit(); err != nil {
return 0, fmt.Errorf("failed to commit transaction: %w", err)
}
@@ -197,7 +234,7 @@ func (s *SQLiteStorage) upsertIssueInTx(ctx context.Context, tx *sql.Tx, issue *
// Check if issue exists
var existingID string
err := tx.QueryRowContext(ctx, `SELECT id FROM issues WHERE id = ?`, issue.ID).Scan(&existingID)
if err == sql.ErrNoRows {
// Issue doesn't exist - insert it
_, err = tx.ExecContext(ctx, `

158
internal/storage/sqlite/multirepo_test.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -72,14 +73,14 @@ func TestHydrateFromMultiRepo(t *testing.T) {
// Create test issue
issue := types.Issue{
ID: "test-1",
Title: "Test Issue",
Status: types.StatusOpen,
Priority: 1,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceRepo: ".",
ID: "test-1",
Title: "Test Issue",
Status: types.StatusOpen,
Priority: 1,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceRepo: ".",
}
issue.ContentHash = issue.ComputeContentHash()
@@ -143,14 +144,14 @@ func TestHydrateFromMultiRepo(t *testing.T) {
// Create test issue
issue := types.Issue{
ID: "test-2",
Title: "Test Issue 2",
Status: types.StatusOpen,
Priority: 1,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceRepo: ".",
ID: "test-2",
Title: "Test Issue 2",
Status: types.StatusOpen,
Priority: 1,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceRepo: ".",
}
issue.ContentHash = issue.ComputeContentHash()
@@ -374,6 +375,131 @@ func TestImportJSONLFile(t *testing.T) {
})
}
func TestImportJSONLFileOutOfOrderDeps(t *testing.T) {
t.Run("handles out-of-order dependencies", func(t *testing.T) {
store, cleanup := setupTestDB(t)
defer cleanup()
// Create test JSONL file with dependency BEFORE its target
tmpDir := t.TempDir()
jsonlPath := filepath.Join(tmpDir, "test.jsonl")
f, err := os.Create(jsonlPath)
if err != nil {
t.Fatalf("failed to create JSONL file: %v", err)
}
// Issue 1 depends on Issue 2, but Issue 1 comes FIRST in the file
// This would fail with FK constraint if not handled properly
issue1 := types.Issue{
ID: "test-1",
Title: "Issue 1 (depends on Issue 2)",
Status: types.StatusOpen,
Priority: 1,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Dependencies: []*types.Dependency{
{
IssueID: "test-1",
DependsOnID: "test-2", // test-2 doesn't exist yet!
Type: types.DepBlocks,
CreatedAt: time.Now(),
CreatedBy: "test",
},
},
SourceRepo: "test",
}
issue1.ContentHash = issue1.ComputeContentHash()
issue2 := types.Issue{
ID: "test-2",
Title: "Issue 2 (dependency target)",
Status: types.StatusOpen,
Priority: 2,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceRepo: "test",
}
issue2.ContentHash = issue2.ComputeContentHash()
enc := json.NewEncoder(f)
enc.Encode(issue1) // Dependent first
enc.Encode(issue2) // Dependency target second
f.Close()
// Import should succeed despite out-of-order dependencies
ctx := context.Background()
count, err := store.importJSONLFile(ctx, jsonlPath, "test")
if err != nil {
t.Fatalf("importJSONLFile() error = %v", err)
}
if count != 2 {
t.Errorf("expected 2 issues imported, got %d", count)
}
// Verify dependency was created
deps, err := store.GetDependencies(ctx, "test-1")
if err != nil {
t.Fatalf("failed to get dependencies: %v", err)
}
if len(deps) != 1 {
t.Errorf("expected 1 dependency, got %d", len(deps))
}
if len(deps) > 0 && deps[0].ID != "test-2" {
t.Errorf("expected dependency on test-2, got %s", deps[0].ID)
}
})
t.Run("detects orphaned dependencies in corrupted data", func(t *testing.T) {
store, cleanup := setupTestDB(t)
defer cleanup()
// Create test JSONL with orphaned dependency (target doesn't exist)
tmpDir := t.TempDir()
jsonlPath := filepath.Join(tmpDir, "test.jsonl")
f, err := os.Create(jsonlPath)
if err != nil {
t.Fatalf("failed to create JSONL file: %v", err)
}
issue := types.Issue{
ID: "test-orphan",
Title: "Issue with orphaned dependency",
Status: types.StatusOpen,
Priority: 1,
IssueType: types.TypeTask,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Dependencies: []*types.Dependency{
{
IssueID: "test-orphan",
DependsOnID: "nonexistent-issue", // This issue doesn't exist
Type: types.DepBlocks,
CreatedAt: time.Now(),
CreatedBy: "test",
},
},
SourceRepo: "test",
}
issue.ContentHash = issue.ComputeContentHash()
enc := json.NewEncoder(f)
enc.Encode(issue)
f.Close()
// Import should fail due to FK violation
ctx := context.Background()
_, err = store.importJSONLFile(ctx, jsonlPath, "test")
if err == nil {
t.Error("expected error for orphaned dependency, got nil")
}
if err != nil && !strings.Contains(err.Error(), "foreign key violation") {
t.Errorf("expected foreign key violation error, got: %v", err)
}
})
}
func TestExportToMultiRepo(t *testing.T) {
t.Run("returns nil in single-repo mode", func(t *testing.T) {
store, cleanup := setupTestDB(t)