feat(dolt): bootstrap routes.jsonl and interactions.jsonl (bd-lbdy7)

Add support for importing routes.jsonl and interactions.jsonl during
Dolt bootstrap. Previously only issues.jsonl was imported.

Changes:
- Add routes and interactions tables to Dolt schema
- Import routes before issues (no dependencies)
- Import interactions after issues (may reference issue_id)
- Reuse audit.Entry type instead of duplicating
- Add tests for multi-file bootstrap

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
beads/crew/wickham
2026-01-20 20:47:49 -08:00
committed by Steve Yegge
parent 938a17cda5
commit 90344b9939
3 changed files with 314 additions and 6 deletions

View File

@@ -12,7 +12,9 @@ import (
"strings"
"time"
"github.com/steveyegge/beads/internal/audit"
"github.com/steveyegge/beads/internal/lockfile"
"github.com/steveyegge/beads/internal/routing"
"github.com/steveyegge/beads/internal/types"
"github.com/steveyegge/beads/internal/utils"
)
@@ -21,6 +23,8 @@ import (
type BootstrapResult struct {
IssuesImported int
IssuesSkipped int
RoutesImported int
InteractionsImported int
ParseErrors []ParseError
PrefixDetected string
}
@@ -174,11 +178,12 @@ func releaseBootstrapLock(f *os.File, lockPath string) {
_ = os.Remove(lockPath)
}
// performBootstrap performs the actual bootstrap from JSONL
// performBootstrap performs the actual bootstrap from JSONL files.
// Import order: routes -> issues -> interactions (dependencies require issues to exist)
func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string) (*BootstrapResult, error) {
result := &BootstrapResult{}
// Parse JSONL with graceful error handling
// Parse issues JSONL with graceful error handling
issues, parseErrors := parseJSONLWithErrors(jsonlPath)
result.ParseErrors = parseErrors
@@ -218,6 +223,14 @@ func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string
}
}
// Import routes first (no dependencies)
routesImported, err := importRoutesBootstrap(ctx, store, cfg.BeadsDir)
if err != nil {
// Non-fatal - routes.jsonl may not exist
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import routes: %v\n", err)
}
result.RoutesImported = routesImported
// Import issues in a transaction
imported, skipped, err := importIssuesBootstrap(ctx, store, issues)
if err != nil {
@@ -227,6 +240,15 @@ func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string
result.IssuesImported = imported
result.IssuesSkipped = skipped
// Import interactions (after issues, since interactions may reference issue_id)
interactionsPath := filepath.Join(cfg.BeadsDir, "interactions.jsonl")
interactionsImported, err := importInteractionsBootstrap(ctx, store, interactionsPath)
if err != nil {
// Non-fatal - interactions.jsonl may not exist
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interactions: %v\n", err)
}
result.InteractionsImported = interactionsImported
// Commit the bootstrap
if err := store.Commit(ctx, "Bootstrap from JSONL"); err != nil {
// Non-fatal - data is still in the database
@@ -432,3 +454,105 @@ func importIssuesBootstrap(ctx context.Context, store *DoltStore, issues []*type
return imported, skipped, nil
}
// importRoutesBootstrap imports routes from routes.jsonl during bootstrap
// Returns the number of routes imported
func importRoutesBootstrap(ctx context.Context, store *DoltStore, beadsDir string) (int, error) {
routes, err := routing.LoadRoutes(beadsDir)
if err != nil {
return 0, err
}
if len(routes) == 0 {
return 0, nil // No routes to import
}
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
return 0, fmt.Errorf("failed to begin transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
imported := 0
for _, route := range routes {
_, err := tx.ExecContext(ctx, `
INSERT INTO routes (prefix, path, created_at)
VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE path = VALUES(path)
`, route.Prefix, route.Path, time.Now().UTC())
if err != nil {
return imported, fmt.Errorf("failed to insert route %s: %w", route.Prefix, err)
}
imported++
}
if err := tx.Commit(); err != nil {
return imported, fmt.Errorf("failed to commit routes: %w", err)
}
return imported, nil
}
// importInteractionsBootstrap imports interactions from interactions.jsonl during bootstrap
// Returns the number of interactions imported
func importInteractionsBootstrap(ctx context.Context, store *DoltStore, interactionsPath string) (int, error) {
// #nosec G304 - controlled path
f, err := os.Open(interactionsPath)
if err != nil {
if os.IsNotExist(err) {
return 0, nil // No interactions file is not an error
}
return 0, err
}
defer func() { _ = f.Close() }()
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
return 0, fmt.Errorf("failed to begin transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
imported := 0
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 1024), 2*1024*1024) // 2MB buffer for large lines
for scanner.Scan() {
line := scanner.Text()
if strings.TrimSpace(line) == "" {
continue
}
var entry audit.Entry
if err := json.Unmarshal([]byte(line), &entry); err != nil {
// Skip malformed lines during bootstrap
continue
}
// Convert extra map to JSON (default to empty object for valid JSON)
extraJSON := []byte("{}")
if entry.Extra != nil {
extraJSON, _ = json.Marshal(entry.Extra)
}
_, err := tx.ExecContext(ctx, `
INSERT INTO interactions (id, kind, created_at, actor, issue_id, model, prompt, response, error, tool_name, exit_code, parent_id, label, reason, extra)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE kind = kind
`, entry.ID, entry.Kind, entry.CreatedAt, entry.Actor, entry.IssueID, entry.Model, entry.Prompt, entry.Response, entry.Error, entry.ToolName, entry.ExitCode, entry.ParentID, entry.Label, entry.Reason, extraJSON)
if err != nil && !strings.Contains(err.Error(), "Duplicate entry") {
// Non-fatal - skip individual failures
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interaction %s: %v\n", entry.ID, err)
continue
}
imported++
}
if err := scanner.Err(); err != nil {
return imported, fmt.Errorf("scanner error: %w", err)
}
if err := tx.Commit(); err != nil {
return imported, fmt.Errorf("failed to commit interactions: %w", err)
}
return imported, nil
}

View File

@@ -347,3 +347,156 @@ func TestDetectPrefixFromIssues(t *testing.T) {
t.Errorf("expected prefix 'proj', got '%s'", prefix)
}
}
func TestBootstrapWithRoutesAndInteractions(t *testing.T) {
// Create temp directory structure
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create test issues JSONL
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
issue := types.Issue{
ID: "test-001",
Title: "Test issue",
Status: types.StatusOpen,
}
data, _ := json.Marshal(issue)
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
t.Fatalf("failed to write issues JSONL: %v", err)
}
// Create test routes JSONL
routesPath := filepath.Join(beadsDir, "routes.jsonl")
routesContent := `{"prefix":"test-","path":"."}
{"prefix":"other-","path":"other/rig"}
`
if err := os.WriteFile(routesPath, []byte(routesContent), 0644); err != nil {
t.Fatalf("failed to write routes JSONL: %v", err)
}
// Create test interactions JSONL
interactionsPath := filepath.Join(beadsDir, "interactions.jsonl")
interactionsContent := `{"id":"int-001","kind":"llm_call","created_at":"2025-01-20T10:00:00Z","actor":"test-agent","model":"claude-3"}
{"id":"int-002","kind":"tool_call","created_at":"2025-01-20T10:01:00Z","actor":"test-agent","tool_name":"bash","exit_code":0}
`
if err := os.WriteFile(interactionsPath, []byte(interactionsContent), 0644); err != nil {
t.Fatalf("failed to write interactions JSONL: %v", err)
}
// Perform bootstrap
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if !bootstrapped {
t.Fatal("expected bootstrap to be performed")
}
if result == nil {
t.Fatal("expected non-nil result")
}
// Verify issues imported
if result.IssuesImported != 1 {
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
}
// Verify routes imported
if result.RoutesImported != 2 {
t.Errorf("expected 2 routes imported, got %d", result.RoutesImported)
}
// Verify interactions imported
if result.InteractionsImported != 2 {
t.Errorf("expected 2 interactions imported, got %d", result.InteractionsImported)
}
// Open store and verify data
store, err := New(ctx, &Config{Path: doltDir})
if err != nil {
t.Fatalf("failed to open store: %v", err)
}
defer store.Close()
// Verify routes table
var routeCount int
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM routes").Scan(&routeCount)
if err != nil {
t.Fatalf("failed to count routes: %v", err)
}
if routeCount != 2 {
t.Errorf("expected 2 routes in table, got %d", routeCount)
}
// Verify interactions table
var interactionCount int
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM interactions").Scan(&interactionCount)
if err != nil {
t.Fatalf("failed to count interactions: %v", err)
}
if interactionCount != 2 {
t.Errorf("expected 2 interactions in table, got %d", interactionCount)
}
}
func TestBootstrapWithoutOptionalFiles(t *testing.T) {
// Test that bootstrap succeeds when routes.jsonl and interactions.jsonl don't exist
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create only issues JSONL
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
issue := types.Issue{
ID: "test-001",
Title: "Test issue",
Status: types.StatusOpen,
}
data, _ := json.Marshal(issue)
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
t.Fatalf("failed to write issues JSONL: %v", err)
}
// Perform bootstrap - should succeed even without routes.jsonl and interactions.jsonl
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if !bootstrapped {
t.Fatal("expected bootstrap to be performed")
}
if result == nil {
t.Fatal("expected non-nil result")
}
// Verify issues imported
if result.IssuesImported != 1 {
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
}
// Routes and interactions should be 0 (files don't exist)
if result.RoutesImported != 0 {
t.Errorf("expected 0 routes imported, got %d", result.RoutesImported)
}
if result.InteractionsImported != 0 {
t.Errorf("expected 0 interactions imported, got %d", result.InteractionsImported)
}
}

View File

@@ -203,6 +203,37 @@ CREATE TABLE IF NOT EXISTS repo_mtimes (
last_checked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_repo_mtimes_checked (last_checked)
);
-- Routes table (prefix-to-path routing configuration)
CREATE TABLE IF NOT EXISTS routes (
prefix VARCHAR(32) PRIMARY KEY,
path VARCHAR(512) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
);
-- Interactions table (agent audit log)
CREATE TABLE IF NOT EXISTS interactions (
id VARCHAR(32) PRIMARY KEY,
kind VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL,
actor VARCHAR(255),
issue_id VARCHAR(255),
model VARCHAR(255),
prompt TEXT,
response TEXT,
error TEXT,
tool_name VARCHAR(255),
exit_code INT,
parent_id VARCHAR(32),
label VARCHAR(64),
reason TEXT,
extra JSON,
INDEX idx_interactions_kind (kind),
INDEX idx_interactions_created_at (created_at),
INDEX idx_interactions_issue_id (issue_id),
INDEX idx_interactions_parent_id (parent_id)
);
`
// defaultConfig contains the default configuration values