feat(dolt): bootstrap routes.jsonl and interactions.jsonl (bd-lbdy7)
Add support for importing routes.jsonl and interactions.jsonl during Dolt bootstrap. Previously only issues.jsonl was imported. Changes: - Add routes and interactions tables to Dolt schema - Import routes before issues (no dependencies) - Import interactions after issues (may reference issue_id) - Reuse audit.Entry type instead of duplicating - Add tests for multi-file bootstrap Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
committed by
Steve Yegge
parent
938a17cda5
commit
90344b9939
@@ -12,7 +12,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/audit"
|
||||
"github.com/steveyegge/beads/internal/lockfile"
|
||||
"github.com/steveyegge/beads/internal/routing"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
"github.com/steveyegge/beads/internal/utils"
|
||||
)
|
||||
@@ -21,6 +23,8 @@ import (
|
||||
type BootstrapResult struct {
|
||||
IssuesImported int
|
||||
IssuesSkipped int
|
||||
RoutesImported int
|
||||
InteractionsImported int
|
||||
ParseErrors []ParseError
|
||||
PrefixDetected string
|
||||
}
|
||||
@@ -174,11 +178,12 @@ func releaseBootstrapLock(f *os.File, lockPath string) {
|
||||
_ = os.Remove(lockPath)
|
||||
}
|
||||
|
||||
// performBootstrap performs the actual bootstrap from JSONL
|
||||
// performBootstrap performs the actual bootstrap from JSONL files.
|
||||
// Import order: routes -> issues -> interactions (dependencies require issues to exist)
|
||||
func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string) (*BootstrapResult, error) {
|
||||
result := &BootstrapResult{}
|
||||
|
||||
// Parse JSONL with graceful error handling
|
||||
// Parse issues JSONL with graceful error handling
|
||||
issues, parseErrors := parseJSONLWithErrors(jsonlPath)
|
||||
result.ParseErrors = parseErrors
|
||||
|
||||
@@ -218,6 +223,14 @@ func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string
|
||||
}
|
||||
}
|
||||
|
||||
// Import routes first (no dependencies)
|
||||
routesImported, err := importRoutesBootstrap(ctx, store, cfg.BeadsDir)
|
||||
if err != nil {
|
||||
// Non-fatal - routes.jsonl may not exist
|
||||
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import routes: %v\n", err)
|
||||
}
|
||||
result.RoutesImported = routesImported
|
||||
|
||||
// Import issues in a transaction
|
||||
imported, skipped, err := importIssuesBootstrap(ctx, store, issues)
|
||||
if err != nil {
|
||||
@@ -227,6 +240,15 @@ func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string
|
||||
result.IssuesImported = imported
|
||||
result.IssuesSkipped = skipped
|
||||
|
||||
// Import interactions (after issues, since interactions may reference issue_id)
|
||||
interactionsPath := filepath.Join(cfg.BeadsDir, "interactions.jsonl")
|
||||
interactionsImported, err := importInteractionsBootstrap(ctx, store, interactionsPath)
|
||||
if err != nil {
|
||||
// Non-fatal - interactions.jsonl may not exist
|
||||
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interactions: %v\n", err)
|
||||
}
|
||||
result.InteractionsImported = interactionsImported
|
||||
|
||||
// Commit the bootstrap
|
||||
if err := store.Commit(ctx, "Bootstrap from JSONL"); err != nil {
|
||||
// Non-fatal - data is still in the database
|
||||
@@ -432,3 +454,105 @@ func importIssuesBootstrap(ctx context.Context, store *DoltStore, issues []*type
|
||||
|
||||
return imported, skipped, nil
|
||||
}
|
||||
|
||||
// importRoutesBootstrap imports routes from routes.jsonl during bootstrap
|
||||
// Returns the number of routes imported
|
||||
func importRoutesBootstrap(ctx context.Context, store *DoltStore, beadsDir string) (int, error) {
|
||||
routes, err := routing.LoadRoutes(beadsDir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(routes) == 0 {
|
||||
return 0, nil // No routes to import
|
||||
}
|
||||
|
||||
tx, err := store.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
imported := 0
|
||||
for _, route := range routes {
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO routes (prefix, path, created_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE path = VALUES(path)
|
||||
`, route.Prefix, route.Path, time.Now().UTC())
|
||||
if err != nil {
|
||||
return imported, fmt.Errorf("failed to insert route %s: %w", route.Prefix, err)
|
||||
}
|
||||
imported++
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return imported, fmt.Errorf("failed to commit routes: %w", err)
|
||||
}
|
||||
|
||||
return imported, nil
|
||||
}
|
||||
|
||||
// importInteractionsBootstrap imports interactions from interactions.jsonl during bootstrap
|
||||
// Returns the number of interactions imported
|
||||
func importInteractionsBootstrap(ctx context.Context, store *DoltStore, interactionsPath string) (int, error) {
|
||||
// #nosec G304 - controlled path
|
||||
f, err := os.Open(interactionsPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil // No interactions file is not an error
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
tx, err := store.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
imported := 0
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 0, 1024), 2*1024*1024) // 2MB buffer for large lines
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var entry audit.Entry
|
||||
if err := json.Unmarshal([]byte(line), &entry); err != nil {
|
||||
// Skip malformed lines during bootstrap
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert extra map to JSON (default to empty object for valid JSON)
|
||||
extraJSON := []byte("{}")
|
||||
if entry.Extra != nil {
|
||||
extraJSON, _ = json.Marshal(entry.Extra)
|
||||
}
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO interactions (id, kind, created_at, actor, issue_id, model, prompt, response, error, tool_name, exit_code, parent_id, label, reason, extra)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE kind = kind
|
||||
`, entry.ID, entry.Kind, entry.CreatedAt, entry.Actor, entry.IssueID, entry.Model, entry.Prompt, entry.Response, entry.Error, entry.ToolName, entry.ExitCode, entry.ParentID, entry.Label, entry.Reason, extraJSON)
|
||||
if err != nil && !strings.Contains(err.Error(), "Duplicate entry") {
|
||||
// Non-fatal - skip individual failures
|
||||
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interaction %s: %v\n", entry.ID, err)
|
||||
continue
|
||||
}
|
||||
imported++
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return imported, fmt.Errorf("scanner error: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return imported, fmt.Errorf("failed to commit interactions: %w", err)
|
||||
}
|
||||
|
||||
return imported, nil
|
||||
}
|
||||
|
||||
@@ -347,3 +347,156 @@ func TestDetectPrefixFromIssues(t *testing.T) {
|
||||
t.Errorf("expected prefix 'proj', got '%s'", prefix)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapWithRoutesAndInteractions(t *testing.T) {
|
||||
// Create temp directory structure
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
doltDir := filepath.Join(beadsDir, "dolt")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatalf("failed to create beads dir: %v", err)
|
||||
}
|
||||
|
||||
// Create test issues JSONL
|
||||
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
|
||||
issue := types.Issue{
|
||||
ID: "test-001",
|
||||
Title: "Test issue",
|
||||
Status: types.StatusOpen,
|
||||
}
|
||||
data, _ := json.Marshal(issue)
|
||||
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
|
||||
t.Fatalf("failed to write issues JSONL: %v", err)
|
||||
}
|
||||
|
||||
// Create test routes JSONL
|
||||
routesPath := filepath.Join(beadsDir, "routes.jsonl")
|
||||
routesContent := `{"prefix":"test-","path":"."}
|
||||
{"prefix":"other-","path":"other/rig"}
|
||||
`
|
||||
if err := os.WriteFile(routesPath, []byte(routesContent), 0644); err != nil {
|
||||
t.Fatalf("failed to write routes JSONL: %v", err)
|
||||
}
|
||||
|
||||
// Create test interactions JSONL
|
||||
interactionsPath := filepath.Join(beadsDir, "interactions.jsonl")
|
||||
interactionsContent := `{"id":"int-001","kind":"llm_call","created_at":"2025-01-20T10:00:00Z","actor":"test-agent","model":"claude-3"}
|
||||
{"id":"int-002","kind":"tool_call","created_at":"2025-01-20T10:01:00Z","actor":"test-agent","tool_name":"bash","exit_code":0}
|
||||
`
|
||||
if err := os.WriteFile(interactionsPath, []byte(interactionsContent), 0644); err != nil {
|
||||
t.Fatalf("failed to write interactions JSONL: %v", err)
|
||||
}
|
||||
|
||||
// Perform bootstrap
|
||||
ctx := context.Background()
|
||||
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
|
||||
BeadsDir: beadsDir,
|
||||
DoltPath: doltDir,
|
||||
LockTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("bootstrap failed: %v", err)
|
||||
}
|
||||
if !bootstrapped {
|
||||
t.Fatal("expected bootstrap to be performed")
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("expected non-nil result")
|
||||
}
|
||||
|
||||
// Verify issues imported
|
||||
if result.IssuesImported != 1 {
|
||||
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
|
||||
}
|
||||
|
||||
// Verify routes imported
|
||||
if result.RoutesImported != 2 {
|
||||
t.Errorf("expected 2 routes imported, got %d", result.RoutesImported)
|
||||
}
|
||||
|
||||
// Verify interactions imported
|
||||
if result.InteractionsImported != 2 {
|
||||
t.Errorf("expected 2 interactions imported, got %d", result.InteractionsImported)
|
||||
}
|
||||
|
||||
// Open store and verify data
|
||||
store, err := New(ctx, &Config{Path: doltDir})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Verify routes table
|
||||
var routeCount int
|
||||
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM routes").Scan(&routeCount)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count routes: %v", err)
|
||||
}
|
||||
if routeCount != 2 {
|
||||
t.Errorf("expected 2 routes in table, got %d", routeCount)
|
||||
}
|
||||
|
||||
// Verify interactions table
|
||||
var interactionCount int
|
||||
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM interactions").Scan(&interactionCount)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count interactions: %v", err)
|
||||
}
|
||||
if interactionCount != 2 {
|
||||
t.Errorf("expected 2 interactions in table, got %d", interactionCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapWithoutOptionalFiles(t *testing.T) {
|
||||
// Test that bootstrap succeeds when routes.jsonl and interactions.jsonl don't exist
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
doltDir := filepath.Join(beadsDir, "dolt")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatalf("failed to create beads dir: %v", err)
|
||||
}
|
||||
|
||||
// Create only issues JSONL
|
||||
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
|
||||
issue := types.Issue{
|
||||
ID: "test-001",
|
||||
Title: "Test issue",
|
||||
Status: types.StatusOpen,
|
||||
}
|
||||
data, _ := json.Marshal(issue)
|
||||
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
|
||||
t.Fatalf("failed to write issues JSONL: %v", err)
|
||||
}
|
||||
|
||||
// Perform bootstrap - should succeed even without routes.jsonl and interactions.jsonl
|
||||
ctx := context.Background()
|
||||
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
|
||||
BeadsDir: beadsDir,
|
||||
DoltPath: doltDir,
|
||||
LockTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("bootstrap failed: %v", err)
|
||||
}
|
||||
if !bootstrapped {
|
||||
t.Fatal("expected bootstrap to be performed")
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("expected non-nil result")
|
||||
}
|
||||
|
||||
// Verify issues imported
|
||||
if result.IssuesImported != 1 {
|
||||
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
|
||||
}
|
||||
|
||||
// Routes and interactions should be 0 (files don't exist)
|
||||
if result.RoutesImported != 0 {
|
||||
t.Errorf("expected 0 routes imported, got %d", result.RoutesImported)
|
||||
}
|
||||
if result.InteractionsImported != 0 {
|
||||
t.Errorf("expected 0 interactions imported, got %d", result.InteractionsImported)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,6 +203,37 @@ CREATE TABLE IF NOT EXISTS repo_mtimes (
|
||||
last_checked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
INDEX idx_repo_mtimes_checked (last_checked)
|
||||
);
|
||||
|
||||
-- Routes table (prefix-to-path routing configuration)
|
||||
CREATE TABLE IF NOT EXISTS routes (
|
||||
prefix VARCHAR(32) PRIMARY KEY,
|
||||
path VARCHAR(512) NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Interactions table (agent audit log)
|
||||
CREATE TABLE IF NOT EXISTS interactions (
|
||||
id VARCHAR(32) PRIMARY KEY,
|
||||
kind VARCHAR(64) NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
actor VARCHAR(255),
|
||||
issue_id VARCHAR(255),
|
||||
model VARCHAR(255),
|
||||
prompt TEXT,
|
||||
response TEXT,
|
||||
error TEXT,
|
||||
tool_name VARCHAR(255),
|
||||
exit_code INT,
|
||||
parent_id VARCHAR(32),
|
||||
label VARCHAR(64),
|
||||
reason TEXT,
|
||||
extra JSON,
|
||||
INDEX idx_interactions_kind (kind),
|
||||
INDEX idx_interactions_created_at (created_at),
|
||||
INDEX idx_interactions_issue_id (issue_id),
|
||||
INDEX idx_interactions_parent_id (parent_id)
|
||||
);
|
||||
`
|
||||
|
||||
// defaultConfig contains the default configuration values
|
||||
|
||||
Reference in New Issue
Block a user