feat(dolt): bootstrap routes.jsonl and interactions.jsonl (bd-lbdy7)
Add support for importing routes.jsonl and interactions.jsonl during Dolt bootstrap. Previously only issues.jsonl was imported. Changes: - Add routes and interactions tables to Dolt schema - Import routes before issues (no dependencies) - Import interactions after issues (may reference issue_id) - Reuse audit.Entry type instead of duplicating - Add tests for multi-file bootstrap Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
committed by
Steve Yegge
parent
938a17cda5
commit
90344b9939
@@ -12,17 +12,21 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/steveyegge/beads/internal/audit"
|
||||||
"github.com/steveyegge/beads/internal/lockfile"
|
"github.com/steveyegge/beads/internal/lockfile"
|
||||||
|
"github.com/steveyegge/beads/internal/routing"
|
||||||
"github.com/steveyegge/beads/internal/types"
|
"github.com/steveyegge/beads/internal/types"
|
||||||
"github.com/steveyegge/beads/internal/utils"
|
"github.com/steveyegge/beads/internal/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BootstrapResult contains statistics about the bootstrap operation
|
// BootstrapResult contains statistics about the bootstrap operation
|
||||||
type BootstrapResult struct {
|
type BootstrapResult struct {
|
||||||
IssuesImported int
|
IssuesImported int
|
||||||
IssuesSkipped int
|
IssuesSkipped int
|
||||||
ParseErrors []ParseError
|
RoutesImported int
|
||||||
PrefixDetected string
|
InteractionsImported int
|
||||||
|
ParseErrors []ParseError
|
||||||
|
PrefixDetected string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseError describes a JSONL parsing error
|
// ParseError describes a JSONL parsing error
|
||||||
@@ -174,11 +178,12 @@ func releaseBootstrapLock(f *os.File, lockPath string) {
|
|||||||
_ = os.Remove(lockPath)
|
_ = os.Remove(lockPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// performBootstrap performs the actual bootstrap from JSONL
|
// performBootstrap performs the actual bootstrap from JSONL files.
|
||||||
|
// Import order: routes -> issues -> interactions (dependencies require issues to exist)
|
||||||
func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string) (*BootstrapResult, error) {
|
func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string) (*BootstrapResult, error) {
|
||||||
result := &BootstrapResult{}
|
result := &BootstrapResult{}
|
||||||
|
|
||||||
// Parse JSONL with graceful error handling
|
// Parse issues JSONL with graceful error handling
|
||||||
issues, parseErrors := parseJSONLWithErrors(jsonlPath)
|
issues, parseErrors := parseJSONLWithErrors(jsonlPath)
|
||||||
result.ParseErrors = parseErrors
|
result.ParseErrors = parseErrors
|
||||||
|
|
||||||
@@ -218,6 +223,14 @@ func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Import routes first (no dependencies)
|
||||||
|
routesImported, err := importRoutesBootstrap(ctx, store, cfg.BeadsDir)
|
||||||
|
if err != nil {
|
||||||
|
// Non-fatal - routes.jsonl may not exist
|
||||||
|
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import routes: %v\n", err)
|
||||||
|
}
|
||||||
|
result.RoutesImported = routesImported
|
||||||
|
|
||||||
// Import issues in a transaction
|
// Import issues in a transaction
|
||||||
imported, skipped, err := importIssuesBootstrap(ctx, store, issues)
|
imported, skipped, err := importIssuesBootstrap(ctx, store, issues)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -227,6 +240,15 @@ func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string
|
|||||||
result.IssuesImported = imported
|
result.IssuesImported = imported
|
||||||
result.IssuesSkipped = skipped
|
result.IssuesSkipped = skipped
|
||||||
|
|
||||||
|
// Import interactions (after issues, since interactions may reference issue_id)
|
||||||
|
interactionsPath := filepath.Join(cfg.BeadsDir, "interactions.jsonl")
|
||||||
|
interactionsImported, err := importInteractionsBootstrap(ctx, store, interactionsPath)
|
||||||
|
if err != nil {
|
||||||
|
// Non-fatal - interactions.jsonl may not exist
|
||||||
|
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interactions: %v\n", err)
|
||||||
|
}
|
||||||
|
result.InteractionsImported = interactionsImported
|
||||||
|
|
||||||
// Commit the bootstrap
|
// Commit the bootstrap
|
||||||
if err := store.Commit(ctx, "Bootstrap from JSONL"); err != nil {
|
if err := store.Commit(ctx, "Bootstrap from JSONL"); err != nil {
|
||||||
// Non-fatal - data is still in the database
|
// Non-fatal - data is still in the database
|
||||||
@@ -432,3 +454,105 @@ func importIssuesBootstrap(ctx context.Context, store *DoltStore, issues []*type
|
|||||||
|
|
||||||
return imported, skipped, nil
|
return imported, skipped, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// importRoutesBootstrap imports routes from routes.jsonl during bootstrap
|
||||||
|
// Returns the number of routes imported
|
||||||
|
func importRoutesBootstrap(ctx context.Context, store *DoltStore, beadsDir string) (int, error) {
|
||||||
|
routes, err := routing.LoadRoutes(beadsDir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if len(routes) == 0 {
|
||||||
|
return 0, nil // No routes to import
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := store.db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
|
imported := 0
|
||||||
|
for _, route := range routes {
|
||||||
|
_, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT INTO routes (prefix, path, created_at)
|
||||||
|
VALUES (?, ?, ?)
|
||||||
|
ON DUPLICATE KEY UPDATE path = VALUES(path)
|
||||||
|
`, route.Prefix, route.Path, time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return imported, fmt.Errorf("failed to insert route %s: %w", route.Prefix, err)
|
||||||
|
}
|
||||||
|
imported++
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return imported, fmt.Errorf("failed to commit routes: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return imported, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// importInteractionsBootstrap imports interactions from interactions.jsonl during bootstrap
|
||||||
|
// Returns the number of interactions imported
|
||||||
|
func importInteractionsBootstrap(ctx context.Context, store *DoltStore, interactionsPath string) (int, error) {
|
||||||
|
// #nosec G304 - controlled path
|
||||||
|
f, err := os.Open(interactionsPath)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return 0, nil // No interactions file is not an error
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
|
||||||
|
tx, err := store.db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
|
imported := 0
|
||||||
|
scanner := bufio.NewScanner(f)
|
||||||
|
scanner.Buffer(make([]byte, 0, 1024), 2*1024*1024) // 2MB buffer for large lines
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if strings.TrimSpace(line) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var entry audit.Entry
|
||||||
|
if err := json.Unmarshal([]byte(line), &entry); err != nil {
|
||||||
|
// Skip malformed lines during bootstrap
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert extra map to JSON (default to empty object for valid JSON)
|
||||||
|
extraJSON := []byte("{}")
|
||||||
|
if entry.Extra != nil {
|
||||||
|
extraJSON, _ = json.Marshal(entry.Extra)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT INTO interactions (id, kind, created_at, actor, issue_id, model, prompt, response, error, tool_name, exit_code, parent_id, label, reason, extra)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON DUPLICATE KEY UPDATE kind = kind
|
||||||
|
`, entry.ID, entry.Kind, entry.CreatedAt, entry.Actor, entry.IssueID, entry.Model, entry.Prompt, entry.Response, entry.Error, entry.ToolName, entry.ExitCode, entry.ParentID, entry.Label, entry.Reason, extraJSON)
|
||||||
|
if err != nil && !strings.Contains(err.Error(), "Duplicate entry") {
|
||||||
|
// Non-fatal - skip individual failures
|
||||||
|
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interaction %s: %v\n", entry.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
imported++
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return imported, fmt.Errorf("scanner error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return imported, fmt.Errorf("failed to commit interactions: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return imported, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -347,3 +347,156 @@ func TestDetectPrefixFromIssues(t *testing.T) {
|
|||||||
t.Errorf("expected prefix 'proj', got '%s'", prefix)
|
t.Errorf("expected prefix 'proj', got '%s'", prefix)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBootstrapWithRoutesAndInteractions(t *testing.T) {
|
||||||
|
// Create temp directory structure
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||||
|
doltDir := filepath.Join(beadsDir, "dolt")
|
||||||
|
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||||
|
t.Fatalf("failed to create beads dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test issues JSONL
|
||||||
|
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
|
||||||
|
issue := types.Issue{
|
||||||
|
ID: "test-001",
|
||||||
|
Title: "Test issue",
|
||||||
|
Status: types.StatusOpen,
|
||||||
|
}
|
||||||
|
data, _ := json.Marshal(issue)
|
||||||
|
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write issues JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test routes JSONL
|
||||||
|
routesPath := filepath.Join(beadsDir, "routes.jsonl")
|
||||||
|
routesContent := `{"prefix":"test-","path":"."}
|
||||||
|
{"prefix":"other-","path":"other/rig"}
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(routesPath, []byte(routesContent), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write routes JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test interactions JSONL
|
||||||
|
interactionsPath := filepath.Join(beadsDir, "interactions.jsonl")
|
||||||
|
interactionsContent := `{"id":"int-001","kind":"llm_call","created_at":"2025-01-20T10:00:00Z","actor":"test-agent","model":"claude-3"}
|
||||||
|
{"id":"int-002","kind":"tool_call","created_at":"2025-01-20T10:01:00Z","actor":"test-agent","tool_name":"bash","exit_code":0}
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(interactionsPath, []byte(interactionsContent), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write interactions JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform bootstrap
|
||||||
|
ctx := context.Background()
|
||||||
|
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
|
||||||
|
BeadsDir: beadsDir,
|
||||||
|
DoltPath: doltDir,
|
||||||
|
LockTimeout: 10 * time.Second,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("bootstrap failed: %v", err)
|
||||||
|
}
|
||||||
|
if !bootstrapped {
|
||||||
|
t.Fatal("expected bootstrap to be performed")
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
t.Fatal("expected non-nil result")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify issues imported
|
||||||
|
if result.IssuesImported != 1 {
|
||||||
|
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify routes imported
|
||||||
|
if result.RoutesImported != 2 {
|
||||||
|
t.Errorf("expected 2 routes imported, got %d", result.RoutesImported)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify interactions imported
|
||||||
|
if result.InteractionsImported != 2 {
|
||||||
|
t.Errorf("expected 2 interactions imported, got %d", result.InteractionsImported)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open store and verify data
|
||||||
|
store, err := New(ctx, &Config{Path: doltDir})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to open store: %v", err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
|
||||||
|
// Verify routes table
|
||||||
|
var routeCount int
|
||||||
|
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM routes").Scan(&routeCount)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to count routes: %v", err)
|
||||||
|
}
|
||||||
|
if routeCount != 2 {
|
||||||
|
t.Errorf("expected 2 routes in table, got %d", routeCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify interactions table
|
||||||
|
var interactionCount int
|
||||||
|
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM interactions").Scan(&interactionCount)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to count interactions: %v", err)
|
||||||
|
}
|
||||||
|
if interactionCount != 2 {
|
||||||
|
t.Errorf("expected 2 interactions in table, got %d", interactionCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBootstrapWithoutOptionalFiles(t *testing.T) {
|
||||||
|
// Test that bootstrap succeeds when routes.jsonl and interactions.jsonl don't exist
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||||
|
doltDir := filepath.Join(beadsDir, "dolt")
|
||||||
|
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||||
|
t.Fatalf("failed to create beads dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create only issues JSONL
|
||||||
|
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
|
||||||
|
issue := types.Issue{
|
||||||
|
ID: "test-001",
|
||||||
|
Title: "Test issue",
|
||||||
|
Status: types.StatusOpen,
|
||||||
|
}
|
||||||
|
data, _ := json.Marshal(issue)
|
||||||
|
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write issues JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform bootstrap - should succeed even without routes.jsonl and interactions.jsonl
|
||||||
|
ctx := context.Background()
|
||||||
|
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
|
||||||
|
BeadsDir: beadsDir,
|
||||||
|
DoltPath: doltDir,
|
||||||
|
LockTimeout: 10 * time.Second,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("bootstrap failed: %v", err)
|
||||||
|
}
|
||||||
|
if !bootstrapped {
|
||||||
|
t.Fatal("expected bootstrap to be performed")
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
t.Fatal("expected non-nil result")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify issues imported
|
||||||
|
if result.IssuesImported != 1 {
|
||||||
|
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Routes and interactions should be 0 (files don't exist)
|
||||||
|
if result.RoutesImported != 0 {
|
||||||
|
t.Errorf("expected 0 routes imported, got %d", result.RoutesImported)
|
||||||
|
}
|
||||||
|
if result.InteractionsImported != 0 {
|
||||||
|
t.Errorf("expected 0 interactions imported, got %d", result.InteractionsImported)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -203,6 +203,37 @@ CREATE TABLE IF NOT EXISTS repo_mtimes (
|
|||||||
last_checked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
last_checked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
INDEX idx_repo_mtimes_checked (last_checked)
|
INDEX idx_repo_mtimes_checked (last_checked)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Routes table (prefix-to-path routing configuration)
|
||||||
|
CREATE TABLE IF NOT EXISTS routes (
|
||||||
|
prefix VARCHAR(32) PRIMARY KEY,
|
||||||
|
path VARCHAR(512) NOT NULL,
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Interactions table (agent audit log)
|
||||||
|
CREATE TABLE IF NOT EXISTS interactions (
|
||||||
|
id VARCHAR(32) PRIMARY KEY,
|
||||||
|
kind VARCHAR(64) NOT NULL,
|
||||||
|
created_at DATETIME NOT NULL,
|
||||||
|
actor VARCHAR(255),
|
||||||
|
issue_id VARCHAR(255),
|
||||||
|
model VARCHAR(255),
|
||||||
|
prompt TEXT,
|
||||||
|
response TEXT,
|
||||||
|
error TEXT,
|
||||||
|
tool_name VARCHAR(255),
|
||||||
|
exit_code INT,
|
||||||
|
parent_id VARCHAR(32),
|
||||||
|
label VARCHAR(64),
|
||||||
|
reason TEXT,
|
||||||
|
extra JSON,
|
||||||
|
INDEX idx_interactions_kind (kind),
|
||||||
|
INDEX idx_interactions_created_at (created_at),
|
||||||
|
INDEX idx_interactions_issue_id (issue_id),
|
||||||
|
INDEX idx_interactions_parent_id (parent_id)
|
||||||
|
);
|
||||||
`
|
`
|
||||||
|
|
||||||
// defaultConfig contains the default configuration values
|
// defaultConfig contains the default configuration values
|
||||||
|
|||||||
Reference in New Issue
Block a user