feat(storage): add --backend flag for Dolt backend selection

Phase 2 of Dolt integration - enables runtime backend selection:

- Add --backend flag to bd init (sqlite|dolt)
- Create storage factory for backend instantiation
- Update daemon and main.go to use factory with config detection
- Update database discovery to find Dolt backends via metadata.json
- Fix Dolt schema init to split statements for MySQL compatibility
- Add ReadOnly mode to skip schema init for read-only commands

Usage: bd init --backend dolt --prefix myproject

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
mayor
2026-01-14 21:42:31 -08:00
committed by gastown/crew/dennis
parent e861a667fc
commit 669ea40684
8 changed files with 1939 additions and 1299 deletions

File diff suppressed because one or more lines are too long

View File

@@ -16,6 +16,7 @@ import (
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/daemon"
"github.com/steveyegge/beads/internal/rpc"
"github.com/steveyegge/beads/internal/storage/factory"
"github.com/steveyegge/beads/internal/storage/sqlite"
"github.com/steveyegge/beads/internal/syncbranch"
)
@@ -403,17 +404,22 @@ func runDaemonLoop(interval time.Duration, autoCommit, autoPush, autoPull, local
log.Warn("could not remove daemon-error file", "error", err)
}
store, err := sqlite.New(ctx, daemonDBPath)
store, err := factory.NewFromConfig(ctx, beadsDir)
if err != nil {
log.Error("cannot open database", "error", err)
return // Use return instead of os.Exit to allow defers to run
}
defer func() { _ = store.Close() }()
// Enable freshness checking to detect external database file modifications
// Enable freshness checking for SQLite backend to detect external database file modifications
// (e.g., when git merge replaces the database file)
store.EnableFreshnessChecking()
log.Info("database opened", "path", daemonDBPath, "freshness_checking", true)
// Dolt doesn't need this since it handles versioning natively.
if sqliteStore, ok := store.(*sqlite.SQLiteStorage); ok {
sqliteStore.EnableFreshnessChecking()
log.Info("database opened", "path", store.Path(), "backend", "sqlite", "freshness_checking", true)
} else {
log.Info("database opened", "path", store.Path(), "backend", "dolt")
}
// Auto-upgrade .beads/.gitignore if outdated
gitignoreCheck := doctor.CheckGitignore()
@@ -426,14 +432,16 @@ func runDaemonLoop(interval time.Duration, autoCommit, autoPush, autoPull, local
}
}
// Hydrate from multi-repo if configured
if results, err := store.HydrateFromMultiRepo(ctx); err != nil {
log.Error("multi-repo hydration failed", "error", err)
return // Use return instead of os.Exit to allow defers to run
} else if results != nil {
log.Info("multi-repo hydration complete")
for repo, count := range results {
log.Info("hydrated issues", "repo", repo, "count", count)
// Hydrate from multi-repo if configured (SQLite only)
if sqliteStore, ok := store.(*sqlite.SQLiteStorage); ok {
if results, err := sqliteStore.HydrateFromMultiRepo(ctx); err != nil {
log.Error("multi-repo hydration failed", "error", err)
return // Use return instead of os.Exit to allow defers to run
} else if results != nil {
log.Info("multi-repo hydration complete")
for repo, count := range results {
log.Info("hydrated issues", "repo", repo, "count", count)
}
}
}
@@ -618,13 +626,13 @@ func runDaemonLoop(interval time.Duration, autoCommit, autoPush, autoPull, local
// Note: The individual auto-commit/auto-push settings are deprecated.
// Use auto-sync for read/write mode, auto-pull for read-only mode.
func loadDaemonAutoSettings(cmd *cobra.Command, autoCommit, autoPush, autoPull bool) (bool, bool, bool) {
dbPath := beads.FindDatabasePath()
if dbPath == "" {
beadsDir := beads.FindBeadsDir()
if beadsDir == "" {
return autoCommit, autoPush, autoPull
}
ctx := context.Background()
store, err := sqlite.New(ctx, dbPath)
store, err := factory.NewFromConfig(ctx, beadsDir)
if err != nil {
return autoCommit, autoPush, autoPull
}

View File

@@ -15,6 +15,8 @@ import (
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/git"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/storage/dolt"
"github.com/steveyegge/beads/internal/storage/sqlite"
"github.com/steveyegge/beads/internal/syncbranch"
"github.com/steveyegge/beads/internal/types"
@@ -43,6 +45,7 @@ With --stealth: configures per-repository git settings for invisible beads usage
prefix, _ := cmd.Flags().GetString("prefix")
quiet, _ := cmd.Flags().GetBool("quiet")
branch, _ := cmd.Flags().GetString("branch")
backend, _ := cmd.Flags().GetString("backend")
contributor, _ := cmd.Flags().GetBool("contributor")
team, _ := cmd.Flags().GetBool("team")
stealth, _ := cmd.Flags().GetBool("stealth")
@@ -51,6 +54,15 @@ With --stealth: configures per-repository git settings for invisible beads usage
force, _ := cmd.Flags().GetBool("force")
fromJSONL, _ := cmd.Flags().GetBool("from-jsonl")
// Validate backend flag
if backend != "" && backend != configfile.BackendSQLite && backend != configfile.BackendDolt {
fmt.Fprintf(os.Stderr, "Error: invalid backend '%s' (must be 'sqlite' or 'dolt')\n", backend)
os.Exit(1)
}
if backend == "" {
backend = configfile.BackendSQLite // Default to SQLite
}
// Initialize config (PersistentPreRun doesn't run for init command)
if err := config.Initialize(); err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to initialize config: %v\n", err)
@@ -280,9 +292,20 @@ With --stealth: configures per-repository git settings for invisible beads usage
}
ctx := rootCtx
store, err := sqlite.New(ctx, initDBPath)
// Create storage backend based on --backend flag
var storagePath string
var store storage.Storage
if backend == configfile.BackendDolt {
// Dolt uses a directory, not a file
storagePath = filepath.Join(beadsDir, "dolt")
store, err = dolt.New(ctx, &dolt.Config{Path: storagePath})
} else {
storagePath = initDBPath
store, err = sqlite.New(ctx, storagePath)
}
if err != nil {
fmt.Fprintf(os.Stderr, "Error: failed to create database: %v\n", err)
fmt.Fprintf(os.Stderr, "Error: failed to create %s database: %v\n", backend, err)
os.Exit(1)
}
@@ -361,6 +384,12 @@ With --stealth: configures per-repository git settings for invisible beads usage
}
}
}
// Save backend choice (only store if non-default to keep metadata.json clean)
if backend != configfile.BackendSQLite {
cfg.Backend = backend
}
if err := cfg.Save(beadsDir); err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to create metadata.json: %v\n", err)
// Non-fatal - continue anyway
@@ -508,7 +537,8 @@ With --stealth: configures per-repository git settings for invisible beads usage
}
fmt.Printf("\n%s bd initialized successfully!\n\n", ui.RenderPass("✓"))
fmt.Printf(" Database: %s\n", ui.RenderAccent(initDBPath))
fmt.Printf(" Backend: %s\n", ui.RenderAccent(backend))
fmt.Printf(" Database: %s\n", ui.RenderAccent(storagePath))
fmt.Printf(" Issue prefix: %s\n", ui.RenderAccent(prefix))
fmt.Printf(" Issues will be named: %s\n\n", ui.RenderAccent(prefix+"-<hash> (e.g., "+prefix+"-a3f2dd)"))
fmt.Printf("Run %s to get started.\n\n", ui.RenderAccent("bd quickstart"))
@@ -540,6 +570,7 @@ func init() {
initCmd.Flags().StringP("prefix", "p", "", "Issue prefix (default: current directory name)")
initCmd.Flags().BoolP("quiet", "q", false, "Suppress output (quiet mode)")
initCmd.Flags().StringP("branch", "b", "", "Git branch for beads commits (default: current branch)")
initCmd.Flags().String("backend", "", "Storage backend: sqlite (default) or dolt (version-controlled)")
initCmd.Flags().Bool("contributor", false, "Run OSS contributor setup wizard")
initCmd.Flags().Bool("team", false, "Run team workflow setup wizard")
initCmd.Flags().Bool("stealth", false, "Enable stealth mode: global gitattributes and gitignore, no local repo tracking")

View File

@@ -18,13 +18,14 @@ import (
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/debug"
"github.com/steveyegge/beads/internal/hooks"
"github.com/steveyegge/beads/internal/molecules"
"github.com/steveyegge/beads/internal/rpc"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/storage/factory"
"github.com/steveyegge/beads/internal/storage/memory"
"github.com/steveyegge/beads/internal/storage/sqlite"
"github.com/steveyegge/beads/internal/utils"
)
@@ -753,22 +754,36 @@ var rootCmd = &cobra.Command{
// Fall back to direct storage access
var err error
var needsBootstrap bool // Track if DB needs initial import (GH#b09)
if useReadOnly {
// Read-only mode: prevents file modifications (GH#804)
store, err = sqlite.NewReadOnlyWithTimeout(rootCtx, dbPath, lockTimeout)
if err != nil {
beadsDir := filepath.Dir(dbPath)
// Detect backend from metadata.json
backend := factory.GetBackendFromConfig(beadsDir)
// Create storage with appropriate options
opts := factory.Options{
ReadOnly: useReadOnly,
LockTimeout: lockTimeout,
}
if backend == configfile.BackendDolt {
// For Dolt, use the dolt subdirectory
doltPath := filepath.Join(beadsDir, "dolt")
store, err = factory.NewWithOptions(rootCtx, backend, doltPath, opts)
} else {
// SQLite backend
store, err = factory.NewWithOptions(rootCtx, backend, dbPath, opts)
if err != nil && useReadOnly {
// If read-only fails (e.g., DB doesn't exist), fall back to read-write
// This handles the case where user runs "bd list" before "bd init"
debug.Logf("read-only open failed, falling back to read-write: %v", err)
store, err = sqlite.NewWithTimeout(rootCtx, dbPath, lockTimeout)
opts.ReadOnly = false
store, err = factory.NewWithOptions(rootCtx, backend, dbPath, opts)
needsBootstrap = true // New DB needs auto-import (GH#b09)
}
} else {
store, err = sqlite.NewWithTimeout(rootCtx, dbPath, lockTimeout)
}
if err != nil {
// Check for fresh clone scenario
beadsDir := filepath.Dir(dbPath)
if handleFreshCloneError(err, beadsDir) {
os.Exit(1)
}

View File

@@ -215,7 +215,9 @@ func findLocalBeadsDir() string {
// findDatabaseInBeadsDir searches for a database file within a .beads directory.
// It implements the standard search order:
// 1. Check config.json first (single source of truth)
// 1. Check metadata.json first (single source of truth)
// - For SQLite backend: returns path to .db file
// - For Dolt backend: returns path to dolt/ directory
// 2. Fall back to canonical beads.db
// 3. Search for *.db files, filtering out backups and vc.db
//
@@ -225,11 +227,21 @@ func findLocalBeadsDir() string {
//
// Returns empty string if no database is found.
func findDatabaseInBeadsDir(beadsDir string, warnOnIssues bool) string {
// Check for config.json first (single source of truth)
// Check for metadata.json first (single source of truth)
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
dbPath := cfg.DatabasePath(beadsDir)
if _, err := os.Stat(dbPath); err == nil {
return dbPath
backend := cfg.GetBackend()
if backend == configfile.BackendDolt {
// For Dolt, check if the dolt directory exists
doltPath := filepath.Join(beadsDir, "dolt")
if info, err := os.Stat(doltPath); err == nil && info.IsDir() {
return doltPath
}
} else {
// For SQLite, check if the .db file exists
dbPath := cfg.DatabasePath(beadsDir)
if _, err := os.Stat(dbPath); err == nil {
return dbPath
}
}
}

View File

@@ -12,6 +12,7 @@ const ConfigFileName = "metadata.json"
type Config struct {
Database string `json:"database"`
JSONLExport string `json:"jsonl_export,omitempty"`
Backend string `json:"backend,omitempty"` // "sqlite" (default) or "dolt"
// Deletions configuration
DeletionsRetentionDays int `json:"deletions_retention_days,omitempty"` // 0 means use default (3 days)
@@ -113,3 +114,17 @@ func (c *Config) GetDeletionsRetentionDays() int {
}
return c.DeletionsRetentionDays
}
// Backend constants
const (
BackendSQLite = "sqlite"
BackendDolt = "dolt"
)
// GetBackend returns the configured backend type, defaulting to SQLite.
func (c *Config) GetBackend() string {
if c.Backend == "" {
return BackendSQLite
}
return c.Backend
}

View File

@@ -21,6 +21,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
@@ -52,6 +53,7 @@ type Config struct {
CommitterEmail string // Git-style committer email
Remote string // Default remote name (e.g., "origin")
Database string // Database name within Dolt (default: "beads")
ReadOnly bool // Open in read-only mode (skip schema init)
}
// New creates a new Dolt storage backend
@@ -85,8 +87,25 @@ func New(ctx context.Context, cfg *Config) (*DoltStore, error) {
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
// Build Dolt connection string
// Format: file:///path/to/db?commitname=Name&commitemail=email&database=dbname
// First, connect without specifying a database to create it if needed
initConnStr := fmt.Sprintf(
"file://%s?commitname=%s&commitemail=%s",
cfg.Path, cfg.CommitterName, cfg.CommitterEmail)
initDB, err := sql.Open("dolt", initConnStr)
if err != nil {
return nil, fmt.Errorf("failed to open Dolt for initialization: %w", err)
}
// Create the database if it doesn't exist
_, err = initDB.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", cfg.Database))
if err != nil {
initDB.Close()
return nil, fmt.Errorf("failed to create database: %w", err)
}
initDB.Close()
// Now connect with the database specified
connStr := fmt.Sprintf(
"file://%s?commitname=%s&commitemail=%s&database=%s",
cfg.Path, cfg.CommitterName, cfg.CommitterEmail, cfg.Database)
@@ -121,11 +140,14 @@ func New(ctx context.Context, cfg *Config) (*DoltStore, error) {
committerEmail: cfg.CommitterEmail,
remote: cfg.Remote,
branch: "main",
readOnly: cfg.ReadOnly,
}
// Initialize schema
if err := store.initSchema(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize schema: %w", err)
// Initialize schema (skip for read-only mode)
if !cfg.ReadOnly {
if err := store.initSchema(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize schema: %w", err)
}
}
return store, nil
@@ -133,14 +155,34 @@ func New(ctx context.Context, cfg *Config) (*DoltStore, error) {
// initSchema creates all tables if they don't exist
func (s *DoltStore) initSchema(ctx context.Context) error {
// Execute schema creation
if _, err := s.db.ExecContext(ctx, schema); err != nil {
return fmt.Errorf("failed to create schema: %w", err)
// Execute schema creation - split into individual statements
// because MySQL/Dolt doesn't support multiple statements in one Exec
for _, stmt := range splitStatements(schema) {
stmt = strings.TrimSpace(stmt)
if stmt == "" {
continue
}
// Skip pure comment-only statements, but execute statements that start with comments
if isOnlyComments(stmt) {
continue
}
if _, err := s.db.ExecContext(ctx, stmt); err != nil {
return fmt.Errorf("failed to create schema: %w\nStatement: %s", err, truncateForError(stmt))
}
}
// Insert default config values
if _, err := s.db.ExecContext(ctx, defaultConfig); err != nil {
return fmt.Errorf("failed to insert default config: %w", err)
for _, stmt := range splitStatements(defaultConfig) {
stmt = strings.TrimSpace(stmt)
if stmt == "" {
continue
}
if isOnlyComments(stmt) {
continue
}
if _, err := s.db.ExecContext(ctx, stmt); err != nil {
return fmt.Errorf("failed to insert default config: %w", err)
}
}
// Create views
@@ -154,6 +196,74 @@ func (s *DoltStore) initSchema(ctx context.Context) error {
return nil
}
// splitStatements splits a SQL script into individual statements
func splitStatements(script string) []string {
var statements []string
var current strings.Builder
inString := false
stringChar := byte(0)
for i := 0; i < len(script); i++ {
c := script[i]
if inString {
current.WriteByte(c)
if c == stringChar && (i == 0 || script[i-1] != '\\') {
inString = false
}
continue
}
if c == '\'' || c == '"' || c == '`' {
inString = true
stringChar = c
current.WriteByte(c)
continue
}
if c == ';' {
stmt := strings.TrimSpace(current.String())
if stmt != "" {
statements = append(statements, stmt)
}
current.Reset()
continue
}
current.WriteByte(c)
}
// Handle last statement without semicolon
stmt := strings.TrimSpace(current.String())
if stmt != "" {
statements = append(statements, stmt)
}
return statements
}
// truncateForError truncates a string for use in error messages
func truncateForError(s string) string {
if len(s) > 100 {
return s[:100] + "..."
}
return s
}
// isOnlyComments returns true if the statement contains only SQL comments
func isOnlyComments(stmt string) bool {
lines := strings.Split(stmt, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "--") {
continue
}
// Found a non-comment, non-empty line
return false
}
return true
}
// Close closes the database connection
func (s *DoltStore) Close() error {
s.closed.Store(true)

View File

@@ -0,0 +1,86 @@
// Package factory provides functions for creating storage backends based on configuration.
package factory
import (
"context"
"fmt"
"path/filepath"
"time"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/storage/dolt"
"github.com/steveyegge/beads/internal/storage/sqlite"
)
// Options configures how the storage backend is opened
type Options struct {
ReadOnly bool
LockTimeout time.Duration
}
// New creates a storage backend based on the backend type.
// For SQLite, path should be the full path to the .db file.
// For Dolt, path should be the directory containing the Dolt database.
func New(ctx context.Context, backend, path string) (storage.Storage, error) {
return NewWithOptions(ctx, backend, path, Options{})
}
// NewWithOptions creates a storage backend with the specified options.
func NewWithOptions(ctx context.Context, backend, path string, opts Options) (storage.Storage, error) {
switch backend {
case configfile.BackendSQLite, "":
if opts.ReadOnly {
if opts.LockTimeout > 0 {
return sqlite.NewReadOnlyWithTimeout(ctx, path, opts.LockTimeout)
}
return sqlite.NewReadOnly(ctx, path)
}
if opts.LockTimeout > 0 {
return sqlite.NewWithTimeout(ctx, path, opts.LockTimeout)
}
return sqlite.New(ctx, path)
case configfile.BackendDolt:
return dolt.New(ctx, &dolt.Config{Path: path, ReadOnly: opts.ReadOnly})
default:
return nil, fmt.Errorf("unknown storage backend: %s (supported: sqlite, dolt)", backend)
}
}
// NewFromConfig creates a storage backend based on the metadata.json configuration.
// beadsDir is the path to the .beads directory.
func NewFromConfig(ctx context.Context, beadsDir string) (storage.Storage, error) {
return NewFromConfigWithOptions(ctx, beadsDir, Options{})
}
// NewFromConfigWithOptions creates a storage backend with options from metadata.json.
func NewFromConfigWithOptions(ctx context.Context, beadsDir string, opts Options) (storage.Storage, error) {
cfg, err := configfile.Load(beadsDir)
if err != nil {
return nil, fmt.Errorf("loading config: %w", err)
}
if cfg == nil {
cfg = configfile.DefaultConfig()
}
backend := cfg.GetBackend()
switch backend {
case configfile.BackendSQLite:
return NewWithOptions(ctx, backend, cfg.DatabasePath(beadsDir), opts)
case configfile.BackendDolt:
// For Dolt, use a subdirectory to store the Dolt database
doltPath := filepath.Join(beadsDir, "dolt")
return NewWithOptions(ctx, backend, doltPath, opts)
default:
return nil, fmt.Errorf("unknown storage backend in config: %s", backend)
}
}
// GetBackendFromConfig returns the backend type from metadata.json
func GetBackendFromConfig(beadsDir string) string {
cfg, err := configfile.Load(beadsDir)
if err != nil || cfg == nil {
return configfile.BackendSQLite
}
return cfg.GetBackend()
}