johno/beads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

doctor: add JSONL integrity check/fix and harden repairs

Browse Source 
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Jordan Hubbard
2025-12-26 08:18:25 -04:00
parent 1a4f06ef8c
commit 8166207eb4
10 changed files with 663 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
cmd/bd/doctor/database.go
View File

@@ -301,15 +301,30 @@ func CheckDatabaseIntegrity(path string) DoctorCheck {
// CheckDatabaseJSONLSync checks if database and JSONL are in sync
func CheckDatabaseJSONLSync(path string) DoctorCheck {
beadsDir := filepath.Join(path, ".beads")
dbPath := filepath.Join(beadsDir, beads.CanonicalDatabaseName)
// Find JSONL file
var jsonlPath string
for _, name := range []string{"issues.jsonl", "beads.jsonl"} {
testPath := filepath.Join(beadsDir, name)
if _, err := os.Stat(testPath); err == nil {
jsonlPath = testPath
break
// Resolve database path (respects metadata.json override).
dbPath := filepath.Join(beadsDir, beads.CanonicalDatabaseName)
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.Database != "" {
dbPath = cfg.DatabasePath(beadsDir)
}
// Find JSONL file (respects metadata.json override when set).
jsonlPath := ""
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
if cfg.JSONLExport != "" && !isSystemJSONLFilename(cfg.JSONLExport) {
p := cfg.JSONLPath(beadsDir)
if _, err := os.Stat(p); err == nil {
jsonlPath = p
}
}
}
if jsonlPath == "" {
for _, name := range []string{"issues.jsonl", "beads.jsonl"} {
testPath := filepath.Join(beadsDir, name)
if _, err := os.Stat(testPath); err == nil {
jsonlPath = testPath
break
}
}
}
@@ -392,11 +407,16 @@ func CheckDatabaseJSONLSync(path string) DoctorCheck {
// Use JSONL error if we got it earlier
if jsonlErr != nil {
fixMsg := "Run 'bd doctor --fix' to attempt recovery"
if strings.Contains(jsonlErr.Error(), "malformed") {
fixMsg = "Run 'bd doctor --fix' to back up and regenerate the JSONL from the database"
}
return DoctorCheck{
Name: "DB-JSONL Sync",
Status: StatusWarning,
Message: "Unable to read JSONL file",
Detail: jsonlErr.Error(),
Fix: fixMsg,
}
}

26
cmd/bd/doctor/fix/database_integrity.go
View File

@@ -28,6 +28,9 @@ func DatabaseIntegrity(path string) error {
beadsDir := filepath.Join(absPath, ".beads")
// Best-effort: stop any running daemon to reduce the chance of DB file locks.
_ = Daemon(absPath)
// Resolve database path (respects metadata.json database override).
var dbPath string
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.Database != "" {
@@ -39,9 +42,11 @@ func DatabaseIntegrity(path string) error {
// Find JSONL source of truth.
jsonlPath := ""
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
candidate := cfg.JSONLPath(beadsDir)
if _, err := os.Stat(candidate); err == nil {
jsonlPath = candidate
if cfg.JSONLExport != "" && !isSystemJSONLFilename(cfg.JSONLExport) {
candidate := cfg.JSONLPath(beadsDir)
if _, err := os.Stat(candidate); err == nil {
jsonlPath = candidate
}
}
}
if jsonlPath == "" {
@@ -61,7 +66,12 @@ func DatabaseIntegrity(path string) error {
ts := time.Now().UTC().Format("20060102T150405Z")
backupDB := dbPath + "." + ts + ".corrupt.backup.db"
if err := os.Rename(dbPath, backupDB); err != nil {
return fmt.Errorf("failed to back up database: %w", err)
// Retry once after attempting to kill daemons again (helps on platforms with strict file locks).
_ = Daemon(absPath)
if err2 := os.Rename(dbPath, backupDB); err2 != nil {
// Prefer the original error (more likely root cause).
return fmt.Errorf("failed to back up database: %w", err)
}
}
for _, suffix := range []string{"-wal", "-shm", "-journal"} {
sidecar := dbPath + suffix
@@ -84,7 +94,7 @@ func DatabaseIntegrity(path string) error {
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
// Best-effort rollback: attempt to restore the backup, preserving any partial init output.
// Best-effort rollback: attempt to restore the original DB, while preserving the backup.
failedTS := time.Now().UTC().Format("20060102T150405Z")
if _, statErr := os.Stat(dbPath); statErr == nil {
failedDB := dbPath + "." + failedTS + ".failed.init.db"
@@ -93,9 +103,11 @@ func DatabaseIntegrity(path string) error {
_ = os.Rename(dbPath+suffix, failedDB+suffix)
}
}
_ = os.Rename(backupDB, dbPath)
_ = copyFile(backupDB, dbPath)
for _, suffix := range []string{"-wal", "-shm", "-journal"} {
_ = os.Rename(backupDB+suffix, dbPath+suffix)
if _, statErr := os.Stat(backupDB + suffix); statErr == nil {
_ = copyFile(backupDB+suffix, dbPath+suffix)
}
}
return fmt.Errorf("failed to rebuild database from JSONL: %w (backup: %s)", err, backupDB)
}

105
cmd/bd/doctor/fix/jsonl_integrity.go Normal file
View File

@@ -0,0 +1,105 @@
package fix
import (
"fmt"
"io"
"os"
"path/filepath"
"time"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/utils"
)
// JSONLIntegrity backs up a malformed JSONL export and regenerates it from the database.
// This is safe only when a database exists and is readable.
func JSONLIntegrity(path string) error {
if err := validateBeadsWorkspace(path); err != nil {
return err
}
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("failed to resolve path: %w", err)
}
beadsDir := filepath.Join(absPath, ".beads")
// Resolve db path.
dbPath := filepath.Join(beadsDir, beads.CanonicalDatabaseName)
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.Database != "" {
dbPath = cfg.DatabasePath(beadsDir)
}
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
return fmt.Errorf("cannot auto-repair JSONL: no database found")
}
// Resolve JSONL export path.
jsonlPath := ""
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
if cfg.JSONLExport != "" && !isSystemJSONLFilename(cfg.JSONLExport) {
p := cfg.JSONLPath(beadsDir)
if _, err := os.Stat(p); err == nil {
jsonlPath = p
}
}
}
if jsonlPath == "" {
p := utils.FindJSONLInDir(beadsDir)
if _, err := os.Stat(p); err == nil {
jsonlPath = p
}
}
if jsonlPath == "" {
return fmt.Errorf("cannot auto-repair JSONL: no JSONL file found")
}
// Back up the JSONL.
ts := time.Now().UTC().Format("20060102T150405Z")
backup := jsonlPath + "." + ts + ".corrupt.backup.jsonl"
if err := os.Rename(jsonlPath, backup); err != nil {
return fmt.Errorf("failed to back up JSONL: %w", err)
}
binary, err := getBdBinary()
if err != nil {
_ = os.Rename(backup, jsonlPath)
return err
}
// Re-export from DB.
cmd := newBdCmd(binary, "--db", dbPath, "export", "-o", jsonlPath, "--force")
cmd.Dir = absPath
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
// Best-effort rollback: restore the original JSONL, but keep the backup.
failedTS := time.Now().UTC().Format("20060102T150405Z")
if _, statErr := os.Stat(jsonlPath); statErr == nil {
failed := jsonlPath + "." + failedTS + ".failed.regen.jsonl"
_ = os.Rename(jsonlPath, failed)
}
_ = copyFile(backup, jsonlPath)
return fmt.Errorf("failed to regenerate JSONL from database: %w (backup: %s)", err, backup)
}
return nil
}
func copyFile(src, dst string) error {
in, err := os.Open(src) // #nosec G304 -- src is within the workspace
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer func() { _ = out.Close() }()
if _, err := io.Copy(out, in); err != nil {
return err
}
return out.Close()
}

12
cmd/bd/doctor/fix/migrate.go
View File

@@ -4,6 +4,9 @@ import (
"fmt"
"os"
"path/filepath"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/configfile"
)
// DatabaseVersion fixes database version mismatches by running bd migrate,
@@ -22,12 +25,15 @@ func DatabaseVersion(path string) error {
// Check if database exists - if not, run init instead of migrate (bd-4h9)
beadsDir := filepath.Join(path, ".beads")
dbPath := filepath.Join(beadsDir, "beads.db")
dbPath := filepath.Join(beadsDir, beads.CanonicalDatabaseName)
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.Database != "" {
dbPath = cfg.DatabasePath(beadsDir)
}
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
// No database - this is a fresh clone, run bd init
fmt.Println("→ No database found, running 'bd init' to hydrate from JSONL...")
cmd := newBdCmd(bdBinary, "init")
cmd := newBdCmd(bdBinary, "--db", dbPath, "init")
cmd.Dir = path
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
@@ -40,7 +46,7 @@ func DatabaseVersion(path string) error {
}
// Database exists - run bd migrate
cmd := newBdCmd(bdBinary, "migrate")
cmd := newBdCmd(bdBinary, "--db", dbPath, "migrate")
cmd.Dir = path // Set working directory without changing process dir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr

30
cmd/bd/doctor/fix/sync.go
View File

@@ -37,13 +37,23 @@ func DBJSONLSync(path string) error {
// Find JSONL file
var jsonlPath string
issuesJSONL := filepath.Join(beadsDir, "issues.jsonl")
beadsJSONL := filepath.Join(beadsDir, "beads.jsonl")
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
if cfg.JSONLExport != "" && !isSystemJSONLFilename(cfg.JSONLExport) {
p := cfg.JSONLPath(beadsDir)
if _, err := os.Stat(p); err == nil {
jsonlPath = p
}
}
}
if jsonlPath == "" {
issuesJSONL := filepath.Join(beadsDir, "issues.jsonl")
beadsJSONL := filepath.Join(beadsDir, "beads.jsonl")
if _, err := os.Stat(issuesJSONL); err == nil {
jsonlPath = issuesJSONL
} else if _, err := os.Stat(beadsJSONL); err == nil {
jsonlPath = beadsJSONL
if _, err := os.Stat(issuesJSONL); err == nil {
jsonlPath = issuesJSONL
} else if _, err := os.Stat(beadsJSONL); err == nil {
jsonlPath = beadsJSONL
}
}
// Check if both database and JSONL exist
@@ -103,8 +113,8 @@ func DBJSONLSync(path string) error {
if syncDirection == "export" {
// Export DB to JSONL file (must specify -o to write to file, not stdout)
jsonlOutputPath := filepath.Join(beadsDir, "issues.jsonl")
exportCmd := newBdCmd(bdBinary, "export", "-o", jsonlOutputPath, "--force")
jsonlOutputPath := jsonlPath
exportCmd := newBdCmd(bdBinary, "--db", dbPath, "export", "-o", jsonlOutputPath, "--force")
exportCmd.Dir = path // Set working directory without changing process dir
exportCmd.Stdout = os.Stdout
exportCmd.Stderr = os.Stderr
@@ -114,7 +124,7 @@ func DBJSONLSync(path string) error {
// Staleness check uses last_import_time. After exporting, JSONL mtime is newer,
// so mark the DB as fresh by running a no-op import (skip existing issues).
markFreshCmd := newBdCmd(bdBinary, "import", "-i", jsonlOutputPath, "--force", "--skip-existing", "--no-git-history")
markFreshCmd := newBdCmd(bdBinary, "--db", dbPath, "import", "-i", jsonlOutputPath, "--force", "--skip-existing", "--no-git-history")
markFreshCmd.Dir = path
markFreshCmd.Stdout = os.Stdout
markFreshCmd.Stderr = os.Stderr
@@ -125,7 +135,7 @@ func DBJSONLSync(path string) error {
return nil
}
importCmd := newBdCmd(bdBinary, "sync", "--import-only")
importCmd := newBdCmd(bdBinary, "--db", dbPath, "sync", "--import-only")
importCmd.Dir = path // Set working directory without changing process dir
importCmd.Stdout = os.Stdout
importCmd.Stderr = os.Stderr

123
cmd/bd/doctor/jsonl_integrity.go Normal file
View File

@@ -0,0 +1,123 @@
package doctor
import (
"bufio"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/utils"
)
func CheckJSONLIntegrity(path string) DoctorCheck {
beadsDir := filepath.Join(path, ".beads")
// Resolve JSONL path.
jsonlPath := ""
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil {
if cfg.JSONLExport != "" && !isSystemJSONLFilename(cfg.JSONLExport) {
p := cfg.JSONLPath(beadsDir)
if _, err := os.Stat(p); err == nil {
jsonlPath = p
}
}
}
if jsonlPath == "" {
// Fall back to a best-effort discovery within .beads/.
p := utils.FindJSONLInDir(beadsDir)
if _, err := os.Stat(p); err == nil {
jsonlPath = p
}
}
if jsonlPath == "" {
return DoctorCheck{Name: "JSONL Integrity", Status: StatusOK, Message: "N/A (no JSONL file)"}
}
// Best-effort scan for malformed lines.
f, err := os.Open(jsonlPath) // #nosec G304 -- jsonlPath is within the workspace
if err != nil {
return DoctorCheck{
Name: "JSONL Integrity",
Status: StatusWarning,
Message: "Unable to read JSONL file",
Detail: err.Error(),
}
}
defer f.Close()
var malformed int
var examples []string
scanner := bufio.NewScanner(f)
lineNo := 0
for scanner.Scan() {
lineNo++
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var v struct {
ID string `json:"id"`
}
if err := json.Unmarshal([]byte(line), &v); err != nil || v.ID == "" {
malformed++
if len(examples) < 5 {
if err != nil {
examples = append(examples, fmt.Sprintf("line %d: %v", lineNo, err))
} else {
examples = append(examples, fmt.Sprintf("line %d: missing id", lineNo))
}
}
}
}
if err := scanner.Err(); err != nil {
return DoctorCheck{
Name: "JSONL Integrity",
Status: StatusWarning,
Message: "Unable to scan JSONL file",
Detail: err.Error(),
}
}
if malformed == 0 {
return DoctorCheck{
Name: "JSONL Integrity",
Status: StatusOK,
Message: fmt.Sprintf("%s looks valid", filepath.Base(jsonlPath)),
}
}
// If we have a database, we can auto-repair by re-exporting from DB.
dbPath := filepath.Join(beadsDir, beads.CanonicalDatabaseName)
if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.Database != "" {
dbPath = cfg.DatabasePath(beadsDir)
}
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
return DoctorCheck{
Name: "JSONL Integrity",
Status: StatusError,
Message: fmt.Sprintf("%s has %d malformed line(s)", filepath.Base(jsonlPath), malformed),
Detail: strings.Join(examples, "\n"),
Fix: "Restore the JSONL file from git or from a backup (no database available for auto-repair).",
}
}
return DoctorCheck{
Name: "JSONL Integrity",
Status: StatusError,
Message: fmt.Sprintf("%s has %d malformed line(s)", filepath.Base(jsonlPath), malformed),
Detail: strings.Join(examples, "\n"),
Fix: "Run 'bd doctor --fix' to back up the JSONL and regenerate it from the database.",
}
}
func isSystemJSONLFilename(name string) bool {
switch name {
case "deletions.jsonl", "interactions.jsonl", "molecules.jsonl":
return true
default:
return false
}
}

43
cmd/bd/doctor/jsonl_integrity_test.go Normal file
View File

@@ -0,0 +1,43 @@
package doctor
import (
"os"
"path/filepath"
"testing"
)
func TestCheckJSONLIntegrity_MalformedLine(t *testing.T) {
ws := t.TempDir()
beadsDir := filepath.Join(ws, ".beads")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatal(err)
}
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
if err := os.WriteFile(jsonlPath, []byte("{\"id\":\"t-1\"}\n{not json}\n"), 0644); err != nil {
t.Fatal(err)
}
// Ensure DB exists so check suggests auto-repair.
if err := os.WriteFile(filepath.Join(beadsDir, "beads.db"), []byte("x"), 0644); err != nil {
t.Fatal(err)
}
check := CheckJSONLIntegrity(ws)
if check.Status != StatusError {
t.Fatalf("expected StatusError, got %v (%s)", check.Status, check.Message)
}
if check.Fix == "" {
t.Fatalf("expected Fix guidance")
}
}
func TestCheckJSONLIntegrity_NoJSONL(t *testing.T) {
ws := t.TempDir()
beadsDir := filepath.Join(ws, ".beads")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatal(err)
}
check := CheckJSONLIntegrity(ws)
if check.Status != StatusOK {
t.Fatalf("expected StatusOK, got %v (%s)", check.Status, check.Message)
}
}