diff --git a/cmd/bd/sync.go b/cmd/bd/sync.go index 989504b1..e95cd042 100644 --- a/cmd/bd/sync.go +++ b/cmd/bd/sync.go @@ -8,10 +8,12 @@ import ( "os/exec" "path/filepath" "sort" + "strconv" "strings" "time" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/deletions" "github.com/steveyegge/beads/internal/rpc" "github.com/steveyegge/beads/internal/syncbranch" "github.com/steveyegge/beads/internal/types" @@ -409,6 +411,12 @@ Use --merge to merge the sync branch back to main branch.`, if dryRun { fmt.Println("\nāœ“ Dry run complete (no changes made)") } else { + // Auto-compact deletions manifest if enabled and threshold exceeded + if err := maybeAutoCompactDeletions(ctx, jsonlPath); err != nil { + // Non-fatal - just log warning + fmt.Fprintf(os.Stderr, "Warning: auto-compact deletions failed: %v\n", err) + } + fmt.Println("\nāœ“ Sync complete") } }, @@ -1105,11 +1113,84 @@ func importFromJSONL(ctx context.Context, jsonlPath string, renameOnImport bool) if err != nil { return fmt.Errorf("import failed: %w\n%s", err, output) } - + // Show output (import command provides the summary) if len(output) > 0 { fmt.Print(string(output)) } - + + return nil +} + +// Default configuration values for auto-compact +const ( + defaultAutoCompact = false + defaultAutoCompactThreshold = 1000 +) + +// maybeAutoCompactDeletions checks if auto-compact is enabled and threshold exceeded, +// and if so, prunes the deletions manifest. +func maybeAutoCompactDeletions(ctx context.Context, jsonlPath string) error { + // Ensure store is initialized for config access + if err := ensureStoreActive(); err != nil { + return nil // Can't access config, skip silently + } + + // Check if auto-compact is enabled (disabled by default) + autoCompactStr, err := store.GetConfig(ctx, "deletions.auto_compact") + if err != nil || autoCompactStr == "" { + return nil // Not configured, skip + } + + autoCompact := autoCompactStr == "true" || autoCompactStr == "1" || autoCompactStr == "yes" + if !autoCompact { + return nil // Disabled, skip + } + + // Get threshold (default 1000) + threshold := defaultAutoCompactThreshold + if thresholdStr, err := store.GetConfig(ctx, "deletions.auto_compact_threshold"); err == nil && thresholdStr != "" { + if parsed, err := strconv.Atoi(thresholdStr); err == nil && parsed > 0 { + threshold = parsed + } + } + + // Get deletions path + beadsDir := filepath.Dir(jsonlPath) + deletionsPath := deletions.DefaultPath(beadsDir) + + // Count current deletions + count, err := deletions.Count(deletionsPath) + if err != nil { + return fmt.Errorf("failed to count deletions: %w", err) + } + + // Check if threshold exceeded + if count <= threshold { + return nil // Below threshold, skip + } + + // Get retention days (default 7) + retentionDays := deletions.DefaultRetentionDays + if retentionStr, err := store.GetConfig(ctx, "deletions.retention_days"); err == nil && retentionStr != "" { + if parsed, err := strconv.Atoi(retentionStr); err == nil && parsed > 0 { + retentionDays = parsed + } + } + + // Prune deletions + fmt.Printf("→ Auto-compacting deletions manifest (%d entries > %d threshold)...\n", count, threshold) + result, err := deletions.PruneDeletions(deletionsPath, retentionDays) + if err != nil { + return fmt.Errorf("failed to prune deletions: %w", err) + } + + if result.PrunedCount > 0 { + fmt.Printf(" Pruned %d entries older than %d days, kept %d entries\n", + result.PrunedCount, retentionDays, result.KeptCount) + } else { + fmt.Printf(" No entries older than %d days to prune\n", retentionDays) + } + return nil } diff --git a/cmd/bd/sync_test.go b/cmd/bd/sync_test.go index df5139c2..0bffbd5b 100644 --- a/cmd/bd/sync_test.go +++ b/cmd/bd/sync_test.go @@ -570,3 +570,221 @@ func TestZFCSkipsExportAfterImport(t *testing.T) { t.Logf("āœ“ ZFC fix verified: DB synced from 100 to 10 issues, JSONL unchanged") } + +func TestMaybeAutoCompactDeletions_Disabled(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + // Create test database + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0755); err != nil { + t.Fatalf("failed to create beads dir: %v", err) + } + + testDBPath := filepath.Join(beadsDir, "beads.db") + jsonlPath := filepath.Join(beadsDir, "beads.jsonl") + + // Create store + testStore, err := sqlite.New(ctx, testDBPath) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + defer testStore.Close() + + // Set global store for maybeAutoCompactDeletions + // Save and restore original values + originalStore := store + originalStoreActive := storeActive + defer func() { + store = originalStore + storeActive = originalStoreActive + }() + + store = testStore + storeActive = true + + // Create empty JSONL file + if err := os.WriteFile(jsonlPath, []byte{}, 0644); err != nil { + t.Fatalf("failed to create JSONL: %v", err) + } + + // Auto-compact is disabled by default, so should return nil + err = maybeAutoCompactDeletions(ctx, jsonlPath) + if err != nil { + t.Errorf("expected no error when auto-compact disabled, got: %v", err) + } +} + +func TestMaybeAutoCompactDeletions_Enabled(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + // Create test database + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0755); err != nil { + t.Fatalf("failed to create beads dir: %v", err) + } + + testDBPath := filepath.Join(beadsDir, "beads.db") + jsonlPath := filepath.Join(beadsDir, "beads.jsonl") + deletionsPath := filepath.Join(beadsDir, "deletions.jsonl") + + // Create store + testStore, err := sqlite.New(ctx, testDBPath) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + defer testStore.Close() + + // Enable auto-compact with low threshold + if err := testStore.SetConfig(ctx, "deletions.auto_compact", "true"); err != nil { + t.Fatalf("failed to set auto_compact config: %v", err) + } + if err := testStore.SetConfig(ctx, "deletions.auto_compact_threshold", "5"); err != nil { + t.Fatalf("failed to set threshold config: %v", err) + } + if err := testStore.SetConfig(ctx, "deletions.retention_days", "1"); err != nil { + t.Fatalf("failed to set retention config: %v", err) + } + + // Set global store for maybeAutoCompactDeletions + // Save and restore original values + originalStore := store + originalStoreActive := storeActive + defer func() { + store = originalStore + storeActive = originalStoreActive + }() + + store = testStore + storeActive = true + + // Create empty JSONL file + if err := os.WriteFile(jsonlPath, []byte{}, 0644); err != nil { + t.Fatalf("failed to create JSONL: %v", err) + } + + // Create deletions file with entries (some old, some recent) + now := time.Now() + deletionsContent := "" + // Add 10 old entries (will be pruned) + for i := 0; i < 10; i++ { + oldTime := now.AddDate(0, 0, -10).Format(time.RFC3339) + deletionsContent += fmt.Sprintf(`{"id":"bd-old-%d","ts":"%s","by":"user"}`, i, oldTime) + "\n" + } + // Add 3 recent entries (will be kept) + for i := 0; i < 3; i++ { + recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339) + deletionsContent += fmt.Sprintf(`{"id":"bd-recent-%d","ts":"%s","by":"user"}`, i, recentTime) + "\n" + } + + if err := os.WriteFile(deletionsPath, []byte(deletionsContent), 0644); err != nil { + t.Fatalf("failed to create deletions file: %v", err) + } + + // Verify initial count + initialCount := strings.Count(deletionsContent, "\n") + if initialCount != 13 { + t.Fatalf("expected 13 initial entries, got %d", initialCount) + } + + // Run auto-compact + err = maybeAutoCompactDeletions(ctx, jsonlPath) + if err != nil { + t.Errorf("auto-compact failed: %v", err) + } + + // Read deletions file and count remaining entries + afterContent, err := os.ReadFile(deletionsPath) + if err != nil { + t.Fatalf("failed to read deletions file: %v", err) + } + + afterLines := strings.Split(strings.TrimSpace(string(afterContent)), "\n") + afterCount := 0 + for _, line := range afterLines { + if line != "" { + afterCount++ + } + } + + // Should have pruned old entries, kept recent ones + if afterCount != 3 { + t.Errorf("expected 3 entries after prune (recent ones), got %d", afterCount) + } +} + +func TestMaybeAutoCompactDeletions_BelowThreshold(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + // Create test database + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0755); err != nil { + t.Fatalf("failed to create beads dir: %v", err) + } + + testDBPath := filepath.Join(beadsDir, "beads.db") + jsonlPath := filepath.Join(beadsDir, "beads.jsonl") + deletionsPath := filepath.Join(beadsDir, "deletions.jsonl") + + // Create store + testStore, err := sqlite.New(ctx, testDBPath) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + defer testStore.Close() + + // Enable auto-compact with high threshold + if err := testStore.SetConfig(ctx, "deletions.auto_compact", "true"); err != nil { + t.Fatalf("failed to set auto_compact config: %v", err) + } + if err := testStore.SetConfig(ctx, "deletions.auto_compact_threshold", "100"); err != nil { + t.Fatalf("failed to set threshold config: %v", err) + } + + // Set global store for maybeAutoCompactDeletions + // Save and restore original values + originalStore := store + originalStoreActive := storeActive + defer func() { + store = originalStore + storeActive = originalStoreActive + }() + + store = testStore + storeActive = true + + // Create empty JSONL file + if err := os.WriteFile(jsonlPath, []byte{}, 0644); err != nil { + t.Fatalf("failed to create JSONL: %v", err) + } + + // Create deletions file with only 5 entries (below threshold of 100) + now := time.Now() + deletionsContent := "" + for i := 0; i < 5; i++ { + ts := now.Add(-1 * time.Hour).Format(time.RFC3339) + deletionsContent += fmt.Sprintf(`{"id":"bd-%d","ts":"%s","by":"user"}`, i, ts) + "\n" + } + + if err := os.WriteFile(deletionsPath, []byte(deletionsContent), 0644); err != nil { + t.Fatalf("failed to create deletions file: %v", err) + } + + // Run auto-compact - should skip because below threshold + err = maybeAutoCompactDeletions(ctx, jsonlPath) + if err != nil { + t.Errorf("auto-compact failed: %v", err) + } + + // Read deletions file - should be unchanged + afterContent, err := os.ReadFile(deletionsPath) + if err != nil { + t.Fatalf("failed to read deletions file: %v", err) + } + + if string(afterContent) != deletionsContent { + t.Error("deletions file should not be modified when below threshold") + } +} diff --git a/internal/deletions/deletions.go b/internal/deletions/deletions.go index a25875ec..af1c1699 100644 --- a/internal/deletions/deletions.go +++ b/internal/deletions/deletions.go @@ -180,6 +180,35 @@ func DefaultPath(beadsDir string) string { return filepath.Join(beadsDir, "deletions.jsonl") } +// Count returns the number of lines in the deletions manifest. +// This is a fast operation that doesn't parse JSON, just counts lines. +// Returns 0 if the file doesn't exist or is empty. +func Count(path string) (int, error) { + f, err := os.Open(path) // #nosec G304 - controlled path from caller + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, fmt.Errorf("failed to open deletions file: %w", err) + } + defer f.Close() + + count := 0 + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + count++ + } + } + + if err := scanner.Err(); err != nil { + return 0, fmt.Errorf("error reading deletions file: %w", err) + } + + return count, nil +} + // DefaultRetentionDays is the default number of days to retain deletion records. const DefaultRetentionDays = 7 diff --git a/internal/deletions/deletions_test.go b/internal/deletions/deletions_test.go index ecb588a3..bfeb6f23 100644 --- a/internal/deletions/deletions_test.go +++ b/internal/deletions/deletions_test.go @@ -546,3 +546,64 @@ func TestPruneDeletions_ZeroRetention(t *testing.T) { t.Errorf("expected 1 pruned with 0 retention, got %d", result.PrunedCount) } } + +func TestCount_Empty(t *testing.T) { + // Non-existent file should return 0 + count, err := Count("/nonexistent/path/deletions.jsonl") + if err != nil { + t.Fatalf("expected no error for non-existent file, got: %v", err) + } + if count != 0 { + t.Errorf("expected 0 count for non-existent file, got %d", count) + } +} + +func TestCount_WithRecords(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "deletions.jsonl") + + now := time.Now() + records := []DeletionRecord{ + {ID: "bd-001", Timestamp: now, Actor: "user1"}, + {ID: "bd-002", Timestamp: now, Actor: "user2"}, + {ID: "bd-003", Timestamp: now, Actor: "user3"}, + } + + for _, r := range records { + if err := AppendDeletion(path, r); err != nil { + t.Fatalf("AppendDeletion failed: %v", err) + } + } + + count, err := Count(path) + if err != nil { + t.Fatalf("Count failed: %v", err) + } + if count != 3 { + t.Errorf("expected 3, got %d", count) + } +} + +func TestCount_WithEmptyLines(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "deletions.jsonl") + + // Write content with empty lines + content := `{"id":"bd-001","ts":"2024-01-01T00:00:00Z","by":"user1"} + +{"id":"bd-002","ts":"2024-01-02T00:00:00Z","by":"user2"} + +` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + count, err := Count(path) + if err != nil { + t.Fatalf("Count failed: %v", err) + } + // Should count only non-empty lines + if count != 2 { + t.Errorf("expected 2 (excluding empty lines), got %d", count) + } +} diff --git a/internal/importer/importer.go b/internal/importer/importer.go index cd8a7487..d54dad35 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "sort" "strings" "time" @@ -862,11 +863,34 @@ func checkGitHistoryForDeletions(beadsDir string, ids []string) []string { return nil } - // Get the repo root directory (parent of .beads) - repoRoot := filepath.Dir(beadsDir) + // Find the actual git repo root using git rev-parse (bd-bhd) + // This handles monorepos and nested projects where .beads isn't at repo root + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "git", "rev-parse", "--show-toplevel") + cmd.Dir = beadsDir + output, err := cmd.Output() + if err != nil { + // Not in a git repo or git not available - can't do history check + return nil + } + repoRoot := strings.TrimSpace(string(output)) + + // Compute relative path from repo root to beads.jsonl + // beadsDir is absolute, compute its path relative to repoRoot + absBeadsDir, err := filepath.Abs(beadsDir) + if err != nil { + return nil + } + + relBeadsDir, err := filepath.Rel(repoRoot, absBeadsDir) + if err != nil { + return nil + } // Build JSONL path relative to repo root - jsonlPath := filepath.Join(".beads", "beads.jsonl") + jsonlPath := filepath.Join(relBeadsDir, "beads.jsonl") var deleted []string @@ -888,15 +912,24 @@ func checkGitHistoryForDeletions(beadsDir string, ids []string) []string { return deleted } +// gitHistoryTimeout is the maximum time to wait for git history searches. +// Prevents hangs on large repositories (bd-f0n). +const gitHistoryTimeout = 30 * time.Second + // wasInGitHistory checks if a single ID was ever in the JSONL via git history. // Returns true if the ID was found in history (meaning it was deleted). func wasInGitHistory(repoRoot, jsonlPath, id string) bool { // git log --all -S "\"id\":\"bd-xxx\"" --oneline -- .beads/beads.jsonl // This searches for commits that added or removed the ID string + // Note: -S uses literal string matching, not regex, so no escaping needed searchPattern := fmt.Sprintf(`"id":"%s"`, id) + // Use context with timeout to prevent hangs on large repos (bd-f0n) + ctx, cancel := context.WithTimeout(context.Background(), gitHistoryTimeout) + defer cancel() + // #nosec G204 - searchPattern is constructed from validated issue IDs - cmd := exec.Command("git", "log", "--all", "-S", searchPattern, "--oneline", "--", jsonlPath) + cmd := exec.CommandContext(ctx, "git", "log", "--all", "-S", searchPattern, "--oneline", "--", jsonlPath) cmd.Dir = repoRoot var stdout bytes.Buffer @@ -904,7 +937,7 @@ func wasInGitHistory(repoRoot, jsonlPath, id string) bool { cmd.Stderr = nil // Ignore stderr if err := cmd.Run(); err != nil { - // Git command failed - could be shallow clone, not a git repo, etc. + // Git command failed - could be shallow clone, not a git repo, timeout, etc. // Conservative: assume issue is local work, don't delete return false } @@ -919,15 +952,21 @@ func wasInGitHistory(repoRoot, jsonlPath, id string) bool { func batchCheckGitHistory(repoRoot, jsonlPath string, ids []string) []string { // Build a regex pattern to match any of the IDs // Pattern: "id":"bd-xxx"|"id":"bd-yyy"|... + // Escape regex special characters in IDs to avoid malformed patterns (bd-bgs) patterns := make([]string, 0, len(ids)) for _, id := range ids { - patterns = append(patterns, fmt.Sprintf(`"id":"%s"`, id)) + escapedID := regexp.QuoteMeta(id) + patterns = append(patterns, fmt.Sprintf(`"id":"%s"`, escapedID)) } searchPattern := strings.Join(patterns, "|") + // Use context with timeout to prevent hangs on large repos (bd-f0n) + ctx, cancel := context.WithTimeout(context.Background(), gitHistoryTimeout) + defer cancel() + // Use git log -G (regex) for batch search // #nosec G204 - searchPattern is constructed from validated issue IDs - cmd := exec.Command("git", "log", "--all", "-G", searchPattern, "-p", "--", jsonlPath) + cmd := exec.CommandContext(ctx, "git", "log", "--all", "-G", searchPattern, "-p", "--", jsonlPath) cmd.Dir = repoRoot var stdout bytes.Buffer @@ -935,7 +974,8 @@ func batchCheckGitHistory(repoRoot, jsonlPath string, ids []string) []string { cmd.Stderr = nil // Ignore stderr if err := cmd.Run(); err != nil { - // Git command failed - fall back to individual checks + // Git command failed (timeout, shallow clone, etc.) - fall back to individual checks + // Individual checks also have timeout protection var deleted []string for _, id := range ids { if wasInGitHistory(repoRoot, jsonlPath, id) {