diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e91d5c30..4c05759c 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -174,13 +174,13 @@ {"id":"bd-256","title":"Implement snapshot creation and restoration","description":"Implement snapshot creation before compaction and restoration capability to undo compaction.","design":"Add to `internal/storage/sqlite/compact.go`:\n\n```go\nfunc (s *SQLiteStorage) CreateSnapshot(ctx context.Context, issue *types.Issue, level int) error\nfunc (s *SQLiteStorage) RestoreFromSnapshot(ctx context.Context, issueID string, level int) error\nfunc (s *SQLiteStorage) GetSnapshots(ctx context.Context, issueID string) ([]*Snapshot, error)\n```\n\nSnapshot JSON structure:\n```json\n{\n \"description\": \"...\",\n \"design\": \"...\",\n \"notes\": \"...\",\n \"acceptance_criteria\": \"...\",\n \"title\": \"...\"\n}\n```","acceptance_criteria":"- Snapshot created atomically with compaction\n- Restore returns exact original content\n- Multiple snapshots per issue supported (Tier 1 → Tier 2)\n- JSON encoding handles UTF-8 and special characters\n- Size calculation is accurate (UTF-8 bytes)","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.231906-07:00","updated_at":"2025-10-15T23:11:31.076796-07:00","closed_at":"2025-10-15T23:11:31.076796-07:00"} {"id":"bd-257","title":"Implement Tier 1 compaction logic","description":"Implement the core Tier 1 compaction process: snapshot → summarize → update.","design":"Add to `internal/compact/compactor.go`:\n\n```go\ntype Compactor struct {\n store storage.Storage\n haiku *HaikuClient\n config *CompactConfig\n}\n\nfunc New(store storage.Storage, apiKey string, config *CompactConfig) (*Compactor, error)\nfunc (c *Compactor) CompactTier1(ctx context.Context, issueID string) error\nfunc (c *Compactor) CompactTier1Batch(ctx context.Context, issueIDs []string) error\n```\n\nProcess:\n1. Verify eligibility\n2. Calculate original size\n3. Create snapshot\n4. Call Haiku for summary\n5. Update issue (description=summary, clear design/notes/criteria)\n6. Set compaction_level=1, compacted_at=now, original_size\n7. Record EventCompacted\n8. Mark dirty for export","acceptance_criteria":"- Single issue compaction works end-to-end\n- Batch processing with parallel workers (5 concurrent)\n- Errors don't corrupt database (transaction rollback)\n- EventCompacted includes size savings\n- Dry-run mode (identify + size estimate only, no API calls)","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.23391-07:00","updated_at":"2025-10-15T23:30:31.967874-07:00","closed_at":"2025-10-15T23:30:31.967874-07:00"} {"id":"bd-258","title":"Implement Tier 2 compaction logic","description":"Implement Tier 2 ultra-compression: more aggressive summarization and optional event pruning.","design":"Add to `internal/compact/compactor.go`:\n\n```go\nfunc (c *Compactor) CompactTier2(ctx context.Context, issueID string) error\nfunc (c *Compactor) CompactTier2Batch(ctx context.Context, issueIDs []string) error\n```\n\nProcess:\n1. Verify issue is at compaction_level = 1\n2. Check Tier 2 eligibility (days, deps, commits/issues)\n3. Create Tier 2 snapshot\n4. Call Haiku with ultra-compression prompt\n5. Update issue (description = single paragraph, clear all other fields)\n6. Set compaction_level = 2\n7. Optionally prune events (keep created/closed, archive rest to snapshot)","acceptance_criteria":"- Requires existing Tier 1 compaction\n- Git commit counting works (with fallback to issue counter)\n- Events optionally pruned (config: compact_events_enabled)\n- Archived events stored in snapshot JSON\n- Size reduction 90-95%","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.23586-07:00","updated_at":"2025-10-15T21:51:23.23586-07:00"} -{"id":"bd-259","title":"Add `bd compact` CLI command","description":"Implement the `bd compact` command with dry-run, batch processing, and progress reporting.","design":"Create `cmd/bd/compact.go`:\n\n```go\nvar compactCmd = \u0026cobra.Command{\n Use: \"compact\",\n Short: \"Compact old closed issues to save space\",\n}\n\nFlags:\n --dry-run Preview without compacting\n --tier int Compaction tier (1 or 2, default: 1)\n --all Process all candidates\n --id string Compact specific issue\n --force Force compact (bypass checks, requires --id)\n --batch-size int Issues per batch\n --workers int Parallel workers\n --json JSON output\n```","acceptance_criteria":"- `--dry-run` shows accurate preview with size estimates\n- `--all` processes all candidates\n- `--id` compacts single issue\n- `--force` bypasses eligibility checks (only with --id)\n- Progress bar for batches (e.g., [████████] 47/47)\n- JSON output with `--json`\n- Exit codes: 0=success, 1=error\n- Shows summary: count, size saved, cost, time","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.238373-07:00","updated_at":"2025-10-15T21:51:23.238373-07:00"} +{"id":"bd-259","title":"Add `bd compact` CLI command","description":"Implement the `bd compact` command with dry-run, batch processing, and progress reporting.","design":"Create `cmd/bd/compact.go`:\n\n```go\nvar compactCmd = \u0026cobra.Command{\n Use: \"compact\",\n Short: \"Compact old closed issues to save space\",\n}\n\nFlags:\n --dry-run Preview without compacting\n --tier int Compaction tier (1 or 2, default: 1)\n --all Process all candidates\n --id string Compact specific issue\n --force Force compact (bypass checks, requires --id)\n --batch-size int Issues per batch\n --workers int Parallel workers\n --json JSON output\n```","acceptance_criteria":"- `--dry-run` shows accurate preview with size estimates\n- `--all` processes all candidates\n- `--id` compacts single issue\n- `--force` bypasses eligibility checks (only with --id)\n- Progress bar for batches (e.g., [████████] 47/47)\n- JSON output with `--json`\n- Exit codes: 0=success, 1=error\n- Shows summary: count, size saved, cost, time","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.238373-07:00","updated_at":"2025-10-16T00:09:21.188292-07:00","closed_at":"2025-10-16T00:09:21.188292-07:00"} {"id":"bd-26","title":"Optimize reference updates to avoid loading all issues into memory","description":"In updateReferences(), we call SearchIssues with no filter to get ALL issues for updating references. For large databases (10k+ issues), this loads everything into memory. Options: 1) Use batched processing with LIMIT/OFFSET, 2) Use SQL UPDATE with REPLACE() directly, 3) Stream results instead of loading all at once. Located in collision.go:266","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-14T14:43:06.911497-07:00","updated_at":"2025-10-15T16:27:22.001829-07:00"} {"id":"bd-260","title":"Add `bd compact --restore` functionality","description":"Implement restore command to undo compaction from snapshots.","design":"Add to `cmd/bd/compact.go`:\n\n```go\nvar compactRestore string\n\ncompactCmd.Flags().StringVar(\u0026compactRestore, \"restore\", \"\", \"Restore issue from snapshot\")\n```\n\nProcess:\n1. Load snapshot for issue\n2. Parse JSON content\n3. Update issue with original content\n4. Set compaction_level = 0, compacted_at = NULL, original_size = NULL\n5. Record event (EventRestored or EventUpdated)\n6. Mark dirty for export","acceptance_criteria":"- Restores exact original content\n- Handles multiple snapshots (use latest by default)\n- `--level` flag to choose specific snapshot\n- Updates compaction_level correctly\n- Exports restored content to JSONL\n- Shows before/after in output","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.240267-07:00","updated_at":"2025-10-15T21:51:23.240267-07:00"} {"id":"bd-261","title":"Add `bd compact --stats` command","description":"Add statistics command showing compaction status and potential savings.","design":"```go\nvar compactStats bool\n\ncompactCmd.Flags().BoolVar(\u0026compactStats, \"stats\", false, \"Show compaction statistics\")\n```\n\nOutput:\n- Total issues, by compaction level (0, 1, 2)\n- Current DB size vs estimated uncompacted size\n- Space savings (KB/MB and %)\n- Candidates for each tier with size estimates\n- Estimated API cost (Haiku pricing)","acceptance_criteria":"- Accurate counts by compaction_level\n- Size calculations include all text fields (UTF-8 bytes)\n- Shows candidates with eligibility reasons\n- Cost estimation based on current Haiku pricing\n- JSON output supported\n- Clear, readable table format","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.242041-07:00","updated_at":"2025-10-15T21:51:23.242041-07:00"} {"id":"bd-262","title":"Add EventCompacted to event system","description":"Add new event type for tracking compaction in audit trail.","design":"1. Add to `internal/types/types.go`:\n```go\nconst EventCompacted EventType = \"compacted\"\n```\n\n2. Record event during compaction:\n```go\neventData := map[string]interface{}{\n \"tier\": tier,\n \"original_size\": originalSize,\n \"compressed_size\": compressedSize,\n \"reduction_pct\": (1 - float64(compressedSize)/float64(originalSize)) * 100,\n}\n```\n\n3. Update event display in `bd show`.","acceptance_criteria":"- Event includes tier, original_size, compressed_size, reduction_pct\n- Shows in event history (`bd events \u003cid\u003e`)\n- Exports to JSONL correctly\n- `bd show` displays compaction status and marker","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.244219-07:00","updated_at":"2025-10-15T21:51:23.244219-07:00"} {"id":"bd-263","title":"Add compaction indicator to `bd show`","description":"Update `bd show` command to display compaction status prominently.","design":"Add to issue display:\n```\nbd-42: Fix authentication bug [CLOSED] 🗜️\n\nStatus: closed (compacted L1)\n...\n\n---\n💾 Restore: bd compact --restore bd-42\n📊 Original: 2,341 bytes | Compressed: 468 bytes (80% reduction)\n🗜️ Compacted: 2025-10-15 (Tier 1)\n```\n\nEmoji indicators:\n- Tier 1: 🗜️\n- Tier 2: 📦","acceptance_criteria":"- Compaction status visible in title line\n- Footer shows size savings when compacted\n- Restore command shown for compacted issues\n- Works with `--json` output (includes compaction fields)\n- Emoji optional (controlled by config or terminal detection)","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.253091-07:00","updated_at":"2025-10-15T21:51:23.253091-07:00"} -{"id":"bd-264","title":"Write compaction tests","description":"Comprehensive test suite for compaction functionality.","design":"Test coverage:\n\n1. **Candidate Identification:**\n - Eligibility by time\n - Dependency depth checking\n - Mixed status dependents\n - Edge cases (no deps, circular deps)\n\n2. **Snapshots:**\n - Create and restore\n - Multiple snapshots per issue\n - Content integrity (UTF-8, special chars)\n\n3. **Tier 1 Compaction:**\n - Single issue compaction\n - Batch processing\n - Error handling (API failures)\n\n4. **Tier 2 Compaction:**\n - Requires Tier 1\n - Events pruning\n - Commit counting fallback\n\n5. **CLI:**\n - All flag combinations\n - Dry-run accuracy\n - JSON output parsing\n\n6. **Integration:**\n - End-to-end flow\n - JSONL export/import\n - Restore verification","acceptance_criteria":"- Test coverage \u003e80%\n- All edge cases covered\n- Mock Haiku API in tests (no real API calls)\n- Integration tests pass\n- `go test ./...` passes\n- Benchmarks for performance-critical paths","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.262504-07:00","updated_at":"2025-10-15T21:51:23.262504-07:00"} +{"id":"bd-264","title":"Write compaction tests","description":"Comprehensive test suite for compaction functionality.","design":"Test coverage:\n\n1. **Candidate Identification:**\n - Eligibility by time\n - Dependency depth checking\n - Mixed status dependents\n - Edge cases (no deps, circular deps)\n\n2. **Snapshots:**\n - Create and restore\n - Multiple snapshots per issue\n - Content integrity (UTF-8, special chars)\n\n3. **Tier 1 Compaction:**\n - Single issue compaction\n - Batch processing\n - Error handling (API failures)\n\n4. **Tier 2 Compaction:**\n - Requires Tier 1\n - Events pruning\n - Commit counting fallback\n\n5. **CLI:**\n - All flag combinations\n - Dry-run accuracy\n - JSON output parsing\n\n6. **Integration:**\n - End-to-end flow\n - JSONL export/import\n - Restore verification","acceptance_criteria":"- Test coverage \u003e80%\n- All edge cases covered\n- Mock Haiku API in tests (no real API calls)\n- Integration tests pass\n- `go test ./...` passes\n- Benchmarks for performance-critical paths","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.262504-07:00","updated_at":"2025-10-16T00:02:11.246331-07:00","closed_at":"2025-10-16T00:02:11.246331-07:00"} {"id":"bd-265","title":"Add compaction documentation","description":"Document compaction feature in README and create detailed COMPACTION.md guide.","design":"**Update README.md:**\n- Add to Features section\n- CLI examples (dry-run, compact, restore, stats)\n- Configuration guide\n- Cost analysis\n\n**Create COMPACTION.md:**\n- How compaction works (architecture overview)\n- When to use each tier\n- Detailed cost analysis with examples\n- Safety mechanisms (snapshots, restore, dry-run)\n- Troubleshooting guide\n- FAQ\n\n**Create examples/compaction/:**\n- `workflow.sh` - Example monthly compaction workflow\n- `cron-compact.sh` - Cron job setup\n- `auto-compact.sh` - Auto-compaction script","acceptance_criteria":"- README.md updated with compaction section\n- COMPACTION.md comprehensive and clear\n- Examples work as documented (tested)\n- Screenshots or ASCII examples included\n- API key setup documented (env var vs config)\n- Covers common questions and issues","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.265589-07:00","updated_at":"2025-10-15T21:51:23.265589-07:00"} {"id":"bd-266","title":"Optional: Implement auto-compaction","description":"Implement automatic compaction triggered by certain operations when enabled via config.","design":"Trigger points (when `auto_compact_enabled = true`):\n1. `bd stats` - check and compact if candidates exist\n2. `bd export` - before exporting\n3. Configurable: on any read operation after N candidates accumulate\n\nAdd:\n```go\nfunc (s *SQLiteStorage) AutoCompact(ctx context.Context) error {\n enabled, _ := s.GetConfig(ctx, \"auto_compact_enabled\")\n if enabled != \"true\" {\n return nil\n }\n\n // Run Tier 1 compaction on all candidates\n // Limit to batch_size to avoid long operations\n // Log activity for transparency\n}\n```","acceptance_criteria":"- Respects auto_compact_enabled config (default: false)\n- Limits batch size to avoid blocking operations\n- Logs compaction activity (visible with --verbose)\n- Can be disabled per-command with `--no-auto-compact` flag\n- Only compacts Tier 1 (Tier 2 remains manual)\n- Doesn't run more than once per hour (rate limiting)","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-15T21:51:23.281006-07:00","updated_at":"2025-10-15T21:51:23.281006-07:00"} {"id":"bd-267","title":"Optional: Add git commit counting","description":"Implement git commit counting for \"project time\" measurement as alternative to calendar time for Tier 2 eligibility.","design":"```go\nfunc getCommitsSince(closedAt time.Time) (int, error) {\n cmd := exec.Command(\"git\", \"rev-list\", \"--count\",\n fmt.Sprintf(\"--since=%s\", closedAt.Format(time.RFC3339)), \"HEAD\")\n output, err := cmd.Output()\n if err != nil {\n return 0, err // Not in git repo or git not available\n }\n return strconv.Atoi(strings.TrimSpace(string(output)))\n}\n```\n\nFallback strategies:\n1. Git commit count (preferred)\n2. Issue counter delta (store counter at close time, compare later)\n3. Pure time-based (90 days)","acceptance_criteria":"- Counts commits since closed_at timestamp\n- Handles git not available gracefully (falls back)\n- Fallback to issue counter delta works\n- Configurable via compact_tier2_commits config key\n- Tested with real git repo\n- Works in non-git environments","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-15T21:51:23.284781-07:00","updated_at":"2025-10-15T21:51:23.284781-07:00"} diff --git a/cmd/bd/compact.go b/cmd/bd/compact.go new file mode 100644 index 00000000..0ac94727 --- /dev/null +++ b/cmd/bd/compact.go @@ -0,0 +1,404 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/compact" + "github.com/steveyegge/beads/internal/storage/sqlite" +) + +var ( + compactDryRun bool + compactTier int + compactAll bool + compactID string + compactForce bool + compactBatch int + compactWorkers int + compactStats bool + compactRestore string +) + +var compactCmd = &cobra.Command{ + Use: "compact", + Short: "Compact old closed issues to save space", + Long: `Compact old closed issues using semantic summarization. + +Compaction reduces database size by summarizing closed issues that are no longer +actively referenced. Full restore capability is maintained via snapshots. + +Tiers: + - Tier 1: Semantic compression (30 days closed, 70% reduction) + - Tier 2: Ultra compression (90 days closed, 95% reduction) + +Examples: + bd compact --dry-run # Preview candidates + bd compact --all # Compact all eligible issues + bd compact --id bd-42 # Compact specific issue + bd compact --id bd-42 --force # Force compact (bypass checks) + bd compact --stats # Show statistics + bd compact --restore bd-42 # Restore from snapshot +`, + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + + sqliteStore, ok := store.(*sqlite.SQLiteStorage) + if !ok { + fmt.Fprintf(os.Stderr, "Error: compact requires SQLite storage\n") + os.Exit(1) + } + + if compactStats { + runCompactStats(ctx, sqliteStore) + return + } + + if compactRestore != "" { + runCompactRestore(ctx, sqliteStore, compactRestore) + return + } + + if compactID != "" && compactAll { + fmt.Fprintf(os.Stderr, "Error: cannot use --id and --all together\n") + os.Exit(1) + } + + if compactForce && compactID == "" { + fmt.Fprintf(os.Stderr, "Error: --force requires --id\n") + os.Exit(1) + } + + if compactID == "" && !compactAll && !compactDryRun { + fmt.Fprintf(os.Stderr, "Error: must specify --all, --id, or --dry-run\n") + os.Exit(1) + } + + apiKey := os.Getenv("ANTHROPIC_API_KEY") + if apiKey == "" && !compactDryRun { + fmt.Fprintf(os.Stderr, "Error: ANTHROPIC_API_KEY environment variable not set\n") + os.Exit(1) + } + + config := &compact.CompactConfig{ + APIKey: apiKey, + Concurrency: compactWorkers, + DryRun: compactDryRun, + } + + compactor, err := compact.New(sqliteStore, apiKey, config) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to create compactor: %v\n", err) + os.Exit(1) + } + + if compactID != "" { + runCompactSingle(ctx, compactor, sqliteStore, compactID) + return + } + + runCompactAll(ctx, compactor, sqliteStore) + }, +} + +func runCompactSingle(ctx context.Context, compactor *compact.Compactor, store *sqlite.SQLiteStorage, issueID string) { + start := time.Now() + + if !compactForce { + eligible, reason, err := store.CheckEligibility(ctx, issueID, compactTier) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to check eligibility: %v\n", err) + os.Exit(1) + } + if !eligible { + fmt.Fprintf(os.Stderr, "Error: %s is not eligible for Tier %d compaction: %s\n", issueID, compactTier, reason) + os.Exit(1) + } + } + + issue, err := store.GetIssue(ctx, issueID) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to get issue: %v\n", err) + os.Exit(1) + } + + originalSize := len(issue.Description) + len(issue.Design) + len(issue.Notes) + len(issue.AcceptanceCriteria) + + if compactDryRun { + if jsonOutput { + output := map[string]interface{}{ + "dry_run": true, + "tier": compactTier, + "issue_id": issueID, + "original_size": originalSize, + "estimated_reduction": "70-80%", + } + outputJSON(output) + return + } + + fmt.Printf("DRY RUN - Tier %d compaction\n\n", compactTier) + fmt.Printf("Issue: %s\n", issueID) + fmt.Printf("Original size: %d bytes\n", originalSize) + fmt.Printf("Estimated reduction: 70-80%%\n") + return + } + + var compactErr error + if compactTier == 1 { + compactErr = compactor.CompactTier1(ctx, issueID) + } else { + fmt.Fprintf(os.Stderr, "Error: Tier 2 compaction not yet implemented\n") + os.Exit(1) + } + + if compactErr != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", compactErr) + os.Exit(1) + } + + issue, err = store.GetIssue(ctx, issueID) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to get updated issue: %v\n", err) + os.Exit(1) + } + + compactedSize := len(issue.Description) + savingBytes := originalSize - compactedSize + elapsed := time.Since(start) + + if jsonOutput { + output := map[string]interface{}{ + "success": true, + "tier": compactTier, + "issue_id": issueID, + "original_size": originalSize, + "compacted_size": compactedSize, + "saved_bytes": savingBytes, + "reduction_pct": float64(savingBytes) / float64(originalSize) * 100, + "elapsed_ms": elapsed.Milliseconds(), + } + outputJSON(output) + return + } + + fmt.Printf("✓ Compacted %s (Tier %d)\n", issueID, compactTier) + fmt.Printf(" %d → %d bytes (saved %d, %.1f%%)\n", + originalSize, compactedSize, savingBytes, + float64(savingBytes)/float64(originalSize)*100) + fmt.Printf(" Time: %v\n", elapsed) +} + +func runCompactAll(ctx context.Context, compactor *compact.Compactor, store *sqlite.SQLiteStorage) { + start := time.Now() + + var candidates []string + if compactTier == 1 { + tier1, err := store.GetTier1Candidates(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to get candidates: %v\n", err) + os.Exit(1) + } + for _, c := range tier1 { + candidates = append(candidates, c.IssueID) + } + } else { + tier2, err := store.GetTier2Candidates(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to get candidates: %v\n", err) + os.Exit(1) + } + for _, c := range tier2 { + candidates = append(candidates, c.IssueID) + } + } + + if len(candidates) == 0 { + if jsonOutput { + outputJSON(map[string]interface{}{ + "success": true, + "count": 0, + "message": "No eligible candidates", + }) + return + } + fmt.Println("No eligible candidates for compaction") + return + } + + if compactDryRun { + totalSize := 0 + for _, id := range candidates { + issue, err := store.GetIssue(ctx, id) + if err != nil { + continue + } + totalSize += len(issue.Description) + len(issue.Design) + len(issue.Notes) + len(issue.AcceptanceCriteria) + } + + if jsonOutput { + output := map[string]interface{}{ + "dry_run": true, + "tier": compactTier, + "candidate_count": len(candidates), + "total_size_bytes": totalSize, + "estimated_reduction": "70-80%", + } + outputJSON(output) + return + } + + fmt.Printf("DRY RUN - Tier %d compaction\n\n", compactTier) + fmt.Printf("Candidates: %d issues\n", len(candidates)) + fmt.Printf("Total size: %d bytes\n", totalSize) + fmt.Printf("Estimated reduction: 70-80%%\n") + return + } + + if !jsonOutput { + fmt.Printf("Compacting %d issues (Tier %d)...\n\n", len(candidates), compactTier) + } + + results, err := compactor.CompactTier1Batch(ctx, candidates) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: batch compaction failed: %v\n", err) + os.Exit(1) + } + + successCount := 0 + failCount := 0 + totalSaved := 0 + totalOriginal := 0 + + for i, result := range results { + if !jsonOutput { + fmt.Printf("[%s] %d/%d\r", progressBar(i+1, len(results)), i+1, len(results)) + } + + if result.Err != nil { + failCount++ + } else { + successCount++ + totalOriginal += result.OriginalSize + totalSaved += (result.OriginalSize - result.CompactedSize) + } + } + + elapsed := time.Since(start) + + if jsonOutput { + output := map[string]interface{}{ + "success": true, + "tier": compactTier, + "total": len(results), + "succeeded": successCount, + "failed": failCount, + "saved_bytes": totalSaved, + "original_size": totalOriginal, + "elapsed_ms": elapsed.Milliseconds(), + } + outputJSON(output) + return + } + + fmt.Printf("\n\nCompleted in %v\n\n", elapsed) + fmt.Printf("Summary:\n") + fmt.Printf(" Succeeded: %d\n", successCount) + fmt.Printf(" Failed: %d\n", failCount) + if totalOriginal > 0 { + fmt.Printf(" Saved: %d bytes (%.1f%%)\n", totalSaved, float64(totalSaved)/float64(totalOriginal)*100) + } +} + +func runCompactStats(ctx context.Context, store *sqlite.SQLiteStorage) { + tier1, err := store.GetTier1Candidates(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to get Tier 1 candidates: %v\n", err) + os.Exit(1) + } + + tier2, err := store.GetTier2Candidates(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to get Tier 2 candidates: %v\n", err) + os.Exit(1) + } + + tier1Size := 0 + for _, c := range tier1 { + tier1Size += c.OriginalSize + } + + tier2Size := 0 + for _, c := range tier2 { + tier2Size += c.OriginalSize + } + + if jsonOutput { + output := map[string]interface{}{ + "tier1": map[string]interface{}{ + "candidates": len(tier1), + "total_size": tier1Size, + }, + "tier2": map[string]interface{}{ + "candidates": len(tier2), + "total_size": tier2Size, + }, + } + outputJSON(output) + return + } + + fmt.Println("Compaction Statistics") + fmt.Printf("Tier 1 (30+ days closed):\n") + fmt.Printf(" Candidates: %d\n", len(tier1)) + fmt.Printf(" Total size: %d bytes\n", tier1Size) + if tier1Size > 0 { + fmt.Printf(" Estimated savings: %d bytes (70%%)\n\n", tier1Size*7/10) + } + + fmt.Printf("Tier 2 (90+ days closed, Tier 1 compacted):\n") + fmt.Printf(" Candidates: %d\n", len(tier2)) + fmt.Printf(" Total size: %d bytes\n", tier2Size) + if tier2Size > 0 { + fmt.Printf(" Estimated savings: %d bytes (95%%)\n", tier2Size*95/100) + } +} + +func runCompactRestore(ctx context.Context, store *sqlite.SQLiteStorage, issueID string) { + fmt.Fprintf(os.Stderr, "Error: --restore not yet implemented\n") + os.Exit(1) +} + +func progressBar(current, total int) string { + const width = 40 + if total == 0 { + return "[" + string(make([]byte, width)) + "]" + } + filled := (current * width) / total + bar := "" + for i := 0; i < width; i++ { + if i < filled { + bar += "█" + } else { + bar += " " + } + } + return "[" + bar + "]" +} + +func init() { + compactCmd.Flags().BoolVar(&compactDryRun, "dry-run", false, "Preview without compacting") + compactCmd.Flags().IntVar(&compactTier, "tier", 1, "Compaction tier (1 or 2)") + compactCmd.Flags().BoolVar(&compactAll, "all", false, "Process all candidates") + compactCmd.Flags().StringVar(&compactID, "id", "", "Compact specific issue") + compactCmd.Flags().BoolVar(&compactForce, "force", false, "Force compact (bypass checks, requires --id)") + compactCmd.Flags().IntVar(&compactBatch, "batch-size", 10, "Issues per batch") + compactCmd.Flags().IntVar(&compactWorkers, "workers", 5, "Parallel workers") + compactCmd.Flags().BoolVar(&compactStats, "stats", false, "Show compaction statistics") + compactCmd.Flags().StringVar(&compactRestore, "restore", "", "Restore issue from snapshot") + + rootCmd.AddCommand(compactCmd) +} diff --git a/internal/compact/compactor_test.go b/internal/compact/compactor_test.go index c8ecbd8f..80de8b7b 100644 --- a/internal/compact/compactor_test.go +++ b/internal/compact/compactor_test.go @@ -370,3 +370,72 @@ func TestCompactTier1Batch_WithAPI(t *testing.T) { } } } + +func TestMockAPI_CompactTier1(t *testing.T) { + store := setupTestStorage(t) + defer store.Close() + + issue := createClosedIssue(t, store, "test-mock") + + c, err := New(store, "", &CompactConfig{DryRun: true, Concurrency: 1}) + if err != nil { + t.Fatalf("failed to create compactor: %v", err) + } + + ctx := context.Background() + err = c.CompactTier1(ctx, issue.ID) + if err == nil || err.Error()[:8] != "dry-run:" { + t.Errorf("expected dry-run error, got: %v", err) + } +} + +func TestBatchOperations_ErrorHandling(t *testing.T) { + store := setupTestStorage(t) + defer store.Close() + + ctx := context.Background() + closedIssue := createClosedIssue(t, store, "test-closed") + openIssue := &types.Issue{ + ID: "test-open", + Title: "Open", + Description: "Open issue", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + if err := store.CreateIssue(ctx, openIssue, "test"); err != nil { + t.Fatalf("failed to create open issue: %v", err) + } + + c, err := New(store, "", &CompactConfig{DryRun: true, Concurrency: 2}) + if err != nil { + t.Fatalf("failed to create compactor: %v", err) + } + + results, err := c.CompactTier1Batch(ctx, []string{closedIssue.ID, openIssue.ID, "nonexistent"}) + if err != nil { + t.Fatalf("batch operation failed: %v", err) + } + + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + var successCount, errorCount int + for _, r := range results { + if r.Err == nil { + successCount++ + } else { + errorCount++ + } + } + + if successCount != 1 { + t.Errorf("expected 1 success, got %d", successCount) + } + if errorCount != 2 { + t.Errorf("expected 2 errors, got %d", errorCount) + } +} diff --git a/internal/storage/sqlite/compact_bench_test.go b/internal/storage/sqlite/compact_bench_test.go new file mode 100644 index 00000000..e30fa750 --- /dev/null +++ b/internal/storage/sqlite/compact_bench_test.go @@ -0,0 +1,199 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +func BenchmarkGetTier1Candidates(b *testing.B) { + store, cleanup := setupBenchDB(b) + defer cleanup() + ctx := context.Background() + + for i := 0; i < 100; i++ { + issue := &types.Issue{ + ID: generateID(b, "bd-", i), + Title: "Benchmark issue", + Description: "Test description for benchmarking", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + b.Fatalf("Failed to create issue: %v", err) + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := store.GetTier1Candidates(ctx) + if err != nil { + b.Fatalf("GetTier1Candidates failed: %v", err) + } + } +} + +func BenchmarkGetTier2Candidates(b *testing.B) { + store, cleanup := setupBenchDB(b) + defer cleanup() + ctx := context.Background() + + for i := 0; i < 50; i++ { + issue := &types.Issue{ + ID: generateID(b, "bd-", i), + Title: "Benchmark issue", + Description: "Test", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now().Add(-100 * 24 * time.Hour)), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + b.Fatalf("Failed to create issue: %v", err) + } + + _, err := store.db.ExecContext(ctx, ` + UPDATE issues + SET compaction_level = 1, + compacted_at = datetime('now', '-95 days'), + original_size = 1000 + WHERE id = ? + `, issue.ID) + if err != nil { + b.Fatalf("Failed to set compaction level: %v", err) + } + + for j := 0; j < 120; j++ { + if err := store.AddComment(ctx, issue.ID, "test", "comment"); err != nil { + b.Fatalf("Failed to add event: %v", err) + } + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := store.GetTier2Candidates(ctx) + if err != nil { + b.Fatalf("GetTier2Candidates failed: %v", err) + } + } +} + +func BenchmarkCheckEligibility(b *testing.B) { + store, cleanup := setupBenchDB(b) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Eligible", + Description: "Test", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + b.Fatalf("Failed to create issue: %v", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := store.CheckEligibility(ctx, issue.ID, 1) + if err != nil { + b.Fatalf("CheckEligibility failed: %v", err) + } + } +} + +func BenchmarkCreateSnapshot(b *testing.B) { + store, cleanup := setupBenchDB(b) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Test Issue", + Description: "Original description with substantial content", + Design: "Design notes with additional context", + Notes: "Additional notes for the issue", + AcceptanceCriteria: "Must meet all requirements", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + b.Fatalf("Failed to create issue: %v", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := store.CreateSnapshot(ctx, issue, i%5+1); err != nil { + b.Fatalf("CreateSnapshot failed: %v", err) + } + } +} + +func BenchmarkGetSnapshots(b *testing.B) { + store, cleanup := setupBenchDB(b) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Test", + Description: "Test description", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + b.Fatalf("Failed to create issue: %v", err) + } + + for i := 1; i <= 5; i++ { + if err := store.CreateSnapshot(ctx, issue, i); err != nil { + b.Fatalf("CreateSnapshot failed: %v", err) + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := store.GetSnapshots(ctx, issue.ID) + if err != nil { + b.Fatalf("GetSnapshots failed: %v", err) + } + } +} + +func generateID(b testing.TB, prefix string, n int) string { + b.Helper() + return prefix + string(rune('0'+n/10)) + string(rune('0'+n%10)) +} + +func setupBenchDB(tb testing.TB) (*SQLiteStorage, func()) { + tb.Helper() + tmpDB := tb.TempDir() + "/test.db" + store, err := New(tmpDB) + if err != nil { + tb.Fatalf("Failed to create storage: %v", err) + } + + ctx := context.Background() + if err := store.SetConfig(ctx, "compact_tier1_days", "30"); err != nil { + tb.Fatalf("Failed to set config: %v", err) + } + if err := store.SetConfig(ctx, "compact_tier1_dep_levels", "2"); err != nil { + tb.Fatalf("Failed to set config: %v", err) + } + + return store, func() { + store.Close() + } +} diff --git a/internal/storage/sqlite/compact_test.go b/internal/storage/sqlite/compact_test.go index 16d1e66e..0102386d 100644 --- a/internal/storage/sqlite/compact_test.go +++ b/internal/storage/sqlite/compact_test.go @@ -2,6 +2,8 @@ package sqlite import ( "context" + "database/sql" + "strings" "testing" "time" @@ -313,6 +315,282 @@ func TestTier1NoCircularDeps(t *testing.T) { } } +func TestCreateSnapshot(t *testing.T) { + store, cleanup := setupTestDB(t) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Test Issue", + Description: "Original description", + Design: "Design notes", + Notes: "Additional notes", + AcceptanceCriteria: "Must work", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Failed to create issue: %v", err) + } + + err := store.CreateSnapshot(ctx, issue, 1) + if err != nil { + t.Fatalf("CreateSnapshot failed: %v", err) + } + + snapshots, err := store.GetSnapshots(ctx, issue.ID) + if err != nil { + t.Fatalf("GetSnapshots failed: %v", err) + } + + if len(snapshots) != 1 { + t.Fatalf("Expected 1 snapshot, got %d", len(snapshots)) + } + + snapshot := snapshots[0] + if snapshot.Description != issue.Description { + t.Errorf("Expected description %q, got %q", issue.Description, snapshot.Description) + } + if snapshot.Design != issue.Design { + t.Errorf("Expected design %q, got %q", issue.Design, snapshot.Design) + } + if snapshot.Notes != issue.Notes { + t.Errorf("Expected notes %q, got %q", issue.Notes, snapshot.Notes) + } + if snapshot.AcceptanceCriteria != issue.AcceptanceCriteria { + t.Errorf("Expected criteria %q, got %q", issue.AcceptanceCriteria, snapshot.AcceptanceCriteria) + } +} + +func TestCreateSnapshotUTF8(t *testing.T) { + store, cleanup := setupTestDB(t) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "UTF-8 Test 🎉", + Description: "Café, résumé, 日本語, emoji 🚀", + Design: "Design with 中文 and émojis 🔥", + Notes: "Notes: ñ, ü, é, à", + AcceptanceCriteria: "Must handle UTF-8 correctly ✅", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Failed to create issue: %v", err) + } + + err := store.CreateSnapshot(ctx, issue, 1) + if err != nil { + t.Fatalf("CreateSnapshot failed: %v", err) + } + + snapshots, err := store.GetSnapshots(ctx, issue.ID) + if err != nil { + t.Fatalf("GetSnapshots failed: %v", err) + } + + if len(snapshots) != 1 { + t.Fatalf("Expected 1 snapshot, got %d", len(snapshots)) + } + + snapshot := snapshots[0] + if snapshot.Title != issue.Title { + t.Errorf("UTF-8 title not preserved: expected %q, got %q", issue.Title, snapshot.Title) + } + if snapshot.Description != issue.Description { + t.Errorf("UTF-8 description not preserved: expected %q, got %q", issue.Description, snapshot.Description) + } + if snapshot.Design != issue.Design { + t.Errorf("UTF-8 design not preserved: expected %q, got %q", issue.Design, snapshot.Design) + } +} + +func TestCreateMultipleSnapshots(t *testing.T) { + store, cleanup := setupTestDB(t) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Test Issue", + Description: "Original", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Failed to create issue: %v", err) + } + + if err := store.CreateSnapshot(ctx, issue, 1); err != nil { + t.Fatalf("CreateSnapshot level 1 failed: %v", err) + } + + issue.Description = "Compacted once" + if err := store.CreateSnapshot(ctx, issue, 2); err != nil { + t.Fatalf("CreateSnapshot level 2 failed: %v", err) + } + + snapshots, err := store.GetSnapshots(ctx, issue.ID) + if err != nil { + t.Fatalf("GetSnapshots failed: %v", err) + } + + if len(snapshots) != 2 { + t.Fatalf("Expected 2 snapshots, got %d", len(snapshots)) + } + + if snapshots[0].CompactionLevel != 1 { + t.Errorf("Expected first snapshot level 1, got %d", snapshots[0].CompactionLevel) + } + if snapshots[1].CompactionLevel != 2 { + t.Errorf("Expected second snapshot level 2, got %d", snapshots[1].CompactionLevel) + } +} + +func TestRestoreFromSnapshot(t *testing.T) { + store, cleanup := setupTestDB(t) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Original Title", + Description: "Original description", + Design: "Original design", + Notes: "Original notes", + AcceptanceCriteria: "Original criteria", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Failed to create issue: %v", err) + } + + if err := store.CreateSnapshot(ctx, issue, 1); err != nil { + t.Fatalf("CreateSnapshot failed: %v", err) + } + + _, err := store.db.ExecContext(ctx, ` + UPDATE issues + SET description = 'Compacted', + design = '', + notes = '', + acceptance_criteria = '', + compaction_level = 1 + WHERE id = ? + `, issue.ID) + if err != nil { + t.Fatalf("Failed to update issue: %v", err) + } + + err = store.RestoreFromSnapshot(ctx, issue.ID, 1) + if err != nil { + t.Fatalf("RestoreFromSnapshot failed: %v", err) + } + + restored, err := store.GetIssue(ctx, issue.ID) + if err != nil { + t.Fatalf("GetIssue failed: %v", err) + } + + if restored.Description != issue.Description { + t.Errorf("Description not restored: expected %q, got %q", issue.Description, restored.Description) + } + if restored.Design != issue.Design { + t.Errorf("Design not restored: expected %q, got %q", issue.Design, restored.Design) + } + if restored.Notes != issue.Notes { + t.Errorf("Notes not restored: expected %q, got %q", issue.Notes, restored.Notes) + } + if restored.AcceptanceCriteria != issue.AcceptanceCriteria { + t.Errorf("Criteria not restored: expected %q, got %q", issue.AcceptanceCriteria, restored.AcceptanceCriteria) + } +} + +func TestRestoreSnapshotNoSnapshot(t *testing.T) { + store, cleanup := setupTestDB(t) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Test", + Description: "Test", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Failed to create issue: %v", err) + } + + err := store.RestoreFromSnapshot(ctx, issue.ID, 1) + if err == nil { + t.Fatal("Expected error when no snapshot exists") + } + if !strings.Contains(err.Error(), "no snapshot found") { + t.Errorf("Expected 'no snapshot found' error, got: %v", err) + } +} + +func TestApplyCompaction(t *testing.T) { + store, cleanup := setupTestDB(t) + defer cleanup() + ctx := context.Background() + + issue := &types.Issue{ + ID: "bd-1", + Title: "Test", + Description: "Original description that is quite long", + Status: "closed", + Priority: 2, + IssueType: "task", + ClosedAt: timePtr(time.Now()), + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("Failed to create issue: %v", err) + } + + originalSize := len(issue.Description) + err := store.ApplyCompaction(ctx, issue.ID, 1, originalSize) + if err != nil { + t.Fatalf("ApplyCompaction failed: %v", err) + } + + var compactionLevel int + var compactedAt sql.NullTime + var storedSize int + err = store.db.QueryRowContext(ctx, ` + SELECT COALESCE(compaction_level, 0), compacted_at, COALESCE(original_size, 0) + FROM issues WHERE id = ? + `, issue.ID).Scan(&compactionLevel, &compactedAt, &storedSize) + if err != nil { + t.Fatalf("Failed to query issue: %v", err) + } + + if compactionLevel != 1 { + t.Errorf("Expected compaction_level 1, got %d", compactionLevel) + } + if !compactedAt.Valid { + t.Error("Expected compacted_at to be set") + } + if storedSize != originalSize { + t.Errorf("Expected original_size %d, got %d", originalSize, storedSize) + } +} + func timePtr(t time.Time) *time.Time { return &t }