Add compaction schema and candidate identification
- Added compaction columns to issues table (compaction_level, compacted_at, original_size) - Created issue_snapshots table for snapshot storage before compaction - Added compaction configuration with opt-in flag (compaction_enabled=false by default) - Implemented GetTier1Candidates and GetTier2Candidates queries - Added CheckEligibility validation function - Comprehensive tests for all compaction queries - Idempotent migrations for existing databases Closes bd-252, bd-253, bd-254 Amp-Thread-ID: https://ampcode.com/threads/T-c4d7acd1-c161-4b80-9d80-a0691e8fa87b Co-authored-by: Amp <amp@ampcode.com>
This commit is contained in:
Steve Yegge
@@ -166,24 +166,24 @@
|
||||
{"id":"bd-249","title":"Test reopen command","description":"","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-15T16:28:49.924381-07:00","updated_at":"2025-10-15T16:28:55.491141-07:00","closed_at":"2025-10-15T16:28:55.491141-07:00"}
|
||||
{"id":"bd-25","title":"Add transaction support to storage layer for atomic multi-operation workflows","description":"Currently each storage method (CreateIssue, UpdateIssue, etc.) starts its own transaction. This makes it impossible to perform atomic multi-step operations like collision resolution. Add support for passing *sql.Tx through the storage interface, or create transaction-aware versions of methods. This would make remapCollisions and other batch operations truly atomic.","status":"closed","priority":4,"issue_type":"feature","created_at":"2025-10-14T14:43:06.910892-07:00","updated_at":"2025-10-15T16:27:22.001363-07:00","closed_at":"2025-10-15T03:01:29.570206-07:00"}
|
||||
{"id":"bd-250","title":"Implement --format flag for bd list (from PR #46)","description":"PR #46 by tmc adds --format flag with Go template support for bd list, including presets for 'digraph' and 'dot' (Graphviz) output with status-based color coding. Unfortunately the PR is based on old main and would delete labels, reopen, and storage tests. Need to reimplement the feature atop current main.\n\nFeatures to implement:\n- --format flag for bd list\n- 'digraph' preset: basic 'from to' format for golang.org/x/tools/cmd/digraph\n- 'dot' preset: Graphviz compatible output with color-coded statuses\n- Custom Go template support with vars: IssueID, DependsOnID, Type, Issue, Dependency\n- Status-based colors: open=white, in_progress=lightyellow, blocked=lightcoral, closed=lightgray\n\nExamples:\n- bd list --format=digraph | digraph nodes\n- bd list --format=dot | dot -Tsvg -o deps.svg\n- bd list --format='{{.IssueID}} -\u003e {{.DependsOnID}} [{{.Type}}]'\n\nOriginal PR: https://github.com/steveyegge/beads/pull/46","status":"open","priority":2,"issue_type":"feature","created_at":"2025-10-15T21:13:11.6698-07:00","updated_at":"2025-10-15T21:13:11.6698-07:00","external_ref":"gh-46"}
|
||||
{"id":"bd-251","title":"Epic: Add intelligent database compaction with Claude Haiku","description":"Implement multi-tier database compaction using Claude Haiku to semantically compress old, closed issues. This keeps the database lightweight and agent-friendly while preserving essential context.\n\nGoals:\n- 70-95% space reduction for eligible issues\n- Full restore capability via snapshots\n- Opt-in with dry-run safety\n- ~$1 per 1,000 issues compacted","acceptance_criteria":"- Schema migration with snapshots table\n- Haiku integration for summarization\n- Two-tier compaction (30d, 90d)\n- CLI with dry-run, restore, stats\n- Full test coverage\n- Documentation complete","status":"open","priority":2,"issue_type":"epic","created_at":"2025-10-15T21:51:23.210339-07:00","updated_at":"2025-10-15T21:51:23.210339-07:00","labels":["---","compaction","epic","haiku","v1.1"]}
|
||||
{"id":"bd-252","title":"Add compaction schema and migrations","description":"Add database schema support for issue compaction tracking and snapshot storage.","design":"Add three columns to `issues` table:\n- `compaction_level INTEGER DEFAULT 0` - 0=original, 1=tier1, 2=tier2\n- `compacted_at DATETIME` - when last compacted\n- `original_size INTEGER` - bytes before first compaction\n\nCreate `issue_snapshots` table:\n```sql\nCREATE TABLE issue_snapshots (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n issue_id TEXT NOT NULL,\n snapshot_time DATETIME NOT NULL,\n compaction_level INTEGER NOT NULL,\n original_size INTEGER NOT NULL,\n compressed_size INTEGER NOT NULL,\n original_content TEXT NOT NULL, -- JSON blob\n archived_events TEXT,\n FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE\n);\n```\n\nAdd indexes:\n- `idx_snapshots_issue` on `issue_id`\n- `idx_snapshots_level` on `compaction_level`\n\nAdd migration functions in `internal/storage/sqlite/sqlite.go`:\n- `migrateCompactionColumns(db *sql.DB) error`\n- `migrateSnapshotsTable(db *sql.DB) error`","acceptance_criteria":"- Existing databases migrate automatically\n- New databases include columns by default\n- Migration is idempotent (safe to run multiple times)\n- No data loss during migration\n- Tests verify migration on fresh and existing DBs","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.216371-07:00","updated_at":"2025-10-15T21:51:23.216371-07:00","labels":["---","compaction","database","migration","schema"]}
|
||||
{"id":"bd-253","title":"Add compaction configuration keys","description":"Add configuration keys for compaction behavior with sensible defaults.","design":"Add to `internal/storage/sqlite/schema.go` initial config:\n```sql\nINSERT OR IGNORE INTO config (key, value) VALUES\n ('compact_tier1_days', '30'),\n ('compact_tier1_dep_levels', '2'),\n ('compact_tier2_days', '90'),\n ('compact_tier2_dep_levels', '5'),\n ('compact_tier2_commits', '100'),\n ('compact_model', 'claude-3-5-haiku-20241022'),\n ('compact_batch_size', '50'),\n ('compact_parallel_workers', '5'),\n ('auto_compact_enabled', 'false');\n```\n\nAdd helper functions for loading config into typed struct.","acceptance_criteria":"- Config keys created on init\n- Existing DBs get defaults on migration\n- `bd config get/set` works with all keys\n- Type validation (days=int, enabled=bool)\n- Documentation in README.md","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.22391-07:00","updated_at":"2025-10-15T21:51:23.22391-07:00","labels":["---","compaction","config","configuration"]}
|
||||
{"id":"bd-254","title":"Implement candidate identification queries","description":"Write SQL queries to identify issues eligible for Tier 1 and Tier 2 compaction based on closure time and dependency status.","design":"Create `internal/storage/sqlite/compact.go` with:\n\n```go\ntype CompactionCandidate struct {\n IssueID string\n ClosedAt time.Time\n OriginalSize int\n EstimatedSize int\n DependentCount int\n}\n\nfunc (s *SQLiteStorage) GetTier1Candidates(ctx context.Context) ([]*CompactionCandidate, error)\nfunc (s *SQLiteStorage) GetTier2Candidates(ctx context.Context) ([]*CompactionCandidate, error)\nfunc (s *SQLiteStorage) CheckEligibility(ctx context.Context, issueID string, tier int) (bool, string, error)\n```\n\nUse recursive CTE for dependency depth checking (similar to ready_issues view).","acceptance_criteria":"- Tier 1 query filters by days and dependency depth\n- Tier 2 query includes commit/issue count checks\n- Dependency checking handles circular deps gracefully\n- Performance: \u003c100ms for 10,000 issue database\n- Tests cover edge cases (no deps, circular deps, mixed status)","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.225835-07:00","updated_at":"2025-10-15T21:51:23.225835-07:00","labels":["---","compaction","dependencies","query","sql"]}
|
||||
{"id":"bd-255","title":"Create Haiku client and prompt templates","description":"Implement Claude Haiku API client with template-based prompts for Tier 1 and Tier 2 summarization.","design":"Create `internal/compact/haiku.go`:\n\n```go\ntype HaikuClient struct {\n client *anthropic.Client\n model string\n}\n\nfunc NewHaikuClient(apiKey string) (*HaikuClient, error)\nfunc (h *HaikuClient) SummarizeTier1(ctx context.Context, issue *types.Issue) (string, error)\nfunc (h *HaikuClient) SummarizeTier2(ctx context.Context, issue *types.Issue) (string, error)\n```\n\nUse text/template for prompt rendering.\n\nTier 1 output format:\n```\n**Summary:** [2-3 sentences]\n**Key Decisions:** [bullet points]\n**Resolution:** [outcome]\n```\n\nTier 2 output format:\n```\nSingle paragraph ≤150 words covering what was built, why it mattered, lasting impact.\n```","acceptance_criteria":"- API key from env var or config (env takes precedence)\n- Prompts render correctly with templates\n- Rate limiting handled gracefully (exponential backoff)\n- Network errors retry up to 3 times\n- Mock tests for API calls","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.229702-07:00","updated_at":"2025-10-15T21:51:23.229702-07:00","labels":["---","api","compaction","haiku","llm"]}
|
||||
{"id":"bd-256","title":"Implement snapshot creation and restoration","description":"Implement snapshot creation before compaction and restoration capability to undo compaction.","design":"Add to `internal/storage/sqlite/compact.go`:\n\n```go\nfunc (s *SQLiteStorage) CreateSnapshot(ctx context.Context, issue *types.Issue, level int) error\nfunc (s *SQLiteStorage) RestoreFromSnapshot(ctx context.Context, issueID string, level int) error\nfunc (s *SQLiteStorage) GetSnapshots(ctx context.Context, issueID string) ([]*Snapshot, error)\n```\n\nSnapshot JSON structure:\n```json\n{\n \"description\": \"...\",\n \"design\": \"...\",\n \"notes\": \"...\",\n \"acceptance_criteria\": \"...\",\n \"title\": \"...\"\n}\n```","acceptance_criteria":"- Snapshot created atomically with compaction\n- Restore returns exact original content\n- Multiple snapshots per issue supported (Tier 1 → Tier 2)\n- JSON encoding handles UTF-8 and special characters\n- Size calculation is accurate (UTF-8 bytes)","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.231906-07:00","updated_at":"2025-10-15T21:51:23.231906-07:00","labels":["---","compaction","restore","safety","snapshot"]}
|
||||
{"id":"bd-257","title":"Implement Tier 1 compaction logic","description":"Implement the core Tier 1 compaction process: snapshot → summarize → update.","design":"Add to `internal/compact/compactor.go`:\n\n```go\ntype Compactor struct {\n store storage.Storage\n haiku *HaikuClient\n config *CompactConfig\n}\n\nfunc New(store storage.Storage, apiKey string, config *CompactConfig) (*Compactor, error)\nfunc (c *Compactor) CompactTier1(ctx context.Context, issueID string) error\nfunc (c *Compactor) CompactTier1Batch(ctx context.Context, issueIDs []string) error\n```\n\nProcess:\n1. Verify eligibility\n2. Calculate original size\n3. Create snapshot\n4. Call Haiku for summary\n5. Update issue (description=summary, clear design/notes/criteria)\n6. Set compaction_level=1, compacted_at=now, original_size\n7. Record EventCompacted\n8. Mark dirty for export","acceptance_criteria":"- Single issue compaction works end-to-end\n- Batch processing with parallel workers (5 concurrent)\n- Errors don't corrupt database (transaction rollback)\n- EventCompacted includes size savings\n- Dry-run mode (identify + size estimate only, no API calls)","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.23391-07:00","updated_at":"2025-10-15T21:51:23.23391-07:00","labels":["---","compaction","core-logic","tier1"]}
|
||||
{"id":"bd-258","title":"Implement Tier 2 compaction logic","description":"Implement Tier 2 ultra-compression: more aggressive summarization and optional event pruning.","design":"Add to `internal/compact/compactor.go`:\n\n```go\nfunc (c *Compactor) CompactTier2(ctx context.Context, issueID string) error\nfunc (c *Compactor) CompactTier2Batch(ctx context.Context, issueIDs []string) error\n```\n\nProcess:\n1. Verify issue is at compaction_level = 1\n2. Check Tier 2 eligibility (days, deps, commits/issues)\n3. Create Tier 2 snapshot\n4. Call Haiku with ultra-compression prompt\n5. Update issue (description = single paragraph, clear all other fields)\n6. Set compaction_level = 2\n7. Optionally prune events (keep created/closed, archive rest to snapshot)","acceptance_criteria":"- Requires existing Tier 1 compaction\n- Git commit counting works (with fallback to issue counter)\n- Events optionally pruned (config: compact_events_enabled)\n- Archived events stored in snapshot JSON\n- Size reduction 90-95%","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.23586-07:00","updated_at":"2025-10-15T21:51:23.23586-07:00","labels":["---","advanced","compaction","tier2"]}
|
||||
{"id":"bd-259","title":"Add `bd compact` CLI command","description":"Implement the `bd compact` command with dry-run, batch processing, and progress reporting.","design":"Create `cmd/bd/compact.go`:\n\n```go\nvar compactCmd = \u0026cobra.Command{\n Use: \"compact\",\n Short: \"Compact old closed issues to save space\",\n}\n\nFlags:\n --dry-run Preview without compacting\n --tier int Compaction tier (1 or 2, default: 1)\n --all Process all candidates\n --id string Compact specific issue\n --force Force compact (bypass checks, requires --id)\n --batch-size int Issues per batch\n --workers int Parallel workers\n --json JSON output\n```","acceptance_criteria":"- `--dry-run` shows accurate preview with size estimates\n- `--all` processes all candidates\n- `--id` compacts single issue\n- `--force` bypasses eligibility checks (only with --id)\n- Progress bar for batches (e.g., [████████] 47/47)\n- JSON output with `--json`\n- Exit codes: 0=success, 1=error\n- Shows summary: count, size saved, cost, time","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.238373-07:00","updated_at":"2025-10-15T21:51:23.238373-07:00","labels":["---","cli","command","compaction"]}
|
||||
{"id":"bd-251","title":"Epic: Add intelligent database compaction with Claude Haiku","description":"Implement multi-tier database compaction using Claude Haiku to semantically compress old, closed issues. This keeps the database lightweight and agent-friendly while preserving essential context.\n\nGoals:\n- 70-95% space reduction for eligible issues\n- Full restore capability via snapshots\n- Opt-in with dry-run safety\n- ~$1 per 1,000 issues compacted","acceptance_criteria":"- Schema migration with snapshots table\n- Haiku integration for summarization\n- Two-tier compaction (30d, 90d)\n- CLI with dry-run, restore, stats\n- Full test coverage\n- Documentation complete","status":"open","priority":2,"issue_type":"epic","created_at":"2025-10-15T21:51:23.210339-07:00","updated_at":"2025-10-15T21:51:23.210339-07:00"}
|
||||
{"id":"bd-252","title":"Add compaction schema and migrations","description":"Add database schema support for issue compaction tracking and snapshot storage.","design":"Add three columns to `issues` table:\n- `compaction_level INTEGER DEFAULT 0` - 0=original, 1=tier1, 2=tier2\n- `compacted_at DATETIME` - when last compacted\n- `original_size INTEGER` - bytes before first compaction\n\nCreate `issue_snapshots` table:\n```sql\nCREATE TABLE issue_snapshots (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n issue_id TEXT NOT NULL,\n snapshot_time DATETIME NOT NULL,\n compaction_level INTEGER NOT NULL,\n original_size INTEGER NOT NULL,\n compressed_size INTEGER NOT NULL,\n original_content TEXT NOT NULL, -- JSON blob\n archived_events TEXT,\n FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE\n);\n```\n\nAdd indexes:\n- `idx_snapshots_issue` on `issue_id`\n- `idx_snapshots_level` on `compaction_level`\n\nAdd migration functions in `internal/storage/sqlite/sqlite.go`:\n- `migrateCompactionColumns(db *sql.DB) error`\n- `migrateSnapshotsTable(db *sql.DB) error`","acceptance_criteria":"- Existing databases migrate automatically\n- New databases include columns by default\n- Migration is idempotent (safe to run multiple times)\n- No data loss during migration\n- Tests verify migration on fresh and existing DBs","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.216371-07:00","updated_at":"2025-10-15T22:02:27.638283-07:00","closed_at":"2025-10-15T22:02:27.638283-07:00"}
|
||||
{"id":"bd-253","title":"Add compaction configuration keys","description":"Add configuration keys for compaction behavior with sensible defaults.","design":"Add to `internal/storage/sqlite/schema.go` initial config:\n```sql\nINSERT OR IGNORE INTO config (key, value) VALUES\n ('compact_tier1_days', '30'),\n ('compact_tier1_dep_levels', '2'),\n ('compact_tier2_days', '90'),\n ('compact_tier2_dep_levels', '5'),\n ('compact_tier2_commits', '100'),\n ('compact_model', 'claude-3-5-haiku-20241022'),\n ('compact_batch_size', '50'),\n ('compact_parallel_workers', '5'),\n ('auto_compact_enabled', 'false');\n```\n\nAdd helper functions for loading config into typed struct.","acceptance_criteria":"- Config keys created on init\n- Existing DBs get defaults on migration\n- `bd config get/set` works with all keys\n- Type validation (days=int, enabled=bool)\n- Documentation in README.md","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.22391-07:00","updated_at":"2025-10-15T22:08:44.984927-07:00","closed_at":"2025-10-15T22:08:44.984927-07:00"}
|
||||
{"id":"bd-254","title":"Implement candidate identification queries","description":"Write SQL queries to identify issues eligible for Tier 1 and Tier 2 compaction based on closure time and dependency status.","design":"Create `internal/storage/sqlite/compact.go` with:\n\n```go\ntype CompactionCandidate struct {\n IssueID string\n ClosedAt time.Time\n OriginalSize int\n EstimatedSize int\n DependentCount int\n}\n\nfunc (s *SQLiteStorage) GetTier1Candidates(ctx context.Context) ([]*CompactionCandidate, error)\nfunc (s *SQLiteStorage) GetTier2Candidates(ctx context.Context) ([]*CompactionCandidate, error)\nfunc (s *SQLiteStorage) CheckEligibility(ctx context.Context, issueID string, tier int) (bool, string, error)\n```\n\nUse recursive CTE for dependency depth checking (similar to ready_issues view).","acceptance_criteria":"- Tier 1 query filters by days and dependency depth\n- Tier 2 query includes commit/issue count checks\n- Dependency checking handles circular deps gracefully\n- Performance: \u003c100ms for 10,000 issue database\n- Tests cover edge cases (no deps, circular deps, mixed status)","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.225835-07:00","updated_at":"2025-10-15T22:16:45.517562-07:00","closed_at":"2025-10-15T22:16:45.517562-07:00"}
|
||||
{"id":"bd-255","title":"Create Haiku client and prompt templates","description":"Implement Claude Haiku API client with template-based prompts for Tier 1 and Tier 2 summarization.","design":"Create `internal/compact/haiku.go`:\n\n```go\ntype HaikuClient struct {\n client *anthropic.Client\n model string\n}\n\nfunc NewHaikuClient(apiKey string) (*HaikuClient, error)\nfunc (h *HaikuClient) SummarizeTier1(ctx context.Context, issue *types.Issue) (string, error)\nfunc (h *HaikuClient) SummarizeTier2(ctx context.Context, issue *types.Issue) (string, error)\n```\n\nUse text/template for prompt rendering.\n\nTier 1 output format:\n```\n**Summary:** [2-3 sentences]\n**Key Decisions:** [bullet points]\n**Resolution:** [outcome]\n```\n\nTier 2 output format:\n```\nSingle paragraph ≤150 words covering what was built, why it mattered, lasting impact.\n```","acceptance_criteria":"- API key from env var or config (env takes precedence)\n- Prompts render correctly with templates\n- Rate limiting handled gracefully (exponential backoff)\n- Network errors retry up to 3 times\n- Mock tests for API calls","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.229702-07:00","updated_at":"2025-10-15T21:51:23.229702-07:00"}
|
||||
{"id":"bd-256","title":"Implement snapshot creation and restoration","description":"Implement snapshot creation before compaction and restoration capability to undo compaction.","design":"Add to `internal/storage/sqlite/compact.go`:\n\n```go\nfunc (s *SQLiteStorage) CreateSnapshot(ctx context.Context, issue *types.Issue, level int) error\nfunc (s *SQLiteStorage) RestoreFromSnapshot(ctx context.Context, issueID string, level int) error\nfunc (s *SQLiteStorage) GetSnapshots(ctx context.Context, issueID string) ([]*Snapshot, error)\n```\n\nSnapshot JSON structure:\n```json\n{\n \"description\": \"...\",\n \"design\": \"...\",\n \"notes\": \"...\",\n \"acceptance_criteria\": \"...\",\n \"title\": \"...\"\n}\n```","acceptance_criteria":"- Snapshot created atomically with compaction\n- Restore returns exact original content\n- Multiple snapshots per issue supported (Tier 1 → Tier 2)\n- JSON encoding handles UTF-8 and special characters\n- Size calculation is accurate (UTF-8 bytes)","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.231906-07:00","updated_at":"2025-10-15T21:51:23.231906-07:00"}
|
||||
{"id":"bd-257","title":"Implement Tier 1 compaction logic","description":"Implement the core Tier 1 compaction process: snapshot → summarize → update.","design":"Add to `internal/compact/compactor.go`:\n\n```go\ntype Compactor struct {\n store storage.Storage\n haiku *HaikuClient\n config *CompactConfig\n}\n\nfunc New(store storage.Storage, apiKey string, config *CompactConfig) (*Compactor, error)\nfunc (c *Compactor) CompactTier1(ctx context.Context, issueID string) error\nfunc (c *Compactor) CompactTier1Batch(ctx context.Context, issueIDs []string) error\n```\n\nProcess:\n1. Verify eligibility\n2. Calculate original size\n3. Create snapshot\n4. Call Haiku for summary\n5. Update issue (description=summary, clear design/notes/criteria)\n6. Set compaction_level=1, compacted_at=now, original_size\n7. Record EventCompacted\n8. Mark dirty for export","acceptance_criteria":"- Single issue compaction works end-to-end\n- Batch processing with parallel workers (5 concurrent)\n- Errors don't corrupt database (transaction rollback)\n- EventCompacted includes size savings\n- Dry-run mode (identify + size estimate only, no API calls)","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.23391-07:00","updated_at":"2025-10-15T21:51:23.23391-07:00"}
|
||||
{"id":"bd-258","title":"Implement Tier 2 compaction logic","description":"Implement Tier 2 ultra-compression: more aggressive summarization and optional event pruning.","design":"Add to `internal/compact/compactor.go`:\n\n```go\nfunc (c *Compactor) CompactTier2(ctx context.Context, issueID string) error\nfunc (c *Compactor) CompactTier2Batch(ctx context.Context, issueIDs []string) error\n```\n\nProcess:\n1. Verify issue is at compaction_level = 1\n2. Check Tier 2 eligibility (days, deps, commits/issues)\n3. Create Tier 2 snapshot\n4. Call Haiku with ultra-compression prompt\n5. Update issue (description = single paragraph, clear all other fields)\n6. Set compaction_level = 2\n7. Optionally prune events (keep created/closed, archive rest to snapshot)","acceptance_criteria":"- Requires existing Tier 1 compaction\n- Git commit counting works (with fallback to issue counter)\n- Events optionally pruned (config: compact_events_enabled)\n- Archived events stored in snapshot JSON\n- Size reduction 90-95%","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.23586-07:00","updated_at":"2025-10-15T21:51:23.23586-07:00"}
|
||||
{"id":"bd-259","title":"Add `bd compact` CLI command","description":"Implement the `bd compact` command with dry-run, batch processing, and progress reporting.","design":"Create `cmd/bd/compact.go`:\n\n```go\nvar compactCmd = \u0026cobra.Command{\n Use: \"compact\",\n Short: \"Compact old closed issues to save space\",\n}\n\nFlags:\n --dry-run Preview without compacting\n --tier int Compaction tier (1 or 2, default: 1)\n --all Process all candidates\n --id string Compact specific issue\n --force Force compact (bypass checks, requires --id)\n --batch-size int Issues per batch\n --workers int Parallel workers\n --json JSON output\n```","acceptance_criteria":"- `--dry-run` shows accurate preview with size estimates\n- `--all` processes all candidates\n- `--id` compacts single issue\n- `--force` bypasses eligibility checks (only with --id)\n- Progress bar for batches (e.g., [████████] 47/47)\n- JSON output with `--json`\n- Exit codes: 0=success, 1=error\n- Shows summary: count, size saved, cost, time","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.238373-07:00","updated_at":"2025-10-15T21:51:23.238373-07:00"}
|
||||
{"id":"bd-26","title":"Optimize reference updates to avoid loading all issues into memory","description":"In updateReferences(), we call SearchIssues with no filter to get ALL issues for updating references. For large databases (10k+ issues), this loads everything into memory. Options: 1) Use batched processing with LIMIT/OFFSET, 2) Use SQL UPDATE with REPLACE() directly, 3) Stream results instead of loading all at once. Located in collision.go:266","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-14T14:43:06.911497-07:00","updated_at":"2025-10-15T16:27:22.001829-07:00"}
|
||||
{"id":"bd-260","title":"Add `bd compact --restore` functionality","description":"Implement restore command to undo compaction from snapshots.","design":"Add to `cmd/bd/compact.go`:\n\n```go\nvar compactRestore string\n\ncompactCmd.Flags().StringVar(\u0026compactRestore, \"restore\", \"\", \"Restore issue from snapshot\")\n```\n\nProcess:\n1. Load snapshot for issue\n2. Parse JSON content\n3. Update issue with original content\n4. Set compaction_level = 0, compacted_at = NULL, original_size = NULL\n5. Record event (EventRestored or EventUpdated)\n6. Mark dirty for export","acceptance_criteria":"- Restores exact original content\n- Handles multiple snapshots (use latest by default)\n- `--level` flag to choose specific snapshot\n- Updates compaction_level correctly\n- Exports restored content to JSONL\n- Shows before/after in output","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.240267-07:00","updated_at":"2025-10-15T21:51:23.240267-07:00","labels":["---","cli","compaction","restore"]}
|
||||
{"id":"bd-261","title":"Add `bd compact --stats` command","description":"Add statistics command showing compaction status and potential savings.","design":"```go\nvar compactStats bool\n\ncompactCmd.Flags().BoolVar(\u0026compactStats, \"stats\", false, \"Show compaction statistics\")\n```\n\nOutput:\n- Total issues, by compaction level (0, 1, 2)\n- Current DB size vs estimated uncompacted size\n- Space savings (KB/MB and %)\n- Candidates for each tier with size estimates\n- Estimated API cost (Haiku pricing)","acceptance_criteria":"- Accurate counts by compaction_level\n- Size calculations include all text fields (UTF-8 bytes)\n- Shows candidates with eligibility reasons\n- Cost estimation based on current Haiku pricing\n- JSON output supported\n- Clear, readable table format","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.242041-07:00","updated_at":"2025-10-15T21:51:23.242041-07:00","labels":["---","compaction","reporting","stats"]}
|
||||
{"id":"bd-262","title":"Add EventCompacted to event system","description":"Add new event type for tracking compaction in audit trail.","design":"1. Add to `internal/types/types.go`:\n```go\nconst EventCompacted EventType = \"compacted\"\n```\n\n2. Record event during compaction:\n```go\neventData := map[string]interface{}{\n \"tier\": tier,\n \"original_size\": originalSize,\n \"compressed_size\": compressedSize,\n \"reduction_pct\": (1 - float64(compressedSize)/float64(originalSize)) * 100,\n}\n```\n\n3. Update event display in `bd show`.","acceptance_criteria":"- Event includes tier, original_size, compressed_size, reduction_pct\n- Shows in event history (`bd events \u003cid\u003e`)\n- Exports to JSONL correctly\n- `bd show` displays compaction status and marker","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.244219-07:00","updated_at":"2025-10-15T21:51:23.244219-07:00","labels":["---","audit","compaction","events"]}
|
||||
{"id":"bd-263","title":"Add compaction indicator to `bd show`","description":"Update `bd show` command to display compaction status prominently.","design":"Add to issue display:\n```\nbd-42: Fix authentication bug [CLOSED] 🗜️\n\nStatus: closed (compacted L1)\n...\n\n---\n💾 Restore: bd compact --restore bd-42\n📊 Original: 2,341 bytes | Compressed: 468 bytes (80% reduction)\n🗜️ Compacted: 2025-10-15 (Tier 1)\n```\n\nEmoji indicators:\n- Tier 1: 🗜️\n- Tier 2: 📦","acceptance_criteria":"- Compaction status visible in title line\n- Footer shows size savings when compacted\n- Restore command shown for compacted issues\n- Works with `--json` output (includes compaction fields)\n- Emoji optional (controlled by config or terminal detection)","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.253091-07:00","updated_at":"2025-10-15T21:51:23.253091-07:00","labels":["---","compaction","display","ui"]}
|
||||
{"id":"bd-264","title":"Write compaction tests","description":"Comprehensive test suite for compaction functionality.","design":"Test coverage:\n\n1. **Candidate Identification:**\n - Eligibility by time\n - Dependency depth checking\n - Mixed status dependents\n - Edge cases (no deps, circular deps)\n\n2. **Snapshots:**\n - Create and restore\n - Multiple snapshots per issue\n - Content integrity (UTF-8, special chars)\n\n3. **Tier 1 Compaction:**\n - Single issue compaction\n - Batch processing\n - Error handling (API failures)\n\n4. **Tier 2 Compaction:**\n - Requires Tier 1\n - Events pruning\n - Commit counting fallback\n\n5. **CLI:**\n - All flag combinations\n - Dry-run accuracy\n - JSON output parsing\n\n6. **Integration:**\n - End-to-end flow\n - JSONL export/import\n - Restore verification","acceptance_criteria":"- Test coverage \u003e80%\n- All edge cases covered\n- Mock Haiku API in tests (no real API calls)\n- Integration tests pass\n- `go test ./...` passes\n- Benchmarks for performance-critical paths","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.262504-07:00","updated_at":"2025-10-15T21:51:23.262504-07:00","labels":["---","compaction","quality","testing"]}
|
||||
{"id":"bd-265","title":"Add compaction documentation","description":"Document compaction feature in README and create detailed COMPACTION.md guide.","design":"**Update README.md:**\n- Add to Features section\n- CLI examples (dry-run, compact, restore, stats)\n- Configuration guide\n- Cost analysis\n\n**Create COMPACTION.md:**\n- How compaction works (architecture overview)\n- When to use each tier\n- Detailed cost analysis with examples\n- Safety mechanisms (snapshots, restore, dry-run)\n- Troubleshooting guide\n- FAQ\n\n**Create examples/compaction/:**\n- `workflow.sh` - Example monthly compaction workflow\n- `cron-compact.sh` - Cron job setup\n- `auto-compact.sh` - Auto-compaction script","acceptance_criteria":"- README.md updated with compaction section\n- COMPACTION.md comprehensive and clear\n- Examples work as documented (tested)\n- Screenshots or ASCII examples included\n- API key setup documented (env var vs config)\n- Covers common questions and issues","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.265589-07:00","updated_at":"2025-10-15T21:51:23.265589-07:00","labels":["---","compaction","docs","documentation","examples"]}
|
||||
{"id":"bd-266","title":"Optional: Implement auto-compaction","description":"Implement automatic compaction triggered by certain operations when enabled via config.","design":"Trigger points (when `auto_compact_enabled = true`):\n1. `bd stats` - check and compact if candidates exist\n2. `bd export` - before exporting\n3. Configurable: on any read operation after N candidates accumulate\n\nAdd:\n```go\nfunc (s *SQLiteStorage) AutoCompact(ctx context.Context) error {\n enabled, _ := s.GetConfig(ctx, \"auto_compact_enabled\")\n if enabled != \"true\" {\n return nil\n }\n\n // Run Tier 1 compaction on all candidates\n // Limit to batch_size to avoid long operations\n // Log activity for transparency\n}\n```","acceptance_criteria":"- Respects auto_compact_enabled config (default: false)\n- Limits batch size to avoid blocking operations\n- Logs compaction activity (visible with --verbose)\n- Can be disabled per-command with `--no-auto-compact` flag\n- Only compacts Tier 1 (Tier 2 remains manual)\n- Doesn't run more than once per hour (rate limiting)","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-15T21:51:23.281006-07:00","updated_at":"2025-10-15T21:51:23.281006-07:00","labels":["---","automation","compaction","optional","v1.2"]}
|
||||
{"id":"bd-267","title":"Optional: Add git commit counting","description":"Implement git commit counting for \"project time\" measurement as alternative to calendar time for Tier 2 eligibility.","design":"```go\nfunc getCommitsSince(closedAt time.Time) (int, error) {\n cmd := exec.Command(\"git\", \"rev-list\", \"--count\",\n fmt.Sprintf(\"--since=%s\", closedAt.Format(time.RFC3339)), \"HEAD\")\n output, err := cmd.Output()\n if err != nil {\n return 0, err // Not in git repo or git not available\n }\n return strconv.Atoi(strings.TrimSpace(string(output)))\n}\n```\n\nFallback strategies:\n1. Git commit count (preferred)\n2. Issue counter delta (store counter at close time, compare later)\n3. Pure time-based (90 days)","acceptance_criteria":"- Counts commits since closed_at timestamp\n- Handles git not available gracefully (falls back)\n- Fallback to issue counter delta works\n- Configurable via compact_tier2_commits config key\n- Tested with real git repo\n- Works in non-git environments","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-15T21:51:23.284781-07:00","updated_at":"2025-10-15T21:51:23.284781-07:00","labels":["compaction","git","optional","tier2"]}
|
||||
{"id":"bd-260","title":"Add `bd compact --restore` functionality","description":"Implement restore command to undo compaction from snapshots.","design":"Add to `cmd/bd/compact.go`:\n\n```go\nvar compactRestore string\n\ncompactCmd.Flags().StringVar(\u0026compactRestore, \"restore\", \"\", \"Restore issue from snapshot\")\n```\n\nProcess:\n1. Load snapshot for issue\n2. Parse JSON content\n3. Update issue with original content\n4. Set compaction_level = 0, compacted_at = NULL, original_size = NULL\n5. Record event (EventRestored or EventUpdated)\n6. Mark dirty for export","acceptance_criteria":"- Restores exact original content\n- Handles multiple snapshots (use latest by default)\n- `--level` flag to choose specific snapshot\n- Updates compaction_level correctly\n- Exports restored content to JSONL\n- Shows before/after in output","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.240267-07:00","updated_at":"2025-10-15T21:51:23.240267-07:00"}
|
||||
{"id":"bd-261","title":"Add `bd compact --stats` command","description":"Add statistics command showing compaction status and potential savings.","design":"```go\nvar compactStats bool\n\ncompactCmd.Flags().BoolVar(\u0026compactStats, \"stats\", false, \"Show compaction statistics\")\n```\n\nOutput:\n- Total issues, by compaction level (0, 1, 2)\n- Current DB size vs estimated uncompacted size\n- Space savings (KB/MB and %)\n- Candidates for each tier with size estimates\n- Estimated API cost (Haiku pricing)","acceptance_criteria":"- Accurate counts by compaction_level\n- Size calculations include all text fields (UTF-8 bytes)\n- Shows candidates with eligibility reasons\n- Cost estimation based on current Haiku pricing\n- JSON output supported\n- Clear, readable table format","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.242041-07:00","updated_at":"2025-10-15T21:51:23.242041-07:00"}
|
||||
{"id":"bd-262","title":"Add EventCompacted to event system","description":"Add new event type for tracking compaction in audit trail.","design":"1. Add to `internal/types/types.go`:\n```go\nconst EventCompacted EventType = \"compacted\"\n```\n\n2. Record event during compaction:\n```go\neventData := map[string]interface{}{\n \"tier\": tier,\n \"original_size\": originalSize,\n \"compressed_size\": compressedSize,\n \"reduction_pct\": (1 - float64(compressedSize)/float64(originalSize)) * 100,\n}\n```\n\n3. Update event display in `bd show`.","acceptance_criteria":"- Event includes tier, original_size, compressed_size, reduction_pct\n- Shows in event history (`bd events \u003cid\u003e`)\n- Exports to JSONL correctly\n- `bd show` displays compaction status and marker","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.244219-07:00","updated_at":"2025-10-15T21:51:23.244219-07:00"}
|
||||
{"id":"bd-263","title":"Add compaction indicator to `bd show`","description":"Update `bd show` command to display compaction status prominently.","design":"Add to issue display:\n```\nbd-42: Fix authentication bug [CLOSED] 🗜️\n\nStatus: closed (compacted L1)\n...\n\n---\n💾 Restore: bd compact --restore bd-42\n📊 Original: 2,341 bytes | Compressed: 468 bytes (80% reduction)\n🗜️ Compacted: 2025-10-15 (Tier 1)\n```\n\nEmoji indicators:\n- Tier 1: 🗜️\n- Tier 2: 📦","acceptance_criteria":"- Compaction status visible in title line\n- Footer shows size savings when compacted\n- Restore command shown for compacted issues\n- Works with `--json` output (includes compaction fields)\n- Emoji optional (controlled by config or terminal detection)","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.253091-07:00","updated_at":"2025-10-15T21:51:23.253091-07:00"}
|
||||
{"id":"bd-264","title":"Write compaction tests","description":"Comprehensive test suite for compaction functionality.","design":"Test coverage:\n\n1. **Candidate Identification:**\n - Eligibility by time\n - Dependency depth checking\n - Mixed status dependents\n - Edge cases (no deps, circular deps)\n\n2. **Snapshots:**\n - Create and restore\n - Multiple snapshots per issue\n - Content integrity (UTF-8, special chars)\n\n3. **Tier 1 Compaction:**\n - Single issue compaction\n - Batch processing\n - Error handling (API failures)\n\n4. **Tier 2 Compaction:**\n - Requires Tier 1\n - Events pruning\n - Commit counting fallback\n\n5. **CLI:**\n - All flag combinations\n - Dry-run accuracy\n - JSON output parsing\n\n6. **Integration:**\n - End-to-end flow\n - JSONL export/import\n - Restore verification","acceptance_criteria":"- Test coverage \u003e80%\n- All edge cases covered\n- Mock Haiku API in tests (no real API calls)\n- Integration tests pass\n- `go test ./...` passes\n- Benchmarks for performance-critical paths","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-15T21:51:23.262504-07:00","updated_at":"2025-10-15T21:51:23.262504-07:00"}
|
||||
{"id":"bd-265","title":"Add compaction documentation","description":"Document compaction feature in README and create detailed COMPACTION.md guide.","design":"**Update README.md:**\n- Add to Features section\n- CLI examples (dry-run, compact, restore, stats)\n- Configuration guide\n- Cost analysis\n\n**Create COMPACTION.md:**\n- How compaction works (architecture overview)\n- When to use each tier\n- Detailed cost analysis with examples\n- Safety mechanisms (snapshots, restore, dry-run)\n- Troubleshooting guide\n- FAQ\n\n**Create examples/compaction/:**\n- `workflow.sh` - Example monthly compaction workflow\n- `cron-compact.sh` - Cron job setup\n- `auto-compact.sh` - Auto-compaction script","acceptance_criteria":"- README.md updated with compaction section\n- COMPACTION.md comprehensive and clear\n- Examples work as documented (tested)\n- Screenshots or ASCII examples included\n- API key setup documented (env var vs config)\n- Covers common questions and issues","status":"open","priority":2,"issue_type":"task","created_at":"2025-10-15T21:51:23.265589-07:00","updated_at":"2025-10-15T21:51:23.265589-07:00"}
|
||||
{"id":"bd-266","title":"Optional: Implement auto-compaction","description":"Implement automatic compaction triggered by certain operations when enabled via config.","design":"Trigger points (when `auto_compact_enabled = true`):\n1. `bd stats` - check and compact if candidates exist\n2. `bd export` - before exporting\n3. Configurable: on any read operation after N candidates accumulate\n\nAdd:\n```go\nfunc (s *SQLiteStorage) AutoCompact(ctx context.Context) error {\n enabled, _ := s.GetConfig(ctx, \"auto_compact_enabled\")\n if enabled != \"true\" {\n return nil\n }\n\n // Run Tier 1 compaction on all candidates\n // Limit to batch_size to avoid long operations\n // Log activity for transparency\n}\n```","acceptance_criteria":"- Respects auto_compact_enabled config (default: false)\n- Limits batch size to avoid blocking operations\n- Logs compaction activity (visible with --verbose)\n- Can be disabled per-command with `--no-auto-compact` flag\n- Only compacts Tier 1 (Tier 2 remains manual)\n- Doesn't run more than once per hour (rate limiting)","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-15T21:51:23.281006-07:00","updated_at":"2025-10-15T21:51:23.281006-07:00"}
|
||||
{"id":"bd-267","title":"Optional: Add git commit counting","description":"Implement git commit counting for \"project time\" measurement as alternative to calendar time for Tier 2 eligibility.","design":"```go\nfunc getCommitsSince(closedAt time.Time) (int, error) {\n cmd := exec.Command(\"git\", \"rev-list\", \"--count\",\n fmt.Sprintf(\"--since=%s\", closedAt.Format(time.RFC3339)), \"HEAD\")\n output, err := cmd.Output()\n if err != nil {\n return 0, err // Not in git repo or git not available\n }\n return strconv.Atoi(strings.TrimSpace(string(output)))\n}\n```\n\nFallback strategies:\n1. Git commit count (preferred)\n2. Issue counter delta (store counter at close time, compare later)\n3. Pure time-based (90 days)","acceptance_criteria":"- Counts commits since closed_at timestamp\n- Handles git not available gracefully (falls back)\n- Fallback to issue counter delta works\n- Configurable via compact_tier2_commits config key\n- Tested with real git repo\n- Works in non-git environments","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-15T21:51:23.284781-07:00","updated_at":"2025-10-15T21:51:23.284781-07:00"}
|
||||
{"id":"bd-27","title":"Cache compiled regexes in replaceIDReferences for performance","description":"replaceIDReferences() compiles the same regex patterns on every call. With 100 issues and 10 ID mappings, that's 1000 regex compilations. Pre-compile regexes once and reuse. Can use a struct with compiled regex, placeholder, and newID. Located in collision.go:329. Estimated performance improvement: 10-100x for large batches.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-14T14:43:06.911892-07:00","updated_at":"2025-10-15T16:27:22.002496-07:00","closed_at":"2025-10-15T03:01:29.570955-07:00"}
|
||||
{"id":"bd-28","title":"Improve error handling in dependency removal during remapping","description":"In updateDependencyReferences(), RemoveDependency errors are caught and ignored with continue (line 392). Comment says 'if dependency doesn't exist' but this catches ALL errors including real failures. Should check error type with errors.Is(err, ErrDependencyNotFound) and only ignore not-found errors, returning other errors properly.","status":"open","priority":3,"issue_type":"bug","created_at":"2025-10-14T14:43:06.912228-07:00","updated_at":"2025-10-15T16:27:22.003145-07:00"}
|
||||
{"id":"bd-29","title":"Use safer placeholder pattern in replaceIDReferences","description":"Currently uses __PLACEHOLDER_0__ which could theoretically collide with user text. Use a truly unique placeholder like null bytes: \\x00REMAP\\x00_0_\\x00 which are unlikely to appear in normal text. Located in collision.go:324. Very low probability issue but worth fixing for completeness.","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-14T14:43:06.912567-07:00","updated_at":"2025-10-15T16:27:22.003668-07:00"}
|
||||
|
||||
277
internal/storage/sqlite/compact.go
Normal file
277
internal/storage/sqlite/compact.go
Normal file
@@ -0,0 +1,277 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CompactionCandidate represents an issue eligible for compaction
|
||||
type CompactionCandidate struct {
|
||||
IssueID string
|
||||
ClosedAt time.Time
|
||||
OriginalSize int
|
||||
EstimatedSize int
|
||||
DependentCount int
|
||||
}
|
||||
|
||||
// GetTier1Candidates returns issues eligible for Tier 1 compaction.
|
||||
// Criteria:
|
||||
// - Status = closed
|
||||
// - Closed for at least compact_tier1_days
|
||||
// - No open dependents within compact_tier1_dep_levels depth
|
||||
// - Not already compacted (compaction_level = 0)
|
||||
func (s *SQLiteStorage) GetTier1Candidates(ctx context.Context) ([]*CompactionCandidate, error) {
|
||||
// Get configuration
|
||||
daysStr, err := s.GetConfig(ctx, "compact_tier1_days")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get compact_tier1_days: %w", err)
|
||||
}
|
||||
if daysStr == "" {
|
||||
daysStr = "30"
|
||||
}
|
||||
|
||||
depthStr, err := s.GetConfig(ctx, "compact_tier1_dep_levels")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get compact_tier1_dep_levels: %w", err)
|
||||
}
|
||||
if depthStr == "" {
|
||||
depthStr = "2"
|
||||
}
|
||||
|
||||
query := `
|
||||
WITH RECURSIVE
|
||||
-- Find all issues that depend on (are blocked by) other issues
|
||||
dependent_tree AS (
|
||||
-- Base case: direct dependents
|
||||
SELECT
|
||||
d.depends_on_id as issue_id,
|
||||
i.id as dependent_id,
|
||||
i.status as dependent_status,
|
||||
0 as depth
|
||||
FROM dependencies d
|
||||
JOIN issues i ON d.issue_id = i.id
|
||||
WHERE d.type = 'blocks'
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Recursive case: parent-child relationships
|
||||
SELECT
|
||||
dt.issue_id,
|
||||
i.id as dependent_id,
|
||||
i.status as dependent_status,
|
||||
dt.depth + 1
|
||||
FROM dependent_tree dt
|
||||
JOIN dependencies d ON d.depends_on_id = dt.dependent_id
|
||||
JOIN issues i ON d.issue_id = i.id
|
||||
WHERE d.type = 'parent-child'
|
||||
AND dt.depth < ?
|
||||
)
|
||||
SELECT
|
||||
i.id,
|
||||
i.closed_at,
|
||||
COALESCE(i.original_size, LENGTH(i.description) + LENGTH(i.design) + LENGTH(i.notes) + LENGTH(i.acceptance_criteria)) as original_size,
|
||||
0 as estimated_size,
|
||||
COUNT(DISTINCT dt.dependent_id) as dependent_count
|
||||
FROM issues i
|
||||
LEFT JOIN dependent_tree dt ON i.id = dt.issue_id
|
||||
AND dt.dependent_status IN ('open', 'in_progress', 'blocked')
|
||||
AND dt.depth <= ?
|
||||
WHERE i.status = 'closed'
|
||||
AND i.closed_at IS NOT NULL
|
||||
AND i.closed_at <= datetime('now', '-' || CAST(? AS INTEGER) || ' days')
|
||||
AND COALESCE(i.compaction_level, 0) = 0
|
||||
AND dt.dependent_id IS NULL -- No open dependents
|
||||
GROUP BY i.id
|
||||
ORDER BY i.closed_at ASC
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, depthStr, depthStr, daysStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query tier1 candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var candidates []*CompactionCandidate
|
||||
for rows.Next() {
|
||||
var c CompactionCandidate
|
||||
if err := rows.Scan(&c.IssueID, &c.ClosedAt, &c.OriginalSize, &c.EstimatedSize, &c.DependentCount); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan candidate: %w", err)
|
||||
}
|
||||
candidates = append(candidates, &c)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("rows iteration error: %w", err)
|
||||
}
|
||||
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// GetTier2Candidates returns issues eligible for Tier 2 compaction.
|
||||
// Criteria:
|
||||
// - Status = closed
|
||||
// - Closed for at least compact_tier2_days
|
||||
// - No open dependents within compact_tier2_dep_levels depth
|
||||
// - Already at compaction_level = 1
|
||||
// - Either has many commits (compact_tier2_commits) or many dependent issues
|
||||
func (s *SQLiteStorage) GetTier2Candidates(ctx context.Context) ([]*CompactionCandidate, error) {
|
||||
// Get configuration
|
||||
daysStr, err := s.GetConfig(ctx, "compact_tier2_days")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get compact_tier2_days: %w", err)
|
||||
}
|
||||
if daysStr == "" {
|
||||
daysStr = "90"
|
||||
}
|
||||
|
||||
depthStr, err := s.GetConfig(ctx, "compact_tier2_dep_levels")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get compact_tier2_dep_levels: %w", err)
|
||||
}
|
||||
if depthStr == "" {
|
||||
depthStr = "5"
|
||||
}
|
||||
|
||||
commitsStr, err := s.GetConfig(ctx, "compact_tier2_commits")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get compact_tier2_commits: %w", err)
|
||||
}
|
||||
if commitsStr == "" {
|
||||
commitsStr = "100"
|
||||
}
|
||||
|
||||
query := `
|
||||
WITH event_counts AS (
|
||||
SELECT issue_id, COUNT(*) as event_count
|
||||
FROM events
|
||||
GROUP BY issue_id
|
||||
)
|
||||
SELECT
|
||||
i.id,
|
||||
i.closed_at,
|
||||
i.original_size,
|
||||
0 as estimated_size,
|
||||
COALESCE(ec.event_count, 0) as dependent_count
|
||||
FROM issues i
|
||||
LEFT JOIN event_counts ec ON i.id = ec.issue_id
|
||||
WHERE i.status = 'closed'
|
||||
AND i.closed_at IS NOT NULL
|
||||
AND i.closed_at <= datetime('now', '-' || CAST(? AS INTEGER) || ' days')
|
||||
AND i.compaction_level = 1
|
||||
AND COALESCE(ec.event_count, 0) >= CAST(? AS INTEGER)
|
||||
AND NOT EXISTS (
|
||||
-- Check for open dependents
|
||||
SELECT 1 FROM dependencies d
|
||||
JOIN issues dep ON d.issue_id = dep.id
|
||||
WHERE d.depends_on_id = i.id
|
||||
AND d.type = 'blocks'
|
||||
AND dep.status IN ('open', 'in_progress', 'blocked')
|
||||
)
|
||||
ORDER BY i.closed_at ASC
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, daysStr, commitsStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query tier2 candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var candidates []*CompactionCandidate
|
||||
for rows.Next() {
|
||||
var c CompactionCandidate
|
||||
if err := rows.Scan(&c.IssueID, &c.ClosedAt, &c.OriginalSize, &c.EstimatedSize, &c.DependentCount); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan candidate: %w", err)
|
||||
}
|
||||
candidates = append(candidates, &c)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("rows iteration error: %w", err)
|
||||
}
|
||||
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// CheckEligibility checks if a specific issue is eligible for compaction at the given tier.
|
||||
// Returns (eligible, reason, error).
|
||||
// If not eligible, reason explains why.
|
||||
func (s *SQLiteStorage) CheckEligibility(ctx context.Context, issueID string, tier int) (bool, string, error) {
|
||||
// Get the issue
|
||||
var status string
|
||||
var closedAt sql.NullTime
|
||||
var compactionLevel int
|
||||
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT status, closed_at, COALESCE(compaction_level, 0)
|
||||
FROM issues
|
||||
WHERE id = ?
|
||||
`, issueID).Scan(&status, &closedAt, &compactionLevel)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return false, "issue not found", nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("failed to get issue: %w", err)
|
||||
}
|
||||
|
||||
// Check basic requirements
|
||||
if status != "closed" {
|
||||
return false, "issue is not closed", nil
|
||||
}
|
||||
|
||||
if !closedAt.Valid {
|
||||
return false, "issue has no closed_at timestamp", nil
|
||||
}
|
||||
|
||||
if tier == 1 {
|
||||
if compactionLevel != 0 {
|
||||
return false, "issue is already compacted", nil
|
||||
}
|
||||
|
||||
// Check if closed long enough
|
||||
daysStr, err := s.GetConfig(ctx, "compact_tier1_days")
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("failed to get compact_tier1_days: %w", err)
|
||||
}
|
||||
if daysStr == "" {
|
||||
daysStr = "30"
|
||||
}
|
||||
|
||||
// Check if it appears in tier1 candidates
|
||||
candidates, err := s.GetTier1Candidates(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("failed to get tier1 candidates: %w", err)
|
||||
}
|
||||
|
||||
for _, c := range candidates {
|
||||
if c.IssueID == issueID {
|
||||
return true, "", nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, "issue has open dependents or not closed long enough", nil
|
||||
|
||||
} else if tier == 2 {
|
||||
if compactionLevel != 1 {
|
||||
return false, "issue must be at compaction level 1 for tier 2", nil
|
||||
}
|
||||
|
||||
// Check if it appears in tier2 candidates
|
||||
candidates, err := s.GetTier2Candidates(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("failed to get tier2 candidates: %w", err)
|
||||
}
|
||||
|
||||
for _, c := range candidates {
|
||||
if c.IssueID == issueID {
|
||||
return true, "", nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, "issue has open dependents, not closed long enough, or insufficient events", nil
|
||||
}
|
||||
|
||||
return false, fmt.Sprintf("invalid tier: %d", tier), nil
|
||||
}
|
||||
318
internal/storage/sqlite/compact_test.go
Normal file
318
internal/storage/sqlite/compact_test.go
Normal file
@@ -0,0 +1,318 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
func TestGetTier1Candidates(t *testing.T) {
|
||||
store, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create test issues
|
||||
// Old closed issue (eligible)
|
||||
issue1 := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Old closed issue",
|
||||
Description: "This is a test description",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue1, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue1: %v", err)
|
||||
}
|
||||
|
||||
// Recently closed issue (not eligible - too recent)
|
||||
issue2 := &types.Issue{
|
||||
ID: "bd-2",
|
||||
Title: "Recent closed issue",
|
||||
Description: "Recent",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-10 * 24 * time.Hour)),
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue2, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue2: %v", err)
|
||||
}
|
||||
|
||||
// Open issue (not eligible)
|
||||
issue3 := &types.Issue{
|
||||
ID: "bd-3",
|
||||
Title: "Open issue",
|
||||
Description: "Open",
|
||||
Status: "open",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue3, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue3: %v", err)
|
||||
}
|
||||
|
||||
// Old closed issue with open dependent (not eligible)
|
||||
issue4 := &types.Issue{
|
||||
ID: "bd-4",
|
||||
Title: "Has open dependent",
|
||||
Description: "Blocked by open issue",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue4, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue4: %v", err)
|
||||
}
|
||||
|
||||
// Create blocking dependency
|
||||
dep := &types.Dependency{
|
||||
IssueID: "bd-3",
|
||||
DependsOnID: "bd-4",
|
||||
Type: "blocks",
|
||||
}
|
||||
if err := store.AddDependency(ctx, dep, "test"); err != nil {
|
||||
t.Fatalf("Failed to add dependency: %v", err)
|
||||
}
|
||||
|
||||
// Get candidates
|
||||
candidates, err := store.GetTier1Candidates(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTier1Candidates failed: %v", err)
|
||||
}
|
||||
|
||||
// Should only return bd-1 (old and no open dependents)
|
||||
if len(candidates) != 1 {
|
||||
t.Errorf("Expected 1 candidate, got %d", len(candidates))
|
||||
}
|
||||
|
||||
if len(candidates) > 0 && candidates[0].IssueID != "bd-1" {
|
||||
t.Errorf("Expected candidate bd-1, got %s", candidates[0].IssueID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTier2Candidates(t *testing.T) {
|
||||
store, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create old tier1 compacted issue with many events
|
||||
issue1 := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Tier1 compacted with events",
|
||||
Description: "Summary",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-100 * 24 * time.Hour)),
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue1, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue1: %v", err)
|
||||
}
|
||||
|
||||
// Set compaction level to 1
|
||||
_, err := store.db.ExecContext(ctx, `
|
||||
UPDATE issues
|
||||
SET compaction_level = 1,
|
||||
compacted_at = datetime('now', '-95 days'),
|
||||
original_size = 1000
|
||||
WHERE id = ?
|
||||
`, "bd-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to set compaction level: %v", err)
|
||||
}
|
||||
|
||||
// Add many events (simulate high activity)
|
||||
for i := 0; i < 120; i++ {
|
||||
if err := store.AddComment(ctx, "bd-1", "test", "comment"); err != nil {
|
||||
t.Fatalf("Failed to add event: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get tier2 candidates
|
||||
candidates, err := store.GetTier2Candidates(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTier2Candidates failed: %v", err)
|
||||
}
|
||||
|
||||
// Should return bd-1
|
||||
if len(candidates) != 1 {
|
||||
t.Errorf("Expected 1 candidate, got %d", len(candidates))
|
||||
}
|
||||
|
||||
if len(candidates) > 0 && candidates[0].IssueID != "bd-1" {
|
||||
t.Errorf("Expected candidate bd-1, got %s", candidates[0].IssueID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckEligibilityTier1(t *testing.T) {
|
||||
store, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create eligible issue
|
||||
issue1 := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Eligible",
|
||||
Description: "Test",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue1, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue: %v", err)
|
||||
}
|
||||
|
||||
eligible, reason, err := store.CheckEligibility(ctx, "bd-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("CheckEligibility failed: %v", err)
|
||||
}
|
||||
|
||||
if !eligible {
|
||||
t.Errorf("Expected eligible, got not eligible: %s", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckEligibilityOpenIssue(t *testing.T) {
|
||||
store, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
issue := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Open",
|
||||
Description: "Test",
|
||||
Status: "open",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue: %v", err)
|
||||
}
|
||||
|
||||
eligible, reason, err := store.CheckEligibility(ctx, "bd-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("CheckEligibility failed: %v", err)
|
||||
}
|
||||
|
||||
if eligible {
|
||||
t.Error("Expected not eligible for open issue")
|
||||
}
|
||||
|
||||
if reason != "issue is not closed" {
|
||||
t.Errorf("Expected 'issue is not closed', got '%s'", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckEligibilityAlreadyCompacted(t *testing.T) {
|
||||
store, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
issue := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Already compacted",
|
||||
Description: "Test",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue: %v", err)
|
||||
}
|
||||
|
||||
// Mark as compacted
|
||||
_, err := store.db.ExecContext(ctx, `
|
||||
UPDATE issues SET compaction_level = 1 WHERE id = ?
|
||||
`, "bd-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to set compaction level: %v", err)
|
||||
}
|
||||
|
||||
eligible, reason, err := store.CheckEligibility(ctx, "bd-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("CheckEligibility failed: %v", err)
|
||||
}
|
||||
|
||||
if eligible {
|
||||
t.Error("Expected not eligible for already compacted issue")
|
||||
}
|
||||
|
||||
if reason != "issue is already compacted" {
|
||||
t.Errorf("Expected 'issue is already compacted', got '%s'", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTier1NoCircularDeps(t *testing.T) {
|
||||
store, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create three closed issues with circular dependency
|
||||
issue1 := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Issue 1",
|
||||
Description: "Test",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
issue2 := &types.Issue{
|
||||
ID: "bd-2",
|
||||
Title: "Issue 2",
|
||||
Description: "Test",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
issue3 := &types.Issue{
|
||||
ID: "bd-3",
|
||||
Title: "Issue 3",
|
||||
Description: "Test",
|
||||
Status: "closed",
|
||||
Priority: 2,
|
||||
IssueType: "task",
|
||||
ClosedAt: timePtr(time.Now().Add(-40 * 24 * time.Hour)),
|
||||
}
|
||||
|
||||
for _, issue := range []*types.Issue{issue1, issue2, issue3} {
|
||||
if err := store.CreateIssue(ctx, issue, "test"); err != nil {
|
||||
t.Fatalf("Failed to create issue: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create circular dependency: 1->2->3->1
|
||||
// Note: the AddDependency validation should prevent this, but let's test the query handles it
|
||||
_, err := store.db.ExecContext(ctx, `
|
||||
INSERT INTO dependencies (issue_id, depends_on_id, type, created_by) VALUES
|
||||
('bd-1', 'bd-2', 'blocks', 'test'),
|
||||
('bd-2', 'bd-3', 'blocks', 'test'),
|
||||
('bd-3', 'bd-1', 'blocks', 'test')
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create dependencies: %v", err)
|
||||
}
|
||||
|
||||
// Should not crash and should return all three as they're all closed
|
||||
candidates, err := store.GetTier1Candidates(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTier1Candidates failed with circular deps: %v", err)
|
||||
}
|
||||
|
||||
// All should be eligible since all are closed
|
||||
if len(candidates) != 3 {
|
||||
t.Errorf("Expected 3 candidates, got %d", len(candidates))
|
||||
}
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
@@ -18,6 +18,9 @@ CREATE TABLE IF NOT EXISTS issues (
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
closed_at DATETIME,
|
||||
external_ref TEXT,
|
||||
compaction_level INTEGER DEFAULT 0,
|
||||
compacted_at DATETIME,
|
||||
original_size INTEGER,
|
||||
CHECK ((status = 'closed') = (closed_at IS NOT NULL))
|
||||
);
|
||||
|
||||
@@ -74,6 +77,19 @@ CREATE TABLE IF NOT EXISTS config (
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Default compaction configuration
|
||||
INSERT OR IGNORE INTO config (key, value) VALUES
|
||||
('compaction_enabled', 'false'),
|
||||
('compact_tier1_days', '30'),
|
||||
('compact_tier1_dep_levels', '2'),
|
||||
('compact_tier2_days', '90'),
|
||||
('compact_tier2_dep_levels', '5'),
|
||||
('compact_tier2_commits', '100'),
|
||||
('compact_model', 'claude-3-5-haiku-20241022'),
|
||||
('compact_batch_size', '50'),
|
||||
('compact_parallel_workers', '5'),
|
||||
('auto_compact_enabled', 'false');
|
||||
|
||||
-- Metadata table (for storing internal state like import hashes)
|
||||
CREATE TABLE IF NOT EXISTS metadata (
|
||||
key TEXT PRIMARY KEY,
|
||||
@@ -96,6 +112,22 @@ CREATE TABLE IF NOT EXISTS issue_counters (
|
||||
last_id INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- Issue snapshots table (for compaction)
|
||||
CREATE TABLE IF NOT EXISTS issue_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
issue_id TEXT NOT NULL,
|
||||
snapshot_time DATETIME NOT NULL,
|
||||
compaction_level INTEGER NOT NULL,
|
||||
original_size INTEGER NOT NULL,
|
||||
compressed_size INTEGER NOT NULL,
|
||||
original_content TEXT NOT NULL,
|
||||
archived_events TEXT,
|
||||
FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_snapshots_issue ON issue_snapshots(issue_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_snapshots_level ON issue_snapshots(compaction_level);
|
||||
|
||||
-- Ready work view (with hierarchical blocking)
|
||||
-- Uses recursive CTE to propagate blocking through parent-child hierarchy
|
||||
CREATE VIEW IF NOT EXISTS ready_issues AS
|
||||
|
||||
@@ -72,6 +72,21 @@ func New(path string) (*SQLiteStorage, error) {
|
||||
return nil, fmt.Errorf("failed to migrate closed_at constraint: %w", err)
|
||||
}
|
||||
|
||||
// Migrate existing databases to add compaction columns
|
||||
if err := migrateCompactionColumns(db); err != nil {
|
||||
return nil, fmt.Errorf("failed to migrate compaction columns: %w", err)
|
||||
}
|
||||
|
||||
// Migrate existing databases to add issue_snapshots table
|
||||
if err := migrateSnapshotsTable(db); err != nil {
|
||||
return nil, fmt.Errorf("failed to migrate snapshots table: %w", err)
|
||||
}
|
||||
|
||||
// Migrate existing databases to add compaction config defaults
|
||||
if err := migrateCompactionConfig(db); err != nil {
|
||||
return nil, fmt.Errorf("failed to migrate compaction config: %w", err)
|
||||
}
|
||||
|
||||
return &SQLiteStorage{
|
||||
db: db,
|
||||
}, nil
|
||||
@@ -290,6 +305,102 @@ func migrateClosedAtConstraint(db *sql.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateCompactionColumns adds compaction_level, compacted_at, and original_size columns to the issues table.
|
||||
// This migration is idempotent and safe to run multiple times.
|
||||
func migrateCompactionColumns(db *sql.DB) error {
|
||||
// Check if compaction_level column exists
|
||||
var columnExists bool
|
||||
err := db.QueryRow(`
|
||||
SELECT COUNT(*) > 0
|
||||
FROM pragma_table_info('issues')
|
||||
WHERE name = 'compaction_level'
|
||||
`).Scan(&columnExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check compaction_level column: %w", err)
|
||||
}
|
||||
|
||||
if columnExists {
|
||||
// Columns already exist, nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add the three compaction columns
|
||||
_, err = db.Exec(`
|
||||
ALTER TABLE issues ADD COLUMN compaction_level INTEGER DEFAULT 0;
|
||||
ALTER TABLE issues ADD COLUMN compacted_at DATETIME;
|
||||
ALTER TABLE issues ADD COLUMN original_size INTEGER;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add compaction columns: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateSnapshotsTable creates the issue_snapshots table if it doesn't exist.
|
||||
// This migration is idempotent and safe to run multiple times.
|
||||
func migrateSnapshotsTable(db *sql.DB) error {
|
||||
// Check if issue_snapshots table exists
|
||||
var tableExists bool
|
||||
err := db.QueryRow(`
|
||||
SELECT COUNT(*) > 0
|
||||
FROM sqlite_master
|
||||
WHERE type='table' AND name='issue_snapshots'
|
||||
`).Scan(&tableExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check issue_snapshots table: %w", err)
|
||||
}
|
||||
|
||||
if tableExists {
|
||||
// Table already exists, nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create the table and indexes
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE issue_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
issue_id TEXT NOT NULL,
|
||||
snapshot_time DATETIME NOT NULL,
|
||||
compaction_level INTEGER NOT NULL,
|
||||
original_size INTEGER NOT NULL,
|
||||
compressed_size INTEGER NOT NULL,
|
||||
original_content TEXT NOT NULL,
|
||||
archived_events TEXT,
|
||||
FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX idx_snapshots_issue ON issue_snapshots(issue_id);
|
||||
CREATE INDEX idx_snapshots_level ON issue_snapshots(compaction_level);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create issue_snapshots table: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateCompactionConfig adds default compaction configuration values.
|
||||
// This migration is idempotent and safe to run multiple times (INSERT OR IGNORE).
|
||||
func migrateCompactionConfig(db *sql.DB) error {
|
||||
_, err := db.Exec(`
|
||||
INSERT OR IGNORE INTO config (key, value) VALUES
|
||||
('compaction_enabled', 'false'),
|
||||
('compact_tier1_days', '30'),
|
||||
('compact_tier1_dep_levels', '2'),
|
||||
('compact_tier2_days', '90'),
|
||||
('compact_tier2_dep_levels', '5'),
|
||||
('compact_tier2_commits', '100'),
|
||||
('compact_model', 'claude-3-5-haiku-20241022'),
|
||||
('compact_batch_size', '50'),
|
||||
('compact_parallel_workers', '5'),
|
||||
('auto_compact_enabled', 'false')
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add compaction config defaults: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getNextIDForPrefix atomically generates the next ID for a given prefix
|
||||
// Uses the issue_counters table for atomic, cross-process ID generation
|
||||
func (s *SQLiteStorage) getNextIDForPrefix(ctx context.Context, prefix string) (int, error) {
|
||||
|
||||
Reference in New Issue
Block a user