From eeef37f37bc71ac32a7a93205c0ad919e0548b99 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Fri, 7 Nov 2025 20:07:59 -0800 Subject: [PATCH] Fix bd-srwk: Add ID-based staleness detection to bd export Prevents data loss when exporting stale database by comparing issue IDs, not just counts. Detects both scenarios: - Database has fewer issues than JSONL - Database has different issues than JSONL (same count) Shows specific missing issue IDs in error. Allows override with --force. Includes comprehensive tests for all scenarios. --- .beads/beads.jsonl | 8 +-- cmd/bd/export.go | 106 +++++++++++++++++++++++---- cmd/bd/export_staleness_test.go | 124 ++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 18 deletions(-) create mode 100644 cmd/bd/export_staleness_test.go diff --git a/.beads/beads.jsonl b/.beads/beads.jsonl index 29397fda..87a172ce 100644 --- a/.beads/beads.jsonl +++ b/.beads/beads.jsonl @@ -33,7 +33,7 @@ {"id":"bd-1vup","content_hash":"99df92e0d2e6f2998d9ef52c60ae1db55a2e52b84c3e935bf371517e5154ad77","title":"Test FK constraint via close","description":"","status":"closed","priority":4,"issue_type":"task","created_at":"2025-11-07T15:06:10.324045-08:00","updated_at":"2025-11-07T15:06:14.289835-08:00","closed_at":"2025-11-07T15:06:14.289835-08:00","source_repo":"."} {"id":"bd-1vv","content_hash":"1db907ddb55edaf7a4c06a566c4e1b8244fcd9ba5d7e2fca4d5c053e424ac515","title":"Add WebSocket support","description":"## Feature Request\n\n[Describe the desired feature]\n\n## Motivation\n\n[Why is this feature needed? What problem does it solve?]\n\n## Use Cases\n\n1. **Use Case 1**: [description]\n2. **Use Case 2**: [description]\n\n## Proposed Solution\n\n[High-level approach to implementing this feature]\n\n## Alternatives Considered\n\n- **Alternative 1**: [description and why not chosen]\n- **Alternative 2**: [description and why not chosen]\n","design":"## Technical Design\n\n[Detailed technical approach]\n\n## API Changes\n\n[New commands, flags, or APIs]\n\n## Data Model Changes\n\n[Database schema changes if any]\n\n## Implementation Notes\n\n- Note 1\n- Note 2\n\n## Testing Strategy\n\n- Unit tests: [scope]\n- Integration tests: [scope]\n- Manual testing: [steps]\n","acceptance_criteria":"- [ ] Feature implements all described use cases\n- [ ] All tests pass\n- [ ] Documentation updated (README, commands)\n- [ ] Examples added if applicable\n- [ ] No performance regressions\n","status":"open","priority":2,"issue_type":"feature","created_at":"2025-11-03T19:56:41.271215-08:00","updated_at":"2025-11-03T19:56:41.271215-08:00","source_repo":".","labels":["feature"]} {"id":"bd-1yi5","content_hash":"11044802d32a77f8ca574ca5902939bcc7e252880437bea7281aaf445db8db84","title":"Use -short flag in CI for PR checks","description":"Update CI configuration to use -short flag for PR checks, run full tests nightly.\n\nThe slow tests already support testing.Short() and will be skipped.\n\nExpected savings: ~20 seconds for PR checks (fast tests only)\n\nImplementation:\n- Update .github/workflows/ci.yml to add -short flag for PR tests\n- Create/update nightly workflow for full test runs\n- Update README/docs about test strategy\n\nFile: .github/workflows/ci.yml:30","status":"closed","priority":1,"issue_type":"task","created_at":"2025-11-04T01:24:17.279618-08:00","updated_at":"2025-11-04T10:25:10.616119-08:00","closed_at":"2025-11-04T10:25:10.616119-08:00","source_repo":".","dependencies":[{"issue_id":"bd-1yi5","depends_on_id":"bd-l5gq","type":"blocks","created_at":"2025-11-04T01:24:17.280453-08:00","created_by":"daemon"}]} -{"id":"bd-23a8","content_hash":"7c54bea4624429ff0842a192489979e0a1eafdd872027a0934b1a1d9b0e80d33","title":"Test simple issue","description":"","status":"closed","priority":2,"issue_type":"task","created_at":"2025-11-02T17:11:04.464726-08:00","updated_at":"2025-11-04T11:10:23.529727-08:00","closed_at":"2025-11-04T11:10:23.529731-08:00","source_repo":".","comments":[{"id":1,"issue_id":"bd-23a8","author":"stevey","text":"Testing the new bd comment alias!","created_at":"2025-11-05T08:38:46Z"},{"id":2,"issue_id":"bd-23a8","author":"stevey","text":"Another test with JSON output","created_at":"2025-11-05T08:38:46Z"},{"id":3,"issue_id":"bd-23a8","author":"stevey","text":"Test comment from file\n","created_at":"2025-11-05T08:38:46Z"}]} +{"id":"bd-23a8","content_hash":"7c54bea4624429ff0842a192489979e0a1eafdd872027a0934b1a1d9b0e80d33","title":"Test simple issue","description":"","status":"closed","priority":2,"issue_type":"task","created_at":"2025-11-02T17:11:04.464726-08:00","updated_at":"2025-11-04T11:10:23.529727-08:00","closed_at":"2025-11-04T11:10:23.529731-08:00","source_repo":"."} {"id":"bd-248bdc3e","content_hash":"8eaeb2dbef1ed6b25fc1bcf3bc5cd1b38a5cf5a487772558ba9fe12a149978f3","title":"Add optional post-merge git hook example for bd sync","description":"Create example git hook that auto-runs bd sync after git pull/merge.\n\nAdd to examples/git-hooks/:\n- post-merge hook that checks if .beads/issues.jsonl changed\n- If changed: run `bd sync` automatically\n- Make it optional/documented (not auto-installed)\n\nBenefits:\n- Zero-friction sync after git pull\n- Complements auto-detection as belt-and-suspenders\n\nNote: post-merge hook already exists for pre-commit/post-merge. Extend it to support sync.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-25T22:47:14.668842-07:00","updated_at":"2025-11-06T19:51:37.787964-08:00","closed_at":"2025-11-06T19:51:37.787964-08:00","source_repo":"."} {"id":"bd-2530","content_hash":"ac8bc0c4d75082ac76200d092b3c1c476fd1523c9e83b890f829ee3dd7dcbb3d","title":"Issue with labels","description":"This is a description","design":"Use MVC pattern","acceptance_criteria":"All tests pass","status":"closed","priority":0,"issue_type":"feature","created_at":"2025-10-31T21:40:34.630173-07:00","updated_at":"2025-11-01T11:11:57.93151-07:00","closed_at":"2025-11-01T11:11:57.93151-07:00","source_repo":".","labels":["bug","critical"]} {"id":"bd-27ea","content_hash":"6fed2225c017a7f060eef560279cf166c7dd4965657de0c036d6ed5db13803eb","title":"Improve cmd/bd test coverage from 21% to 40% (multi-session effort)","description":"Current coverage: 21.0% of statements in cmd/bd\nTarget: 40%\nThis is a multi-session incremental effort.\n\nFocus areas:\n- Command handler tests (create, update, close, list, etc.)\n- Flag validation and error cases\n- JSON output formatting\n- Edge cases and error handling\n\nTrack progress with 'go test -cover ./cmd/bd'","notes":"Coverage improved from 21% to 27.4% (package) and 42.9% (total function coverage).\n\nAdded tests for:\n- compact.go test coverage (eligibility checks, dry run scenarios)\n- epic.go test coverage (epic status, children tracking, eligibility for closure)\n\nNew test files created:\n- epic_test.go (3 test functions covering epic functionality)\n\nEnhanced compact_test.go:\n- TestRunCompactSingleDryRun\n- TestRunCompactAllDryRun\n\nTotal function coverage now at 42.9%, exceeding the 40% target.","status":"closed","priority":0,"issue_type":"task","created_at":"2025-10-31T19:35:57.558346-07:00","updated_at":"2025-11-01T12:23:39.158922-07:00","closed_at":"2025-11-01T12:23:39.158926-07:00","source_repo":"."} @@ -98,7 +98,7 @@ {"id":"bd-5iv","content_hash":"229ad9764bd3eb8b09441adefce960aede63fd1b5466d52cc74f112f5bb610ac","title":"Test Epic","description":"## Overview\n\n[Describe the high-level goal and scope of this epic]\n\n## Success Criteria\n\n- [ ] Criteria 1\n- [ ] Criteria 2\n- [ ] Criteria 3\n\n## Background\n\n[Provide context and motivation]\n\n## Scope\n\n**In Scope:**\n- Item 1\n- Item 2\n\n**Out of Scope:**\n- Item 1\n- Item 2\n","design":"## Architecture\n\n[Describe the overall architecture and approach]\n\n## Components\n\n- Component 1: [description]\n- Component 2: [description]\n\n## Dependencies\n\n[List external dependencies or constraints]\n","acceptance_criteria":"- [ ] All child issues are completed\n- [ ] Integration tests pass\n- [ ] Documentation is updated\n- [ ] Code review completed\n","status":"closed","priority":1,"issue_type":"epic","created_at":"2025-11-03T20:15:03.864229-08:00","updated_at":"2025-11-05T00:25:06.538749-08:00","closed_at":"2025-11-05T00:25:06.538749-08:00","source_repo":".","labels":["epic"]} {"id":"bd-5ohb","content_hash":"c7432e58e6518764c9e88b92b16becbb545e179f187becf13aa980f1b93c8192","title":"Issue to reopen with reason","description":"","status":"open","priority":1,"issue_type":"bug","created_at":"2025-11-07T19:00:18.583903-08:00","updated_at":"2025-11-07T19:00:18.629189-08:00","source_repo":"."} {"id":"bd-5ots","content_hash":"197fdfe95399e1becb827c6d1ab57d8ccdca19a84206d30b20102686f5384726","title":"SearchIssues N+1 query causes context timeout with GetLabels","description":"scanIssues() calls GetLabels in a loop for every issue, causing N+1 queries and context deadline exceeded errors when used with short timeouts or in-memory databases. This is especially problematic since SearchIssues already supports label filtering via SQL WHERE clauses.","acceptance_criteria":"- Optimize scanIssues to batch-load labels for all issues in one query\n- Or make label loading optional/lazy\n- Add test that calls SearchIssues repeatedly with label filters and short context timeouts","status":"closed","priority":1,"issue_type":"bug","created_at":"2025-11-05T19:12:02.245879-08:00","updated_at":"2025-11-05T19:22:11.668682-08:00","closed_at":"2025-11-05T19:22:11.668682-08:00","source_repo":"."} -{"id":"bd-6049","content_hash":"16c54bc547f4ab180aee39efbb197709a47a39047f5bc2dd59e6e6b57ca8bc87","title":"bd doctor --json flag not working","description":"The --json flag on bd doctor command doesn't produce JSON output. It continues to show human-readable output instead. The flag is registered locally on doctorCmd but the code uses the global jsonOutput variable set by PersistentPreRun. Need to investigate why the flag isn't being honored.","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-11-02T17:08:18.170428-08:00","updated_at":"2025-11-02T18:41:01.376783-08:00","closed_at":"2025-11-02T18:41:01.376786-08:00","source_repo":".","comments":[{"id":4,"issue_id":"bd-6049","author":"stevey","text":"Fixed by removing the local --json flag definition in doctor.go that was shadowing the persistent --json flag from main.go. The doctor command now correctly uses the global jsonOutput variable.","created_at":"2025-11-05T08:38:46Z"}]} +{"id":"bd-6049","content_hash":"16c54bc547f4ab180aee39efbb197709a47a39047f5bc2dd59e6e6b57ca8bc87","title":"bd doctor --json flag not working","description":"The --json flag on bd doctor command doesn't produce JSON output. It continues to show human-readable output instead. The flag is registered locally on doctorCmd but the code uses the global jsonOutput variable set by PersistentPreRun. Need to investigate why the flag isn't being honored.","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-11-02T17:08:18.170428-08:00","updated_at":"2025-11-02T18:41:01.376783-08:00","closed_at":"2025-11-02T18:41:01.376786-08:00","source_repo":"."} {"id":"bd-627d","content_hash":"5b3d3d69ceac28dcbfbc2c7ea2f7a6ff2a3a02bc58ce02dcf6b05f8469e8bddc","title":"AI-supervised database migrations for safer schema evolution","description":"## Problem\n\nDatabase migrations can lose user data through edge cases that are hard to anticipate (e.g., GH #201 where bd migrate failed to set issue_prefix, or bd-d355a07d false positive data loss warnings). Since beads is designed to be run by AI agents, we should leverage AI to make migrations safer.\n\n## Current State\n\nMigrations run blindly with:\n- No pre-flight validation\n- No data integrity verification\n- No rollback on failure\n- Limited post-migration testing\n\nRecent issues:\n- GH #201: Migration didn't set issue_prefix config, breaking commands\n- bd-d355a07d: False positive \"data loss\" warnings on collision resolution\n- Users reported migration data loss (fixed but broader problem remains)\n\n## Proposal: AI-Supervised Migration Framework\n\nUse AI to supervise migrations through structured verification:\n\n### 1. Pre-Migration Analysis\n- AI reads migration code and current schema\n- Identifies potential data loss scenarios\n- Generates validation queries to verify assumptions\n- Creates snapshot queries for before/after comparison\n\n### 2. Migration Execution\n- Take database backup/snapshot\n- Run validation queries (pre-state)\n- Execute migration in transaction\n- Run validation queries (post-state)\n\n### 3. Post-Migration Verification\n- AI compares pre/post snapshots\n- Verifies data integrity invariants\n- Checks for unexpected data loss\n- Validates config completeness (like issue_prefix)\n\n### 4. Rollback on Anomalies\n- If AI detects data loss, rollback transaction\n- Present human-readable error report\n- Suggest fix before retrying\n\n## Example Flow\n\n```\n$ bd migrate\n\n→ Analyzing migration plan...\n→ AI identified 3 potential data loss scenarios\n→ Generating validation queries...\n→ Creating pre-migration snapshot...\n→ Running migration in transaction...\n→ Verifying post-migration state...\n✓ All 247 issues accounted for\n✓ Config table complete (issue_prefix: \"mcp\")\n✓ Dependencies intact (342 relationships verified)\n→ Migration successful!\n```\n\nIf something goes wrong:\n```\n$ bd migrate\n\n→ Analyzing migration plan...\n→ AI identified issue: Missing issue_prefix config after migration\n→ Recommendation: Add prefix detection step\n→ Aborting migration - database unchanged\n```\n\n## Implementation Ideas\n\n### A. Migration Validator Tool\nCreate `bd migrate --validate` that:\n- Simulates migration on copy of database\n- Uses AI to verify data integrity\n- Reports potential issues before real migration\n\n### B. Migration Test Generator\nAI generates test cases for migrations:\n- Edge cases (empty DB, large DB, missing config)\n- Data integrity checks\n- Regression tests\n\n### C. Migration Invariants\nDefine invariants that AI checks:\n- Issue count should not decrease (unless collision resolution)\n- All required config keys present\n- Foreign key relationships intact\n- No orphaned dependencies\n\n### D. Self-Healing Migrations\nAI detects incomplete migrations and suggests fixes:\n- Missing config values (like GH #201)\n- Orphaned data\n- Index inconsistencies\n\n## Benefits\n\n1. **Catch edge cases**: AI explores scenarios humans miss\n2. **Self-documenting**: AI explains what migration does\n3. **Agent-friendly**: Agents can run migrations confidently\n4. **Fewer rollbacks**: Detect issues before committing\n5. **Better testing**: AI generates comprehensive test suites\n\n## Open Questions\n\n1. Which AI model? (Fast: Haiku, Thorough: Sonnet/GPT-4)\n2. How to balance safety vs migration speed?\n3. Should AI validation be required or optional?\n4. How to handle offline scenarios (no API access)?\n5. What invariants should always be checked?\n\n## Related Work\n\n- bd-b245: Migration registry (makes migrations introspectable)\n- GH #201: issue_prefix migration bug (motivating example)\n- bd-d355a07d: False positive data loss warnings","design":"## Architecture: Agent-Supervised Migrations (Inversion of Control)\n\n**Key principle:** Beads provides observability and validation primitives. AI agents supervise using their own reasoning. Beads NEVER makes AI API calls.\n\n## Phase 1: Migration Invariants (Pure Validation)\n\nCreate `internal/storage/sqlite/migration_invariants.go`:\n\n```go\ntype MigrationInvariant struct {\n Name string\n Description string\n Check func(*sql.DB, *Snapshot) error\n}\n\ntype Snapshot struct {\n IssueCount int\n ConfigKeys []string\n DependencyCount int\n LabelCount int\n}\n\nvar invariants = []MigrationInvariant{\n {\n Name: \"required_config_present\",\n Description: \"Required config keys must exist\",\n Check: checkRequiredConfig, // Would have caught GH #201\n },\n {\n Name: \"foreign_keys_valid\",\n Description: \"No orphaned dependencies or labels\",\n Check: checkForeignKeys,\n },\n {\n Name: \"issue_count_stable\",\n Description: \"Issue count should not decrease unexpectedly\",\n Check: checkIssueCount,\n },\n}\n\nfunc checkRequiredConfig(db *sql.DB, snapshot *Snapshot) error {\n required := []string{\"issue_prefix\", \"schema_version\"}\n for _, key := range required {\n var value string\n err := db.QueryRow(\"SELECT value FROM config WHERE key = ?\", key).Scan(\u0026value)\n if err != nil || value == \"\" {\n return fmt.Errorf(\"required config key missing: %s\", key)\n }\n }\n return nil\n}\n```\n\n## Phase 2: Dry-Run \u0026 Inspection Tools\n\nAdd `bd migrate --dry-run --json`:\n\n```json\n{\n \"pending_migrations\": [\n {\"name\": \"dirty_issues_table\", \"description\": \"Adds dirty_issues table\"},\n {\"name\": \"content_hash_column\", \"description\": \"Adds content_hash for collision resolution\"}\n ],\n \"current_state\": {\n \"schema_version\": \"0.9.9\",\n \"issue_count\": 247,\n \"config\": {\"schema_version\": \"0.9.9\"},\n \"missing_config\": [\"issue_prefix\"]\n },\n \"warnings\": [\n \"issue_prefix config not set - may break commands after migration\"\n ],\n \"invariants_to_check\": [\n \"required_config_present\",\n \"foreign_keys_valid\",\n \"issue_count_stable\"\n ]\n}\n```\n\nAdd `bd info --schema --json`:\n\n```json\n{\n \"tables\": [\"issues\", \"dependencies\", \"labels\", \"config\"],\n \"schema_version\": \"0.9.9\",\n \"config\": {},\n \"sample_issue_ids\": [\"mcp-1\", \"mcp-2\"],\n \"detected_prefix\": \"mcp\"\n}\n```\n\n## Phase 3: Pre/Post Snapshots with Rollback\n\nUpdate `RunMigrations()`:\n\n```go\nfunc RunMigrations(db *sql.DB) error {\n // Capture pre-migration snapshot\n snapshot := captureSnapshot(db)\n \n // Run migrations in transaction\n tx, err := db.Begin()\n if err != nil {\n return err\n }\n defer tx.Rollback()\n \n for _, migration := range migrations {\n if err := migration.Func(tx); err != nil {\n return fmt.Errorf(\"migration %s failed: %w\", migration.Name, err)\n }\n }\n \n // Verify invariants before commit\n if err := verifyInvariants(tx, snapshot); err != nil {\n return fmt.Errorf(\"post-migration validation failed (rolled back): %w\", err)\n }\n \n return tx.Commit()\n}\n```\n\n## Phase 4: MCP Tools for Agent Supervision\n\nAdd to beads-mcp:\n\n```python\n@server.tool()\nasync def inspect_migration(workspace_root: str) -\u003e dict:\n \"\"\"Get migration plan and current state for agent analysis.\n \n Agent should:\n 1. Review pending migrations\n 2. Check for warnings (missing config, etc.)\n 3. Verify invariants will pass\n 4. Decide whether to run bd migrate\n \"\"\"\n result = run_bd([\"migrate\", \"--dry-run\", \"--json\"], workspace_root)\n return json.loads(result.stdout)\n\n@server.tool() \nasync def get_schema_info(workspace_root: str) -\u003e dict:\n \"\"\"Get current database schema for migration analysis.\"\"\"\n result = run_bd([\"info\", \"--schema\", \"--json\"], workspace_root)\n return json.loads(result.stdout)\n```\n\n## Agent Workflow Example\n\n```python\n# Agent detects user wants to migrate\nmigration_plan = inspect_migration(\"/path/to/workspace\")\n\n# Agent analyzes (using its own reasoning, no API calls from beads)\nif \"issue_prefix\" in migration_plan[\"missing_config\"]:\n schema = get_schema_info(\"/path/to/workspace\")\n detected_prefix = schema[\"detected_prefix\"]\n \n # Agent fixes issue before migration\n run_bd([\"config\", \"set\", \"issue_prefix\", detected_prefix])\n \n# Now safe to migrate\nrun_bd([\"migrate\"])\n```\n\n## What Beads Provides\n\n✅ Deterministic validation (invariants)\n✅ Structured inspection (--dry-run, --explain)\n✅ Rollback on invariant failure\n✅ JSON output for agent parsing\n\n## What Beads Does NOT Do\n\n❌ No AI API calls\n❌ No external model access\n❌ No agent invocation\n\nAgents supervise migrations using their own reasoning and the inspection tools beads provides.","acceptance_criteria":"Phase 1: Migration invariants implemented and tested, checked after every migration, clear error messages when invariants fail.\n\nPhase 2: Snapshot capture before migrations, comparison after, rollback on verification failure.\n\nPhase 3 (stretch): AI validation optional flag implemented, AI can analyze migration code and generate custom validation queries.\n\nPhase 4 (stretch): Migration test fixtures created, all fixtures pass migrations, CI runs migration tests.","notes":"## Progress\n\n### ✅ Phase 1: Migration Invariants (COMPLETED)\n\n**Implemented:**\n- Created internal/storage/sqlite/migration_invariants.go with 3 invariants\n- Updated RunMigrations() to verify invariants after migrations\n- All tests pass ✓\n\n### ✅ Phase 2: Inspection Tools (COMPLETED \u0026 PUSHED)\n\n**Commit:** 1abe4e7 - \"Add migration inspection tools for AI agents (bd-627d Phase 2)\"\n\n**Implemented:**\n1. ✅ bd migrate --inspect --json - Shows migration plan\n2. ✅ bd info --schema --json - Returns schema details\n3. ✅ Migration warnings system\n4. ✅ Documentation updated in AGENTS.md\n5. ✅ All tests pass\n\n### ✅ Phase 3: MCP Tools (COMPLETED \u0026 PUSHED)\n\n**Commit:** 2493693 - \"Add MCP tools for migration inspection (bd-627d Phase 3)\"\n\n**Implemented:**\n1. ✅ inspect_migration(workspace_root) tool in beads-mcp\n2. ✅ get_schema_info(workspace_root) tool in beads-mcp\n3. ✅ Abstract methods in BdClientBase\n4. ✅ CLI client implementations\n5. ✅ All tests pass\n\n**All phases complete!** Migration inspection fully integrated into MCP server.","status":"closed","priority":1,"issue_type":"epic","created_at":"2025-11-02T12:57:10.722048-08:00","updated_at":"2025-11-02T14:31:25.095296-08:00","closed_at":"2025-11-02T14:31:25.095308-08:00","source_repo":"."} {"id":"bd-62a0","content_hash":"b8b2a58a86211a19aed9d21ec5215b4f14ef341ee95d4ed845e1412840d00fd7","title":"Create WASM build infrastructure (Makefile, scripts)","description":"Set up build tooling for WASM compilation:\n- Add GOOS=js GOARCH=wasm build target\n- Copy wasm_exec.js from Go distribution\n- Create wrapper script for Node.js execution\n- Add build task to Makefile or build script","status":"closed","priority":1,"issue_type":"task","created_at":"2025-11-02T21:58:07.286826-08:00","updated_at":"2025-11-02T22:23:49.376789-08:00","closed_at":"2025-11-02T22:23:49.376789-08:00","source_repo":".","dependencies":[{"issue_id":"bd-62a0","depends_on_id":"bd-44d0","type":"parent-child","created_at":"2025-11-02T22:23:49.423064-08:00","created_by":"stevey"}]} {"id":"bd-63e9","content_hash":"7c709804b6d15ce63897344b0674dfae6a4fe97e3ae2768585e2a3407484bad0","title":"Fix Nix flake build test failures","description":"Nix build is failing during test phase with same test errors as Windows.\n\n**Error:**\n```\nerror: Cannot build '/nix/store/rgyi1j44dm6ylrzlg2h3z97axmfq9hzr-beads-0.9.9.drv'.\nReason: builder failed with exit code 1.\nFAIL github.com/steveyegge/beads/cmd/bd 16.141s\n```\n\nThis may be related to test environment setup or the same issues affecting Windows tests.","status":"closed","priority":2,"issue_type":"bug","created_at":"2025-11-02T09:29:37.2851-08:00","updated_at":"2025-11-04T11:10:23.531386-08:00","closed_at":"2025-11-04T11:10:23.531389-08:00","source_repo":".","dependencies":[{"issue_id":"bd-63e9","depends_on_id":"bd-1231","type":"blocks","created_at":"2025-11-02T09:29:37.28618-08:00","created_by":"stevey"}]} @@ -143,7 +143,7 @@ {"id":"bd-85d1","content_hash":"a82c0064b840eacb4896f68e73650a3e99aaeaffbb2a7269a857b6c4245b5572","title":"Add integration tests for multi-repo sync","description":"Test: Clone A deletes issue, Clone B imports Clone A's JSONL. Verify Clone B handles deletion gracefully with resurrection. Test concurrent imports with same orphans (should be idempotent). Test round-trip fidelity (export→delete parent→import→verify structure).","status":"closed","priority":1,"issue_type":"task","created_at":"2025-11-04T12:32:21.410318-08:00","updated_at":"2025-11-05T00:25:06.707471-08:00","closed_at":"2025-11-05T00:25:06.707471-08:00","source_repo":"."} {"id":"bd-86nf","content_hash":"9b3ca05bf503906514baace406ea211b309e8870a716711cc6db9609ecba01e2","title":"Assigned issue","description":"","status":"open","priority":1,"issue_type":"task","assignee":"testuser","created_at":"2025-11-07T19:07:19.017805-08:00","updated_at":"2025-11-07T19:07:19.017805-08:00","source_repo":"."} {"id":"bd-879d","content_hash":"9716c230d9b2793bd1e51d9e3c380c06caf7b3e9a0dd20253764af19e3de7ac8","title":"Test issue 1","description":"","status":"closed","priority":2,"issue_type":"task","created_at":"2025-11-02T09:44:12.538697729Z","updated_at":"2025-11-02T09:45:20.76214671Z","closed_at":"2025-11-02T09:45:20.76214671Z","source_repo":".","dependencies":[{"issue_id":"bd-879d","depends_on_id":"bd-d3e5","type":"discovered-from","created_at":"2025-11-02T09:44:22.103468321Z","created_by":"mrdavidlaing"}]} -{"id":"bd-87a0","content_hash":"b6c322852ff360ade9f0d46bb2af29a7cf3d3acc8b7469dcbb5d98bf48050240","title":"Publish @beads/bd package to npm registry","description":"Publish the npm package to the public npm registry:\n\n## Prerequisites\n- npm account created\n- Organization @beads created (or use different namespace)\n- npm login completed locally\n- Package tested locally (bd-f282 completed)\n\n## Publishing steps\n1. Verify package.json version matches current bd version\n2. Run npm pack and inspect tarball contents\n3. Test installation from tarball one more time\n4. Run npm publish --access public\n5. Verify package appears on https://www.npmjs.com/package/@beads/bd\n6. Test installation from registry: npm install -g @beads/bd\n\n## Post-publish\n- Add npm badge to README.md\n- Update CHANGELOG.md with npm package release\n- Announce in release notes\n\n## Note\n- May need to choose different name if @beads namespace unavailable\n- Alternative: beads-cli, bd-cli, or unscoped beads-issue-tracker","status":"closed","priority":1,"issue_type":"task","created_at":"2025-11-02T23:40:25.263569-08:00","updated_at":"2025-11-03T10:39:41.772338-08:00","closed_at":"2025-11-03T10:39:41.772338-08:00","source_repo":".","dependencies":[{"issue_id":"bd-87a0","depends_on_id":"bd-febc","type":"parent-child","created_at":"2025-11-02T23:40:33.014043-08:00","created_by":"daemon"}],"comments":[{"id":5,"issue_id":"bd-87a0","author":"stevey","text":"Package is ready to publish. All code complete and tested locally. Next steps: 1) npm login, 2) create @beads org if needed, 3) npm publish --access public. See npm-package/PUBLISHING.md for complete instructions.","created_at":"2025-11-05T08:38:46Z"}]} +{"id":"bd-87a0","content_hash":"b6c322852ff360ade9f0d46bb2af29a7cf3d3acc8b7469dcbb5d98bf48050240","title":"Publish @beads/bd package to npm registry","description":"Publish the npm package to the public npm registry:\n\n## Prerequisites\n- npm account created\n- Organization @beads created (or use different namespace)\n- npm login completed locally\n- Package tested locally (bd-f282 completed)\n\n## Publishing steps\n1. Verify package.json version matches current bd version\n2. Run npm pack and inspect tarball contents\n3. Test installation from tarball one more time\n4. Run npm publish --access public\n5. Verify package appears on https://www.npmjs.com/package/@beads/bd\n6. Test installation from registry: npm install -g @beads/bd\n\n## Post-publish\n- Add npm badge to README.md\n- Update CHANGELOG.md with npm package release\n- Announce in release notes\n\n## Note\n- May need to choose different name if @beads namespace unavailable\n- Alternative: beads-cli, bd-cli, or unscoped beads-issue-tracker","status":"closed","priority":1,"issue_type":"task","created_at":"2025-11-02T23:40:25.263569-08:00","updated_at":"2025-11-03T10:39:41.772338-08:00","closed_at":"2025-11-03T10:39:41.772338-08:00","source_repo":".","dependencies":[{"issue_id":"bd-87a0","depends_on_id":"bd-febc","type":"parent-child","created_at":"2025-11-02T23:40:33.014043-08:00","created_by":"daemon"}]} {"id":"bd-8931","content_hash":"409c16d9e6c83c2bf6cccfa6ee6cb18e1e1eee032b522fb99500bb40f2a05649","title":"Daemon gets stuck when auto-import blocked by git conflicts","description":"CRITICAL: The daemon enters a corrupt state that breaks RPC commands when auto-import is triggered but git pull fails due to uncommitted changes.\n\nImpact: This is a data integrity and usability issue that could cause users to lose trust in Beads. The daemon silently fails for certain commands while appearing healthy.\n\nReproduction:\n1. Make local changes to issues (creates uncommitted .beads/beads.jsonl)\n2. Remote has updates (JSONL newer, triggers auto-import)\n3. Daemon tries to pull but fails: 'cannot pull with rebase: You have unstaged changes'\n4. Daemon enters bad state - 'bd show' and other commands return EOF\n5. 'bd list' still works, daemon process is running, no errors logged\n\nTechnical details:\n- Auto-import check runs in handleRequest() before processing RPC commands\n- When import is blocked, it appears to corrupt daemon state\n- Likely: deadlock, unclosed transaction, or storage handle corruption\n- Panic recovery (server_lifecycle_conn.go:183) didn't catch anything - not a panic\n\nRequired fix:\n- Auto-import must not block RPC command execution\n- Handle git pull failures gracefully without corrupting state\n- Consider: skip auto-import if git is dirty, queue import for later, or use separate goroutine\n- Add timeout/circuit breaker for import operations\n- Log clear warnings when auto-import is skipped\n\nWithout this fix, users in collaborative environments will frequently encounter mysterious EOF errors that require daemon restarts.","design":"Options to fix:\n\n1. Skip auto-import when git is dirty (safest, simplest)\n - Check git status before pull\n - Log warning and continue without import\n - User must manually import after cleaning git state\n\n2. Async import with timeout (better UX)\n - Run auto-import in background goroutine\n - Don't block RPC command execution\n - Timeout after 5s, log error if stuck\n - Use sync.Once or similar to prevent concurrent imports\n\n3. Transactional import with rollback\n - Wrap import in database transaction\n - Rollback if git operations fail\n - Ensure storage is never left in bad state\n\nRecommended: Combine #1 and #2\n- Check git status first, skip if dirty\n- If clean, do async import with timeout\n- Add metrics to track import success/failure rates","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-11-02T17:15:25.181425-08:00","updated_at":"2025-11-03T12:08:12.949061-08:00","closed_at":"2025-11-03T12:08:12.949064-08:00","source_repo":".","dependencies":[{"issue_id":"bd-8931","depends_on_id":"bd-1048","type":"blocks","created_at":"2025-11-02T17:15:25.181857-08:00","created_by":"stevey"}]} {"id":"bd-897a","content_hash":"ae488407bf5e71242535f4c35b59b0981d2b8b338d1701f19acba2c8e93049f0","title":"Add UNIQUE constraint on external_ref column","description":"The external_ref column should have a UNIQUE constraint to prevent multiple issues from having the same external reference. This ensures data integrity when syncing from external systems (Jira, GitHub, Linear).\n\nCurrent behavior:\n- Multiple issues can have the same external_ref\n- GetIssueByExternalRef returns first match (non-deterministic with duplicates)\n\nProposed solution:\n- Add UNIQUE constraint to external_ref column\n- Add migration to check for and resolve existing duplicates\n- Update tests to verify constraint enforcement\n\nRelated: bd-1022","status":"closed","priority":2,"issue_type":"task","created_at":"2025-11-02T15:31:54.718005-08:00","updated_at":"2025-11-02T16:01:45.742666-08:00","closed_at":"2025-11-02T16:01:45.742666-08:00","source_repo":"."} {"id":"bd-89e2","content_hash":"c7da1d6c0d28b2fe9a5a1166a7b6712a870b5271fdf21558e4ef639e2675f92f","title":"Daemon race condition: stale export overwrites recent DB changes","description":"**Symptom:**\nMerged bd-fc2d into bd-fb05 in ~/src/beads (commit ce4d756), pushed to remote. The ~/src/fred/beads daemon then exported its stale DB state and committed (8cc1bb4), reverting bd-fc2d back to \"open\" status.\n\n**Timeline:**\n1. 21:45:12 - Merge committed from ~/src/beads (ce4d756): bd-fc2d closed\n2. 21:49:42 - Daemon in ~/src/fred/beads exported stale state (8cc1bb4): bd-fc2d open again\n\n**Root cause:**\nThe fred/beads daemon had a stale database (bd-fc2d still open) and didn't auto-import the newer JSONL before exporting. When it exported, it overwrote the merge with its stale state.\n\n**Expected behavior:**\nDaemon should detect that JSONL is newer than its last export and import before exporting.\n\n**Actual behavior:**\nDaemon exported stale DB state, creating a conflicting commit that reverted upstream changes.\n\n**Impact:**\nMulti-workspace setups with daemons can silently lose changes if one daemon has stale state and exports.","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-11-01T21:53:07.930819-07:00","updated_at":"2025-11-01T22:01:25.54126-07:00","closed_at":"2025-11-01T22:01:25.54126-07:00","source_repo":"."} @@ -317,7 +317,7 @@ {"id":"bd-ry1u","content_hash":"013464efc3cf53d47c2a170040c66099ce77b13d3d1b34c3729e6a5208122799","title":"Publish official devcontainer configuration","description":"","design":"Problem: GH issue #229 shows git hooks aren't available in devcontainers because bd CLI isn't installed. Solution: Provide official .devcontainer/devcontainer.json that installs Go, builds bd from source, runs bd init --quiet, and installs hooks automatically. Benefits: New contributors start immediately, AI assistants work out-of-box, no manual setup. Implementation: Create .devcontainer/devcontainer.json with Go feature, postCreateCommand to build bd and install hooks, environment variables. Acceptance: Container builds with bd installed, hooks functional, documentation updated, tested with Codespaces and VSCode.","notes":"Devcontainer configuration implemented. Manual testing required in actual devcontainer environment (Codespaces or VSCode Remote Containers). All code changes complete, tests pass, linting clean.","status":"closed","priority":2,"issue_type":"epic","created_at":"2025-11-05T15:02:21.783666-08:00","updated_at":"2025-11-05T17:52:53.019917-08:00","closed_at":"2025-11-05T17:46:42.70998-08:00","source_repo":"."} {"id":"bd-s02","content_hash":"911d456e4dabae028dd615b643c99058ef12e55ea523cb81cc933783c7b13546","title":"Manual task","description":"","status":"open","priority":2,"issue_type":"task","created_at":"2025-11-03T20:15:10.022202-08:00","updated_at":"2025-11-03T20:15:10.022202-08:00","source_repo":"."} {"id":"bd-sjmr","content_hash":"31947009877c3246620f5ca4af142234e4e8f43d5a350e80a8b3e37898e81f6a","title":"Fix inconsistent error handling in multi-repo deletion tracking","description":"From bd-xo6b code review: Multi-repo deletion tracking has mixed failure modes that can leave system in inconsistent state.\n\n**Current behavior (daemon_sync.go):**\n- Snapshot capture (L505-514): Hard fail → aborts sync\n- Merge/prune (L575-584): Hard fail → aborts sync \n- Base snapshot update (L613-619): Soft fail → logs warning, continues\n\n**Critical problem:**\nIf merge fails on repo 3 of 5:\n- Repos 1-2 have already merged and deleted issues (irreversible)\n- Repos 3-5 are untouched\n- Database is in partially-updated state\n- No rollback mechanism\n\n**Real-world scenario:**\n```\nSync with repos [A, B, C]:\n1. Capture snapshots A ✓, B ✓, C ✗ → ABORT (good)\n2. Merge A ✓, B ✗ → ABORT but A already deleted issues (BAD - no rollback)\n3. Update base A ⚠, B ⚠ → Warnings only (inconsistent with 1 \u0026 2)\n```\n\n**Solution options:**\n1. **Two-phase commit:**\n - Phase 1: Validate all repos (check files exist, readable, parseable)\n - Phase 2: Apply changes atomically (or fail entirely before any mutations)\n\n2. **Fail-fast validation:**\n - Before any snapshot/merge operations, validate all repos upfront\n - Abort entire sync if any repo fails validation\n\n3. **Make base snapshot update consistent:**\n - Either make it hard-fail like the others, or make all soft-fail\n\n**Files:**\n- cmd/bd/daemon_sync.go:505-514 (snapshot capture)\n- cmd/bd/daemon_sync.go:575-584 (merge/prune)\n- cmd/bd/daemon_sync.go:613-619 (base snapshot update)\n\n**Recommendation:** Use option 1 (two-phase) or option 2 (fail-fast validation) + fix base snapshot inconsistency.","status":"closed","priority":1,"issue_type":"bug","created_at":"2025-11-06T19:31:29.538092-08:00","updated_at":"2025-11-06T19:35:41.268584-08:00","closed_at":"2025-11-06T19:35:41.268584-08:00","source_repo":".","dependencies":[{"issue_id":"bd-sjmr","depends_on_id":"bd-xo6b","type":"discovered-from","created_at":"2025-11-06T19:32:12.310033-08:00","created_by":"daemon"}]} -{"id":"bd-srwk","content_hash":"9bf65cf1bc02d15a26da0948692b28f8e57e08e09ca63d3f7ba035fde047d96f","title":"bd export should detect and prevent stale database exports","description":"## Problem\n\nWhen `bd export` is run with a stale database (older than issues.jsonl), it silently overwrites the JSONL file with stale data, causing data loss.\n\n## What Happened (vc project)\n\n1. Agent A created 4 new issues and exported to issues.jsonl (commit 99a9d58)\n2. Agent A closed an issue and exported again (commit 58b4613) - JSONL now has 4 epics\n3. Agent B had stale database (from before step 1)\n4. Agent B worked on unrelated issue and exported (commit 0609233)\n5. Agent B's export **overwrote issues.jsonl**, removing the 4 epics created by Agent A\n6. Required manual recovery by re-exporting from Agent A's correct database\n\n## Expected Behavior\n\n`bd export` should detect that the database is stale and either:\n- **Refuse to export** with error message explaining the issue\n- **Warn prominently** and require explicit --force flag to override\n- **Auto-import first** to sync database before exporting\n\n## How to Detect Staleness\n\nCompare modification times (similar to VC's ValidateDatabaseFreshness):\n1. Check .db, .db-wal, .db-shm timestamps (use newest for WAL mode)\n2. Check issues.jsonl timestamp\n3. If JSONL is newer by \u003e1 second: database is stale\n\n## Suggested Fix\n\nAdd staleness check in `bd export`:\n\n```go\nfunc Export(dbPath, jsonlPath string, force bool) error {\n // Check if database is stale\n if !force {\n if err := checkDatabaseFreshness(dbPath, jsonlPath); err != nil {\n return fmt.Errorf(\"database is stale: %w\\n\" +\n \"Run 'bd import %s' first to sync, or use --force to override\",\n err, jsonlPath)\n }\n }\n \n // Proceed with export...\n}\n```\n\n## Impact\n\n- **Severity**: High (silent data loss)\n- **Frequency**: Happens in multi-agent workflows when agents don't sync\n- **Workaround**: Manual recovery (re-export from correct database)\n\n## References\n\n- VC issue tracker: commits 58b4613 -\u003e 0609233 -\u003e c41c638\n- VC has similar check: `storage.ValidateDatabaseFreshness()`\n- Tolerance: 1 second (handles filesystem timestamp precision)","acceptance_criteria":"1. bd export detects stale database (JSONL newer than DB)\n2. Refuses to export with clear error message\n3. Suggests running 'bd import' first\n4. --force flag allows override with warning\n5. Tolerates filesystem timestamp precision (1 second)\n6. Works correctly with WAL mode (.db-wal files)\n7. Test coverage for staleness detection\n8. Documentation updated with export safety warnings","status":"open","priority":1,"issue_type":"bug","created_at":"2025-11-07T19:39:24.172154-08:00","updated_at":"2025-11-07T19:39:24.172154-08:00","source_repo":"."} +{"id":"bd-srwk","content_hash":"77494cfb4c2b8f2746367239f800f0e36c93959a47385c7dc2a9d4ac54f4256d","title":"bd export should detect and prevent stale database exports","description":"## Problem\n\nWhen `bd export` is run with a stale database (older than issues.jsonl), it silently overwrites the JSONL file with stale data, causing data loss.\n\n## What Happened (vc project)\n\n1. Agent A created 4 new issues and exported to issues.jsonl (commit 99a9d58)\n2. Agent A closed an issue and exported again (commit 58b4613) - JSONL now has 4 epics\n3. Agent B had stale database (from before step 1)\n4. Agent B worked on unrelated issue and exported (commit 0609233)\n5. Agent B's export **overwrote issues.jsonl**, removing the 4 epics created by Agent A\n6. Required manual recovery by re-exporting from Agent A's correct database\n\n## Expected Behavior\n\n`bd export` should detect that the database is stale and either:\n- **Refuse to export** with error message explaining the issue\n- **Warn prominently** and require explicit --force flag to override\n- **Auto-import first** to sync database before exporting\n\n## How to Detect Staleness\n\nCompare modification times (similar to VC's ValidateDatabaseFreshness):\n1. Check .db, .db-wal, .db-shm timestamps (use newest for WAL mode)\n2. Check issues.jsonl timestamp\n3. If JSONL is newer by \u003e1 second: database is stale\n\n## Suggested Fix\n\nAdd staleness check in `bd export`:\n\n```go\nfunc Export(dbPath, jsonlPath string, force bool) error {\n // Check if database is stale\n if !force {\n if err := checkDatabaseFreshness(dbPath, jsonlPath); err != nil {\n return fmt.Errorf(\"database is stale: %w\\n\" +\n \"Run 'bd import %s' first to sync, or use --force to override\",\n err, jsonlPath)\n }\n }\n \n // Proceed with export...\n}\n```\n\n## Impact\n\n- **Severity**: High (silent data loss)\n- **Frequency**: Happens in multi-agent workflows when agents don't sync\n- **Workaround**: Manual recovery (re-export from correct database)\n\n## References\n\n- VC issue tracker: commits 58b4613 -\u003e 0609233 -\u003e c41c638\n- VC has similar check: `storage.ValidateDatabaseFreshness()`\n- Tolerance: 1 second (handles filesystem timestamp precision)","acceptance_criteria":"1. bd export detects stale database (JSONL newer than DB)\n2. Refuses to export with clear error message\n3. Suggests running 'bd import' first\n4. --force flag allows override with warning\n5. Tolerates filesystem timestamp precision (1 second)\n6. Works correctly with WAL mode (.db-wal files)\n7. Test coverage for staleness detection\n8. Documentation updated with export safety warnings","notes":"Fixed with ID-based comparison instead of just count. Now detects:\n1. DB has fewer issues than JSONL (count check)\n2. DB has different issues than JSONL (ID comparison)\n\nBoth scenarios now properly refuse export unless --force is used.\n\nImplementation uses getIssueIDsFromJSONL() to build a set of IDs from JSONL, then checks if any JSONL IDs are missing from DB. Shows specific missing issue IDs in error message.","status":"closed","priority":1,"issue_type":"bug","assignee":"amp","created_at":"2025-11-07T19:39:24.172154-08:00","updated_at":"2025-11-07T20:05:13.649736-08:00","closed_at":"2025-11-07T19:58:43.300177-08:00","source_repo":"."} {"id":"bd-tmdx","content_hash":"3ecb07e74eaa468c9fc7a1c6e1c4547182267785b9dbc7998f031d71552842cf","title":"Investigate database pollution - unexpected issue count increases","description":"Two repositories showing unexpected issue counts:\n- ~/src/beads: 280 issues (expected ~209-220)\n- ~/src/dave/beads: 895 issues (675 open, 149 closed)\n\nThis suggests database pollution - issues from one repository leaking into another. Need to investigate:\n1. Run bd detect-pollution on both repos\n2. Check for cross-repo contamination\n3. Identify source of pollution (daemon? multi-repo config? import issues?)\n4. Clean up polluted databases\n5. Prevent future pollution","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-11-06T22:50:16.957689-08:00","updated_at":"2025-11-07T00:05:38.994405-08:00","closed_at":"2025-11-07T00:05:38.994405-08:00","source_repo":"."} {"id":"bd-tuqd","content_hash":"06ac95944f03d871a6f58d2cd63796828873e92ef7c9b897eb639d28860a458e","title":"bd init overwrites existing git hooks without detection or chaining","description":"GH #254: bd init silently overwrites existing git hooks (like pre-commit framework) without detecting them, backing them up, or offering to chain. This breaks workflows and can result in committed code with failing tests.\n\nFix: Detect existing hooks, prompt user with options to chain/overwrite/skip.","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-11-07T15:51:17.582882-08:00","updated_at":"2025-11-07T15:55:01.330531-08:00","closed_at":"2025-11-07T15:55:01.330531-08:00","source_repo":"."} {"id":"bd-twlr","content_hash":"85bb87aa0a7988bf3399b1ebfe3115302aa9e487097415cb8573cd8aad7392c2","title":"Add bd init --team wizard","description":"Interactive wizard for team workflow setup. Guides user through: branch workflow configuration, shared repo setup, team member onboarding, examples of team collaboration patterns.","status":"closed","priority":1,"issue_type":"task","created_at":"2025-11-05T18:04:30.013645-08:00","updated_at":"2025-11-05T18:56:03.004161-08:00","closed_at":"2025-11-05T18:56:03.004161-08:00","source_repo":".","dependencies":[{"issue_id":"bd-twlr","depends_on_id":"bd-8rd","type":"parent-child","created_at":"2025-11-05T18:04:39.164445-08:00","created_by":"daemon"}]} diff --git a/cmd/bd/export.go b/cmd/bd/export.go index 3967994b..be194229 100644 --- a/cmd/bd/export.go +++ b/cmd/bd/export.go @@ -46,6 +46,37 @@ func countIssuesInJSONL(path string) (int, error) { return count, nil } +// getIssueIDsFromJSONL reads a JSONL file and returns a set of issue IDs +func getIssueIDsFromJSONL(path string) (map[string]bool, error) { + // #nosec G304 - controlled path from config + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { + if err := file.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to close file: %v\n", err) + } + }() + + ids := make(map[string]bool) + decoder := json.NewDecoder(file) + lineNum := 0 + for { + var issue types.Issue + if err := decoder.Decode(&issue); err != nil { + if err.Error() == "EOF" { + break + } + // Return error for corrupt/invalid JSON + return ids, fmt.Errorf("invalid JSON at line %d: %w", lineNum+1, err) + } + ids[issue.ID] = true + lineNum++ + } + return ids, nil +} + // validateExportPath checks if the output path is safe to write to func validateExportPath(path string) error { // Get absolute path to normalize it @@ -90,6 +121,8 @@ Output to stdout by default, or use -o flag for file output.`, output, _ := cmd.Flags().GetString("output") statusFilter, _ := cmd.Flags().GetString("status") force, _ := cmd.Flags().GetBool("force") + + debug.Logf("Debug: export flags - output=%q, force=%v\n", output, force) if format != "jsonl" { fmt.Fprintf(os.Stderr, "Error: only 'jsonl' format is currently supported\n") @@ -104,6 +137,10 @@ Output to stdout by default, or use -o flag for file output.`, daemonClient = nil } + // Note: We used to check database file timestamps here, but WAL files + // get created when opening the DB, making timestamp checks unreliable. + // Instead, we check issue counts after loading (see below). + // Ensure we have a direct store connection if store == nil { var err error @@ -153,20 +190,61 @@ Output to stdout by default, or use -o flag for file output.`, } } - // Warning: check if export would lose >50% of issues - if output != "" { - existingCount, err := countIssuesInJSONL(output) - if err == nil && existingCount > 0 { - lossPercent := float64(existingCount-len(issues)) / float64(existingCount) * 100 - if lossPercent > 50 { - fmt.Fprintf(os.Stderr, "WARNING: Export would lose %.1f%% of issues!\n", lossPercent) - fmt.Fprintf(os.Stderr, " Existing JSONL: %d issues\n", existingCount) - fmt.Fprintf(os.Stderr, " Database: %d issues\n", len(issues)) - fmt.Fprintf(os.Stderr, " This suggests database staleness or corruption.\n") - fmt.Fprintf(os.Stderr, "Press Ctrl+C to abort, or Enter to continue: ") - // Read a line from stdin to wait for user confirmation - var response string - _, _ = fmt.Scanln(&response) // ignore EOF on empty input + // Safety check: prevent exporting stale database that would lose issues + if output != "" && !force { + debug.Logf("Debug: checking staleness - output=%s, force=%v\n", output, force) + + // Read existing JSONL to get issue IDs + jsonlIDs, err := getIssueIDsFromJSONL(output) + if err != nil && !os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "Warning: failed to read existing JSONL for staleness check: %v\n", err) + } + + if err == nil && len(jsonlIDs) > 0 { + // Build set of DB issue IDs + dbIDs := make(map[string]bool) + for _, issue := range issues { + dbIDs[issue.ID] = true + } + + // Check if JSONL has any issues that DB doesn't have + var missingIDs []string + for id := range jsonlIDs { + if !dbIDs[id] { + missingIDs = append(missingIDs, id) + } + } + + debug.Logf("Debug: JSONL has %d issues, DB has %d issues, missing %d\n", + len(jsonlIDs), len(issues), len(missingIDs)) + + if len(missingIDs) > 0 { + sort.Strings(missingIDs) + fmt.Fprintf(os.Stderr, "Error: refusing to export stale database that would lose issues\n") + fmt.Fprintf(os.Stderr, " Database has %d issues\n", len(issues)) + fmt.Fprintf(os.Stderr, " JSONL has %d issues\n", len(jsonlIDs)) + fmt.Fprintf(os.Stderr, " Export would lose %d issue(s):\n", len(missingIDs)) + + // Show first 10 missing issues + showCount := len(missingIDs) + if showCount > 10 { + showCount = 10 + } + for i := 0; i < showCount; i++ { + fmt.Fprintf(os.Stderr, " - %s\n", missingIDs[i]) + } + if len(missingIDs) > 10 { + fmt.Fprintf(os.Stderr, " ... and %d more\n", len(missingIDs)-10) + } + + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "This usually means:\n") + fmt.Fprintf(os.Stderr, " 1. You need to run 'bd import -i %s' to sync the latest changes\n", output) + fmt.Fprintf(os.Stderr, " 2. Or another workspace added issues that weren't synced to this database\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "To force export anyway (will lose these issues):\n") + fmt.Fprintf(os.Stderr, " bd export -o %s --force\n", output) + os.Exit(1) } } } diff --git a/cmd/bd/export_staleness_test.go b/cmd/bd/export_staleness_test.go new file mode 100644 index 00000000..f2cd1fe7 --- /dev/null +++ b/cmd/bd/export_staleness_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// TestExportStaleness_DBHasFewerIssues tests that export refuses when database +// has fewer issues than JSONL (indicating staleness) +func TestExportStaleness_DBHasFewerIssues(t *testing.T) { + tmpDir := t.TempDir() + + // Create JSONL with 3 issues + jsonlPath := filepath.Join(tmpDir, "test.jsonl") + jsonlContent := `{"id":"test-1","title":"Issue 1","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +{"id":"test-2","title":"Issue 2","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +{"id":"test-3","title":"Issue 3","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +` + if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0600); err != nil { + t.Fatalf("Failed to create JSONL: %v", err) + } + + // Verify count function works + count, err := countIssuesInJSONL(jsonlPath) + if err != nil { + t.Fatalf("Failed to count issues: %v", err) + } + if count != 3 { + t.Errorf("Expected 3 issues in JSONL, got %d", count) + } +} + +// TestExportStaleness_DBHasSameIssues tests that export succeeds when database +// has same number of issues as JSONL +func TestExportStaleness_DBHasSameIssues(t *testing.T) { + tmpDir := t.TempDir() + + // Create JSONL with 2 issues + jsonlPath := filepath.Join(tmpDir, "test.jsonl") + jsonlContent := `{"id":"test-1","title":"Issue 1","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +{"id":"test-2","title":"Issue 2","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +` + if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0600); err != nil { + t.Fatalf("Failed to create JSONL: %v", err) + } + + // Verify count + count, err := countIssuesInJSONL(jsonlPath) + if err != nil { + t.Fatalf("Failed to count issues: %v", err) + } + if count != 2 { + t.Errorf("Expected 2 issues in JSONL, got %d", count) + } +} + +// TestExportStaleness_NoJSONL tests that export succeeds when JSONL doesn't exist +func TestExportStaleness_NoJSONL(t *testing.T) { + tmpDir := t.TempDir() + jsonlPath := filepath.Join(tmpDir, "nonexistent.jsonl") + + // Should not error when file doesn't exist + _, err := countIssuesInJSONL(jsonlPath) + if err == nil { + t.Error("Expected error when JSONL doesn't exist") + } + if !os.IsNotExist(err) { + t.Errorf("Expected IsNotExist error, got: %v", err) + } +} + +// TestExportStaleness_DifferentIssues tests that export refuses when database +// has different issues than JSONL (even with same count) +func TestExportStaleness_DifferentIssues(t *testing.T) { + tmpDir := t.TempDir() + + // Create JSONL with issues test-1, test-2, test-3 + jsonlPath := filepath.Join(tmpDir, "test.jsonl") + jsonlContent := `{"id":"test-1","title":"Issue 1","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +{"id":"test-2","title":"Issue 2","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +{"id":"test-3","title":"Issue 3","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +` + if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0600); err != nil { + t.Fatalf("Failed to create JSONL: %v", err) + } + + // Verify getIssueIDsFromJSONL function + ids, err := getIssueIDsFromJSONL(jsonlPath) + if err != nil { + t.Fatalf("Failed to get issue IDs: %v", err) + } + if len(ids) != 3 { + t.Errorf("Expected 3 issue IDs, got %d", len(ids)) + } + if !ids["test-1"] || !ids["test-2"] || !ids["test-3"] { + t.Errorf("Missing expected issue IDs, got: %v", ids) + } +} + +// TestGetIssueIDsFromJSONL_InvalidJSON tests error handling for corrupt JSONL +func TestGetIssueIDsFromJSONL_InvalidJSON(t *testing.T) { + tmpDir := t.TempDir() + jsonlPath := filepath.Join(tmpDir, "corrupt.jsonl") + + // Create JSONL with invalid JSON on second line + jsonlContent := `{"id":"test-1","title":"Issue 1","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +this is not valid JSON +{"id":"test-3","title":"Issue 3","status":"open","priority":1,"issue_type":"task","created_at":"2025-01-01T00:00:00Z","updated_at":"2025-01-01T00:00:00Z"} +` + if err := os.WriteFile(jsonlPath, []byte(jsonlContent), 0600); err != nil { + t.Fatalf("Failed to create JSONL: %v", err) + } + + // Should return error with first valid issue ID read + ids, err := getIssueIDsFromJSONL(jsonlPath) + if err == nil { + t.Error("Expected error for invalid JSON") + } + // Should have read the first line before hitting the error + if len(ids) != 1 || !ids["test-1"] { + t.Errorf("Expected to have read test-1 before error, got: %v", ids) + } +}