From aada5d9ac67402cc51e64e4e33fa276e16d67a9b Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Sat, 25 Oct 2025 16:37:54 -0700 Subject: [PATCH] Fix bd-144: Update main .db file timestamp after import (WAL mode) - Added CheckpointWAL method to SQLite storage - Import now checkpoints WAL after completion - Updates main .db file modification time for staleness detection - PRAGMA wal_checkpoint(FULL) flushes WAL to main database --- .beads/beads.jsonl | 4 ++-- cmd/bd/import_shared.go | 8 ++++++++ internal/storage/sqlite/sqlite.go | 9 +++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.beads/beads.jsonl b/.beads/beads.jsonl index fdc3d108..00cad086 100644 --- a/.beads/beads.jsonl +++ b/.beads/beads.jsonl @@ -47,8 +47,8 @@ {"id":"bd-140","title":"Add integration tests for multi-project MCP switching","description":"Comprehensive tests to verify multi-project isolation, concurrency, and edge cases.\n\nEXPANDED TEST COVERAGE (per architectural review):\n\n**Concurrency tests (CRITICAL):**\n- asyncio.gather() with calls to different workspace_root values\n- Verify no cross-project data leakage\n- Verify pool lock prevents race conditions\n\n**Edge case tests:**\n- Submodule handling: Parent repo vs submodule with own .beads\n- Symlink deduplication: Same physical path via different symlinks\n- Stale socket recovery: Kill daemon, verify retry on failure\n- Missing .beads directory handling\n\n**Isolation tests:**\n- Create 2+ temp repos with bd init\n- Verify operations in project A don't affect project B\n- Stress test: many parallel calls across 3-5 repos\n\nEstimated effort: M (1-2 days) including fixtures for temp repos and daemon process management","design":"Test structure:\n\n1. test_concurrent_multi_project.py:\n - asyncio.gather with 2+ projects\n - Verify pool lock prevents corruption\n \n2. test_path_canonicalization.py:\n - Submodule edge case (check .beads first)\n - Symlink deduplication (realpath normalization)\n \n3. test_stale_socket_recovery.py:\n - Kill daemon mid-session\n - Verify retry-on-failure works\n \n4. test_cross_project_isolation.py:\n - Create issues in project A\n - List from project B, verify empty\n - No data leakage\n\nUse pytest fixtures for temp repos and daemon lifecycle.","acceptance_criteria":"- All concurrency tests pass with asyncio.gather\n- Submodule edge case handled correctly\n- Symlinks deduplicated to same connection\n- Stale socket retry works\n- No cross-project data leakage in stress tests","status":"closed","priority":1,"issue_type":"task","assignee":"amp","created_at":"2025-10-25T14:00:27.896623-07:00","updated_at":"2025-10-25T14:35:13.09686-07:00","closed_at":"2025-10-25T14:35:13.09686-07:00","dependencies":[{"issue_id":"bd-140","depends_on_id":"bd-135","type":"parent-child","created_at":"2025-10-25T14:00:27.90028-07:00","created_by":"daemon"}]} {"id":"bd-141","title":"Update MCP multi-project documentation","description":"Update documentation for multi-project workflow in README.md, AGENTS.md, and MCP integration docs.\n\nEXPANDED SECTIONS (per architectural review):\n\n**Usage examples:**\n- Per-request workspace_root parameter usage\n- Concurrent multi-project queries with asyncio.gather\n- Migration from set_context() to workspace_root parameter\n\n**Architecture notes:**\n- Connection pooling behavior (no limits initially)\n- set_context() as default fallback (still supported)\n- Library users NOT affected (all changes in MCP layer)\n\n**Concurrency gotchas (CRITICAL):**\n- ContextVar doesn't propagate to asyncio.create_task()\n- Do NOT spawn background tasks in tool implementations\n- All tool calls should be synchronous/sequential\n\n**Troubleshooting:**\n- Stale sockets (retry once on failure)\n- Version mismatches (auto-detected since v0.16.0)\n- Path aliasing via symlinks (deduplicated by realpath)\n- Submodules with own .beads (handled correctly)\n\n**Use cases:**\n- Multi-organization collaboration (GH#145)\n- Parallel project management scripts\n- Cross-project queries","design":"Documentation structure:\n\n1. integrations/beads-mcp/README.md:\n - Add \"Multi-Project Support\" section\n - workspace_root parameter examples\n - Connection pool behavior\n \n2. AGENTS.md:\n - Update MCP section with workspace_root usage\n - Add concurrency warning (no spawned tasks)\n - Document library non-impact\n \n3. New: docs/MCP_MULTI_PROJECT.md:\n - Detailed architecture explanation\n - Migration guide from set_context()\n - Troubleshooting guide\n - Edge cases (submodules, symlinks)","status":"closed","priority":2,"issue_type":"task","assignee":"amp","created_at":"2025-10-25T14:00:27.897025-07:00","updated_at":"2025-10-25T14:35:55.392654-07:00","closed_at":"2025-10-25T14:35:55.392654-07:00","dependencies":[{"issue_id":"bd-141","depends_on_id":"bd-135","type":"parent-child","created_at":"2025-10-25T14:00:27.901495-07:00","created_by":"daemon"}]} {"id":"bd-142","title":"Add health checks and reconnection logic for stale sockets","description":"⚠️ DEFERRED TO PHASE 2 (per architectural review)\n\nAdd health checks and reconnection logic for stale sockets ONLY IF monitoring shows it's needed.\n\nSIMPLIFIED APPROACH:\n- NO preemptive health checks (adds latency to every call)\n- NO periodic ping before use (daemon restarts are rare)\n- YES: Single retry on connection failure (DaemonConnectionError)\n- YES: Evict stale client from pool on failure\n\nRationale:\n- Stale sockets are rare (daemon auto-restart is uncommon)\n- Preemptive checks add latency with little benefit\n- Retry-on-failure is sufficient for most cases\n\nImplementation (if needed later):\n- Wrap tool calls with try/except\n- On DaemonConnectionError: evict from pool, retry once\n- Log failures for monitoring\n\nMonitor after Phase 1 launch:\n- Frequency of stale socket errors\n- User reports of connection issues\n- Decision point: Add if \u003e1% of calls fail","design":"Simplified retry wrapper (implement only if monitoring shows need):\n\n```python\nasync def _robust_client_call(func):\n try:\n client = await _get_client()\n return await func(client)\n except (DaemonConnectionError, DaemonNotRunningError):\n # Evict stale client and retry once\n workspace = current_workspace.get()\n canonical = _canonicalize_path(workspace)\n async with _pool_lock:\n _connection_pool.pop(canonical, None)\n # Retry\n client = await _get_client()\n return await func(client)\n```\n\nNO bounded backoff, NO health checks, NO version validation pings.","notes":"DEFERRED - Not needed for MVP.\n\nAdd to Phase 2 roadmap only if monitoring shows:\n- Stale socket errors \u003e1% of calls\n- User complaints about connection issues\n- Long-running MCP servers experiencing problems","status":"closed","priority":3,"issue_type":"task","created_at":"2025-10-25T14:00:36.252409-07:00","updated_at":"2025-10-25T14:35:55.394617-07:00","closed_at":"2025-10-25T14:35:55.394617-07:00","dependencies":[{"issue_id":"bd-142","depends_on_id":"bd-135","type":"parent-child","created_at":"2025-10-25T14:00:42.132775-07:00","created_by":"daemon"}]} -{"id":"bd-143","title":"bd daemon auto-sync can wipe out issues.jsonl when database is empty","description":"During dogfooding session, bd daemon auto-sync exported empty database to JSONL, losing all 177 issues. Had to git restore to recover.\n\nRoot cause: bd export doesn't check if database is empty before exporting. When daemon has empty/wrong database, it wipes out valid JSONL file.\n\nImpact: DATA LOSS","design":"Add safeguard in bd export:\n1. Count total issues in database before export\n2. If count is 0, refuse to export and show error\n3. Provide --force flag to override if truly want empty export\n\nAlternative: Check if target JSONL exists and has issues, warn if about to replace with empty export","acceptance_criteria":"- bd export refuses to export when database has 0 issues\n- Clear error message: \"Refusing to export empty database (0 issues). Use --force to override.\"\n- --force flag allows override for intentional empty exports\n- Test: export with empty db fails, export with --force succeeds","status":"in_progress","priority":0,"issue_type":"bug","created_at":"2025-10-25T16:29:16.045548-07:00","updated_at":"2025-10-25T16:30:16.559585-07:00"} -{"id":"bd-144","title":"bd import doesn't update database modification time (WAL mode)","description":"When running bd import in WAL mode, the -wal file is updated but main .db file timestamp stays old. This breaks staleness detection which only checks main .db file.\n\nDiscovered during dogfooding when import didn't trigger staleness refresh.\n\nImpact: Staleness checks fail to detect that database is newer than expected","design":"Two options:\n1. Checkpoint WAL after import to flush changes to main .db file\n2. Update staleness detection to check both .db and -wal file timestamps\n\nOption 1 is simpler and safer - just add PRAGMA wal_checkpoint(FULL) after import completes","acceptance_criteria":"- After bd import, main .db file modification time is updated\n- Staleness detection correctly sees database as fresh\n- Test: import, check .db mtime, verify it's recent","status":"open","priority":1,"issue_type":"bug","created_at":"2025-10-25T16:29:16.048176-07:00","updated_at":"2025-10-25T16:29:16.048176-07:00"} +{"id":"bd-143","title":"bd daemon auto-sync can wipe out issues.jsonl when database is empty","description":"During dogfooding session, bd daemon auto-sync exported empty database to JSONL, losing all 177 issues. Had to git restore to recover.\n\nRoot cause: bd export doesn't check if database is empty before exporting. When daemon has empty/wrong database, it wipes out valid JSONL file.\n\nImpact: DATA LOSS","design":"Add safeguard in bd export:\n1. Count total issues in database before export\n2. If count is 0, refuse to export and show error\n3. Provide --force flag to override if truly want empty export\n\nAlternative: Check if target JSONL exists and has issues, warn if about to replace with empty export","acceptance_criteria":"- bd export refuses to export when database has 0 issues\n- Clear error message: \"Refusing to export empty database (0 issues). Use --force to override.\"\n- --force flag allows override for intentional empty exports\n- Test: export with empty db fails, export with --force succeeds","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-10-25T16:29:16.045548-07:00","updated_at":"2025-10-25T16:35:38.233384-07:00","closed_at":"2025-10-25T16:35:38.233384-07:00"} +{"id":"bd-144","title":"bd import doesn't update database modification time (WAL mode)","description":"When running bd import in WAL mode, the -wal file is updated but main .db file timestamp stays old. This breaks staleness detection which only checks main .db file.\n\nDiscovered during dogfooding when import didn't trigger staleness refresh.\n\nImpact: Staleness checks fail to detect that database is newer than expected","design":"Two options:\n1. Checkpoint WAL after import to flush changes to main .db file\n2. Update staleness detection to check both .db and -wal file timestamps\n\nOption 1 is simpler and safer - just add PRAGMA wal_checkpoint(FULL) after import completes","acceptance_criteria":"- After bd import, main .db file modification time is updated\n- Staleness detection correctly sees database as fresh\n- Test: import, check .db mtime, verify it's recent","status":"in_progress","priority":1,"issue_type":"bug","created_at":"2025-10-25T16:29:16.048176-07:00","updated_at":"2025-10-25T16:36:29.60624-07:00"} {"id":"bd-145","title":"bd should show which database file it's using","description":"During dogfooding, bd showed \"0 issues\" when correct database had 177 issues. Confusion arose from which database path was being used (daemon default vs explicit --db flag).\n\nUsers need clear feedback about which database file bd is actually using, especially when daemon is involved.\n\nImpact: User confusion, working with wrong database unknowingly","design":"Add database path to verbose output or as a bd info command:\n1. bd info shows current database path, daemon status\n2. OR: bd ready/list/etc --verbose shows \"Using database: /path/to/.beads/beads.db\"\n3. Consider adding to bd status output\n\nWhen database path differs from expected, show warning","acceptance_criteria":"- User can easily determine which database file bd is using\n- bd info or similar command shows full database path\n- When using unexpected database (e.g., daemon vs explicit --db), show clear indication\n- Documentation updated with how to check database path","status":"open","priority":1,"issue_type":"feature","created_at":"2025-10-25T16:29:16.059118-07:00","updated_at":"2025-10-25T16:29:16.059118-07:00"} {"id":"bd-15","title":"Make merge command idempotent for safe retry after partial failures","description":"The merge command currently performs 3 operations without an outer transaction:\n1. Migrate dependencies from source → target\n2. Update text references across all issues\n3. Close source issues\n\nIf merge fails mid-operation (network issue, daemon crash, etc.), a retry will fail or produce incorrect results because some operations already succeeded.\n\n**Goal:** Make merge idempotent so retrying after partial failure is safe and completes the remaining work.\n\n**Idempotency checks needed:**\n- Skip dependency migration if target already has the dependency\n- Skip text reference updates if already updated\n- Skip closing source issues if already closed\n- Report which operations were skipped vs performed\n\n**Example output:**\n```\n✓ Merged 2 issue(s) into bd-63\n - Dependencies: 3 migrated, 2 already existed\n - Text references: 5 updated, 0 already correct\n - Source issues: 1 closed, 1 already closed\n```\n\n**Related:** bd-115 originally requested transaction support, but idempotency is a better solution for this use case since individual operations are already atomic.","design":"Current merge code already has some idempotency:\n- Dependency migration checks `alreadyExists` before adding (line ~145-151 in merge.go)\n- Text reference updates are naturally idempotent (replacing bd-X with bd-Y twice has same result)\n\nMissing idempotency:\n- CloseIssue fails if source already closed\n- Error messages don't distinguish \"already done\" from \"real failure\"\n\nImplementation:\n1. Check source issue status before closing - skip if already closed\n2. Track which operations succeeded/skipped\n3. Return detailed results for user visibility\n4. Consider adding --dry-run output showing what would be done vs skipped","status":"closed","priority":2,"issue_type":"feature","created_at":"2025-10-22T00:47:43.165434-07:00","updated_at":"2025-10-24T13:51:54.437619-07:00","closed_at":"2025-10-22T11:56:36.526276-07:00"} {"id":"bd-16","title":"Global daemon should warn/reject --auto-commit and --auto-push","description":"When user runs 'bd daemon --global --auto-commit', it's unclear which repo the daemon will commit to (especially after fixing bd-62 where global daemon won't open a DB).\n\nOptions:\n1. Warn and ignore the flags in global mode\n2. Error out with clear message\n\nLine 87-91 already checks autoPush, but should skip check entirely for global mode. Add user-friendly messaging about flag incompatibility.","status":"closed","priority":3,"issue_type":"feature","created_at":"2025-10-22T00:47:43.165645-07:00","updated_at":"2025-10-24T13:51:54.437812-07:00","closed_at":"2025-10-17T23:04:30.223432-07:00"} diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index f4845caa..cb7935ec 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "sort" "strings" @@ -242,6 +243,13 @@ func importIssuesCore(ctx context.Context, dbPath string, store storage.Storage, return nil, err } + // Phase 8: Checkpoint WAL to update main .db file timestamp + // This ensures staleness detection sees the database as fresh + if err := sqliteStore.CheckpointWAL(ctx); err != nil { + // Non-fatal - just log warning + fmt.Fprintf(os.Stderr, "Warning: failed to checkpoint WAL: %v\n", err) + } + return result, nil } diff --git a/internal/storage/sqlite/sqlite.go b/internal/storage/sqlite/sqlite.go index a168102e..7339517b 100644 --- a/internal/storage/sqlite/sqlite.go +++ b/internal/storage/sqlite/sqlite.go @@ -2062,3 +2062,12 @@ func (s *SQLiteStorage) UnderlyingDB() *sql.DB { func (s *SQLiteStorage) UnderlyingConn(ctx context.Context) (*sql.Conn, error) { return s.db.Conn(ctx) } + +// CheckpointWAL checkpoints the WAL file to flush changes to the main database file. +// This updates the main .db file's modification time, which is important for staleness detection. +// In WAL mode, writes go to the -wal file, leaving the main .db file untouched. +// Checkpointing flushes the WAL to the main database file. +func (s *SQLiteStorage) CheckpointWAL(ctx context.Context) error { + _, err := s.db.ExecContext(ctx, "PRAGMA wal_checkpoint(FULL)") + return err +}