Fix bd-160: Implement JSONL integrity validation and prevent export deduplication data loss
## Problem Export deduplication feature broke when JSONL and export_hashes diverged (e.g., after git pull/reset). This caused exports to skip issues that weren't actually in the file, leading to silent data loss. ## Solution 1. JSONL integrity validation before every export - Store JSONL file hash after export - Validate hash before export, clear export_hashes if mismatch - Automatically recovers from git operations changing JSONL 2. Clear export_hashes on all imports - Prevents stale hashes from causing future export failures - Import operations invalidate export_hashes state 3. Add Storage interface methods: - GetJSONLFileHash/SetJSONLFileHash for integrity tracking - ClearAllExportHashes for recovery ## Tests Added - TestJSONLIntegrityValidation: Unit tests for validation logic - TestImportClearsExportHashes: Verifies imports clear hashes - TestExportIntegrityAfterJSONLTruncation: Simulates git reset (would have caught bd-160) - TestExportIntegrityAfterJSONLDeletion: Tests recovery from file deletion - TestMultipleExportsStayConsistent: Tests repeated export integrity ## Follow-up Created bd-179 epic for remaining integration test gaps (multi-repo sync, daemon auto-sync, corruption recovery tests). Closes bd-160
This commit is contained in:
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -273,6 +275,17 @@ Output to stdout by default, or use -o flag for file output.`,
|
||||
// Clear auto-flush state since we just manually exported
|
||||
// This cancels any pending auto-flush timer and marks DB as clean
|
||||
clearAutoFlushState()
|
||||
|
||||
// Store JSONL file hash for integrity validation (bd-160)
|
||||
jsonlData, err := os.ReadFile(finalPath)
|
||||
if err == nil {
|
||||
hasher := sha256.New()
|
||||
hasher.Write(jsonlData)
|
||||
fileHash := hex.EncodeToString(hasher.Sum(nil))
|
||||
if err := store.SetJSONLFileHash(ctx, fileHash); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to update jsonl_file_hash: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If writing to file, atomically replace the target file
|
||||
|
||||
Reference in New Issue
Block a user