Fix bd-160: Implement JSONL integrity validation and prevent export deduplication data loss
## Problem Export deduplication feature broke when JSONL and export_hashes diverged (e.g., after git pull/reset). This caused exports to skip issues that weren't actually in the file, leading to silent data loss. ## Solution 1. JSONL integrity validation before every export - Store JSONL file hash after export - Validate hash before export, clear export_hashes if mismatch - Automatically recovers from git operations changing JSONL 2. Clear export_hashes on all imports - Prevents stale hashes from causing future export failures - Import operations invalidate export_hashes state 3. Add Storage interface methods: - GetJSONLFileHash/SetJSONLFileHash for integrity tracking - ClearAllExportHashes for recovery ## Tests Added - TestJSONLIntegrityValidation: Unit tests for validation logic - TestImportClearsExportHashes: Verifies imports clear hashes - TestExportIntegrityAfterJSONLTruncation: Simulates git reset (would have caught bd-160) - TestExportIntegrityAfterJSONLDeletion: Tests recovery from file deletion - TestMultipleExportsStayConsistent: Tests repeated export integrity ## Follow-up Created bd-179 epic for remaining integration test gaps (multi-repo sync, daemon auto-sync, corruption recovery tests). Closes bd-160
This commit is contained in:
Steve Yegge
237
cmd/bd/jsonl_integrity_test.go
Normal file
237
cmd/bd/jsonl_integrity_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage/sqlite"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
const testActor = "test"
|
||||
|
||||
// TestJSONLIntegrityValidation tests the JSONL integrity validation (bd-160)
|
||||
func TestJSONLIntegrityValidation(t *testing.T) {
|
||||
// Create temp directory
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, ".beads", "beads.db")
|
||||
jsonlPath := filepath.Join(tmpDir, ".beads", "issues.jsonl")
|
||||
|
||||
// Ensure .beads directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(dbPath), 0755); err != nil {
|
||||
t.Fatalf("failed to create .beads directory: %v", err)
|
||||
}
|
||||
|
||||
// Create database
|
||||
testStore, err := sqlite.New(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer testStore.Close()
|
||||
|
||||
// Set global store for validateJSONLIntegrity
|
||||
oldStore := store
|
||||
store = testStore
|
||||
defer func() { store = oldStore }()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Initialize database with prefix
|
||||
if err := testStore.SetConfig(ctx, "issue_prefix", "bd"); err != nil {
|
||||
t.Fatalf("failed to set issue prefix: %v", err)
|
||||
}
|
||||
|
||||
// Create a test issue
|
||||
issue := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Test issue",
|
||||
Description: "Test description",
|
||||
Status: types.StatusOpen,
|
||||
Priority: 1,
|
||||
IssueType: types.TypeTask,
|
||||
}
|
||||
|
||||
if err := testStore.CreateIssue(ctx, issue, testActor); err != nil {
|
||||
t.Fatalf("failed to create issue: %v", err)
|
||||
}
|
||||
|
||||
// Export to JSONL
|
||||
issues := []*types.Issue{issue}
|
||||
exportedIDs, err := writeJSONLAtomic(jsonlPath, issues)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write JSONL: %v", err)
|
||||
}
|
||||
|
||||
if len(exportedIDs) != 1 {
|
||||
t.Fatalf("expected 1 exported ID, got %d", len(exportedIDs))
|
||||
}
|
||||
|
||||
// Compute and store JSONL file hash
|
||||
jsonlData, err := os.ReadFile(jsonlPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read JSONL: %v", err)
|
||||
}
|
||||
hasher := sha256.New()
|
||||
hasher.Write(jsonlData)
|
||||
fileHash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
if err := testStore.SetJSONLFileHash(ctx, fileHash); err != nil {
|
||||
t.Fatalf("failed to set JSONL file hash: %v", err)
|
||||
}
|
||||
|
||||
// Test 1: Validate with matching hash (should succeed)
|
||||
t.Run("MatchingHash", func(t *testing.T) {
|
||||
if err := validateJSONLIntegrity(ctx, jsonlPath); err != nil {
|
||||
t.Fatalf("validation failed with matching hash: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 2: Modify JSONL file (simulating git pull) and validate
|
||||
t.Run("MismatchedHash", func(t *testing.T) {
|
||||
// Modify the JSONL file
|
||||
if err := os.WriteFile(jsonlPath, []byte(`{"id":"bd-1","title":"Modified"}`+"\n"), 0644); err != nil {
|
||||
t.Fatalf("failed to modify JSONL: %v", err)
|
||||
}
|
||||
|
||||
// Add an export hash to verify it gets cleared
|
||||
if err := testStore.SetExportHash(ctx, "bd-1", "dummy-hash"); err != nil {
|
||||
t.Fatalf("failed to set export hash: %v", err)
|
||||
}
|
||||
|
||||
// Validate should detect mismatch and clear export_hashes
|
||||
if err := validateJSONLIntegrity(ctx, jsonlPath); err != nil {
|
||||
t.Fatalf("validation failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify export_hashes were cleared
|
||||
hash, err := testStore.GetExportHash(ctx, "bd-1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get export hash: %v", err)
|
||||
}
|
||||
if hash != "" {
|
||||
t.Fatalf("expected export hash to be cleared, got %q", hash)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 3: Missing JSONL file
|
||||
t.Run("MissingJSONL", func(t *testing.T) {
|
||||
// Store a hash to simulate previous export
|
||||
if err := testStore.SetJSONLFileHash(ctx, "some-hash"); err != nil {
|
||||
t.Fatalf("failed to set JSONL file hash: %v", err)
|
||||
}
|
||||
|
||||
// Add an export hash
|
||||
if err := testStore.SetExportHash(ctx, "bd-1", "dummy-hash"); err != nil {
|
||||
t.Fatalf("failed to set export hash: %v", err)
|
||||
}
|
||||
|
||||
// Remove JSONL file
|
||||
if err := os.Remove(jsonlPath); err != nil {
|
||||
t.Fatalf("failed to remove JSONL: %v", err)
|
||||
}
|
||||
|
||||
// Validate should detect missing file and clear export_hashes
|
||||
if err := validateJSONLIntegrity(ctx, jsonlPath); err != nil {
|
||||
t.Fatalf("validation failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify export_hashes were cleared
|
||||
hash, err := testStore.GetExportHash(ctx, "bd-1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get export hash: %v", err)
|
||||
}
|
||||
if hash != "" {
|
||||
t.Fatalf("expected export hash to be cleared, got %q", hash)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestImportClearsExportHashes tests that imports clear export_hashes (bd-160)
|
||||
func TestImportClearsExportHashes(t *testing.T) {
|
||||
// Create temp directory
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, ".beads", "beads.db")
|
||||
|
||||
// Ensure .beads directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(dbPath), 0755); err != nil {
|
||||
t.Fatalf("failed to create .beads directory: %v", err)
|
||||
}
|
||||
|
||||
// Create database
|
||||
testStore, err := sqlite.New(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer testStore.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Initialize database with prefix
|
||||
if err := testStore.SetConfig(ctx, "issue_prefix", "bd"); err != nil {
|
||||
t.Fatalf("failed to set issue prefix: %v", err)
|
||||
}
|
||||
|
||||
// Create a test issue
|
||||
issue := &types.Issue{
|
||||
ID: "bd-1",
|
||||
Title: "Test issue",
|
||||
Description: "Test description",
|
||||
Status: types.StatusOpen,
|
||||
Priority: 1,
|
||||
IssueType: types.TypeTask,
|
||||
}
|
||||
|
||||
if err := testStore.CreateIssue(ctx, issue, testActor); err != nil {
|
||||
t.Fatalf("failed to create issue: %v", err)
|
||||
}
|
||||
|
||||
// Set an export hash
|
||||
if err := testStore.SetExportHash(ctx, "bd-1", "dummy-hash"); err != nil {
|
||||
t.Fatalf("failed to set export hash: %v", err)
|
||||
}
|
||||
|
||||
// Verify hash is set
|
||||
hash, err := testStore.GetExportHash(ctx, "bd-1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get export hash: %v", err)
|
||||
}
|
||||
if hash != "dummy-hash" {
|
||||
t.Fatalf("expected hash 'dummy-hash', got %q", hash)
|
||||
}
|
||||
|
||||
// Import another issue (should clear export_hashes)
|
||||
issue2 := &types.Issue{
|
||||
ID: "bd-2",
|
||||
Title: "Another issue",
|
||||
Description: "Another description",
|
||||
Status: types.StatusOpen,
|
||||
Priority: 2,
|
||||
IssueType: types.TypeTask,
|
||||
}
|
||||
|
||||
opts := ImportOptions{
|
||||
ResolveCollisions: false,
|
||||
DryRun: false,
|
||||
SkipUpdate: false,
|
||||
Strict: false,
|
||||
SkipPrefixValidation: true,
|
||||
}
|
||||
|
||||
_, err = importIssuesCore(ctx, dbPath, testStore, []*types.Issue{issue2}, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("import failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify export_hashes were cleared
|
||||
hash, err = testStore.GetExportHash(ctx, "bd-1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get export hash after import: %v", err)
|
||||
}
|
||||
if hash != "" {
|
||||
t.Fatalf("expected export hash to be cleared after import, got %q", hash)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user