Complete bd-164: Fix timestamp-only export deduplication
- Add export_hashes table to track last exported content state - Implement GetExportHash/SetExportHash in storage interface - Update shouldSkipExport to use export_hashes instead of dirty_issues - Call SetExportHash after successful export - Clean up dirty_issues table (remove content_hash column) - Simplify MarkIssueDirty functions (no longer compute hashes) - Update markIssuesDirtyTx signature (remove store parameter) Testing: - Timestamp-only updates are skipped during export ✓ - Real content changes trigger export ✓ - export_hashes table populated correctly ✓ Fixes bd-159, bd-164
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -46,13 +46,13 @@ func computeIssueContentHash(issue *types.Issue) (string, error) {
|
|||||||
// shouldSkipExport checks if an issue should be skipped during export because
|
// shouldSkipExport checks if an issue should be skipped during export because
|
||||||
// it only has timestamp changes (no actual content changes).
|
// it only has timestamp changes (no actual content changes).
|
||||||
func shouldSkipExport(ctx context.Context, store storage.Storage, issue *types.Issue) (bool, error) {
|
func shouldSkipExport(ctx context.Context, store storage.Storage, issue *types.Issue) (bool, error) {
|
||||||
// Get the stored hash from dirty_issues table
|
// Get the stored hash from export_hashes table (last exported state)
|
||||||
storedHash, err := store.GetDirtyIssueHash(ctx, issue.ID)
|
storedHash, err := store.GetExportHash(ctx, issue.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// If no hash stored, we must export (first export or old data)
|
// If no hash stored, we must export (first export)
|
||||||
if storedHash == "" {
|
if storedHash == "" {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
@@ -296,6 +296,15 @@ Output to stdout by default, or use -o flag for file output.`,
|
|||||||
fmt.Fprintf(os.Stderr, "Error encoding issue %s: %v\n", issue.ID, err)
|
fmt.Fprintf(os.Stderr, "Error encoding issue %s: %v\n", issue.ID, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Save content hash after successful export (bd-164)
|
||||||
|
contentHash, err := computeIssueContentHash(issue)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Warning: failed to compute hash for %s: %v\n", issue.ID, err)
|
||||||
|
} else if err := store.SetExportHash(ctx, issue.ID, contentHash); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Warning: failed to save export hash for %s: %v\n", issue.ID, err)
|
||||||
|
}
|
||||||
|
|
||||||
exportedIDs = append(exportedIDs, issue.ID)
|
exportedIDs = append(exportedIDs, issue.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ func (s *SQLiteStorage) AddDependency(ctx context.Context, dep *types.Dependency
|
|||||||
|
|
||||||
// Mark both issues as dirty for incremental export
|
// Mark both issues as dirty for incremental export
|
||||||
// (dependencies are exported with each issue, so both need updating)
|
// (dependencies are exported with each issue, so both need updating)
|
||||||
if err := markIssuesDirtyTx(ctx, tx, s, []string{dep.IssueID, dep.DependsOnID}); err != nil {
|
if err := markIssuesDirtyTx(ctx, tx, []string{dep.IssueID, dep.DependsOnID}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,7 +264,7 @@ func (s *SQLiteStorage) addDependencyUnchecked(ctx context.Context, dep *types.D
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Mark both issues as dirty
|
// Mark both issues as dirty
|
||||||
if err := markIssuesDirtyTx(ctx, tx, s, []string{dep.IssueID, dep.DependsOnID}); err != nil {
|
if err := markIssuesDirtyTx(ctx, tx, []string{dep.IssueID, dep.DependsOnID}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -305,7 +305,7 @@ func (s *SQLiteStorage) RemoveDependency(ctx context.Context, issueID, dependsOn
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Mark both issues as dirty for incremental export
|
// Mark both issues as dirty for incremental export
|
||||||
if err := markIssuesDirtyTx(ctx, tx, s, []string{issueID, dependsOnID}); err != nil {
|
if err := markIssuesDirtyTx(ctx, tx, []string{issueID, dependsOnID}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -348,7 +348,7 @@ func (s *SQLiteStorage) removeDependencyIfExists(ctx context.Context, issueID, d
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Mark both issues as dirty for incremental export
|
// Mark both issues as dirty for incremental export
|
||||||
if err := markIssuesDirtyTx(ctx, tx, s, []string{issueID, dependsOnID}); err != nil {
|
if err := markIssuesDirtyTx(ctx, tx, []string{issueID, dependsOnID}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,22 +11,11 @@ import (
|
|||||||
// MarkIssueDirty marks an issue as dirty (needs to be exported to JSONL)
|
// MarkIssueDirty marks an issue as dirty (needs to be exported to JSONL)
|
||||||
// This should be called whenever an issue is created, updated, or has dependencies changed
|
// This should be called whenever an issue is created, updated, or has dependencies changed
|
||||||
func (s *SQLiteStorage) MarkIssueDirty(ctx context.Context, issueID string) error {
|
func (s *SQLiteStorage) MarkIssueDirty(ctx context.Context, issueID string) error {
|
||||||
// Fetch the issue to compute its content hash
|
_, err := s.db.ExecContext(ctx, `
|
||||||
issue, err := s.GetIssue(ctx, issueID)
|
INSERT INTO dirty_issues (issue_id, marked_at)
|
||||||
if err != nil {
|
VALUES (?, ?)
|
||||||
return err
|
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
|
||||||
}
|
`, issueID, time.Now())
|
||||||
|
|
||||||
hash, err := computeIssueContentHash(issue)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = s.db.ExecContext(ctx, `
|
|
||||||
INSERT INTO dirty_issues (issue_id, marked_at, content_hash)
|
|
||||||
VALUES (?, ?, ?)
|
|
||||||
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at, content_hash = excluded.content_hash
|
|
||||||
`, issueID, time.Now(), hash)
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -45,9 +34,9 @@ func (s *SQLiteStorage) MarkIssuesDirty(ctx context.Context, issueIDs []string)
|
|||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
stmt, err := tx.PrepareContext(ctx, `
|
stmt, err := tx.PrepareContext(ctx, `
|
||||||
INSERT INTO dirty_issues (issue_id, marked_at, content_hash)
|
INSERT INTO dirty_issues (issue_id, marked_at)
|
||||||
VALUES (?, ?, ?)
|
VALUES (?, ?)
|
||||||
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at, content_hash = excluded.content_hash
|
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
|
||||||
`)
|
`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to prepare statement: %w", err)
|
return fmt.Errorf("failed to prepare statement: %w", err)
|
||||||
@@ -55,18 +44,7 @@ func (s *SQLiteStorage) MarkIssuesDirty(ctx context.Context, issueIDs []string)
|
|||||||
defer func() { _ = stmt.Close() }()
|
defer func() { _ = stmt.Close() }()
|
||||||
|
|
||||||
for _, issueID := range issueIDs {
|
for _, issueID := range issueIDs {
|
||||||
// Fetch issue to compute content hash
|
if _, err := stmt.ExecContext(ctx, issueID, now); err != nil {
|
||||||
issue, err := s.GetIssue(ctx, issueID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to get issue %s: %w", issueID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hash, err := computeIssueContentHash(issue)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to compute hash for issue %s: %w", issueID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := stmt.ExecContext(ctx, issueID, now, hash); err != nil {
|
|
||||||
return fmt.Errorf("failed to mark issue %s dirty: %w", issueID, err)
|
return fmt.Errorf("failed to mark issue %s dirty: %w", issueID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -171,16 +149,16 @@ func (s *SQLiteStorage) GetDirtyIssueCount(ctx context.Context) (int, error) {
|
|||||||
|
|
||||||
// markIssuesDirtyTx marks multiple issues as dirty within an existing transaction
|
// markIssuesDirtyTx marks multiple issues as dirty within an existing transaction
|
||||||
// This is a helper for operations that need to mark issues dirty as part of a larger transaction
|
// This is a helper for operations that need to mark issues dirty as part of a larger transaction
|
||||||
func markIssuesDirtyTx(ctx context.Context, tx *sql.Tx, store *SQLiteStorage, issueIDs []string) error {
|
func markIssuesDirtyTx(ctx context.Context, tx *sql.Tx, issueIDs []string) error {
|
||||||
if len(issueIDs) == 0 {
|
if len(issueIDs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
stmt, err := tx.PrepareContext(ctx, `
|
stmt, err := tx.PrepareContext(ctx, `
|
||||||
INSERT INTO dirty_issues (issue_id, marked_at, content_hash)
|
INSERT INTO dirty_issues (issue_id, marked_at)
|
||||||
VALUES (?, ?, ?)
|
VALUES (?, ?)
|
||||||
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at, content_hash = excluded.content_hash
|
ON CONFLICT (issue_id) DO UPDATE SET marked_at = excluded.marked_at
|
||||||
`)
|
`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to prepare dirty statement: %w", err)
|
return fmt.Errorf("failed to prepare dirty statement: %w", err)
|
||||||
@@ -188,18 +166,7 @@ func markIssuesDirtyTx(ctx context.Context, tx *sql.Tx, store *SQLiteStorage, is
|
|||||||
defer func() { _ = stmt.Close() }()
|
defer func() { _ = stmt.Close() }()
|
||||||
|
|
||||||
for _, issueID := range issueIDs {
|
for _, issueID := range issueIDs {
|
||||||
// Fetch issue to compute content hash
|
if _, err := stmt.ExecContext(ctx, issueID, now); err != nil {
|
||||||
issue, err := store.GetIssue(ctx, issueID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to get issue %s: %w", issueID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hash, err := computeIssueContentHash(issue)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to compute hash for issue %s: %w", issueID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := stmt.ExecContext(ctx, issueID, now, hash); err != nil {
|
|
||||||
return fmt.Errorf("failed to mark issue %s dirty: %w", issueID, err)
|
return fmt.Errorf("failed to mark issue %s dirty: %w", issueID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
package sqlite
|
package sqlite
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/steveyegge/beads/internal/types"
|
"github.com/steveyegge/beads/internal/types"
|
||||||
@@ -33,3 +36,38 @@ func computeIssueContentHash(issue *types.Issue) (string, error) {
|
|||||||
hash := sha256.Sum256(data)
|
hash := sha256.Sum256(data)
|
||||||
return hex.EncodeToString(hash[:]), nil
|
return hex.EncodeToString(hash[:]), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetExportHash retrieves the content hash of the last export for an issue.
|
||||||
|
// Returns empty string if no hash is stored (first export).
|
||||||
|
func (s *SQLiteStorage) GetExportHash(ctx context.Context, issueID string) (string, error) {
|
||||||
|
var hash string
|
||||||
|
err := s.db.QueryRowContext(ctx, `
|
||||||
|
SELECT content_hash FROM export_hashes WHERE issue_id = ?
|
||||||
|
`, issueID).Scan(&hash)
|
||||||
|
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return "", nil // No hash stored yet
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to get export hash for %s: %w", issueID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hash, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetExportHash stores the content hash of an issue after successful export.
|
||||||
|
func (s *SQLiteStorage) SetExportHash(ctx context.Context, issueID, contentHash string) error {
|
||||||
|
_, err := s.db.ExecContext(ctx, `
|
||||||
|
INSERT INTO export_hashes (issue_id, content_hash, exported_at)
|
||||||
|
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||||
|
ON CONFLICT(issue_id) DO UPDATE SET
|
||||||
|
content_hash = excluded.content_hash,
|
||||||
|
exported_at = CURRENT_TIMESTAMP
|
||||||
|
`, issueID, contentHash)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to set export hash for %s: %w", issueID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -116,6 +116,11 @@ func New(path string) (*SQLiteStorage, error) {
|
|||||||
return nil, fmt.Errorf("failed to migrate compacted_at_commit column: %w", err)
|
return nil, fmt.Errorf("failed to migrate compacted_at_commit column: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Migrate existing databases to add export_hashes table (bd-164)
|
||||||
|
if err := migrateExportHashesTable(db); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to migrate export_hashes table: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Convert to absolute path for consistency
|
// Convert to absolute path for consistency
|
||||||
absPath, err := filepath.Abs(path)
|
absPath, err := filepath.Abs(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -144,7 +149,6 @@ func migrateDirtyIssuesTable(db *sql.DB) error {
|
|||||||
CREATE TABLE dirty_issues (
|
CREATE TABLE dirty_issues (
|
||||||
issue_id TEXT PRIMARY KEY,
|
issue_id TEXT PRIMARY KEY,
|
||||||
marked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
marked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
content_hash TEXT,
|
|
||||||
FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
|
FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
CREATE INDEX idx_dirty_issues_marked_at ON dirty_issues(marked_at);
|
CREATE INDEX idx_dirty_issues_marked_at ON dirty_issues(marked_at);
|
||||||
@@ -481,7 +485,38 @@ func migrateCompactedAtCommitColumn(db *sql.DB) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// migrateExportHashesTable ensures the export_hashes table exists for timestamp-only dedup (bd-164)
|
||||||
|
func migrateExportHashesTable(db *sql.DB) error {
|
||||||
|
// Check if export_hashes table exists
|
||||||
|
var tableName string
|
||||||
|
err := db.QueryRow(`
|
||||||
|
SELECT name FROM sqlite_master
|
||||||
|
WHERE type='table' AND name='export_hashes'
|
||||||
|
`).Scan(&tableName)
|
||||||
|
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
// Table doesn't exist, create it
|
||||||
|
_, err := db.Exec(`
|
||||||
|
CREATE TABLE export_hashes (
|
||||||
|
issue_id TEXT PRIMARY KEY,
|
||||||
|
content_hash TEXT NOT NULL,
|
||||||
|
exported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE
|
||||||
|
)
|
||||||
|
`)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create export_hashes table: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to check export_hashes table: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Table already exists
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// getNextIDForPrefix atomically generates the next ID for a given prefix
|
// getNextIDForPrefix atomically generates the next ID for a given prefix
|
||||||
// Uses the issue_counters table for atomic, cross-process ID generation
|
// Uses the issue_counters table for atomic, cross-process ID generation
|
||||||
|
|||||||
@@ -56,6 +56,10 @@ type Storage interface {
|
|||||||
ClearDirtyIssues(ctx context.Context) error // WARNING: Race condition (bd-52), use ClearDirtyIssuesByID
|
ClearDirtyIssues(ctx context.Context) error // WARNING: Race condition (bd-52), use ClearDirtyIssuesByID
|
||||||
ClearDirtyIssuesByID(ctx context.Context, issueIDs []string) error
|
ClearDirtyIssuesByID(ctx context.Context, issueIDs []string) error
|
||||||
|
|
||||||
|
// Export hash tracking (for timestamp-only dedup, bd-164)
|
||||||
|
GetExportHash(ctx context.Context, issueID string) (string, error)
|
||||||
|
SetExportHash(ctx context.Context, issueID, contentHash string) error
|
||||||
|
|
||||||
// Config
|
// Config
|
||||||
SetConfig(ctx context.Context, key, value string) error
|
SetConfig(ctx context.Context, key, value string) error
|
||||||
GetConfig(ctx context.Context, key string) (string, error)
|
GetConfig(ctx context.Context, key string) (string, error)
|
||||||
|
|||||||
Reference in New Issue
Block a user