feat(sync): add incremental export for large repos
For repos with 1000+ issues where less than 20% are dirty, incremental export reads the existing JSONL, merges only changed issues, and writes back - avoiding full re-export. - Add exportToJSONLIncrementalDeferred as new default export path - Add shouldUseIncrementalExport to check thresholds - Add performIncrementalExport for merge-based export - Add readJSONLToMap helper for fast JSONL parsing - Falls back to full export when incremental is not beneficial Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
committed by
Steve Yegge
parent
d3d2326a8b
commit
63c2e50158
@@ -772,8 +772,8 @@ func doExportSync(ctx context.Context, jsonlPath string, force, dryRun bool) err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export to JSONL
|
// Export to JSONL (uses incremental export for large repos)
|
||||||
result, err := exportToJSONLDeferred(ctx, jsonlPath)
|
result, err := exportToJSONLIncrementalDeferred(ctx, jsonlPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("exporting: %w", err)
|
return fmt.Errorf("exporting: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"cmp"
|
"cmp"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -18,6 +19,15 @@ import (
|
|||||||
"github.com/steveyegge/beads/internal/validation"
|
"github.com/steveyegge/beads/internal/validation"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Incremental export thresholds
|
||||||
|
const (
|
||||||
|
// incrementalThreshold is the minimum total issue count to consider incremental export
|
||||||
|
incrementalThreshold = 1000
|
||||||
|
// incrementalDirtyRatio is the max ratio of dirty/total issues for incremental export
|
||||||
|
// If more than 20% of issues are dirty, full export is likely faster
|
||||||
|
incrementalDirtyRatio = 0.20
|
||||||
|
)
|
||||||
|
|
||||||
// ExportResult contains information needed to finalize an export after git commit.
|
// ExportResult contains information needed to finalize an export after git commit.
|
||||||
// This enables atomic sync by deferring metadata updates until after git commit succeeds.
|
// This enables atomic sync by deferring metadata updates until after git commit succeeds.
|
||||||
// See GH#885 for the atomicity gap this fixes.
|
// See GH#885 for the atomicity gap this fixes.
|
||||||
@@ -259,6 +269,295 @@ func exportToJSONLDeferred(ctx context.Context, jsonlPath string) (*ExportResult
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// exportToJSONLIncrementalDeferred performs incremental export for large repos.
|
||||||
|
// It checks if incremental export would be beneficial (large repo, few dirty issues),
|
||||||
|
// and if so, reads the existing JSONL, updates only dirty issues, and writes back.
|
||||||
|
// Falls back to full export when incremental is not beneficial.
|
||||||
|
//
|
||||||
|
// Returns the export result for deferred finalization (same as exportToJSONLDeferred).
|
||||||
|
func exportToJSONLIncrementalDeferred(ctx context.Context, jsonlPath string) (*ExportResult, error) {
|
||||||
|
// If daemon is running, delegate to it (daemon has its own optimization)
|
||||||
|
if daemonClient != nil {
|
||||||
|
return exportToJSONLDeferred(ctx, jsonlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure store is initialized
|
||||||
|
if err := ensureStoreActive(); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to initialize store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if incremental export would be beneficial
|
||||||
|
useIncremental, dirtyIDs, err := shouldUseIncrementalExport(ctx, jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
// On error checking, fall back to full export
|
||||||
|
return exportToJSONLDeferred(ctx, jsonlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !useIncremental {
|
||||||
|
return exportToJSONLDeferred(ctx, jsonlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No dirty issues means nothing to export
|
||||||
|
if len(dirtyIDs) == 0 {
|
||||||
|
// Still need to return a valid result for idempotency
|
||||||
|
contentHash, _ := computeJSONLHash(jsonlPath)
|
||||||
|
return &ExportResult{
|
||||||
|
JSONLPath: jsonlPath,
|
||||||
|
ExportedIDs: []string{},
|
||||||
|
ContentHash: contentHash,
|
||||||
|
ExportTime: time.Now().Format(time.RFC3339Nano),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform incremental export
|
||||||
|
return performIncrementalExport(ctx, jsonlPath, dirtyIDs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shouldUseIncrementalExport determines if incremental export would be beneficial.
|
||||||
|
// Returns (useIncremental, dirtyIDs, error).
|
||||||
|
func shouldUseIncrementalExport(ctx context.Context, jsonlPath string) (bool, []string, error) {
|
||||||
|
// Check if JSONL file exists (can't do incremental without existing file)
|
||||||
|
if _, err := os.Stat(jsonlPath); os.IsNotExist(err) {
|
||||||
|
return false, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get dirty issue IDs
|
||||||
|
dirtyIDs, err := store.GetDirtyIssues(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, nil, fmt.Errorf("failed to get dirty issues: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no dirty issues, we can skip export entirely
|
||||||
|
if len(dirtyIDs) == 0 {
|
||||||
|
return true, dirtyIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get total issue count from existing JSONL (fast line count)
|
||||||
|
totalCount, err := countIssuesInJSONL(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
// Can't read JSONL, fall back to full export
|
||||||
|
return false, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check thresholds:
|
||||||
|
// 1. Total must be above threshold (small repos are fast enough with full export)
|
||||||
|
// 2. Dirty ratio must be below threshold (if most issues changed, full export is faster)
|
||||||
|
if totalCount < incrementalThreshold {
|
||||||
|
return false, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
dirtyRatio := float64(len(dirtyIDs)) / float64(totalCount)
|
||||||
|
if dirtyRatio > incrementalDirtyRatio {
|
||||||
|
return false, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return true, dirtyIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// performIncrementalExport performs the actual incremental export.
|
||||||
|
// It reads the existing JSONL, queries only dirty issues, merges them,
|
||||||
|
// and writes the result.
|
||||||
|
func performIncrementalExport(ctx context.Context, jsonlPath string, dirtyIDs []string) (*ExportResult, error) {
|
||||||
|
// Read existing JSONL into map[id]rawJSON
|
||||||
|
issueMap, allIDs, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
// Fall back to full export on read error
|
||||||
|
return exportToJSONLDeferred(ctx, jsonlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query dirty issues from database and track which IDs were found
|
||||||
|
dirtyIssues := make([]*types.Issue, 0, len(dirtyIDs))
|
||||||
|
issueByID := make(map[string]*types.Issue, len(dirtyIDs))
|
||||||
|
for _, id := range dirtyIDs {
|
||||||
|
issue, err := store.GetIssue(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get dirty issue %s: %w", id, err)
|
||||||
|
}
|
||||||
|
issueByID[id] = issue // Store result (may be nil for deleted issues)
|
||||||
|
if issue != nil {
|
||||||
|
dirtyIssues = append(dirtyIssues, issue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get dependencies for dirty issues only
|
||||||
|
// Note: GetAllDependencyRecords is used because there's no batch method for specific IDs,
|
||||||
|
// but for truly large repos this could be optimized with a targeted query
|
||||||
|
allDeps, err := store.GetAllDependencyRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get dependencies: %w", err)
|
||||||
|
}
|
||||||
|
for _, issue := range dirtyIssues {
|
||||||
|
issue.Dependencies = allDeps[issue.ID]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get labels for dirty issues (batch query)
|
||||||
|
labelsMap, err := store.GetLabelsForIssues(ctx, dirtyIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get labels: %w", err)
|
||||||
|
}
|
||||||
|
for _, issue := range dirtyIssues {
|
||||||
|
issue.Labels = labelsMap[issue.ID]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get comments for dirty issues (batch query)
|
||||||
|
commentsMap, err := store.GetCommentsForIssues(ctx, dirtyIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get comments: %w", err)
|
||||||
|
}
|
||||||
|
for _, issue := range dirtyIssues {
|
||||||
|
issue.Comments = commentsMap[issue.ID]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update map with dirty issues
|
||||||
|
idSet := make(map[string]bool, len(allIDs))
|
||||||
|
for _, id := range allIDs {
|
||||||
|
idSet[id] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, issue := range dirtyIssues {
|
||||||
|
// Skip wisps - they should never be exported
|
||||||
|
if issue.Ephemeral {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serialize issue to JSON
|
||||||
|
data, err := json.Marshal(issue)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to marshal issue %s: %w", issue.ID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
issueMap[issue.ID] = data
|
||||||
|
if !idSet[issue.ID] {
|
||||||
|
allIDs = append(allIDs, issue.ID)
|
||||||
|
idSet[issue.ID] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tombstones and deletions using cached results (no second GetIssue call)
|
||||||
|
for _, id := range dirtyIDs {
|
||||||
|
issue := issueByID[id] // Use cached result
|
||||||
|
if issue == nil {
|
||||||
|
// Issue was fully deleted (not even a tombstone)
|
||||||
|
delete(issueMap, id)
|
||||||
|
} else if issue.Status == types.StatusTombstone {
|
||||||
|
// Issue is a tombstone - keep it in export for propagation
|
||||||
|
if !issue.Ephemeral {
|
||||||
|
data, err := json.Marshal(issue)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to marshal tombstone %s: %w", id, err)
|
||||||
|
}
|
||||||
|
issueMap[id] = data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build sorted list of IDs (excluding deleted ones)
|
||||||
|
finalIDs := make([]string, 0, len(issueMap))
|
||||||
|
for id := range issueMap {
|
||||||
|
finalIDs = append(finalIDs, id)
|
||||||
|
}
|
||||||
|
slices.Sort(finalIDs)
|
||||||
|
|
||||||
|
// Write to temp file, then atomic rename
|
||||||
|
dir := filepath.Dir(jsonlPath)
|
||||||
|
base := filepath.Base(jsonlPath)
|
||||||
|
tempFile, err := os.CreateTemp(dir, base+".tmp.*")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tempPath := tempFile.Name()
|
||||||
|
defer func() {
|
||||||
|
_ = tempFile.Close()
|
||||||
|
_ = os.Remove(tempPath)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Write JSONL in sorted order
|
||||||
|
exportedIDs := make([]string, 0, len(finalIDs))
|
||||||
|
for _, id := range finalIDs {
|
||||||
|
data := issueMap[id]
|
||||||
|
if _, err := tempFile.Write(data); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to write issue %s: %w", id, err)
|
||||||
|
}
|
||||||
|
if _, err := tempFile.WriteString("\n"); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to write newline: %w", err)
|
||||||
|
}
|
||||||
|
exportedIDs = append(exportedIDs, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close and rename
|
||||||
|
_ = tempFile.Close()
|
||||||
|
if err := os.Rename(tempPath, jsonlPath); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to replace JSONL file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set permissions
|
||||||
|
if err := os.Chmod(jsonlPath, 0600); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Warning: failed to set file permissions: %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute hash
|
||||||
|
contentHash, _ := computeJSONLHash(jsonlPath)
|
||||||
|
exportTime := time.Now().Format(time.RFC3339Nano)
|
||||||
|
|
||||||
|
// Note: exportedIDs contains ALL IDs in the file, but we only need to clear
|
||||||
|
// dirty flags for the dirtyIDs (which we received as parameter)
|
||||||
|
return &ExportResult{
|
||||||
|
JSONLPath: jsonlPath,
|
||||||
|
ExportedIDs: dirtyIDs, // Only clear dirty flags for actually dirty issues
|
||||||
|
ContentHash: contentHash,
|
||||||
|
ExportTime: exportTime,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readJSONLToMap reads a JSONL file into a map of id -> raw JSON bytes.
|
||||||
|
// Also returns the list of IDs in original order.
|
||||||
|
func readJSONLToMap(jsonlPath string) (map[string]json.RawMessage, []string, error) {
|
||||||
|
// #nosec G304 - controlled path
|
||||||
|
file, err := os.Open(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = file.Close() }()
|
||||||
|
|
||||||
|
issueMap := make(map[string]json.RawMessage)
|
||||||
|
var ids []string
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
// Use larger buffer for large lines
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Bytes()
|
||||||
|
if len(line) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract ID from JSON without full unmarshal
|
||||||
|
var partial struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(line, &partial); err != nil {
|
||||||
|
// Skip malformed lines
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if partial.ID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store a copy of the line (scanner reuses buffer)
|
||||||
|
lineCopy := make([]byte, len(line))
|
||||||
|
copy(lineCopy, line)
|
||||||
|
issueMap[partial.ID] = json.RawMessage(lineCopy)
|
||||||
|
ids = append(ids, partial.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return issueMap, ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
// validateOpenIssuesForSync validates all open issues against their templates
|
// validateOpenIssuesForSync validates all open issues against their templates
|
||||||
// before export, based on the validation.on-sync config setting.
|
// before export, based on the validation.on-sync config setting.
|
||||||
// Returns an error if validation.on-sync is "error" and issues fail validation.
|
// Returns an error if validation.on-sync is "error" and issues fail validation.
|
||||||
|
|||||||
439
cmd/bd/sync_export_incremental_test.go
Normal file
439
cmd/bd/sync_export_incremental_test.go
Normal file
@@ -0,0 +1,439 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/steveyegge/beads/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReadJSONLToMap(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
jsonlPath := filepath.Join(tmpDir, "test.jsonl")
|
||||||
|
|
||||||
|
// Create test JSONL with 3 issues
|
||||||
|
issues := []types.Issue{
|
||||||
|
{ID: "test-001", Title: "First", Status: types.StatusOpen},
|
||||||
|
{ID: "test-002", Title: "Second", Status: types.StatusInProgress},
|
||||||
|
{ID: "test-003", Title: "Third", Status: types.StatusClosed},
|
||||||
|
}
|
||||||
|
|
||||||
|
var content strings.Builder
|
||||||
|
for _, issue := range issues {
|
||||||
|
data, _ := json.Marshal(issue)
|
||||||
|
content.Write(data)
|
||||||
|
content.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(jsonlPath, []byte(content.String()), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write test file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test readJSONLToMap
|
||||||
|
issueMap, ids, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("readJSONLToMap failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify count
|
||||||
|
if len(issueMap) != 3 {
|
||||||
|
t.Errorf("expected 3 issues in map, got %d", len(issueMap))
|
||||||
|
}
|
||||||
|
if len(ids) != 3 {
|
||||||
|
t.Errorf("expected 3 IDs, got %d", len(ids))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify order preserved
|
||||||
|
expectedOrder := []string{"test-001", "test-002", "test-003"}
|
||||||
|
for i, id := range ids {
|
||||||
|
if id != expectedOrder[i] {
|
||||||
|
t.Errorf("ID order mismatch at %d: expected %s, got %s", i, expectedOrder[i], id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify content can be unmarshaled
|
||||||
|
for id, rawJSON := range issueMap {
|
||||||
|
var issue types.Issue
|
||||||
|
if err := json.Unmarshal(rawJSON, &issue); err != nil {
|
||||||
|
t.Errorf("failed to unmarshal issue %s: %v", id, err)
|
||||||
|
}
|
||||||
|
if issue.ID != id {
|
||||||
|
t.Errorf("ID mismatch: expected %s, got %s", id, issue.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadJSONLToMapWithMalformedLines(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
jsonlPath := filepath.Join(tmpDir, "test.jsonl")
|
||||||
|
|
||||||
|
content := `{"id": "test-001", "title": "Good"}
|
||||||
|
{invalid json}
|
||||||
|
{"id": "test-002", "title": "Also Good"}
|
||||||
|
|
||||||
|
{"id": "", "title": "No ID"}
|
||||||
|
{"id": "test-003", "title": "Third Good"}
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := os.WriteFile(jsonlPath, []byte(content), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write test file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
issueMap, ids, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("readJSONLToMap failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should have 3 valid issues (skipped invalid JSON, empty line, and empty ID)
|
||||||
|
if len(issueMap) != 3 {
|
||||||
|
t.Errorf("expected 3 issues, got %d", len(issueMap))
|
||||||
|
}
|
||||||
|
if len(ids) != 3 {
|
||||||
|
t.Errorf("expected 3 IDs, got %d", len(ids))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShouldUseIncrementalExport(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testDB := filepath.Join(tmpDir, "test.db")
|
||||||
|
s := newTestStore(t, testDB)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
|
||||||
|
|
||||||
|
// Set up global state
|
||||||
|
oldStore := store
|
||||||
|
oldDBPath := dbPath
|
||||||
|
oldRootCtx := rootCtx
|
||||||
|
store = s
|
||||||
|
dbPath = testDB
|
||||||
|
rootCtx = ctx
|
||||||
|
defer func() {
|
||||||
|
store = oldStore
|
||||||
|
dbPath = oldDBPath
|
||||||
|
rootCtx = oldRootCtx
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Run("no JSONL file returns false", func(t *testing.T) {
|
||||||
|
useIncremental, _, err := shouldUseIncrementalExport(ctx, jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if useIncremental {
|
||||||
|
t.Error("expected false when JSONL doesn't exist")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no dirty issues returns true with empty IDs", func(t *testing.T) {
|
||||||
|
// Create JSONL file with some issues
|
||||||
|
var content strings.Builder
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
issue := types.Issue{ID: "test-" + string(rune('a'+i)), Title: "Test"}
|
||||||
|
data, _ := json.Marshal(issue)
|
||||||
|
content.Write(data)
|
||||||
|
content.WriteString("\n")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(jsonlPath, []byte(content.String()), 0644); err != nil {
|
||||||
|
t.Fatalf("failed to write JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No dirty issues in database
|
||||||
|
useIncremental, dirtyIDs, err := shouldUseIncrementalExport(ctx, jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
// With 0 dirty issues, we return true to signal nothing needs export
|
||||||
|
if !useIncremental {
|
||||||
|
t.Error("expected true when no dirty issues (nothing to export)")
|
||||||
|
}
|
||||||
|
if len(dirtyIDs) != 0 {
|
||||||
|
t.Errorf("expected 0 dirty IDs, got %d", len(dirtyIDs))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("small repo with dirty issues returns false", func(t *testing.T) {
|
||||||
|
// Create an issue to make it dirty
|
||||||
|
issue := &types.Issue{Title: "Dirty Issue", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||||
|
if err := s.CreateIssue(ctx, issue, "test"); err != nil {
|
||||||
|
t.Fatalf("failed to create issue: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Small repo (10 issues in JSONL) below 1000 threshold
|
||||||
|
useIncremental, _, err := shouldUseIncrementalExport(ctx, jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if useIncremental {
|
||||||
|
t.Error("expected false for small repo below threshold")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncrementalExportIntegration(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testDB := filepath.Join(tmpDir, "test.db")
|
||||||
|
s := newTestStore(t, testDB)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
|
||||||
|
|
||||||
|
// Set up global state
|
||||||
|
oldStore := store
|
||||||
|
oldDBPath := dbPath
|
||||||
|
oldRootCtx := rootCtx
|
||||||
|
store = s
|
||||||
|
dbPath = testDB
|
||||||
|
rootCtx = ctx
|
||||||
|
defer func() {
|
||||||
|
store = oldStore
|
||||||
|
dbPath = oldDBPath
|
||||||
|
rootCtx = oldRootCtx
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Create initial issues
|
||||||
|
issues := []*types.Issue{
|
||||||
|
{Title: "Issue 1", Description: "Desc 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask},
|
||||||
|
{Title: "Issue 2", Description: "Desc 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask},
|
||||||
|
{Title: "Issue 3", Description: "Desc 3", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, issue := range issues {
|
||||||
|
if err := s.CreateIssue(ctx, issue, "test"); err != nil {
|
||||||
|
t.Fatalf("failed to create issue: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial full export using exportToJSONL (not deferred, which calls ensureStoreActive)
|
||||||
|
if err := exportToJSONL(ctx, jsonlPath); err != nil {
|
||||||
|
t.Fatalf("initial export failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify initial export worked
|
||||||
|
issueMap, _, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read initial JSONL: %v", err)
|
||||||
|
}
|
||||||
|
if len(issueMap) != 3 {
|
||||||
|
t.Errorf("expected 3 issues after initial export, got %d", len(issueMap))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update one issue
|
||||||
|
if err := s.UpdateIssue(ctx, issues[0].ID, map[string]interface{}{"title": "Updated Issue 1"}, "test"); err != nil {
|
||||||
|
t.Fatalf("failed to update issue: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify dirty count
|
||||||
|
dirtyIDs, err := s.GetDirtyIssues(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get dirty issues: %v", err)
|
||||||
|
}
|
||||||
|
if len(dirtyIDs) != 1 {
|
||||||
|
t.Errorf("expected 1 dirty issue, got %d", len(dirtyIDs))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test incremental export (using performIncrementalExport directly)
|
||||||
|
result, err := performIncrementalExport(ctx, jsonlPath, dirtyIDs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("incremental export failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify result
|
||||||
|
if result == nil {
|
||||||
|
t.Fatal("expected non-nil result")
|
||||||
|
}
|
||||||
|
if len(result.ExportedIDs) != 1 {
|
||||||
|
t.Errorf("expected 1 exported ID (dirty), got %d", len(result.ExportedIDs))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read back JSONL and verify
|
||||||
|
issueMap, ids, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read JSONL: %v", err)
|
||||||
|
}
|
||||||
|
if len(issueMap) != 3 {
|
||||||
|
t.Errorf("expected 3 issues in JSONL, got %d", len(issueMap))
|
||||||
|
}
|
||||||
|
if len(ids) != 3 {
|
||||||
|
t.Errorf("expected 3 IDs, got %d", len(ids))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify updated issue content
|
||||||
|
var updatedIssue types.Issue
|
||||||
|
if err := json.Unmarshal(issueMap[issues[0].ID], &updatedIssue); err != nil {
|
||||||
|
t.Fatalf("failed to unmarshal updated issue: %v", err)
|
||||||
|
}
|
||||||
|
if updatedIssue.Title != "Updated Issue 1" {
|
||||||
|
t.Errorf("expected title 'Updated Issue 1', got '%s'", updatedIssue.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncrementalExportWithDeletion(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testDB := filepath.Join(tmpDir, "test.db")
|
||||||
|
s := newTestStore(t, testDB)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
|
||||||
|
|
||||||
|
// Set up global state
|
||||||
|
oldStore := store
|
||||||
|
oldDBPath := dbPath
|
||||||
|
oldRootCtx := rootCtx
|
||||||
|
store = s
|
||||||
|
dbPath = testDB
|
||||||
|
rootCtx = ctx
|
||||||
|
defer func() {
|
||||||
|
store = oldStore
|
||||||
|
dbPath = oldDBPath
|
||||||
|
rootCtx = oldRootCtx
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Create initial issues
|
||||||
|
issues := []*types.Issue{
|
||||||
|
{Title: "Issue 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask},
|
||||||
|
{Title: "Issue 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, issue := range issues {
|
||||||
|
if err := s.CreateIssue(ctx, issue, "test"); err != nil {
|
||||||
|
t.Fatalf("failed to create issue: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial export
|
||||||
|
if err := exportToJSONL(ctx, jsonlPath); err != nil {
|
||||||
|
t.Fatalf("initial export failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get initial dirty count (should be 0 after export)
|
||||||
|
dirtyIDs, _ := s.GetDirtyIssues(ctx)
|
||||||
|
if len(dirtyIDs) != 0 {
|
||||||
|
t.Errorf("expected 0 dirty issues after export, got %d", len(dirtyIDs))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete one issue (soft delete creates tombstone)
|
||||||
|
if err := s.DeleteIssue(ctx, issues[0].ID); err != nil {
|
||||||
|
t.Fatalf("failed to delete issue: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get dirty IDs after deletion
|
||||||
|
dirtyIDs, err := s.GetDirtyIssues(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get dirty issues: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform incremental export
|
||||||
|
if len(dirtyIDs) > 0 {
|
||||||
|
_, err = performIncrementalExport(ctx, jsonlPath, dirtyIDs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("incremental export failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read back and verify tombstone is present
|
||||||
|
issueMap, _, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should still have 2 entries (one is now tombstone)
|
||||||
|
if len(issueMap) != 2 {
|
||||||
|
t.Errorf("expected 2 issues in JSONL (including tombstone), got %d", len(issueMap))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check tombstone status
|
||||||
|
var tombstone types.Issue
|
||||||
|
if err := json.Unmarshal(issueMap[issues[0].ID], &tombstone); err != nil {
|
||||||
|
t.Fatalf("failed to unmarshal tombstone: %v", err)
|
||||||
|
}
|
||||||
|
if tombstone.Status != types.StatusTombstone {
|
||||||
|
t.Errorf("expected tombstone status, got %s", tombstone.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncrementalExportWithNewIssue(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testDB := filepath.Join(tmpDir, "test.db")
|
||||||
|
s := newTestStore(t, testDB)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
jsonlPath := filepath.Join(tmpDir, "issues.jsonl")
|
||||||
|
|
||||||
|
// Set up global state
|
||||||
|
oldStore := store
|
||||||
|
oldDBPath := dbPath
|
||||||
|
oldRootCtx := rootCtx
|
||||||
|
store = s
|
||||||
|
dbPath = testDB
|
||||||
|
rootCtx = ctx
|
||||||
|
defer func() {
|
||||||
|
store = oldStore
|
||||||
|
dbPath = oldDBPath
|
||||||
|
rootCtx = oldRootCtx
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Create initial issue
|
||||||
|
issue1 := &types.Issue{Title: "Issue 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||||
|
if err := s.CreateIssue(ctx, issue1, "test"); err != nil {
|
||||||
|
t.Fatalf("failed to create issue: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial export
|
||||||
|
if err := exportToJSONL(ctx, jsonlPath); err != nil {
|
||||||
|
t.Fatalf("initial export failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create new issue
|
||||||
|
issue2 := &types.Issue{Title: "Issue 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||||
|
if err := s.CreateIssue(ctx, issue2, "test"); err != nil {
|
||||||
|
t.Fatalf("failed to create issue 2: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get dirty IDs
|
||||||
|
dirtyIDs, err := s.GetDirtyIssues(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get dirty issues: %v", err)
|
||||||
|
}
|
||||||
|
if len(dirtyIDs) != 1 {
|
||||||
|
t.Errorf("expected 1 dirty issue (new one), got %d", len(dirtyIDs))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform incremental export
|
||||||
|
_, err = performIncrementalExport(ctx, jsonlPath, dirtyIDs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("incremental export failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read back and verify new issue was added
|
||||||
|
issueMap, ids, err := readJSONLToMap(jsonlPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read JSONL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(issueMap) != 2 {
|
||||||
|
t.Errorf("expected 2 issues in JSONL, got %d", len(issueMap))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify both issues exist
|
||||||
|
if _, ok := issueMap[issue1.ID]; !ok {
|
||||||
|
t.Errorf("issue 1 missing from JSONL")
|
||||||
|
}
|
||||||
|
if _, ok := issueMap[issue2.ID]; !ok {
|
||||||
|
t.Errorf("issue 2 missing from JSONL")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify sorted order
|
||||||
|
for i := 0; i < len(ids)-1; i++ {
|
||||||
|
if ids[i] > ids[i+1] {
|
||||||
|
t.Errorf("IDs not sorted: %s > %s", ids[i], ids[i+1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user