From 63c2e5015845fc5d272d0c345867d8d4a2aeffd0 Mon Sep 17 00:00:00 2001 From: beads/crew/lydia Date: Tue, 20 Jan 2026 21:26:08 -0800 Subject: [PATCH] feat(sync): add incremental export for large repos For repos with 1000+ issues where less than 20% are dirty, incremental export reads the existing JSONL, merges only changed issues, and writes back - avoiding full re-export. - Add exportToJSONLIncrementalDeferred as new default export path - Add shouldUseIncrementalExport to check thresholds - Add performIncrementalExport for merge-based export - Add readJSONLToMap helper for fast JSONL parsing - Falls back to full export when incremental is not beneficial Co-Authored-By: Claude Opus 4.5 --- cmd/bd/sync.go | 4 +- cmd/bd/sync_export.go | 299 +++++++++++++++++ cmd/bd/sync_export_incremental_test.go | 439 +++++++++++++++++++++++++ 3 files changed, 740 insertions(+), 2 deletions(-) create mode 100644 cmd/bd/sync_export_incremental_test.go diff --git a/cmd/bd/sync.go b/cmd/bd/sync.go index 58bbddb4..9598b970 100644 --- a/cmd/bd/sync.go +++ b/cmd/bd/sync.go @@ -772,8 +772,8 @@ func doExportSync(ctx context.Context, jsonlPath string, force, dryRun bool) err } } - // Export to JSONL - result, err := exportToJSONLDeferred(ctx, jsonlPath) + // Export to JSONL (uses incremental export for large repos) + result, err := exportToJSONLIncrementalDeferred(ctx, jsonlPath) if err != nil { return fmt.Errorf("exporting: %w", err) } diff --git a/cmd/bd/sync_export.go b/cmd/bd/sync_export.go index ce1d573c..2dcc09f5 100644 --- a/cmd/bd/sync_export.go +++ b/cmd/bd/sync_export.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "cmp" "context" "encoding/json" @@ -18,6 +19,15 @@ import ( "github.com/steveyegge/beads/internal/validation" ) +// Incremental export thresholds +const ( + // incrementalThreshold is the minimum total issue count to consider incremental export + incrementalThreshold = 1000 + // incrementalDirtyRatio is the max ratio of dirty/total issues for incremental export + // If more than 20% of issues are dirty, full export is likely faster + incrementalDirtyRatio = 0.20 +) + // ExportResult contains information needed to finalize an export after git commit. // This enables atomic sync by deferring metadata updates until after git commit succeeds. // See GH#885 for the atomicity gap this fixes. @@ -259,6 +269,295 @@ func exportToJSONLDeferred(ctx context.Context, jsonlPath string) (*ExportResult }, nil } +// exportToJSONLIncrementalDeferred performs incremental export for large repos. +// It checks if incremental export would be beneficial (large repo, few dirty issues), +// and if so, reads the existing JSONL, updates only dirty issues, and writes back. +// Falls back to full export when incremental is not beneficial. +// +// Returns the export result for deferred finalization (same as exportToJSONLDeferred). +func exportToJSONLIncrementalDeferred(ctx context.Context, jsonlPath string) (*ExportResult, error) { + // If daemon is running, delegate to it (daemon has its own optimization) + if daemonClient != nil { + return exportToJSONLDeferred(ctx, jsonlPath) + } + + // Ensure store is initialized + if err := ensureStoreActive(); err != nil { + return nil, fmt.Errorf("failed to initialize store: %w", err) + } + + // Check if incremental export would be beneficial + useIncremental, dirtyIDs, err := shouldUseIncrementalExport(ctx, jsonlPath) + if err != nil { + // On error checking, fall back to full export + return exportToJSONLDeferred(ctx, jsonlPath) + } + + if !useIncremental { + return exportToJSONLDeferred(ctx, jsonlPath) + } + + // No dirty issues means nothing to export + if len(dirtyIDs) == 0 { + // Still need to return a valid result for idempotency + contentHash, _ := computeJSONLHash(jsonlPath) + return &ExportResult{ + JSONLPath: jsonlPath, + ExportedIDs: []string{}, + ContentHash: contentHash, + ExportTime: time.Now().Format(time.RFC3339Nano), + }, nil + } + + // Perform incremental export + return performIncrementalExport(ctx, jsonlPath, dirtyIDs) +} + +// shouldUseIncrementalExport determines if incremental export would be beneficial. +// Returns (useIncremental, dirtyIDs, error). +func shouldUseIncrementalExport(ctx context.Context, jsonlPath string) (bool, []string, error) { + // Check if JSONL file exists (can't do incremental without existing file) + if _, err := os.Stat(jsonlPath); os.IsNotExist(err) { + return false, nil, nil + } + + // Get dirty issue IDs + dirtyIDs, err := store.GetDirtyIssues(ctx) + if err != nil { + return false, nil, fmt.Errorf("failed to get dirty issues: %w", err) + } + + // If no dirty issues, we can skip export entirely + if len(dirtyIDs) == 0 { + return true, dirtyIDs, nil + } + + // Get total issue count from existing JSONL (fast line count) + totalCount, err := countIssuesInJSONL(jsonlPath) + if err != nil { + // Can't read JSONL, fall back to full export + return false, nil, nil + } + + // Check thresholds: + // 1. Total must be above threshold (small repos are fast enough with full export) + // 2. Dirty ratio must be below threshold (if most issues changed, full export is faster) + if totalCount < incrementalThreshold { + return false, nil, nil + } + + dirtyRatio := float64(len(dirtyIDs)) / float64(totalCount) + if dirtyRatio > incrementalDirtyRatio { + return false, nil, nil + } + + return true, dirtyIDs, nil +} + +// performIncrementalExport performs the actual incremental export. +// It reads the existing JSONL, queries only dirty issues, merges them, +// and writes the result. +func performIncrementalExport(ctx context.Context, jsonlPath string, dirtyIDs []string) (*ExportResult, error) { + // Read existing JSONL into map[id]rawJSON + issueMap, allIDs, err := readJSONLToMap(jsonlPath) + if err != nil { + // Fall back to full export on read error + return exportToJSONLDeferred(ctx, jsonlPath) + } + + // Query dirty issues from database and track which IDs were found + dirtyIssues := make([]*types.Issue, 0, len(dirtyIDs)) + issueByID := make(map[string]*types.Issue, len(dirtyIDs)) + for _, id := range dirtyIDs { + issue, err := store.GetIssue(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get dirty issue %s: %w", id, err) + } + issueByID[id] = issue // Store result (may be nil for deleted issues) + if issue != nil { + dirtyIssues = append(dirtyIssues, issue) + } + } + + // Get dependencies for dirty issues only + // Note: GetAllDependencyRecords is used because there's no batch method for specific IDs, + // but for truly large repos this could be optimized with a targeted query + allDeps, err := store.GetAllDependencyRecords(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get dependencies: %w", err) + } + for _, issue := range dirtyIssues { + issue.Dependencies = allDeps[issue.ID] + } + + // Get labels for dirty issues (batch query) + labelsMap, err := store.GetLabelsForIssues(ctx, dirtyIDs) + if err != nil { + return nil, fmt.Errorf("failed to get labels: %w", err) + } + for _, issue := range dirtyIssues { + issue.Labels = labelsMap[issue.ID] + } + + // Get comments for dirty issues (batch query) + commentsMap, err := store.GetCommentsForIssues(ctx, dirtyIDs) + if err != nil { + return nil, fmt.Errorf("failed to get comments: %w", err) + } + for _, issue := range dirtyIssues { + issue.Comments = commentsMap[issue.ID] + } + + // Update map with dirty issues + idSet := make(map[string]bool, len(allIDs)) + for _, id := range allIDs { + idSet[id] = true + } + + for _, issue := range dirtyIssues { + // Skip wisps - they should never be exported + if issue.Ephemeral { + continue + } + + // Serialize issue to JSON + data, err := json.Marshal(issue) + if err != nil { + return nil, fmt.Errorf("failed to marshal issue %s: %w", issue.ID, err) + } + + issueMap[issue.ID] = data + if !idSet[issue.ID] { + allIDs = append(allIDs, issue.ID) + idSet[issue.ID] = true + } + } + + // Handle tombstones and deletions using cached results (no second GetIssue call) + for _, id := range dirtyIDs { + issue := issueByID[id] // Use cached result + if issue == nil { + // Issue was fully deleted (not even a tombstone) + delete(issueMap, id) + } else if issue.Status == types.StatusTombstone { + // Issue is a tombstone - keep it in export for propagation + if !issue.Ephemeral { + data, err := json.Marshal(issue) + if err != nil { + return nil, fmt.Errorf("failed to marshal tombstone %s: %w", id, err) + } + issueMap[id] = data + } + } + } + + // Build sorted list of IDs (excluding deleted ones) + finalIDs := make([]string, 0, len(issueMap)) + for id := range issueMap { + finalIDs = append(finalIDs, id) + } + slices.Sort(finalIDs) + + // Write to temp file, then atomic rename + dir := filepath.Dir(jsonlPath) + base := filepath.Base(jsonlPath) + tempFile, err := os.CreateTemp(dir, base+".tmp.*") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + tempPath := tempFile.Name() + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempPath) + }() + + // Write JSONL in sorted order + exportedIDs := make([]string, 0, len(finalIDs)) + for _, id := range finalIDs { + data := issueMap[id] + if _, err := tempFile.Write(data); err != nil { + return nil, fmt.Errorf("failed to write issue %s: %w", id, err) + } + if _, err := tempFile.WriteString("\n"); err != nil { + return nil, fmt.Errorf("failed to write newline: %w", err) + } + exportedIDs = append(exportedIDs, id) + } + + // Close and rename + _ = tempFile.Close() + if err := os.Rename(tempPath, jsonlPath); err != nil { + return nil, fmt.Errorf("failed to replace JSONL file: %w", err) + } + + // Set permissions + if err := os.Chmod(jsonlPath, 0600); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to set file permissions: %v\n", err) + } + + // Compute hash + contentHash, _ := computeJSONLHash(jsonlPath) + exportTime := time.Now().Format(time.RFC3339Nano) + + // Note: exportedIDs contains ALL IDs in the file, but we only need to clear + // dirty flags for the dirtyIDs (which we received as parameter) + return &ExportResult{ + JSONLPath: jsonlPath, + ExportedIDs: dirtyIDs, // Only clear dirty flags for actually dirty issues + ContentHash: contentHash, + ExportTime: exportTime, + }, nil +} + +// readJSONLToMap reads a JSONL file into a map of id -> raw JSON bytes. +// Also returns the list of IDs in original order. +func readJSONLToMap(jsonlPath string) (map[string]json.RawMessage, []string, error) { + // #nosec G304 - controlled path + file, err := os.Open(jsonlPath) + if err != nil { + return nil, nil, err + } + defer func() { _ = file.Close() }() + + issueMap := make(map[string]json.RawMessage) + var ids []string + + scanner := bufio.NewScanner(file) + // Use larger buffer for large lines + scanner.Buffer(make([]byte, 0, 64*1024), 2*1024*1024) + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + // Extract ID from JSON without full unmarshal + var partial struct { + ID string `json:"id"` + } + if err := json.Unmarshal(line, &partial); err != nil { + // Skip malformed lines + continue + } + if partial.ID == "" { + continue + } + + // Store a copy of the line (scanner reuses buffer) + lineCopy := make([]byte, len(line)) + copy(lineCopy, line) + issueMap[partial.ID] = json.RawMessage(lineCopy) + ids = append(ids, partial.ID) + } + + if err := scanner.Err(); err != nil { + return nil, nil, err + } + + return issueMap, ids, nil +} + // validateOpenIssuesForSync validates all open issues against their templates // before export, based on the validation.on-sync config setting. // Returns an error if validation.on-sync is "error" and issues fail validation. diff --git a/cmd/bd/sync_export_incremental_test.go b/cmd/bd/sync_export_incremental_test.go new file mode 100644 index 00000000..d33a153d --- /dev/null +++ b/cmd/bd/sync_export_incremental_test.go @@ -0,0 +1,439 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +func TestReadJSONLToMap(t *testing.T) { + tmpDir := t.TempDir() + jsonlPath := filepath.Join(tmpDir, "test.jsonl") + + // Create test JSONL with 3 issues + issues := []types.Issue{ + {ID: "test-001", Title: "First", Status: types.StatusOpen}, + {ID: "test-002", Title: "Second", Status: types.StatusInProgress}, + {ID: "test-003", Title: "Third", Status: types.StatusClosed}, + } + + var content strings.Builder + for _, issue := range issues { + data, _ := json.Marshal(issue) + content.Write(data) + content.WriteString("\n") + } + + if err := os.WriteFile(jsonlPath, []byte(content.String()), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + // Test readJSONLToMap + issueMap, ids, err := readJSONLToMap(jsonlPath) + if err != nil { + t.Fatalf("readJSONLToMap failed: %v", err) + } + + // Verify count + if len(issueMap) != 3 { + t.Errorf("expected 3 issues in map, got %d", len(issueMap)) + } + if len(ids) != 3 { + t.Errorf("expected 3 IDs, got %d", len(ids)) + } + + // Verify order preserved + expectedOrder := []string{"test-001", "test-002", "test-003"} + for i, id := range ids { + if id != expectedOrder[i] { + t.Errorf("ID order mismatch at %d: expected %s, got %s", i, expectedOrder[i], id) + } + } + + // Verify content can be unmarshaled + for id, rawJSON := range issueMap { + var issue types.Issue + if err := json.Unmarshal(rawJSON, &issue); err != nil { + t.Errorf("failed to unmarshal issue %s: %v", id, err) + } + if issue.ID != id { + t.Errorf("ID mismatch: expected %s, got %s", id, issue.ID) + } + } +} + +func TestReadJSONLToMapWithMalformedLines(t *testing.T) { + tmpDir := t.TempDir() + jsonlPath := filepath.Join(tmpDir, "test.jsonl") + + content := `{"id": "test-001", "title": "Good"} +{invalid json} +{"id": "test-002", "title": "Also Good"} + +{"id": "", "title": "No ID"} +{"id": "test-003", "title": "Third Good"} +` + + if err := os.WriteFile(jsonlPath, []byte(content), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + issueMap, ids, err := readJSONLToMap(jsonlPath) + if err != nil { + t.Fatalf("readJSONLToMap failed: %v", err) + } + + // Should have 3 valid issues (skipped invalid JSON, empty line, and empty ID) + if len(issueMap) != 3 { + t.Errorf("expected 3 issues, got %d", len(issueMap)) + } + if len(ids) != 3 { + t.Errorf("expected 3 IDs, got %d", len(ids)) + } +} + +func TestShouldUseIncrementalExport(t *testing.T) { + tmpDir := t.TempDir() + testDB := filepath.Join(tmpDir, "test.db") + s := newTestStore(t, testDB) + defer s.Close() + + ctx := context.Background() + jsonlPath := filepath.Join(tmpDir, "issues.jsonl") + + // Set up global state + oldStore := store + oldDBPath := dbPath + oldRootCtx := rootCtx + store = s + dbPath = testDB + rootCtx = ctx + defer func() { + store = oldStore + dbPath = oldDBPath + rootCtx = oldRootCtx + }() + + t.Run("no JSONL file returns false", func(t *testing.T) { + useIncremental, _, err := shouldUseIncrementalExport(ctx, jsonlPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if useIncremental { + t.Error("expected false when JSONL doesn't exist") + } + }) + + t.Run("no dirty issues returns true with empty IDs", func(t *testing.T) { + // Create JSONL file with some issues + var content strings.Builder + for i := 0; i < 10; i++ { + issue := types.Issue{ID: "test-" + string(rune('a'+i)), Title: "Test"} + data, _ := json.Marshal(issue) + content.Write(data) + content.WriteString("\n") + } + if err := os.WriteFile(jsonlPath, []byte(content.String()), 0644); err != nil { + t.Fatalf("failed to write JSONL: %v", err) + } + + // No dirty issues in database + useIncremental, dirtyIDs, err := shouldUseIncrementalExport(ctx, jsonlPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // With 0 dirty issues, we return true to signal nothing needs export + if !useIncremental { + t.Error("expected true when no dirty issues (nothing to export)") + } + if len(dirtyIDs) != 0 { + t.Errorf("expected 0 dirty IDs, got %d", len(dirtyIDs)) + } + }) + + t.Run("small repo with dirty issues returns false", func(t *testing.T) { + // Create an issue to make it dirty + issue := &types.Issue{Title: "Dirty Issue", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := s.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("failed to create issue: %v", err) + } + + // Small repo (10 issues in JSONL) below 1000 threshold + useIncremental, _, err := shouldUseIncrementalExport(ctx, jsonlPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if useIncremental { + t.Error("expected false for small repo below threshold") + } + }) +} + +func TestIncrementalExportIntegration(t *testing.T) { + tmpDir := t.TempDir() + testDB := filepath.Join(tmpDir, "test.db") + s := newTestStore(t, testDB) + defer s.Close() + + ctx := context.Background() + jsonlPath := filepath.Join(tmpDir, "issues.jsonl") + + // Set up global state + oldStore := store + oldDBPath := dbPath + oldRootCtx := rootCtx + store = s + dbPath = testDB + rootCtx = ctx + defer func() { + store = oldStore + dbPath = oldDBPath + rootCtx = oldRootCtx + }() + + // Create initial issues + issues := []*types.Issue{ + {Title: "Issue 1", Description: "Desc 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + {Title: "Issue 2", Description: "Desc 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + {Title: "Issue 3", Description: "Desc 3", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } + + for _, issue := range issues { + if err := s.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("failed to create issue: %v", err) + } + } + + // Initial full export using exportToJSONL (not deferred, which calls ensureStoreActive) + if err := exportToJSONL(ctx, jsonlPath); err != nil { + t.Fatalf("initial export failed: %v", err) + } + + // Verify initial export worked + issueMap, _, err := readJSONLToMap(jsonlPath) + if err != nil { + t.Fatalf("failed to read initial JSONL: %v", err) + } + if len(issueMap) != 3 { + t.Errorf("expected 3 issues after initial export, got %d", len(issueMap)) + } + + // Update one issue + if err := s.UpdateIssue(ctx, issues[0].ID, map[string]interface{}{"title": "Updated Issue 1"}, "test"); err != nil { + t.Fatalf("failed to update issue: %v", err) + } + + // Verify dirty count + dirtyIDs, err := s.GetDirtyIssues(ctx) + if err != nil { + t.Fatalf("failed to get dirty issues: %v", err) + } + if len(dirtyIDs) != 1 { + t.Errorf("expected 1 dirty issue, got %d", len(dirtyIDs)) + } + + // Test incremental export (using performIncrementalExport directly) + result, err := performIncrementalExport(ctx, jsonlPath, dirtyIDs) + if err != nil { + t.Fatalf("incremental export failed: %v", err) + } + + // Verify result + if result == nil { + t.Fatal("expected non-nil result") + } + if len(result.ExportedIDs) != 1 { + t.Errorf("expected 1 exported ID (dirty), got %d", len(result.ExportedIDs)) + } + + // Read back JSONL and verify + issueMap, ids, err := readJSONLToMap(jsonlPath) + if err != nil { + t.Fatalf("failed to read JSONL: %v", err) + } + if len(issueMap) != 3 { + t.Errorf("expected 3 issues in JSONL, got %d", len(issueMap)) + } + if len(ids) != 3 { + t.Errorf("expected 3 IDs, got %d", len(ids)) + } + + // Verify updated issue content + var updatedIssue types.Issue + if err := json.Unmarshal(issueMap[issues[0].ID], &updatedIssue); err != nil { + t.Fatalf("failed to unmarshal updated issue: %v", err) + } + if updatedIssue.Title != "Updated Issue 1" { + t.Errorf("expected title 'Updated Issue 1', got '%s'", updatedIssue.Title) + } +} + +func TestIncrementalExportWithDeletion(t *testing.T) { + tmpDir := t.TempDir() + testDB := filepath.Join(tmpDir, "test.db") + s := newTestStore(t, testDB) + defer s.Close() + + ctx := context.Background() + jsonlPath := filepath.Join(tmpDir, "issues.jsonl") + + // Set up global state + oldStore := store + oldDBPath := dbPath + oldRootCtx := rootCtx + store = s + dbPath = testDB + rootCtx = ctx + defer func() { + store = oldStore + dbPath = oldDBPath + rootCtx = oldRootCtx + }() + + // Create initial issues + issues := []*types.Issue{ + {Title: "Issue 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + {Title: "Issue 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } + + for _, issue := range issues { + if err := s.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("failed to create issue: %v", err) + } + } + + // Initial export + if err := exportToJSONL(ctx, jsonlPath); err != nil { + t.Fatalf("initial export failed: %v", err) + } + + // Get initial dirty count (should be 0 after export) + dirtyIDs, _ := s.GetDirtyIssues(ctx) + if len(dirtyIDs) != 0 { + t.Errorf("expected 0 dirty issues after export, got %d", len(dirtyIDs)) + } + + // Delete one issue (soft delete creates tombstone) + if err := s.DeleteIssue(ctx, issues[0].ID); err != nil { + t.Fatalf("failed to delete issue: %v", err) + } + + // Get dirty IDs after deletion + dirtyIDs, err := s.GetDirtyIssues(ctx) + if err != nil { + t.Fatalf("failed to get dirty issues: %v", err) + } + + // Perform incremental export + if len(dirtyIDs) > 0 { + _, err = performIncrementalExport(ctx, jsonlPath, dirtyIDs) + if err != nil { + t.Fatalf("incremental export failed: %v", err) + } + + // Read back and verify tombstone is present + issueMap, _, err := readJSONLToMap(jsonlPath) + if err != nil { + t.Fatalf("failed to read JSONL: %v", err) + } + + // Should still have 2 entries (one is now tombstone) + if len(issueMap) != 2 { + t.Errorf("expected 2 issues in JSONL (including tombstone), got %d", len(issueMap)) + } + + // Check tombstone status + var tombstone types.Issue + if err := json.Unmarshal(issueMap[issues[0].ID], &tombstone); err != nil { + t.Fatalf("failed to unmarshal tombstone: %v", err) + } + if tombstone.Status != types.StatusTombstone { + t.Errorf("expected tombstone status, got %s", tombstone.Status) + } + } +} + +func TestIncrementalExportWithNewIssue(t *testing.T) { + tmpDir := t.TempDir() + testDB := filepath.Join(tmpDir, "test.db") + s := newTestStore(t, testDB) + defer s.Close() + + ctx := context.Background() + jsonlPath := filepath.Join(tmpDir, "issues.jsonl") + + // Set up global state + oldStore := store + oldDBPath := dbPath + oldRootCtx := rootCtx + store = s + dbPath = testDB + rootCtx = ctx + defer func() { + store = oldStore + dbPath = oldDBPath + rootCtx = oldRootCtx + }() + + // Create initial issue + issue1 := &types.Issue{Title: "Issue 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := s.CreateIssue(ctx, issue1, "test"); err != nil { + t.Fatalf("failed to create issue: %v", err) + } + + // Initial export + if err := exportToJSONL(ctx, jsonlPath); err != nil { + t.Fatalf("initial export failed: %v", err) + } + + // Create new issue + issue2 := &types.Issue{Title: "Issue 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := s.CreateIssue(ctx, issue2, "test"); err != nil { + t.Fatalf("failed to create issue 2: %v", err) + } + + // Get dirty IDs + dirtyIDs, err := s.GetDirtyIssues(ctx) + if err != nil { + t.Fatalf("failed to get dirty issues: %v", err) + } + if len(dirtyIDs) != 1 { + t.Errorf("expected 1 dirty issue (new one), got %d", len(dirtyIDs)) + } + + // Perform incremental export + _, err = performIncrementalExport(ctx, jsonlPath, dirtyIDs) + if err != nil { + t.Fatalf("incremental export failed: %v", err) + } + + // Read back and verify new issue was added + issueMap, ids, err := readJSONLToMap(jsonlPath) + if err != nil { + t.Fatalf("failed to read JSONL: %v", err) + } + + if len(issueMap) != 2 { + t.Errorf("expected 2 issues in JSONL, got %d", len(issueMap)) + } + + // Verify both issues exist + if _, ok := issueMap[issue1.ID]; !ok { + t.Errorf("issue 1 missing from JSONL") + } + if _, ok := issueMap[issue2.ID]; !ok { + t.Errorf("issue 2 missing from JSONL") + } + + // Verify sorted order + for i := 0; i < len(ids)-1; i++ { + if ids[i] > ids[i+1] { + t.Errorf("IDs not sorted: %s > %s", ids[i], ids[i+1]) + } + } +}