perf: batch external dep checks by project (bd-687v)
Optimize CheckExternalDeps to group refs by project and open each external DB only once, checking all capabilities in a single query. Before: If 10 issues depend on external:gastown:cap1, we opened gastown's DB 10 times. After: We open each external project's DB once, query for all capabilities needed from that project, then close. Also added deduplication in filterByExternalDeps to collect all unique refs before checking, avoiding redundant checks for the same ref across multiple issues. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
committed by
Steve Yegge
parent
939f222c35
commit
32a4a9e060
@@ -120,16 +120,172 @@ func CheckExternalDep(ctx context.Context, ref string) *ExternalDepStatus {
|
||||
return status
|
||||
}
|
||||
|
||||
// CheckExternalDeps checks multiple external dependencies.
|
||||
// CheckExternalDeps checks multiple external dependencies with batching optimization.
|
||||
// Groups refs by project and opens each external DB only once, checking all
|
||||
// capabilities for that project in a single query. This avoids O(N) DB opens
|
||||
// when multiple issues depend on the same external project.
|
||||
// Returns a map of ref -> status.
|
||||
func CheckExternalDeps(ctx context.Context, refs []string) map[string]*ExternalDepStatus {
|
||||
results := make(map[string]*ExternalDepStatus)
|
||||
|
||||
// Parse and group refs by project
|
||||
// Key: project name, Value: map of capability -> list of original refs
|
||||
// (multiple refs might have same project:capability, we dedupe)
|
||||
projectCaps := make(map[string]map[string][]string)
|
||||
|
||||
for _, ref := range refs {
|
||||
results[ref] = CheckExternalDep(ctx, ref)
|
||||
parsed := parseExternalRef(ref)
|
||||
if parsed == nil {
|
||||
results[ref] = &ExternalDepStatus{
|
||||
Ref: ref,
|
||||
Satisfied: false,
|
||||
Reason: "invalid external reference format",
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if projectCaps[parsed.project] == nil {
|
||||
projectCaps[parsed.project] = make(map[string][]string)
|
||||
}
|
||||
projectCaps[parsed.project][parsed.capability] = append(
|
||||
projectCaps[parsed.project][parsed.capability], ref)
|
||||
}
|
||||
|
||||
// Check each project's capabilities in batch
|
||||
for project, caps := range projectCaps {
|
||||
capList := make([]string, 0, len(caps))
|
||||
for cap := range caps {
|
||||
capList = append(capList, cap)
|
||||
}
|
||||
|
||||
// Check all capabilities for this project in one DB open
|
||||
satisfied := checkProjectCapabilities(ctx, project, capList)
|
||||
|
||||
// Map results back to original refs
|
||||
for cap, refList := range caps {
|
||||
isSatisfied := satisfied[cap]
|
||||
reason := "capability shipped"
|
||||
if !isSatisfied {
|
||||
reason = "capability not shipped (no closed issue with provides:" + cap + " label)"
|
||||
}
|
||||
|
||||
for _, ref := range refList {
|
||||
results[ref] = &ExternalDepStatus{
|
||||
Ref: ref,
|
||||
Project: project,
|
||||
Capability: cap,
|
||||
Satisfied: isSatisfied,
|
||||
Reason: reason,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// parsedRef holds parsed components of an external reference
|
||||
type parsedRef struct {
|
||||
project string
|
||||
capability string
|
||||
}
|
||||
|
||||
// parseExternalRef parses "external:project:capability" into components.
|
||||
// Returns nil if the format is invalid.
|
||||
func parseExternalRef(ref string) *parsedRef {
|
||||
if !strings.HasPrefix(ref, "external:") {
|
||||
return nil
|
||||
}
|
||||
parts := strings.SplitN(ref, ":", 3)
|
||||
if len(parts) != 3 || parts[1] == "" || parts[2] == "" {
|
||||
return nil
|
||||
}
|
||||
return &parsedRef{project: parts[1], capability: parts[2]}
|
||||
}
|
||||
|
||||
// checkProjectCapabilities opens a project's beads DB once and checks
|
||||
// multiple capabilities in a single query. Returns map of capability -> satisfied.
|
||||
func checkProjectCapabilities(ctx context.Context, project string, capabilities []string) map[string]bool {
|
||||
result := make(map[string]bool)
|
||||
for _, cap := range capabilities {
|
||||
result[cap] = false // default to unsatisfied
|
||||
}
|
||||
|
||||
if len(capabilities) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
// Look up project path from config
|
||||
projectPath := config.ResolveExternalProjectPath(project)
|
||||
if projectPath == "" {
|
||||
return result // all unsatisfied - project not configured
|
||||
}
|
||||
|
||||
// Find the beads database in the project
|
||||
beadsDir := filepath.Join(projectPath, ".beads")
|
||||
cfg, err := configfile.Load(beadsDir)
|
||||
if err != nil || cfg == nil {
|
||||
return result // all unsatisfied - no beads database
|
||||
}
|
||||
|
||||
dbPath := cfg.DatabasePath(beadsDir)
|
||||
|
||||
// Verify database file exists
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
return result // all unsatisfied - database not found
|
||||
}
|
||||
|
||||
// Open the external database once for all capability checks
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return result // all unsatisfied - cannot open
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
return result // all unsatisfied - cannot connect
|
||||
}
|
||||
|
||||
// Build query to check all capabilities at once
|
||||
// SELECT label FROM labels WHERE label IN ('provides:cap1', 'provides:cap2', ...)
|
||||
// AND EXISTS closed issue with that label
|
||||
placeholders := make([]string, len(capabilities))
|
||||
args := make([]interface{}, len(capabilities))
|
||||
for i, cap := range capabilities {
|
||||
placeholders[i] = "?"
|
||||
args[i] = "provides:" + cap
|
||||
}
|
||||
|
||||
// Query returns which provides: labels exist on closed issues
|
||||
query := `
|
||||
SELECT DISTINCT l.label FROM labels l
|
||||
JOIN issues i ON l.issue_id = i.id
|
||||
WHERE i.status = 'closed'
|
||||
AND l.label IN (` + strings.Join(placeholders, ",") + `)
|
||||
`
|
||||
|
||||
rows, err := db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return result // all unsatisfied - query failed
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
// Mark satisfied capabilities
|
||||
for rows.Next() {
|
||||
var label string
|
||||
if err := rows.Scan(&label); err != nil {
|
||||
continue
|
||||
}
|
||||
// Extract capability from "provides:capability"
|
||||
if strings.HasPrefix(label, "provides:") {
|
||||
cap := strings.TrimPrefix(label, "provides:")
|
||||
result[cap] = true
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetUnsatisfiedExternalDeps returns external dependencies that are not satisfied.
|
||||
func GetUnsatisfiedExternalDeps(ctx context.Context, refs []string) []string {
|
||||
var unsatisfied []string
|
||||
|
||||
Reference in New Issue
Block a user