wip: add sync cmd and update models

lukaszcl · lukaszcl · commit da405fd5c009 · 2025-04-02T16:02:57.000+02:00
diff --git a/tools/flakeguard/cmd/create_jira_tickets.go b/tools/flakeguard/cmd/create_jira_tickets.go
@@ -560,8 +560,8 @@ func (m tmodel) View() string {
 	// Assignee
 	var assigneeLine string
 	var assigneeDisplayValue string
-	if t.AssigneeName != "" {
-		assigneeDisplayValue = fmt.Sprintf("%s (%s)", t.AssigneeName, t.AssigneeId)
+	if t.AssigneeId != "" {
+		assigneeDisplayValue = fmt.Sprintf("%s (%s)", t.AssigneeId, t.AssigneeId)
 	} else {
 		assigneeDisplayValue = t.AssigneeId
 	}
diff --git a/tools/flakeguard/cmd/sync_jira.go b/tools/flakeguard/cmd/sync_jira.go
@@ -0,0 +1,172 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/andygrunwald/go-jira"
+	"github.com/rs/zerolog/log"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/jirautils"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/localdb"
+	"github.com/spf13/cobra"
+)
+
+var (
+	syncJiraSearchLabel string
+	syncTestDBPath      string
+	syncDryRun          bool
+	updateAssignees     bool
+)
+
+var SyncJiraCmd = &cobra.Command{
+	Use:   "sync-jira",
+	Short: "Sync Jira tickets with local database",
+	Long: `Searches for all flaky test tickets in Jira that aren't yet tracked in the local database.
+This command will:
+1. Search Jira for all tickets with the flaky_test label
+2. Compare against the local database
+3. Add any missing tickets to the local database
+4. Update assignee information from Jira
+5. Optionally show which tickets were added/updated (--dry-run)`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// 1) Set default label if not provided
+		if syncJiraSearchLabel == "" {
+			syncJiraSearchLabel = "flaky_test"
+		}
+
+		// 2) Load local DB
+		db, err := localdb.LoadDBWithPath(syncTestDBPath)
+		if err != nil {
+			log.Warn().Err(err).Msg("Failed to load local DB; continuing with empty DB.")
+			db = localdb.NewDB()
+		}
+
+		// 3) Get Jira client
+		client, err := jirautils.GetJiraClient()
+		if err != nil {
+			log.Error().Err(err).Msg("Failed to create Jira client")
+			return err
+		}
+
+		// 4) Search for all flaky test tickets in Jira
+		jql := fmt.Sprintf(`labels = "%s" ORDER BY created DESC`, syncJiraSearchLabel)
+		var startAt int
+		var allIssues []jira.Issue
+
+		for {
+			issues, resp, err := client.Issue.SearchWithContext(context.Background(), jql, &jira.SearchOptions{
+				StartAt:    startAt,
+				MaxResults: 50,                              // Fetch in batches of 50
+				Fields:     []string{"summary", "assignee"}, // Include assignee field
+			})
+			if err != nil {
+				return fmt.Errorf("error searching Jira: %w (resp: %v)", err, resp)
+			}
+			if len(issues) == 0 {
+				break
+			}
+			allIssues = append(allIssues, issues...)
+			startAt += len(issues)
+		}
+
+		// 5) Process each issue
+		var added int
+		var updated int
+		var skipped int
+		var assigneeUpdated int
+
+		// Get all entries for efficient updating
+		entries := db.GetAllEntries()
+		entriesMap := make(map[string]*localdb.Entry) // map by Jira ticket key
+		for i := range entries {
+			entriesMap[entries[i].JiraTicket] = &entries[i]
+		}
+
+		for _, issue := range allIssues {
+			// Extract test name from summary
+			summary := issue.Fields.Summary
+			testName := extractTestName(summary)
+			if testName == "" {
+				log.Warn().Msgf("Could not extract test name from summary: %s", summary)
+				skipped++
+				continue
+			}
+
+			// Get assignee ID if available
+			var assigneeID string
+			if issue.Fields.Assignee != nil {
+				assigneeID = issue.Fields.Assignee.AccountID
+			}
+
+			// Check if this ticket is already in the local DB
+			if entry, exists := entriesMap[issue.Key]; exists {
+				if assigneeID != "" && entry.AssigneeID != assigneeID {
+					if !syncDryRun {
+						entry.AssigneeID = assigneeID
+						// Update the entry in the DB
+						db.UpdateEntry(*entry)
+					}
+					assigneeUpdated++
+					log.Info().Msgf("Updated assignee for ticket %s to %s", issue.Key, assigneeID)
+				}
+				updated++
+			} else {
+				if !syncDryRun {
+					// Create new entry with assignee information
+					entry := localdb.Entry{
+						TestPackage: "", // Empty as we can't reliably extract it
+						TestName:    testName,
+						JiraTicket:  issue.Key,
+						AssigneeID:  assigneeID,
+					}
+					db.AddEntry(entry)
+				}
+				added++
+				log.Info().Msgf("Added ticket %s for test %s (assignee: %s)", issue.Key, testName, assigneeID)
+			}
+		}
+
+		// 6) Save DB if not in dry run mode
+		if !syncDryRun && (added > 0 || assigneeUpdated > 0) {
+			if err := db.Save(); err != nil {
+				log.Error().Err(err).Msg("Failed to save local DB")
+				return err
+			}
+			log.Info().Msgf("Local DB has been updated at: %s", db.FilePath())
+		}
+
+		// 7) Print summary
+		fmt.Printf("\nSummary:\n")
+		fmt.Printf("Total Jira tickets found: %d\n", len(allIssues))
+		fmt.Printf("New tickets added to DB: %d\n", added)
+		fmt.Printf("Existing tickets found: %d\n", updated)
+		fmt.Printf("Assignees updated: %d\n", assigneeUpdated)
+		fmt.Printf("Skipped (could not parse): %d\n", skipped)
+		if syncDryRun {
+			fmt.Printf("\nThis was a dry run. No changes were made to the local database.\n")
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	SyncJiraCmd.Flags().StringVar(&syncJiraSearchLabel, "jira-search-label", "", "Jira label to filter existing tickets (default: flaky_test)")
+	SyncJiraCmd.Flags().StringVar(&syncTestDBPath, "test-db-path", "", "Path to the flaky test JSON database (default: ~/.flaky_test_db.json)")
+	SyncJiraCmd.Flags().BoolVar(&syncDryRun, "dry-run", false, "If true, only show what would be added without making changes")
+	InitCommonFlags(SyncJiraCmd)
+}
+
+// extractTestName attempts to extract the test name from a ticket summary
+func extractTestName(summary string) string {
+	// Expected format: "Fix Flaky Test: TestName (X% flake rate)"
+	parts := strings.Split(summary, ": ")
+	if len(parts) != 2 {
+		return ""
+	}
+	testPart := parts[1]
+	// Split on " (" to remove the flake rate part
+	testName := strings.Split(testPart, " (")[0]
+	return strings.TrimSpace(testName)
+}
diff --git a/tools/flakeguard/cmd/tickets.go b/tools/flakeguard/cmd/tickets.go
@@ -4,9 +4,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/andygrunwald/go-jira"
+	"github.com/briandowns/spinner"
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/charmbracelet/lipgloss"
 	"github.com/rs/zerolog/log"
@@ -22,6 +24,7 @@ var (
 	jiraComment     bool // if true, post a comment when marking as skipped
 	ticketsDryRun   bool // if true, do not send anything to Jira
 	hideSkipped     bool // if true, do not show skipped tests
+	missingPillars  bool // if true, only show tickets with missing pillar names
 )
 
 // TicketsCmd is the new CLI command for managing tickets.
@@ -99,11 +102,13 @@ Actions:
 				SkippedAt:       entry.SkippedAt,
 			}
 
-			// Map user ID based on test package pattern
-			if userID := model.MapTestPackageToUser(entry.TestPackage, patternToUserID); userID != "" {
-				if userMapping, exists := userMap[userID]; exists {
-					tickets[i].AssigneeId = userID
-					tickets[i].AssigneeName = userMapping.UserName
+			// Map user based on assignee ID from local DB
+			if entry.AssigneeID != "" {
+				if _, exists := userMap[entry.AssigneeID]; exists {
+					tickets[i].AssigneeId = entry.AssigneeID
+				} else {
+					tickets[i].AssigneeId = entry.AssigneeID
+					tickets[i].MissingUserMapping = true
 				}
 			}
 		}
@@ -126,6 +131,77 @@ Actions:
 			jiraClient = nil
 		}
 
+		// Fetch pillar names with spinner
+		if jiraClient != nil {
+			s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)
+			s.Suffix = " Fetching pillar names from Jira..."
+			s.Start()
+
+			// Collect all Jira keys that need pillar names
+			var jiraKeys []string
+			for _, t := range tickets {
+				if t.ExistingJiraKey != "" {
+					jiraKeys = append(jiraKeys, t.ExistingJiraKey)
+				}
+			}
+
+			// Process tickets in batches of 50 (Jira's recommended batch size)
+			batchSize := 50
+			for i := 0; i < len(jiraKeys); i += batchSize {
+				end := i + batchSize
+				if end > len(jiraKeys) {
+					end = len(jiraKeys)
+				}
+				batch := jiraKeys[i:end]
+
+				// Create JQL query for the batch
+				jql := fmt.Sprintf("key IN (%s)", strings.Join(batch, ","))
+
+				// Fetch issues in batch
+				issues, _, err := jiraClient.Issue.Search(jql, &jira.SearchOptions{
+					Fields:     []string{"key", "customfield_11016"},
+					MaxResults: batchSize,
+				})
+
+				if err != nil {
+					log.Warn().Err(err).Msgf("Failed to fetch pillar names for batch of tickets")
+					continue
+				}
+
+				// Update tickets with pillar names
+				for _, issue := range issues {
+					// Find the corresponding ticket
+					for j := range tickets {
+						if tickets[j].ExistingJiraKey == issue.Key {
+							if issue.Fields != nil {
+								if pillarField, ok := issue.Fields.Unknowns["customfield_11016"].(map[string]interface{}); ok {
+									if value, ok := pillarField["value"].(string); ok {
+										tickets[j].PillarName = value
+									}
+								}
+							}
+							break
+						}
+					}
+				}
+
+				// Update spinner progress
+				s.Suffix = fmt.Sprintf(" Fetching pillar names from Jira... (%d/%d)", end, len(jiraKeys))
+			}
+			s.Stop()
+		}
+
+		// Filter tickets with missing pillars if the flag is set
+		if missingPillars {
+			filtered := make([]model.FlakyTicket, 0, len(tickets))
+			for _, t := range tickets {
+				if t.PillarName == "" {
+					filtered = append(filtered, t)
+				}
+			}
+			tickets = filtered
+		}
+
 		// 6) Initialize the Bubble Tea model.
 		m := initialTicketsModel(tickets, userMap, testPatternMap)
 		m.JiraClient = jiraClient
@@ -153,7 +229,8 @@ func init() {
 	TicketsCmd.Flags().StringVar(&ticketsJSONPath, "test-db-path", "flaky_test_db.json", "Path to the JSON file containing tickets")
 	TicketsCmd.Flags().BoolVar(&jiraComment, "jira-comment", true, "If true, post a comment to the Jira ticket when marking as skipped")
 	TicketsCmd.Flags().BoolVar(&ticketsDryRun, "dry-run", false, "If true, do not send anything to Jira")
-	TicketsCmd.Flags().BoolVar(&hideSkipped, "hide-skipped", false, "If true, do not show skipped tests")
+	TicketsCmd.Flags().BoolVar(&hideSkipped, "hide-skipped", false, "If true, dbto not show skipped tests")
+	TicketsCmd.Flags().BoolVar(&missingPillars, "missing-pillars", false, "If true, only show tickets with missing pillar names")
 	TicketsCmd.Flags().StringVar(&userMappingPath, "user-mapping-path", "user_mapping.json", "Path to the JSON file containing user mapping")
 	TicketsCmd.Flags().StringVar(&userTestMappingPath, "user-test-mapping-path", "user_test_mapping.json", "Path to the JSON file containing user test mapping")
 	InitCommonFlags(TicketsCmd)
@@ -328,18 +405,34 @@ func (m ticketModel) View() string {
 		view += fmt.Sprintf("%s %s\n", labelStyle.Render("Jira:"), jirautils.GetJiraLink(t.ExistingJiraKey))
 	}
 
+	// Show assignee information
+	if t.AssigneeId != "" {
+		view += fmt.Sprintf("%s %s\n", labelStyle.Render("Assignee ID:"), t.AssigneeId)
+	}
+
 	// Show status with color.
 	if !t.SkippedAt.IsZero() {
 		view += fmt.Sprintf("%s %s\n", labelStyle.Render("Status:"), fmt.Sprintf("skipped at: %s", t.SkippedAt.UTC().Format(time.RFC822)))
 	} else {
 		view += fmt.Sprintf("%s %s\n", labelStyle.Render("Status:"), infoStyle.Render("not skipped"))
 	}
 
+	view += fmt.Sprintf("%s %s\n", labelStyle.Render("Pillar:"), t.PillarName)
+
 	// Display any info message.
 	if m.infoMessage != "" {
 		view += "\n" + infoStyle.Render(m.infoMessage) + "\n"
 	}
 
-	view += "\n" + actionStyle.Render("Actions:") + " [s] mark as skipped, [u] unskip, [i] set pillar name, [n] next, [q] quit"
+	// Build actions list
+	actions := []string{
+		"[s] mark as skipped",
+		"[u] unskip",
+		"[i] set pillar name",
+		"[n] next",
+		"[q] quit",
+	}
+
+	view += "\n" + actionStyle.Render("Actions:") + " " + strings.Join(actions, ", ")
 	return view
 }
diff --git a/tools/flakeguard/localdb/localdb.go b/tools/flakeguard/localdb/localdb.go
@@ -17,6 +17,7 @@ type Entry struct {
 	TestPackage string    `json:"test_package"`
 	TestName    string    `json:"test_name"`
 	JiraTicket  string    `json:"jira_ticket"`
+	AssigneeID  string    `json:"jira_assignee_id,omitempty"`
 	IsSkipped   bool      `json:"is_skipped,omitempty"`
 	SkippedAt   time.Time `json:"skipped_at,omitempty"`
 }
@@ -158,3 +159,15 @@ func getDefaultDBPath() string {
 func makeKey(pkg, testName string) string {
 	return pkg + "::" + testName
 }
+
+// UpdateEntry updates an existing entry in the DB
+func (db *DB) UpdateEntry(entry Entry) {
+	key := makeKey(entry.TestPackage, entry.TestName)
+	db.data[key] = entry
+}
+
+// AddEntry adds a new entry to the DB
+func (db *DB) AddEntry(entry Entry) {
+	key := makeKey(entry.TestPackage, entry.TestName)
+	db.data[key] = entry
+}
diff --git a/tools/flakeguard/main.go b/tools/flakeguard/main.go
@@ -45,6 +45,7 @@ func init() {
 	rootCmd.AddCommand(cmd.SendToSplunkCmd)
 	rootCmd.AddCommand(cmd.TicketsCmd)
 	rootCmd.AddCommand(cmd.CreateTicketsCmd)
+	rootCmd.AddCommand(cmd.SyncJiraCmd)
 }
 
 func main() {
diff --git a/tools/flakeguard/model/ticket.go b/tools/flakeguard/model/ticket.go
@@ -21,10 +21,11 @@ type FlakyTicket struct {
 	ExistingJiraKey      string
 	ExistingTicketSource string // "localdb" or "jira"
 	AssigneeId           string
-	AssigneeName         string
 	Priority             string
 	FlakeRate            float64
 	SkippedAt            time.Time // timestamp when the ticket was marked as skipped
+	MissingUserMapping   bool      // true if the assignee ID exists but has no mapping in user_mapping.json
+	PillarName           string    // pillar name from Jira customfield_11016
 }
 
 // MapTestPackageToUser maps a test package to a user ID using regex patterns

Original file line number	Diff line number	Diff line change
`@@ -45,6 +45,7 @@ func init() {`
`45`	`45`	`rootCmd.AddCommand(cmd.SendToSplunkCmd)`
`46`	`46`	`rootCmd.AddCommand(cmd.TicketsCmd)`
`47`	`47`	`rootCmd.AddCommand(cmd.CreateTicketsCmd)`
	`48`	`+ rootCmd.AddCommand(cmd.SyncJiraCmd)`
`48`	`49`	`}`
`49`	`50`
`50`	`51`	`func main() {`