Handle low usage tags (#28)

* Added support for tags under min threshold and cleaned up type issues

* Update utils/stackoverflow/populate_discussion.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update utils/stackoverflow/so_explore.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: MonicaG

utils/stackoverflow/populate_discussion.py

@@ -64,7 +64,7 @@ def get_labels(repo):
     """Get all labels in the repository"""
     return {label.name: label for label in repo.get_labels()}
 
-def create_label(repo, name: str, description: str = None):
+def create_label(repo, name: str, description: Optional[str] = None):
     """Create a new label in the repository, ensuring description is < 100 chars"""
     if description and len(description) > 100:
         description = description[:97] + '...'
@@ -472,7 +472,10 @@ def get_tags_under_threshold(min_threshold: int, tags_data: List[Dict[str, Any]]
     Returns:
         List of tag names (strings) for tags with count < min_threshold
     """
-    return [tag['name'] for tag in tags_data if tag.get('count', 0) < min_threshold]
+    logger.info(f"Identifying tags with count below threshold of {min_threshold}")
+    low_count_tags = [tag['name'] for tag in tags_data if tag.get('count', 0) < min_threshold]
+    logger.info(f"Found {len(low_count_tags)} tags below threshold: {low_count_tags}")
+    return low_count_tags
 
 def get_tags_at_or_above_threshold(min_threshold: int, tags_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     """Get tag objects for tags with count at or above the minimum threshold.
@@ -682,14 +685,19 @@ def main():
     github_auth_manager = GitHubAuthManager()
     github_auth_manager.initialize()
 
+    github_client = github_auth_manager.get_client()
+    if github_client is None:
+        logger.error("GitHub client is not initialized. Please check your authentication setup.")
+        raise RuntimeError("GitHub client is not initialized.")
+
     github_graphql = GraphQLHelper(github_auth_manager, rate_limiter)
 
     repo_parts = args.repo.split('/')
     if len(repo_parts) != 2:
         raise ValueError("Repository must be in format 'owner/name'")
 
     owner, name = repo_parts
-    repo = github_auth_manager.get_client().get_repo(f"{owner}/{name}")
+    repo = github_client.get_repo(f"{owner}/{name}")
 
     logger.info(f"Using repo '{repo.full_name}'")
 
@@ -879,7 +887,7 @@ def main():
 class TagsToIgnore:
     """Helper class to manage tags that should be ignored during migration."""
 
-    def __init__(self, tags_to_ignore: list[str] = None):
+    def __init__(self, tags_to_ignore: Optional[List[str]] = None):
         """Initialize tags to ignore.
         
         Args:
@@ -900,7 +908,7 @@ def should_ignore(self, tags: list[str]) -> bool:
             True if any tag should be ignored, False otherwise
         """
 
-        return self.tags_to_ignore and any(t in self.tags_to_ignore for t in tags)
+        return bool(self.tags_to_ignore) and any(t in self.tags_to_ignore for t in tags)
 
 if __name__ == '__main__':
     main()

utils/stackoverflow/populate_discussion_helpers.py

@@ -7,6 +7,7 @@
 import time
 from github import Github, Auth
 import requests
+from typing import Optional
 
 # Setup logging
 logger = logging.getLogger(__name__)
@@ -87,7 +88,9 @@ def get_token(self):
         if not self._initialized:
             raise Exception("GitHub auth not initialized. Call initialize() first.")
 
-        return  self._github_auth.token
+        if self._github_auth is None:
+            raise Exception("GitHub authentication has not been initialized or failed.")
+        return self._github_auth.token
 
     def get_client(self):
         """Get the current GitHub client."""
@@ -104,8 +107,7 @@ def is_initialized(self):
 
 class GraphQLHelper:
     """Helper class for GraphQL operations."""
-    
-    def __init__(self, github_auth_manager: GitHubAuthManager, rate_limiter: RateLimiter = None):
+    def __init__(self, github_auth_manager: GitHubAuthManager, rate_limiter: Optional[RateLimiter] = None):
         """Initialize with GitHub auth manager and optional rate limiter.
         
         Args:

utils/stackoverflow/so_explore.py

@@ -23,6 +23,7 @@
 # dd.sql("select question_id, view_count from './questions_answers_comments.json' as questions where view_count >= 100 order by view_count desc").show()
 # dd.sql("select questions.answers.comments.link, questions.answers.link, questions.answers.share_link, questions.link, questions.share_link from './questions_answers_comments.json' as questions").show()
 # dd.sql("select unnest(questions.comments).link, unnest(questions.answers).share_link, unnest(questions.answers).link, unnest(questions.answers.comments).link, questions.link, questions.share_link from './questions_answers_comments.json' as questions").show()
+dd.sql("select tags.name, tags.count from './tags.json' as tags where tags.count < 5").show()
 
 # This gets all links and share_links from answers, questions and comments and outputs them to a file. User links are ignored
 #dd.sql("Copy (" \

utils/stackoverflow/test_validate_migration.py

@@ -173,6 +173,47 @@ def test_validate_question_content_missing_tags(self):
         self.assertIn("networking", issues[0])
         self.assertIn("troubleshooting", issues[0])
 
+    def test_validate_question_content_has_low_usage_tag(self):
+        """Test validation when a low usage tag is present."""
+        so_question = {
+            "title": "VPN Help",
+            "body": "Need VPN help.",
+            "tags": ["vpn", "networking", "troubleshooting"]
+        }
+
+        gh_discussion = {
+            "title": "VPN Help",
+            "body": "Need VPN help.",
+            "labels": {"nodes": [{"name": "vpn"}, {"name": "troubleshooting"}]}
+        }
+
+        mock_low_usage_tags_data = ["networking", "help"]
+
+        issues = self.validator.validate_question_content(so_question, gh_discussion, mock_low_usage_tags_data)
+        self.assertEqual(len(issues), 0)
+
+    def test_validate_question_content_missing_mixed_missing_and_low_usage_tag(self):
+        """Test validation when a low usage tag is present."""
+        so_question = {
+            "title": "VPN Help",
+            "body": "Need VPN help.",
+            "tags": ["vpn", "networking", "troubleshooting"]
+        }
+
+        gh_discussion = {
+            "title": "VPN Help",
+            "body": "Need VPN help.",
+            "labels": {"nodes": [{"name": "vpn"}]}
+        }
+
+        mock_low_usage_tags_data = ["networking", "help"]
+
+        issues = self.validator.validate_question_content(so_question, gh_discussion, mock_low_usage_tags_data)
+        self.assertEqual(len(issues), 1)    
+        self.assertIn("Missing tags:", issues[0])
+        # Check that both missing tags are mentioned, regardless of order
+        self.assertIn("troubleshooting", issues[0])
+
     def test_validate_question_content_with_images_matching(self):
         """Test validation when images are present and match between SO and GH."""
         so_question = {
@@ -1069,12 +1110,44 @@ def test_validate_migration_with_ignored_tags_integration(self, mock_load_json):
                 "tags": ["current", "deprecated"]
             }
         ]
-        mock_load_json.return_value = mock_so_questions
+        
+        # Mock the tags data (second file parameter)
+        mock_tags_data = [
+            {
+                "count": 145,
+                "name": "openshift"
+            },
+            {
+                "count": 32,
+                "name": "keycloak"
+            },
+            {
+                "count": 26,
+                "name": "security"
+            },
+            {
+                "count": 15,
+                "name": "deprecated"
+            },
+            {
+                "count": 10,
+                "name": "legacy"
+            }
+        ]
+        
+        # Configure mock to return different data based on which file is being loaded
+        def mock_load_json_side_effect(file_path):
+            if "mock_tag_file.json" in file_path:
+                return mock_tags_data
+            else:
+                return mock_so_questions
+        
+        mock_load_json.side_effect = mock_load_json_side_effect
 
         # Mock get_github_discussions to return empty (none migrated)
         validator.get_github_discussions = Mock(return_value=[])
 
-        results = validator.validate_migration("dummy_file.json")
+        results = validator.validate_migration("mock_so_file.json", "mock_tag_file.json")
 
         # Check results
         self.assertEqual(results['total_questions'], 4)

utils/stackoverflow/validate_migration.py

@@ -6,7 +6,7 @@
 from typing import Dict, List, Optional
 from populate_discussion_helpers import GitHubAuthManager, GraphQLHelper
 from populate_discussion import (
-    load_json, decode_html_entities, get_category_id, POPULAR_TAG_NAME
+    load_json, decode_html_entities, get_category_id, POPULAR_TAG_NAME, get_tags_under_threshold
 )
 import duckdb as dd
 
@@ -64,12 +64,13 @@ def replace_html_image(match):
     return text
 
 class MigrationValidator:
-    def __init__(self, auth_manager: GitHubAuthManager, owner: str, name: str, category_name: str, ignored_tags: Optional[List[str]] = None, popular_tag_min_threshold: int = 200):
+    def __init__(self, auth_manager: GitHubAuthManager, owner: str, name: str, category_name: str, ignored_tags: Optional[List[str]] = None, popular_tag_min_threshold: int = 200, tag_min_threshold: int = 1):
         self.owner = owner
         self.name = name
         self.category_name = category_name
         self.ignored_tags = ignored_tags or []
         self.popular_tag_min_threshold = popular_tag_min_threshold
+        self.tag_min_threshold = tag_min_threshold
         self.github_graphql = GraphQLHelper(auth_manager)
         self.popular_gh_questions = set()
         self.validation_results = {
@@ -157,7 +158,7 @@ def get_github_discussions(self) -> List[Dict]:
 
         return discussions
 
-    def validate_question_content(self, so_question: Dict, gh_discussion: Dict) -> List[str]:
+    def validate_question_content(self, so_question: Dict, gh_discussion: Dict, tags_under_min_threshold: Optional[List[str]] = None) -> List[str]:
         """Validate that question content was transferred correctly."""
         issues = []
 
@@ -203,6 +204,8 @@ def validate_question_content(self, so_question: Dict, gh_discussion: Dict) -> L
 
         # Check tags/labels
         so_tags = set(so_question.get('tags', []))
+        if tags_under_min_threshold:
+           so_tags = so_tags - set(tags_under_min_threshold) 
         gh_labels = set(label['name'] for label in gh_discussion['labels']['nodes'])
         missing_tags = so_tags - gh_labels
         if missing_tags:
@@ -335,7 +338,7 @@ def validate_comments(self, so_question: Dict, gh_discussion: Dict) -> List[str]
 
         return issues
 
-    def process_question(self, so_question: Dict, gh_discussions_by_title: Dict) -> None:
+    def process_question(self, so_question: Dict, gh_discussions_by_title: Dict, tags_under_min_threshold: Optional[List[str]] = None) -> None:
         """
         Process a single SO question and update validation results.
         
@@ -350,7 +353,7 @@ def process_question(self, so_question: Dict, gh_discussions_by_title: Dict) ->
             gh_discussion = gh_discussions_by_title[so_title]
 
             # Validate content
-            content_issues = self.validate_question_content(so_question, gh_discussion)
+            content_issues = self.validate_question_content(so_question, gh_discussion, tags_under_min_threshold)
             if content_issues:
                 self.validation_results['content_issues'].append({
                     'id': so_question['question_id'],
@@ -400,49 +403,45 @@ def validate_popular_tags(self, questions_file: str) -> None:
         if tagged_as_popular_but_are_not:
             self.validation_results['popular_question_issues']['tagged_as_popular_but_are_not'] = tagged_as_popular_but_are_not
 
-    def validate_migration(self, questions_file: str) -> Dict:
+    def validate_migration(self, questions_file: str, tags_file: str) -> Dict:
         """Main validation method."""
         logger.info("Starting migration validation...")
 
         # Load Stack Overflow data
         so_questions = load_json(questions_file)
+        tags = load_json(tags_file)
         self.validation_results['total_questions'] = len(so_questions)
 
         # Get GitHub discussions
         gh_discussions = self.get_github_discussions()
         gh_discussions_by_title = {d['title']: d for d in gh_discussions}
-        
+
+        # Get tags that were omitted because of low usage
+        tags_under_min_threshold = get_tags_under_threshold(self.tag_min_threshold, tags)
+
         logger.info(f"Found {len(so_questions)} SO questions and {len(gh_discussions)} GH discussions")
-        
+
+        logger.info("Starting question verification...")
         for so_question in so_questions:
-            self.process_question(so_question, gh_discussions_by_title)
+            self.process_question(so_question, gh_discussions_by_title, tags_under_min_threshold)
 
         self.validate_popular_tags(questions_file)
         return self.validation_results
 
-    def calculate_success_rate(self) -> float:
-        results = self.validation_results
-        if results['total_questions'] == 0:
-            return 0.0
-        
-        successful_questions = results['migrated_questions'] + len(results['ignored_questions'])
-        return (successful_questions / results['total_questions']) * 100
+
 
     def generate_report(self) -> str:
         """Generate a validation report."""
         results = self.validation_results
-        success_rate = self.calculate_success_rate()
-
         report = f"""
 # Migration Validation Report
 
-Success rate is determined by the formula: (migrated questions + ignored questions) / total questions * 100
 
 ## Summary
 - Total SO Questions: {results['total_questions']}
 - Total Migrated Questions: {results['migrated_questions']}
 - Ignored Questions: {len(results['ignored_questions'])}
-- Success Rate: {success_rate:.1f}%
+- Migrated + Ignored Questions: {results['migrated_questions'] + len(results['ignored_questions'])}
 - Missing Questions: {len(results['missing_questions'])}
 - Content Issues: {len(results['content_issues'])}
 - Answer Mismatches: {len(results['answer_mismatches'])}
@@ -515,16 +514,21 @@ def main():
     parser.add_argument('--category', required=True, help='Discussion category name')
     parser.add_argument('--questions-file', default='questions_answers_comments.json',
                         help='Path to questions JSON file')
+    parser.add_argument('--tags-file', default='tags.json', help='Path to tags JSON file')
     parser.add_argument('--output', default='validation_report.md',
                         help='Output file for validation report')
     parser.add_argument('--ignore-tags', 
                      type=str, 
                      nargs='+',
                      help='List of tags that were ignored in the migration process (space-separated). Questions that were tagged with these tag(s) were not migrated.')
+    parser.add_argument('--tag-min-threshold',
+                         type=int,
+                         default=1,
+                         help='The value used in the migration process to determine if a tag would be migrated. (default=1)')
     parser.add_argument('--popular-tag-min-threshold',
                         required=True,
                         type=int,
-                        help='The value used in the migration process to determine popular tags.')
+                        help='The value used in the migration process to determine popular questions.')
 
     args = parser.parse_args()
 
@@ -542,10 +546,10 @@ def main():
     github_auth_manager.initialize()
 
     # Create validator
-    validator = MigrationValidator(github_auth_manager, owner, name, args.category, args.ignore_tags, args.popular_tag_min_threshold)
+    validator = MigrationValidator(github_auth_manager, owner, name, args.category, args.ignore_tags, args.popular_tag_min_threshold, args.tag_min_threshold)
 
     # Run validation
-    validator.validate_migration(args.questions_file)
+    validator.validate_migration(args.questions_file, args.tags_file)
 
     # Generate and save report
     report = validator.generate_report()