check for duplicate topics in for loop

Author: mcantu

crawler.py

@@ -28,54 +28,58 @@
     topics = [t.strip() for t in topic.split(',')]
     organization = os.getenv("ORGANIZATION")
 
-    # Get all repos from organization
-    search_string = "org:{} topic:{}".format(organization, topic)
-    all_repos = gh.search_repositories(search_string)
+    # Create empty list for repos
     repo_list = []
-
-    for repo in all_repos:
-        if repo is not None:
-            print("{0}".format(repo.repository))
-            full_repository = repo.repository.refresh()
-
-            innersource_repo = repo.as_dict()
-            innersource_repo["_InnerSourceMetadata"] = {}
-
-            # fetch innersource.json
-            try:
-                content = repo.repository.file_contents("/innersource.json").content
-                metadata = json.loads(b64decode(content))
-
-                innersource_repo["_InnerSourceMetadata"] = metadata
-            except github3.exceptions.NotFoundError:
-                # innersource.json not found in repository, but it's not required
-                pass
-
-            # fetch repository participation
-            participation = repo.repository.weekly_commit_count()
-            innersource_repo["_InnerSourceMetadata"]["participation"] = participation[
-                "all"
-            ]
-
-            # fetch contributing guidelines
-            try:
-                # if CONTRIBUTING.md exists in the repository, link to that instead of repo root
-                content = repo.repository.file_contents("/CONTRIBUTING.md").content
-                innersource_repo["_InnerSourceMetadata"][
-                    "guidelines"
-                ] = "CONTRIBUTING.md"
-            except github3.exceptions.NotFoundError:
-                # CONTRIBUTING.md not found in repository, but it's not required
-                pass
-
-            # fetch repository topics
-            topics = repo.repository.topics()
-            innersource_repo["_InnerSourceMetadata"]["topics"] = topics.names
-
-            # calculate score
-            innersource_repo["score"] = repo_activity.score.calculate(innersource_repo)
-
-            repo_list.append(innersource_repo)
+    # Set for repos that have already been added to the list
+    repo_set = set()
+    
+    for topic in topics:
+        search_string = "org:{} topic:{}".format(organization, topic)
+        all_repos = gh.search_repositories(search_string)
+        
+        for repo in all_repos:
+            if repo is not None and repo.repository.full_name not in repo_set:
+                repo_set.add(repo.repository.full_name)
+                print("{0}".format(repo.repository))
+
+                innersource_repo = repo.as_dict()
+                innersource_repo["_InnerSourceMetadata"] = {}
+
+                # fetch innersource.json
+                try:
+                    content = repo.repository.file_contents("/innersource.json").content
+                    metadata = json.loads(b64decode(content))
+
+                    innersource_repo["_InnerSourceMetadata"] = metadata
+                except github3.exceptions.NotFoundError:
+                    # innersource.json not found in repository, but it's not required
+                    pass
+
+                # fetch repository participation
+                participation = repo.repository.weekly_commit_count()
+                innersource_repo["_InnerSourceMetadata"]["participation"] = participation[
+                    "all"
+                ]
+
+                # fetch contributing guidelines
+                try:
+                    # if CONTRIBUTING.md exists in the repository, link to that instead of repo root
+                    content = repo.repository.file_contents("/CONTRIBUTING.md").content
+                    innersource_repo["_InnerSourceMetadata"][
+                        "guidelines"
+                    ] = "CONTRIBUTING.md"
+                except github3.exceptions.NotFoundError:
+                    # CONTRIBUTING.md not found in repository, but it's not required
+                    pass
+
+                # fetch repository topics
+                repo_topics = repo.repository.topics()
+                innersource_repo["_InnerSourceMetadata"]["topics"] = repo_topics.names
+
+                # calculate score
+                innersource_repo["score"] = repo_activity.score.calculate(innersource_repo)
+
+                repo_list.append(innersource_repo)
 
     # Write each repository to a repos.json file
     with open("repos.json", "w") as f: