Update enrich issue,pull,repo,star,fork,watch

Signed-off-by: lishengbao <[email protected]>

Author: lishengbao

.gitignore

@@ -4,3 +4,4 @@ utils/cache*
 .pydevproject
 .settings
 __pycache__
+*.egg-info/

grimoire_elk_gitcode/enriched/gitcode.py

@@ -289,7 +289,7 @@ def get_rich_item(self, item):
         if item['category'] in rich_category_switch:
             rich_item = rich_category_switch[item['category']]()
         else:
-            logger.error("[github] rich item not defined for GitHub category {}".format(
+            logger.error("[gitcode] rich item not defined for GitCode category {}".format(
                          item['category']))
 
         self.add_repository_labels(rich_item)
@@ -307,15 +307,20 @@ def __get_rich_pull(self, item):
         # The real data
         pull_request = item['data']
 
+        if pull_request['closed_at'] == '':
+            pull_request['closed_at'] = None
+        if pull_request['merged_at'] == '':
+            pull_request['merged_at'] = None
+
         #close and merge in gitcode are two different status
         if pull_request['state'] == 'merged':
             rich_pr['time_to_close_days'] = \
                 get_time_diff_days(pull_request['created_at'], pull_request['merged_at'])
         else:
-           rich_pr['time_to_close_days'] = \
+            rich_pr['time_to_close_days'] = \
                 get_time_diff_days(pull_request['created_at'], pull_request['closed_at'])
 
-        #merged is not equal to closed in gitcode
+        # merged is not equal to closed in gitcode, pr state have open, merged, closed
         if pull_request['state'] == 'open':
             rich_pr['time_open_days'] = \
                 get_time_diff_days(pull_request['created_at'], datetime_utcnow().replace(tzinfo=None))
@@ -436,11 +441,14 @@ def __get_rich_issue(self, item):
         # The real data
         issue = item['data']
 
+        if issue['finished_at'] == '':
+            issue['finished_at'] = None
+
         rich_issue['time_to_close_days'] = \
             get_time_diff_days(issue['created_at'], issue['finished_at'])
 
-        #issue have four status: open,progressing, closed, rejected.
-        if issue['state'] == 'open' or issue['state'] == 'progressing':
+        #issue have four status: open, closed.
+        if issue['state'] == 'open':
             rich_issue['time_open_days'] = \
                 get_time_diff_days(issue['created_at'], datetime_utcnow().replace(tzinfo=None))
         else:
@@ -468,7 +476,6 @@ def __get_rich_issue(self, item):
 
         assignee = issue.get('assignee_data', None)
         if assignee and assignee is not None:
-            assignee = issue['assignee_data']
             rich_issue['assignee_login'] = assignee['login']
             rich_issue['assignee_name'] = assignee['name']
             rich_issue["assignee_domain"] = self.get_email_domain(assignee['email']) if assignee.get('email', None) else None
@@ -493,16 +500,13 @@ def __get_rich_issue(self, item):
         rich_issue['updated_at'] = issue['updated_at']
         rich_issue['closed_at'] = issue['finished_at']
         rich_issue['url'] = issue['html_url']
-        rich_issue['issue_type'] = issue['issue_type']
+        rich_issue['issue_type'] = None
         labels = []
         [labels.append(label['name']) for label in issue['labels'] if 'labels' in issue]
         rich_issue['labels'] = labels
 
-        rich_issue['pull_request'] = True
-        rich_issue['item_type'] = 'pull request'
-        if 'head' not in issue.keys() and 'pull_request' not in issue.keys():
-            rich_issue['pull_request'] = False
-            rich_issue['item_type'] = 'issue'
+        rich_issue['pull_request'] = False
+        rich_issue['item_type'] = 'issue'
 
         rich_issue['gitcode_repo'] = rich_issue['repository'].replace(GITCODE, '')
         rich_issue['gitcode_repo'] = re.sub('.git$', '', rich_issue['gitcode_repo'])
@@ -573,7 +577,7 @@ def __get_rich_repo(self, item):
         rich_repo['subscribers_count'] = repo['watchers_count']
         rich_repo['stargazers_count'] = repo['stargazers_count']
         rich_repo['fetched_on'] = repo['fetched_on']
-        rich_repo['url'] = repo['html_url']
+        rich_repo['url'] = repo['web_url']
         rich_repo['status'] = repo['status']
         if repo["status"] in "关闭":
             rich_repo['archived'] = True
@@ -587,7 +591,6 @@ def __get_rich_repo(self, item):
         rich_releases = []
         for release in repo['releases'] :
             rich_releases_dict = {}
-            rich_releases_dict['id'] = release['id']
             rich_releases_dict['tag_name'] = release['tag_name']
             rich_releases_dict['target_commitish'] = release['target_commitish']
             rich_releases_dict['prerelease'] = release['prerelease']
@@ -612,7 +615,49 @@ def __get_rich_repo(self, item):
 
         return rich_repo
 
-    def get_event_type(self, action_type):
+    def get_event_type(self, action_type, content):
+        if action_type == "label" and "add" in content:
+            return "LabeledEvent"
+        elif action_type == "label" and "delete" in content:
+            return "UnlabeledEvent"
+        elif action_type == "closed":
+            return "ClosedEvent"
+        elif action_type == "opened":
+            return "ReopenedEvent"
+        elif action_type == "assignee" and "assigned" in content:
+            return "AssignedEvent"
+        elif action_type == "assignee" and "unassigned" in content:
+            return "UnassignedEvent"
+
+        
+
+        switch_event_type = {
+            "add_label": "LabeledEvent",
+            "remove_label": "UnlabeledEvent",
+            "closed_pr": "ClosedEvent",
+            "reopened_pr": "ReopenedEvent",
+            "set_assignee": "AssignedEvent",
+            "setting_assignee": "AssignedEvent",
+            "unset_assignee": "UnassignedEvent",
+            "change_assignee": "UnassignedEvent",
+            "set_milestone": "MilestonedEvent",
+            "setting_milestone": "MilestonedEvent",
+            "unset_milestone": "DemilestonedEvent",
+            "change_milestone": "DemilestonedEvent",
+            "update_title": "RenamedTitleEvent",
+            "change_title": "RenamedTitleEvent",
+            "merged_pr": "MergedEvent",
+            "update_description": "ChangeDescriptionEvent",
+            "change_description": "ChangeDescriptionEvent",
+            "setting_priority": "SettingPriorityEvent",
+            "change_priority": "ChangePriorityEvent"
+        }
+        if action_type in switch_event_type:
+            return switch_event_type[action_type]
+        else:
+            return ''.join(word.capitalize() for word in action_type.split('_')) + "Event"
+
+    def get_event_type(self, action_type, content):
         switch_event_type = {
             "add_label": "LabeledEvent",
             "remove_label": "UnlabeledEvent",
@@ -659,13 +704,13 @@ def __get_rich_event(self, item):
         item['data']['actor'] = actor
 
         rich_event['id'] = event['id']
-        rich_event['icon'] = event['icon']
+        rich_event['icon'] = event.get('icon')
         rich_event['actor_username'] = actor['login']
         rich_event['user_login'] = rich_event['actor_username']
         rich_event['content'] = event['content']
         rich_event['created_at'] = event['created_at']
         rich_event['action_type'] = event['action_type']
-        rich_event['event_type'] = self.get_event_type(event['action_type'])
+        rich_event['event_type'] = self.get_event_type(event['action_type'], )  #待定
         rich_event['repository'] = item["tag"]
         rich_event['pull_request'] = False if 'issue' in event else True
         rich_event['item_type'] = 'issue' if 'issue' in event else 'pull request'
@@ -697,6 +742,25 @@ def __get_rich_event(self, item):
             rich_event['issue_url'] = main_content['html_url']
             rich_event['issue_labels'] = [label['name'] for label in main_content['labels']]
             rich_event["issue_url_id"] = rich_event['gitcode_repo'] + "/issues/" + rich_event['issue_id_in_repo']
+        
+        user = event.get('user_data', None)
+        if user is not None and user:
+            rich_event['user_name'] = user['name']
+            rich_event['author_name'] = user['name']
+            rich_event['user_email'] = user.get('email', None)
+            rich_event["user_domain"] = self.get_email_domain(user['email']) if user.get('email', None) else None
+            rich_event['user_org'] = user.get('company', None)
+            rich_event['user_location'] = user.get('location', None)
+            rich_event['user_geolocation'] = None
+        else:
+            rich_event['user_name'] = None
+            rich_event['user_email'] = None
+            rich_event["user_domain"] = None
+            rich_event['user_org'] = None
+            rich_event['user_location'] = None
+            rich_event['user_geolocation'] = None
+            rich_event['author_name'] = None
+
 
         if self.prjs_map:
             rich_event.update(self.get_item_project(rich_event))
@@ -720,21 +784,37 @@ def __get_rich_stargazer(self, item):
                 rich_stargazer[f] = None
         # The real data
         stargazer = item['data']
-        rich_stargazer["user_id"] = stargazer["id"]
-        rich_stargazer["user_login"] = stargazer["login"]
-        rich_stargazer["user_name"] = stargazer["name"]
-        rich_stargazer["auhtor_name"] = stargazer["name"]
-        rich_stargazer["user_html_url"] = stargazer["html_url"]
-        rich_stargazer['user_email'] = stargazer.get('email', None)
-        rich_stargazer['user_company'] = stargazer.get('company', None)
-        rich_stargazer["user_remark"] = stargazer["remark"]
-        rich_stargazer["user_type"] = stargazer["type"]
-        rich_stargazer["star_at"] = stargazer["star_at"]
-        rich_stargazer["created_at"] = stargazer["star_at"]
+        user = stargazer.get('user_data', None)
+        if user is not None and user:
+            rich_stargazer['user_id'] = user['id']
+            rich_stargazer["user_login"] = user["login"]
+            rich_stargazer["user_name"] = user["name"]
+            rich_stargazer["auhtor_name"] = user["name"]
+            rich_stargazer["user_html_url"] = user["html_url"]
+            rich_stargazer['user_email'] = user.get('email', None)
+            rich_stargazer["user_domain"] = self.get_email_domain(user['email']) if user.get('email', None) else None
+            rich_stargazer['user_company'] = user.get('company', None)
+            rich_stargazer["user_remark"] = user.get("remark", None)
+            rich_stargazer["user_type"] = user["type"]
+            
+        else:
+            rich_stargazer['user_id'] = None
+            rich_stargazer["user_login"] = stargazer['login']
+            rich_stargazer["user_name"] = stargazer['name']
+            rich_stargazer["auhtor_name"] = None
+            rich_stargazer["user_html_url"] = None
+            rich_stargazer['user_email'] = None
+            rich_stargazer["user_domain"] = None
+            rich_stargazer['user_company'] = None
+            rich_stargazer["user_remark"] = None
+            rich_stargazer["user_type"] = None
+
+        rich_stargazer["star_at"] = stargazer["starred_at"]
+        rich_stargazer["created_at"] = stargazer["starred_at"]
 
         if self.prjs_map:
             rich_stargazer.update(self.get_item_project(rich_stargazer))                  
-        rich_stargazer.update(self.get_grimoire_fields(stargazer['star_at'], "stargazer"))
+        rich_stargazer.update(self.get_grimoire_fields(stargazer['starred_at'], "stargazer"))
 
         return rich_stargazer
 
@@ -748,16 +828,31 @@ def __get_rich_fork(self, item):
                 rich_fork[f] = None
         # The real data
         fork = item['data']
-        fork_owner = fork['owner']
-        rich_fork["user_id"] = fork_owner["id"]
-        rich_fork["user_login"] = fork_owner["login"]
-        rich_fork["user_name"] = fork_owner["name"]
-        rich_fork["auhtor_name"] = fork_owner["name"]
-        rich_fork["user_html_url"] = fork_owner["html_url"]
-        rich_fork['user_email'] = fork_owner.get('email', None)
-        rich_fork['user_company'] = fork_owner.get('company', None)
-        rich_fork["user_remark"] = fork_owner["remark"]
-        rich_fork["user_type"] = fork_owner["type"]
+        user = fork.get('user_data', None)
+        if user is not None and user:
+            rich_fork['user_id'] = user['id']
+            rich_fork["user_login"] = user["login"]
+            rich_fork["user_name"] = user["name"]
+            rich_fork["auhtor_name"] = user["name"]
+            rich_fork["user_html_url"] = user["html_url"]
+            rich_fork['user_email'] = user.get('email', None)
+            rich_fork["user_domain"] = self.get_email_domain(user['email']) if user.get('email', None) else None
+            rich_fork['user_company'] = user.get('company', None)
+            rich_fork["user_remark"] = user.get("remark", None)
+            rich_fork["user_type"] = user["type"]
+            
+        else:
+            rich_fork['user_id'] = None
+            rich_fork["user_login"] = fork["owner"]['login']
+            rich_fork["user_name"] = fork["owner"]['name']
+            rich_fork["auhtor_name"] = None
+            rich_fork["user_html_url"] = None
+            rich_fork['user_email'] = None
+            rich_fork['user_domain'] = None
+            rich_fork['user_company'] = None
+            rich_fork["user_remark"] = None
+            rich_fork["user_type"] = None
+
         rich_fork["fork_at"] = fork["created_at"]
         rich_fork["created_at"] = fork["created_at"]
 
@@ -777,15 +872,31 @@ def __get_rich_watch(self, item):
                 rich_watch[f] = None
         # The real data
         watch = item['data']
-        rich_watch["user_id"] = watch["id"]
-        rich_watch["user_login"] = watch["login"]
-        rich_watch["user_name"] = watch["name"]
-        rich_watch["auhtor_name"] = watch["name"]
-        rich_watch["user_html_url"] = watch["html_url"]
-        rich_watch['user_email'] = watch.get('email', None)
-        rich_watch['user_company'] = watch.get('company', None)
-        rich_watch["user_remark"] = watch["remark"]
-        rich_watch["user_type"] = watch["type"]
+        user = watch.get('user_data', None)
+        if user is not None and user:
+            rich_watch['user_id'] = user['id']
+            rich_watch["user_login"] = user["login"]
+            rich_watch["user_name"] = user["name"]
+            rich_watch["auhtor_name"] = user["name"]
+            rich_watch["user_html_url"] = user["html_url"]
+            rich_watch['user_email'] = user.get('email', None)
+            rich_watch["user_domain"] = self.get_email_domain(user['email']) if user.get('email', None) else None
+            rich_watch['user_company'] = user.get('company', None)
+            rich_watch["user_remark"] = user.get("remark", None)
+            rich_watch["user_type"] = user["type"]
+            
+        else:
+            rich_watch['user_id'] = None
+            rich_watch["user_login"] = watch['login']
+            rich_watch["user_name"] = watch['name']
+            rich_watch["auhtor_name"] = None
+            rich_watch["user_html_url"] = None
+            rich_watch['user_email'] = None
+            rich_watch['user_demain'] = None
+            rich_watch['user_company'] = None
+            rich_watch["user_remark"] = None
+            rich_watch["user_type"] = None
+
         rich_watch["watch_at"] = watch["watch_at"]
         rich_watch["created_at"] = watch["watch_at"]