Merge pull request #4770 from linuxfoundation/unicron-4759-fix-missing-installation-id-prod

Unicron 4759 fix missing installation id prod

Author: lukaszgryglicki

.gitignore

@@ -257,3 +257,4 @@ cover.out
 .env
 src.txt
 src.txt.*
+out.json

cla-backend/cla/controllers/github.py

@@ -48,7 +48,7 @@ def get_organization(organization_name):
     try:
         cla.log.debug(f'Loading GitHub by organization name: {organization_name}..')
         org = github_organization.get_organization_by_lower_name(organization_name)
-        cla.log.debug(f'Loaded GitHub by organization name: {org}')
+        cla.log.debug(f'Loaded GitHub organization by name: {org}')
     except DoesNotExist as err:
         cla.log.warning(f'organization name {organization_name} does not exist')
         return {'errors': {'organization_name': str(err)}}
@@ -68,7 +68,7 @@ def get_organization_model(organization_name) -> Optional[GitHubOrg]:
     try:
         cla.log.debug(f'Loading GitHub by organization name: {organization_name}..')
         org = github_organization.get_organization_by_lower_name(organization_name)
-        cla.log.debug(f'Loaded GitHub by organization name: {org}')
+        cla.log.debug(f'Loaded GitHub organization model by name: {org}')
         return org
     except DoesNotExist as err:
         cla.log.warning(f'organization name {organization_name} does not exist, error: {err}')
@@ -138,9 +138,13 @@ def update_organization(organization_name,  # pylint: disable=too-many-arguments
     if organization_sfid:
         github_organization.set_organization_sfid(organization_sfid)
 
-    github_organization.save()
-    cla.log.debug('updated organization: {}'.format(organization_name))
-    return github_organization.to_dict()
+    try:
+        github_organization.save()
+        cla.log.debug('updated organization: {}'.format(organization_name))
+        return github_organization.to_dict()
+    except Exception as err:
+        cla.log.error(f"failed to save organization {organization_name}: {err}")
+        return {"errors": {"organization_name": str(err)}}
 
 
 def delete_organization(auth_user, organization_name):
@@ -296,6 +300,12 @@ def activity(action: str, event_type: str, body: dict):
 def handle_installation_event(action: str, body: dict):
     func_name = 'github.activity.handle_installation_event'
     cla.log.debug(f'{func_name} - processing github [installation] activity callback...')
+    installation_id = None
+    try:
+        installation_id = body['installation']['id']
+    except KeyError:
+        cla.log.warning(f'{func_name} - unable to determine installation id from body: {json.dumps(body)}.')
+    cla.log.debug(f'{func_name} - processing github installation {installation_id}...')
 
     # New Installations
     if action == 'created':
@@ -306,7 +316,7 @@ def handle_installation_event(action: str, body: dict):
             cla.log.warning(f'{func_name} - Unable to determine organization name from the github installation event '
                             f'with action: {action}'
                             f'event body: {json.dumps(body)}')
-            return {'status', f'GitHub installation {action} event malformed.'}
+            return {'status': f'GitHub installation {action} event malformed.'}
 
         cla.log.debug(f'Locating organization using name: {org_name}')
         existing = get_organization(org_name)
@@ -315,23 +325,16 @@ def handle_installation_event(action: str, body: dict):
                             'but the organization is not configured in EasyCLA')
             # TODO: Need a way of keeping track of new organizations that don't have projects yet.
             return {'status': 'Github Organization must be created through the Project Management Console.'}
-        elif not existing['organization_installation_id']:
-            update_organization(
-                existing['organization_name'],
-                existing['organization_sfid'],
-                body['installation']['id'],
-            )
-            cla.log.info(f'{func_name} - Organization enrollment completed: {existing["organization_name"]}')
+        elif not existing.get('organization_installation_id'):
+            cla.log.info(f'{func_name} - Setting installation ID for github organization: {existing.get("organization_name")} to {installation_id}')
+            update_organization(existing.get('organization_name'), existing.get('organization_sfid'), installation_id)
+            cla.log.info(f'{func_name} - Organization enrollment completed: {existing.get("organization_name")}')
             return {'status': 'Organization Enrollment Completed. CLA System is operational'}
         else:
-            cla.log.info(f'{func_name} - Organization already enrolled: {existing["organization_name"]}')
-            cla.log.info(f'{func_name} - Updating installation ID for '
-                         f'github organization: {existing["organization_name"]}')
-            update_organization(
-                existing['organization_name'],
-                existing['organization_sfid'],
-                body['installation']['id'],
-            )
+            cla.log.info(f'{func_name} - Organization already enrolled: {existing.get("organization_name")}')
+            cla.log.info(f'{func_name} - installation ID: {existing.get("organization_installation_id")}')
+            cla.log.info(f'{func_name} - Updating installation ID for github organization: {existing.get("organization_name")} to {installation_id}')
+            update_organization(existing.get('organization_name'), existing.get('organization_sfid'), installation_id)
             return {'status': 'Already Enrolled Organization Updated. CLA System is operational'}
 
     elif action == 'deleted':
@@ -341,7 +344,7 @@ def handle_installation_event(action: str, body: dict):
             cla.log.warning('Unable to determine organization name from the github installation event '
                             f'with action: {action}'
                             f'event body: {json.dumps(body)}')
-            return {'status', f'GitHub installation {action} event malformed.'}
+            return {'status': f'GitHub installation {action} event malformed.'}
         repositories = Repository().get_repositories_by_organization(org_name)
         notify_project_managers(repositories)
         return

cla-backend/cla/models/dynamo_models.py

@@ -4001,10 +4001,20 @@ def __str__(self):
 
     def to_dict(self):
         ret = dict(self.model)
-        if ret["organization_installation_id"] == "null":
+
+        if "organization_installation_id" not in ret:
             ret["organization_installation_id"] = None
-        if ret["organization_sfid"] == "null":
+        if "organization_sfid" not in ret:
             ret["organization_sfid"] = None
+
+        val = ret.get("organization_installation_id")
+        if isinstance(val, str) and val.strip().lower() in ("null", "none", ""):
+            ret["organization_installation_id"] = None
+
+        val = ret.get("organization_sfid")
+        if isinstance(val, str) and val.strip().lower() in ("null", "none", ""):
+            ret["organization_sfid"] = None
+
         return ret
 
     def save(self) -> None:

utils/count_apis.sh

@@ -4,20 +4,34 @@
 
 if [ -z "${1}" ]
 then
-  echo "Usage: $0 <path-to-api-logs>"
+  echo "Usage: $0 <path-to-api-logs> [min-count]"
   echo "Example: $0 api-logs-prod.json"
   exit 1
 fi
-
+N=10
+if [ ! -z "${2}" ]
+then
+  N="${2}"
+fi
 jq -r '
   .[].message
   | capture("LG:api-request-path:(?<p>[^\"[:space:]]+)")?  # find the path
   | select(.)                                             # drop non-matches
   | .p
 ' "${1}" \
+| sed -E 's#/{2,}#/#g' \
+| sed -E 's#/$##' \
+| sed -E 's#\.(png|svg|css|js|json|xml)$#.<asset>#g' \
+| sed -E 's#^/v[0-9]+/swagger\.<asset>$#/v*/swagger#g' \
+| sed -E 's#^/v[0-9]+/api-docs$#/v*/api-docs#g' \
+| sed -E 's#^/v[0-9]+/(api/)?graphql(\.php)?$#/v*/graphql#g' \
+| sed -E 's#^/v[0-9]+/graph(i)?ql(/.*)?$#/v*/graphiql#g' \
+| sed -E 's#^/v[0-9]+/(explorer|console|playground|altair)$#/v*/graphql-ui#g' \
+| sed -E 's#/([A-Za-z0-9]{5,8})(/|$)#/<shortid>\2#g' \
 | sed -E 's/[0-9a-fA-F-]{36}/<uuid>/g' \
 | sed -E ':a;s#/([0-9]{1,})(/|$)#/<id>\2#g;ta' \
 | sed -E 's#/(00|a0)[A-Za-z0-9]{13,16}(/|$)#/<sfid>\2#g' \
 | sed -E 's#/lf[A-Za-z0-9]{16,22}(/|$)#/<lfxid>\1#g' \
 | sed -E 's#/null(/|$)#/<null>\1#g' \
-| sort | uniq -c | sort -nr
+| sort | uniq -c | sort -nr \
+| awk -v N="$N" '$1 >= N'

utils/search_aws_log_group.sh

@@ -39,14 +39,20 @@ if [ -z "${DTFROM}" ]
 then
   export DTFROM="$(date -d '3 days ago' +%s)000"
 else
-  export DTFROM="$(date -d "${DTFROM}" +%s)000"
+  if [[ ! "$DTFROM" =~ ^[0-9]+$ ]]
+  then
+    export DTFROM="$(date -d "${DTFROM}" +%s)000"
+  fi
 fi
 
 if [ -z "${DTTO}" ]
 then
   export DTTO="$(date +%s)000"
 else
-  export DTTO="$(date -d "${DTTO}" +%s)000"
+  if [[ ! "$DTTO" =~ ^[0-9]+$ ]]
+  then
+    export DTTO="$(date -d "${DTTO}" +%s)000"
+  fi
 fi
 
 if [ ! -z "${DEBUG}" ]