Merge pull request #4646 from communitybridge/unicron-add-logs-search-command-line-utility

Update merge queue commit message parsing to get the correct PR number

Author: lukaszgryglicki

cla-backend/cla/tests/unit/test_utils.py

@@ -7,7 +7,7 @@
 import cla
 from cla import utils
 from cla.models.dynamo_models import Project, Signature, User
-from cla.utils import (append_email_help_sign_off_content,
+from cla.utils import (append_email_help_sign_off_content, extract_pull_request_number,
                        append_project_version_to_url, get_email_help_content,
                        get_email_sign_off_content, get_full_sign_url)
 
@@ -283,3 +283,32 @@ def test_append_project_version_to_url():
 
 if __name__ == '__main__':
     unittest.main()
+
+def test_extract_pull_request_number():
+    tests = [
+        ["Merge pull request #232 from sun-test-org/thakurveerendras-patch-26#1\n\nUpdate README.md", 232],
+        ["Merge pull request #234 from sun-test-org/thakurveerendras-patch-26\n\nCreate mqfile2#file2", 234],
+        ["Merge pull request #235 from sun-test-org/branch#2341\n\nMQFileBranch#2342", 235],
+        ["Merge pull request #236 from sun-test-org/thakurveerendras-patch-27\n\nUpdate mqfile2#234", 236],
+        ["Merge pull request #237 from sun-test-org/thakurveerendras-patch-28#123\n\nCreate mqfile3#123", 237],
+        ["Merge pull request #235 from sun-test-org/branch#2341\n\nMQFileBranch#2342", 235],
+        ["Merge pull request #238 from sun-test-org/branch#23456\n\nPR#234567", 238],
+        ["Merge pull request #235 from sun-test-org/branch#2341\n\nMQFileBranch#2342", 235],
+        ["merge pull request #235 from sun-test-org/branch#2341\n\nMQFileBranch#2342", 235],
+        ["Hello world\nThis if for PR #123 fixing issue #112", 123],
+        # ["Hello world\nThis if for Issue #112 - PR #123", 123],
+        ["[mdatagen] Add event type definition (#12822)\n\n#Description\n\nHello, ...", 12822],
+        ["[pt] Update localized content on content/pt/docs/languages/go/exporters.md (#6783)", 6783],
+        ["[chore]: remove testifylint-fix target (#12828)\n\n#### Description\n\ngolangci-lint is now able to apply suggested fixes from testifylint with\ngolangci-lint --fix .\nThis PR removes testifylint-fix target from Makefile.\n\nSigned-off-by: Matthieu MOREL <[email protected]>", 12828],
+        ["[chore] Prepare release 0.125.0 (#933)\n\n* Update version from 0.124.0 to 0.125.0\n\n* update versions in ebpf\n\n---------\n\nCo-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>\nCo-authored-by: Yang Song <[email protected]>", 933],
+        ["Add invoke_agent as a member of gen_ai.operation.name (#2160)", 2160],
+        ["Merge pull request #61 from open-telemetry/renovate/all-patch\n\nfix(deps): update all patch versions", 61],
+        ["Merge pull request #51 from open-telemetry/rollback-deps\n\nchore: roll back major dependency updates", 51],
+        ["fixes #6549 incorrect use of resource constructor (#6707)", 6707],
+        ["", None],
+        ["Add documentation example for xconfmap (#5675) (#12832)\n#### Description\n\nThis PR introduces a simple testable examples to the package\n[confmap](/confmap/xconfmap)", 12832]
+    ]
+    
+    for i, (message, expected) in enumerate(tests, 1):
+        result = extract_pull_request_number(message)
+        assert result == expected

cla-backend/cla/utils.py

@@ -1942,12 +1942,33 @@ def extract_pull_request_number(pull_request_message):
     pull_request_number = None
     try:
         first_line = pull_request_message.splitlines()[0]
-        matches = re.findall(r"#(\d+)", first_line)
+        cla.log.debug(f"{fn} - checking line '{first_line}")
+        # Case 1: "Merge pull request #N"
+        matches = re.match(r'^Merge pull request #(\d+)', first_line)
+        if matches:
+            pull_request_number = int(matches.group(1))
+            cla.log.debug(f"{fn} - extracted PR number {pull_request_number} from merge_queue data: {pull_request_message} by matching 'Merge pull request #N...'")
+            return pull_request_number
+        # Case 2: PR number in last (#N) group on first line, like: "Some text (#whatever) (#N)"
+        matches = re.findall(r'\(#(\d+)\)', first_line)
+        if matches:
+            pull_request_number = int(matches[-1])  # last match
+            cla.log.debug(f"{fn} - extracted PR number {pull_request_number} from merge_queue data: {pull_request_message} by matching '...(#N)'")
+            return pull_request_number
+        # Case 3: PR number in last #N on first line, like: "Some text #N"
+        matches = re.findall(r"\s+#(\d+)", first_line)
         if matches:
             pull_request_number = int(matches[-1])  # last match
+            cla.log.debug(f"{fn} - extracted PR number {pull_request_number} from merge_queue data: {pull_request_message} by matching '... #N'")
+            return pull_request_number
+        # Case 4: PR number in first #N in the entire commit message
+        matches = re.findall(r"#(\d+)", pull_request_message)
+        if matches:
+            pull_request_number = int(matches[0])  # first match
+            cla.log.debug(f"{fn} - extracted PR number {pull_request_number} from merge_queue data: {pull_request_message} by matching first '#N'")
+            return pull_request_number
         else:
             cla.log.warning(f"{fn} - error - unable to extract pull request number from message: {pull_request_message}")
     except Exception as e:
         cla.log.warning(f"{fn} - error - unable to extract pull request number from message: {pull_request_message}, error: {e}")
-    cla.log.debug(f"{fn} - extracted PR number {pull_request_number} from merge_queue data")
     return pull_request_number

utils/search_aws_logs.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# STAGE=dev DEBUG=1 DTFROM='3 days ago' DTTO='2 days ago' ./utils/search_aws_logs.sh 'cla-backend-dev-githubactivity' 'error'
+
+if [ -z "$STAGE" ]
+then
+  export STAGE=dev
+fi
+
+if [ -z "${1}" ]
+then
+  echo "$0: you must specify log group name, for example: 'cla-backend-dev-githubactivity', 'cla-backend-dev-apiv2', 'cla-backend-dev-api-v3-lambda', 'cla-backend-go-api-v4-lambda'"
+  exit 1
+fi
+
+log_group=$(echo "$1" | sed -E "s/\b(dev|prod)\b/${STAGE}/g")
+
+if [ -z "${2}" ]
+then
+  echo "$0: you must specify the search term, for example 'error'"
+  exit 2
+fi
+
+if [ -z "${DTFROM}" ]
+then
+  export DTFROM="$(date -d '3 days ago' +%s)000"
+else
+  export DTFROM="$(date -d "${DTFROM}" +%s)000"
+fi
+
+if [ -z "${DTTO}" ]
+then
+  export DTTO="$(date +%s)000"
+else
+  export DTTO="$(date -d "${DTTO}" +%s)000"
+fi
+
+if [ ! -z "${DEBUG}" ]
+then
+  echo "aws --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${2}\""
+fi
+
+aws --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\""