Merge branch 'trunk' into KIP-1052-PR

Author: matt-welch

.github/actions/gh-api-approve-run/action.yml

@@ -43,9 +43,13 @@ runs:
       shell: bash
       env:
         GH_TOKEN: ${{ inputs.gh-token }}
+        REPO: ${{ inputs.repository }}
+        RUN_ID: ${{ inputs.run_id }}
+        PR_NUMBER: ${{ inputs.pr_number }}
+        COMMIT_SHA: ${{ inputs.commit_sha }}
       run: |
-        echo "Approving workflow run ${{ inputs.run_id }} for PR ${{ inputs.pr_number }} at SHA ${{ inputs.commit_sha }}";
+        echo "Approving workflow run $RUN_ID for PR $PR_NUMBER at SHA $COMMIT_SHA";
         gh api --method POST \
               -H 'Accept: application/vnd.github+json' \
               -H 'X-GitHub-Api-Version: 2022-11-28' \
-              /repos/${{ inputs.repository }}/actions/runs/${{ inputs.run_id }}/approve
+              /repos/$REPO/actions/runs/$RUN_ID/approve

.github/actions/gh-api-update-status/action.yml

@@ -53,9 +53,15 @@ runs:
       shell: bash
       env:
         GH_TOKEN: ${{ inputs.gh-token }}
+        REPO: ${{ inputs.repository }}
+        COMMIT_SHA: ${{ inputs.commit_sha }}
+        STATE: ${{ inputs.state }}
+        URL: ${{ inputs.url }}
+        DESCRIPTION: ${{ inputs.description }}
+        CONTEXT: ${{ inputs.context }}
       run: |
         gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
-        /repos/${{ inputs.repository }}/statuses/${{ inputs.commit_sha }} \
-        -f "state=${{ inputs.state }}" -f "target_url=${{ inputs.url }}" \
-        -f "description=${{ inputs.description }}" \
-        -f "context=${{ inputs.context }}"
+        /repos/$REPO/statuses/$COMMIT_SHA \
+        -f "state=$STATE" -f "target_url=$URL" \
+        -f "description=$DESCRIPTION" \
+        -f "context=$CONTEXT"

.github/configs/labeler.yml

@@ -19,6 +19,8 @@ build:
       - '.github/**'
       - 'gradle/**'
       - '*gradlew*'
+      - 'build.gradle'
+      - 'settings.gradle'
 
 clients:
 - changed-files:
@@ -91,11 +93,12 @@ transactions:
       - any-glob-to-any-file:
           - 'transaction-coordinator/**'
 
-KIP-932:
+kip-932:
   - changed-files:
       - any-glob-to-any-file:
           - 'share/**'
           - 'share-coordinator/**'
+          - 'core/src/*/java/kafka/server/share/**'
 
 docker:
 - changed-files:
@@ -110,12 +113,12 @@ performance:
 consumer:
   - changed-files:
     - any-glob-to-any-file:
-      - 'clients/src/main/java/org/apache/kafka/clients/consumer/**'
+      - 'clients/src/*/java/org/apache/kafka/clients/consumer/**'
 
 producer:
   - changed-files:
     - any-glob-to-any-file:
-      - 'clients/src/main/java/org/apache/kafka/clients/producer/**'
+      - 'clients/src/*/java/org/apache/kafka/clients/producer/**'
 
 kraft:
   - changed-files:

.github/scripts/junit.py

@@ -14,17 +14,22 @@
 # limitations under the License.
 
 import argparse
+from collections import OrderedDict
 import dataclasses
 from functools import partial
 from glob import glob
 import logging
 import os
 import os.path
+import pathlib
+import re
 import sys
 from typing import Tuple, Optional, List, Iterable
 import xml.etree.ElementTree
 import html
 
+import yaml
+
 
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
@@ -78,6 +83,60 @@ class TestSuite:
     passed_tests: List[TestCase]
 
 
+# Java method names can start with alpha, "_", or "$". Following characters can also include digits
+method_matcher = re.compile(r"([a-zA-Z_$][a-zA-Z0-9_$]+).*")
+
+
+def clean_test_name(test_name: str) -> str:
+    cleaned = test_name.strip("\"").rstrip("()")
+    m = method_matcher.match(cleaned)
+    return m.group(1)
+
+
+class TestCatalogExporter:
+    def __init__(self):
+        self.all_tests = {}   # module -> class -> set of methods
+
+    def handle_suite(self, module: str, suite: TestSuite):
+        if module not in self.all_tests:
+            self.all_tests[module] = OrderedDict()
+
+        for test in suite.failed_tests:
+            if test.class_name not in self.all_tests[module]:
+                self.all_tests[module][test.class_name] = set()
+            self.all_tests[module][test.class_name].add(clean_test_name(test.test_name))
+        for test in suite.passed_tests:
+            if test.class_name not in self.all_tests[module]:
+                self.all_tests[module][test.class_name] = set()
+            self.all_tests[module][test.class_name].add(clean_test_name(test.test_name))
+
+    def export(self, out_dir: str):
+        if not os.path.exists(out_dir):
+            logger.debug(f"Creating output directory {out_dir} for test catalog export.")
+            os.makedirs(out_dir)
+
+        total_count = 0
+        for module, module_tests in self.all_tests.items():
+            module_path = os.path.join(out_dir, module)
+            if not os.path.exists(module_path):
+                os.makedirs(module_path)
+
+            sorted_tests = {}
+            count = 0
+            for test_class, methods in module_tests.items():
+                sorted_methods = sorted(methods)
+                count += len(sorted_methods)
+                sorted_tests[test_class] = sorted_methods
+
+            out_path = os.path.join(module_path, f"tests.yaml")
+            logger.debug(f"Writing {count} tests for {module} into {out_path}.")
+            total_count += count
+            with open(out_path, "w") as fp:
+                yaml.dump(sorted_tests, fp)
+
+        logger.debug(f"Wrote {total_count} tests into test catalog.")
+
+
 def parse_report(workspace_path, report_path, fp) -> Iterable[TestSuite]:
     cur_suite: Optional[TestSuite] = None
     partial_test_case = None
@@ -138,6 +197,20 @@ def pretty_time_duration(seconds: float) -> str:
     return time_fmt
 
 
+def module_path_from_report_path(base_path: str, report_path: str) -> str:
+    """
+    Parse a report XML and extract the module path. Test report paths look like:
+
+        build/junit-xml/module[/sub-module]/[suite]/TEST-class.method.xml
+
+    This method strips off a base path and assumes all path segments leading up to the suite name
+    are part of the module path.
+    """
+    rel_report_path = os.path.relpath(report_path, base_path)
+    path_segments = pathlib.Path(rel_report_path).parts
+    return os.path.join(*path_segments[0:-2])
+
+
 if __name__ == "__main__":
     """
     Parse JUnit XML reports and generate GitHub job summary in Markdown format.
@@ -150,16 +223,21 @@ def pretty_time_duration(seconds: float) -> str:
     parser = argparse.ArgumentParser(description="Parse JUnit XML results.")
     parser.add_argument("--path",
                         required=False,
-                        default="build/junit-xml/**/*.xml",
-                        help="Path to XML files. Glob patterns are supported.")
+                        default="build/junit-xml",
+                        help="Base path of JUnit XML files. A glob of **/*.xml will be applied on top of this path.")
+    parser.add_argument("--export-test-catalog",
+                        required=False,
+                        default="",
+                        help="Optional path to dump all tests")
 
     if not os.getenv("GITHUB_WORKSPACE"):
         print("This script is intended to by run by GitHub Actions.")
         exit(1)
 
     args = parser.parse_args()
 
-    reports = glob(pathname=args.path, recursive=True)
+    glob_path = os.path.join(args.path, "**/*.xml")
+    reports = glob(pathname=glob_path, recursive=True)
     logger.info(f"Found {len(reports)} JUnit results")
     workspace_path = get_env("GITHUB_WORKSPACE") # e.g., /home/runner/work/apache/kafka
 
@@ -177,10 +255,13 @@ def pretty_time_duration(seconds: float) -> str:
     flaky_table = []
     skipped_table = []
 
+    exporter = TestCatalogExporter()
+
     logger.debug(f"::group::Parsing {len(reports)} JUnit Report Files")
     for report in reports:
         with open(report, "r") as fp:
-            logger.debug(f"Parsing {report}")
+            module_path = module_path_from_report_path(args.path, report)
+            logger.debug(f"Parsing file: {report}, module: {module_path}")
             for suite in parse_report(workspace_path, report, fp):
                 total_skipped += suite.skipped
                 total_errors += suite.errors
@@ -216,7 +297,17 @@ def pretty_time_duration(seconds: float) -> str:
                     simple_class_name = skipped_test.class_name.split(".")[-1]
                     logger.debug(f"Found skipped test: {skipped_test}")
                     skipped_table.append((simple_class_name, skipped_test.test_name))
+
+                if args.export_test_catalog:
+                    exporter.handle_suite(module_path, suite)
+
     logger.debug("::endgroup::")
+
+    if args.export_test_catalog:
+        logger.debug(f"::group::Generating Test Catalog Files")
+        exporter.export(args.export_test_catalog)
+        logger.debug("::endgroup::")
+
     duration = pretty_time_duration(total_time)
     logger.info(f"Finished processing {len(reports)} reports")
 

.github/scripts/label_small.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LABEL_NAME=small
+MAX_SIZE=100
+
+pr_diff=$(gh pr view $PR_NUM -R $GITHUB_REPOSITORY --json additions,deletions)
+
+additions=$(echo "$pr_diff" | jq -r '.additions')
+deletions=$(echo "$pr_diff" | jq -r '.deletions')
+
+total_changes=$((additions + deletions))
+if [ "$total_changes" -lt "$MAX_SIZE" ]; then
+    gh issue edit $PR_NUM --add-label $LABEL_NAME -R $GITHUB_REPOSITORY
+else
+    gh issue edit $PR_NUM --remove-label $LABEL_NAME -R $GITHUB_REPOSITORY
+fi

…test/resources/junit-platform.properties .github/scripts/requirements.txttools/src/test/resources/junit-platform.properties renamed to .github/scripts/requirements.txt tools/src/test/resources/junit-platform.properties renamed to .github/scripts/requirements.txt

@@ -12,4 +12,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-junit.jupiter.params.displayname.default = "{displayName}.{argumentsWithNames}"
+PyYAML~=6.0

.github/workflows/README.md

@@ -19,6 +19,38 @@ By default, GitHub sends an email for each failed action run. To change this,
 visit https://github.com/settings/notifications and find System -> Actions.
 Here you can change your notification preferences.
 
+## Security
+
+Please read the following GitHub articles before authoring new workflows.
+
+1) https://github.blog/security/supply-chain-security/four-tips-to-keep-your-github-actions-workflows-secure/
+2) https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
+
+### Variable Injection
+
+Any workflows that use the `run` directive should avoid using the `${{ ... }}` syntax.
+Instead, declare all injectable variables as environment variables. For example:
+
+```yaml
+    - name: Copy RC Image to promoted image
+      env:
+        PROMOTED_DOCKER_IMAGE: ${{ github.event.inputs.promoted_docker_image }}
+        RC_DOCKER_IMAGE: ${{ github.event.inputs.rc_docker_image }}
+      run: |
+        docker buildx imagetools create --tag $PROMOTED_DOCKER_IMAGE $RC_DOCKER_IMAGE
+```
+
+This prevents untrusted inputs from doing script injection in the `run` steps.
+
+### `pull_request_target` events
+
+In addition to the above security articles, please review the [official documentation](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target)
+on `pull_request_target`. This event type allows PRs to trigger actions that run
+with elevated permission and access to repository secrets. We should only be 
+using this for very simple tasks such as applying labels or adding comments to PRs.
+
+_We must never run the untrusted PR code in the elevated `pull_request_target` context_
+
 ## GitHub Actions Quirks
 
 ### Composite Actions