Merge branch 'main' into kinesis-source-metrics-config

Signed-off-by: Rashmi <ras.xena@gmail.com>

Author: RashmiRam

.github/scripts/check-license-headers.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+#
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+"""
+License Header Compliance Checker for OpenSearch Data Prepper
+
+This script checks that files contain the required license headers
+as specified in CONTRIBUTING.md.
+
+Usage:
+  python check-license-headers.py file1.java file2.py ...
+  echo "file1.java\nfile2.py" | python check-license-headers.py
+"""
+
+import os
+import sys
+from pathlib import Path
+from typing import List
+
+# File extensions that require license headers
+SUPPORTED_EXTENSIONS = {
+    '.java', '.groovy', '.gradle',  # Java ecosystem
+    '.py',                          # Python
+    '.sh', '.bash', '.zsh',        # Shell scripts
+    '.yaml', '.yml',               # YAML files
+    '.properties',                 # Properties files
+}
+
+def needs_license_header(file_path: str) -> bool:
+    """Check if a file needs a license header based on its extension."""
+    path = Path(file_path)
+    return path.suffix.lower() in SUPPORTED_EXTENSIONS
+
+def check_file_header(file_path: str) -> bool:
+    """Check if a file has the required complete license header."""
+    if not Path(file_path).exists():
+        return True
+        
+    try:
+        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+            # Read first 15 lines to check for license header
+            lines = []
+            for i, line in enumerate(f):
+                if i >= 15:  # Only check first 15 lines
+                    break
+                lines.append(line)
+            
+        content = ''.join(lines)
+        
+        # Check for all 5 required license header components
+        required_components = [
+            'Copyright OpenSearch Contributors',
+            'SPDX-License-Identifier: Apache-2.0',
+            'The OpenSearch Contributors require contributions made to',
+            'this file be licensed under the Apache-2.0 license or a',
+            'compatible open source license.'
+        ]
+        
+        # All components must be present
+        for component in required_components:
+            if component not in content:
+                return False
+                
+        return True
+        
+    except Exception as e:
+        print(f"Error reading file {file_path}: {e}", file=sys.stderr)
+        return True  # Skip files we can't read
+
+def get_files_to_check() -> List[str]:
+    """Get files to check from command line args or stdin."""
+    if len(sys.argv) > 1:
+        # Files provided as command line arguments
+        return sys.argv[1:]
+    else:
+        # Read files from stdin
+        files = []
+        for line in sys.stdin:
+            file_path = line.strip()
+            if file_path:
+                files.append(file_path)
+        return files
+
+def main():
+    """Main function to check license headers."""
+    files_to_check = get_files_to_check()
+    
+    if not files_to_check:
+        print("No files to check", file=sys.stderr)
+        return
+    
+    print(f"Checking {len(files_to_check)} files for license headers.")
+    
+    violations = []
+    
+    for file_path in files_to_check:
+        print(f"Checking: {file_path}")
+        
+        if not Path(file_path).exists():
+            print(f"  File not found: {file_path}")
+            continue
+            
+        # Skip if doesn't need header
+        if not needs_license_header(file_path):
+            print(f"  Skipped (no header needed): {file_path}")
+            continue
+            
+        # Check header
+        if not check_file_header(file_path):
+            violations.append(f"- `{file_path}`")
+            print(f"  ❌ Missing license header: {file_path}")
+        else:
+            print(f"  ✅ Header OK: {file_path}")
+    
+    # Output results
+    if violations:
+        print(f"\n❌ Found {len(violations)} license header violations:")
+        
+        violation_text = '\n'.join(violations)
+        
+        # Set output for GitHub Actions
+        github_output = os.environ.get('GITHUB_OUTPUT')
+        if github_output:
+            with open(github_output, 'a') as f:
+                f.write(f"violations<<EOF\n{violation_text}\nEOF\n")
+        
+        print("\nViolations:")
+        for violation in violations:
+            print(f"  {violation}")
+            
+        sys.exit(1)
+    else:
+        print("\n✅ All files have proper license headers!")
+        # Set empty output for GitHub Actions
+        github_output = os.environ.get('GITHUB_OUTPUT')
+        if github_output:
+            with open(github_output, 'a') as f:
+                f.write("violations=\n")
+
+if __name__ == "__main__":
+    main()

.github/scripts/get-new-files.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+#
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+"""
+Get newly added files from Git.
+
+This script identifies files added in the current PR and outputs them
+one per line to stdout.
+"""
+
+import os
+import subprocess
+import sys
+
+def get_newly_added_files():
+    """Get list of files added in this PR."""
+    try:
+        # Get the base branch (usually main)
+        base_ref = os.environ.get('GITHUB_BASE_REF', 'main')
+        
+        # Get added files in this PR
+        result = subprocess.run([
+            'git', 'diff', '--name-only', '--diff-filter=A', 
+            f'origin/{base_ref}...HEAD'
+        ], capture_output=True, text=True, check=True)
+        
+        files = [f.strip() for f in result.stdout.split('\n') if f.strip()]
+        return files
+        
+    except subprocess.CalledProcessError as e:
+        print(f"Error getting changed files: {e}", file=sys.stderr)
+        return []
+
+def main():
+    """Main function to get newly added files."""
+    files = get_newly_added_files()
+    
+    if not files:
+        print("No newly added files found", file=sys.stderr)
+        sys.exit(0)
+    
+    # Output files one per line
+    for file_path in files:
+        print(file_path)
+
+if __name__ == "__main__":
+    main()

.github/workflows/data-prepper-kafka-backward-compatibility-e2e-tests.yml

@@ -0,0 +1,29 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper Kafka Backward Compatibility End-to-end test with Gradle
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        java: [11, 17, 21, docker]
+      fail-fast: false
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Checkout Data Prepper
+        uses: actions/checkout@v2
+      - name: Run Kafka backward compatibility end-to-end tests with Gradle
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:kafka-buffer-backward-compatibility:kafkaBufferBackwardCompatibilityTest

.github/workflows/license-header-check.yml

@@ -0,0 +1,117 @@
+#
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+# Performs a license header check on new files.
+# It will comment on PRs if it finds violations.
+
+name: License Header Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: Check License Headers on New Files
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.14'
+          
+      - name: Run License Header Check
+        id: license-check
+        run: |
+          python .github/scripts/get-new-files.py | python .github/scripts/check-license-headers.py
+          
+      - name: Comment on PR
+        if: failure() && steps.license-check.outputs.violations != ''
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const violations = process.env.VIOLATIONS;
+            
+            const body = [
+              '## ⚠️ License Header Violations Found',
+              '',
+              'The following newly added files are missing required license headers:',
+              '',
+              violations,
+              '',
+              'Please add the appropriate license header to each file and push your changes.',
+              '',
+              '**See the license header requirements:** https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md#license-headers'
+            ].join('\n');
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            
+            const botComment = comments.find(comment => 
+              comment.user.type === 'Bot' && 
+              comment.body.includes('License Header Violations Found')
+            );
+            
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: body
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: body
+              });
+            }
+        env:
+          VIOLATIONS: ${{ steps.license-check.outputs.violations }}
+          
+      - name: Update PR comment (all violations resolved)
+        if: success()
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            
+            const botComment = comments.find(comment => 
+              comment.user.type === 'Bot' && 
+              (comment.body.includes('License Header Violations Found') || comment.body.includes('License Header Check Passed'))
+            );
+            
+            if (botComment) {
+              const successBody = [
+                '## ✅ License Header Check Passed',
+                '',
+                'All newly added files have proper license headers. Great work! 🎉'
+              ].join('\n');
+              
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: successBody
+              });
+            }

.github/workflows/maven-publish-snapshot.yml

@@ -30,8 +30,14 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Publish snapshots to Maven
         run: |

.github/workflows/release.yml

@@ -47,9 +47,6 @@ jobs:
     - name: Build Maven Artifacts
       run: ./gradlew publishAllPublicationsToMavenRepository
 
-    - name: Build Docker Image
-      run: ./gradlew :release:docker:docker
-
     - name: Upload Archives to Archives Bucket
       run: ./gradlew :release:archives:uploadArchives -Pregion=us-east-1 -Pbucket=${{ secrets.ARCHIVES_BUCKET_NAME }} -Pprofile=default -PbuildNumber=${{ github.run_number }}
 
@@ -63,10 +60,8 @@ jobs:
         registry: public.ecr.aws
       env:
         AWS_REGION: us-east-1
-    - name: Push Image to Staging ECR
-      run: |
-        docker tag opensearch-data-prepper:${{ env.version }} ${{ secrets.ECR_REPOSITORY_URL }}:${{ env.version }}-${{ github.run_number }}
-        docker push ${{ secrets.ECR_REPOSITORY_URL }}:${{ env.version }}-${{ github.run_number }}
+    - name: Build and Push Multi-Architecture Docker Image to Staging ECR
+      run: ./gradlew :release:docker:dockerMultiArchitecture -PdockerRepository=${{ secrets.ECR_REPOSITORY_URL }}:${{ env.version }}-${{ github.run_number }}
 
   validate-docker:
     runs-on: ubuntu-latest