COMMUNITY-15: Obtain RTI Connext bundles using credentials (#631)

* COMMUNITY-15: adapt linux_install to new S3 bucket

* COMMUNITY-15: adapt build.yml to the new linux_install script

* COMMUNITY-15: add requirements.txt for CI/CD

* COMMUNITY-15: add AWS credentials

* COMMUNITY-15: improve scripts safety

* COMMUNITY-15: add double asterisk to create a glob

* COMMUNITY-15: trying with space-checkout

* COMMUNITY-15: reorder checkout for workflow_run

* COMMUNITY-15: reorder checkout for workflow_run x2

* COMMUNITY-15: do not include the git folder in the artifact

* COMMUNITY-15: do not clean in the checkout step

* COMMUNITY-15: do checkout before downloading repo from PR

* COMMUNITY-15: solve deprecation warnings

* COMMUNITY-15: rename steps

* COMMUNITY-15: update comment

* COMMUNITY-15: update VERSION

Author: ManuelJNunez

.github/workflows/build.yml

@@ -18,7 +18,11 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        architecture: [{name: x64Linux4gcc7.3.0, os: ubuntu-20.04}]
+
+    runs-on: ${{ matrix.architecture.os }}
     name: Build examples
 
     # This job will be executed if the "PR handler" finalized successfully or
@@ -33,6 +37,50 @@ jobs:
 
     steps:
 
+      # Download the artifacts generated by "PR Handler" if this workflow was
+      # triggered by a workflow_run event.
+      - name: Download artifact
+        if: ${{ github.event_name == 'workflow_run' }}
+        uses: actions/[email protected]
+        with:
+          script: |
+            var fs = require('fs');
+            var artifacts = await github.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{ github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "targetbranch"
+            })[0];
+            var download = await github.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            fs.writeFileSync('${{github.workspace}}/targetbranch.zip', Buffer.from(download.data));
+
+      # Unzip the artifacts.
+      - name: Unzip artifact
+        if: ${{ github.event_name == 'workflow_run' }}
+        run: unzip targetbranch.zip
+
+        # Save the PR number to send messages to the user that opened it.
+      - name: Get target branch
+        if: ${{ github.event_name == 'workflow_run' }}
+        id: target_branch
+        run: echo "TARGET_BRANCH=$(cat targetbranch)" >> $GITHUB_OUTPUT
+
+      # Sparse-checkout: we clone just the scripts from the target branch for
+      # sefaty reasons.
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.target_branch.outputs.TARGET_BRANCH }}
+          sparse-checkout: resources/ci_cd/
+          sparse-checkout-cone-mode: false
+        if: ${{ github.event_name == 'workflow_run' }}
+
       # Download the artifacts generated by "PR Handler" if this workflow was
       # triggered by a workflow_run event.
       - name: Download artifact
@@ -60,13 +108,13 @@ jobs:
       # Unzip the artifacts.
       - name: Unzip artifact
         if: ${{ github.event_name == 'workflow_run' }}
-        run: unzip pr.zip
+        run: unzip -n pr.zip
 
       # Save the PR number to send messages to the user that opened it.
-      - name: Save PR number
+      - name: Get PR number
         if: ${{ github.event_name == 'workflow_run' }}
         id: pr_number
-        run: echo '::set-output name=PR_NUMBER::'$(cat NR)
+        run: echo "PR_NUMBER=$(cat NR)" >> $GITHUB_OUTPUT
 
       # Send an information message to the user that opened the pull request.
       - name: Send information message
@@ -105,14 +153,17 @@ jobs:
       # Install the dependencies to run the Python scripts.
       - name: Install dependencies
         run: |
-          pip install scan-build
+          pip install -r resources/ci_cd/requirements.txt
 
       # This script downloads the mandatory libraries of RTI Connext DDS for
       # compiling the examples
       - name: Install RTI Connext DDS
-        run: python resources/ci_cd/linux_install.py
+        run: python resources/ci_cd/linux_install.py -a ${{ matrix.architecture.name }}
         env:
-          RTI_MIN_PACKAGE_URL: ${{ secrets.RTI_MIN_PACKAGE_URL }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          RTI_AWS_BUCKET: ${{ secrets.RTI_AWS_BUCKET }}
+          RTI_AWS_PATH: ${{ secrets.RTI_AWS_PATH }}
 
       # This script will compile the examples and execute the static analysis.
       - name: Build all the examples

.github/workflows/prhandler.yml

@@ -25,11 +25,30 @@ jobs:
       - name: Save PR number
         run: echo ${{ github.event.number }} > ${{ github.workspace }}/NR
 
+      # Save the PR number for the next workflow to create the current
+      # workflow artifact along with the repository itself.
+      - name: Save target branch
+        run: echo ${{ github.base_ref }} > ${{ github.workspace }}/targetbranch
+
       # Upload the artifacts to GitHub so that they can be used in the next
       # workflow.
-      - name: Upload generated artifact
+      - name: Upload PR changes
         uses: actions/upload-artifact@v3
         with:
           name: pr
-          path: ${{ github.workspace }}
+          path: |
+            ${{ github.workspace }}
+            !${{ github.workspace }}/resources/ci_cd
+            !${{ github.workspace }}/.git
+            !${{ github.workspace }}/targetbranch
+          if-no-files-found: error
+
+      # Upload the artifacts to GitHub so that they can be used in the next
+      # workflow.
+      - name: Upload target branch name
+        uses: actions/upload-artifact@v3
+        with:
+          name: targetbranch
+          path: |
+            ${{ github.workspace }}/targetbranch
           if-no-files-found: error

VERSION

@@ -0,0 +1 @@
+7.2.0

resources/aws/requirements.txt

@@ -1,3 +1,2 @@
 boto3
 boto3_type_annotations
-pexpect

resources/ci_cd/linux_install.py

@@ -14,46 +14,73 @@
 the minimal installation archive.
 
 """
+import argparse
 import os
 import sys
 import tempfile
-import urllib.request
-
 from distutils.spawn import find_executable
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from subprocess import call
 from urllib.error import URLError
 
+import boto3
+
+ROOT_DIR = Path(__file__).resolve().parents[2]
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse the CLI options."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-a", "--architecture", type=str, required=True)
+    parser.add_argument(
+        "-i",
+        "--install-parent-dir",
+        type=Path,
+        default=os.getenv("RTI_INSTALLATION_PATH") or Path.home(),
+    )
+    parser.add_argument(
+        "-b", "--bucket", type=str, default=os.getenv("RTI_AWS_BUCKET")
+    )
+    parser.add_argument(
+        "-p",
+        "--s3-path",
+        type=PurePosixPath,
+        default=os.getenv("RTI_AWS_PATH"),
+    )
+
+    args = parser.parse_args()
+
+    return args
+
 
 def main():
-    rti_minimal_package_url = os.getenv("RTI_MIN_PACKAGE_URL")
-    temp_dir = Path(tempfile.gettempdir())
-    if not rti_minimal_package_url:
-        sys.exit(
-            "Environment variable RTI_MIN_PACKAGE_URL not found, skipping..."
-        )
+    args = parse_args()
+    version = ROOT_DIR.joinpath("VERSION").read_text()
+    downloaded_zipped_file_path = Path(
+        tempfile.gettempdir(),
+        f"rti_connext_dds-{version}-lm-{args.architecture}.zip",
+    )
+    remote_zipped_file_path = PurePosixPath(
+        args.s3_path, version, downloaded_zipped_file_path.name
+    )
+    s3 = boto3.client("s3")
 
     try:
-        rti_installation_path = Path(
-            os.getenv("RTI_INSTALLATION_PATH") or Path.home()
-        ).resolve(strict=True)
+        rti_install_parent_dir = Path(args.install_parent_dir).resolve(
+            strict=True
+        )
     except FileNotFoundError:
-        sys.exit("The RTI_INSTALLATION_PATH does not exist.")
+        sys.exit(f"File not found: {args.install_parent_dir}")
 
     cmake_command = find_executable("cmake")
     if cmake_command is None:
         sys.exit("CMake must be installed in order to use the script.")
 
     try:
-        rti_zipped_file_name = Path(
-            urllib.request.url2pathname(rti_minimal_package_url)
-        ).name
-        rti_zipped_file_path = temp_dir.joinpath(rti_zipped_file_name)
-        urllib.request.urlretrieve(
-            rti_minimal_package_url, rti_zipped_file_path
-        )
+        with open(downloaded_zipped_file_path, "wb") as f:
+            s3.download_fileobj(args.bucket, str(remote_zipped_file_path), f)
     except URLError as e:
-        sys.exit("Error opening the URL: {}".format(e))
+        sys.exit(f"Error opening the object: {e}")
 
     try:
         return_value = call(
@@ -62,10 +89,10 @@ def main():
                 "-E",
                 "tar",
                 "xf",
-                str(rti_zipped_file_path),
+                str(downloaded_zipped_file_path),
                 "--format=zip",
             ],
-            cwd=rti_installation_path,
+            cwd=rti_install_parent_dir,
         )
     except FileNotFoundError:
         sys.exit("The CMake executable could not be found.")

resources/ci_cd/requirements.txt

@@ -0,0 +1,3 @@
+boto3
+boto3_type_annotations
+scan-build