misc: artifact size metrics actions (#150)

Author: 0marperez

.github/actions/artifact-size-metrics/calculate-and-upload/action.yml

@@ -0,0 +1,24 @@
+name: Calculate Artifact Size
+description: Calculates size for JVM artifacts
+
+inputs:
+  upload:
+    description: Whether the metrics should be uploaded to S3/Cloudwatch
+    type: boolean
+  release_metrics:
+    description: Whether the metrics are coming from a release build
+    type: boolean
+
+runs:
+  using: composite
+  steps:
+    - name: Calculate and upload artifact sizes
+      shell: bash
+      env:
+        GITHUB_REPOSITORY: ${{ github.repository }}
+        IDENTIFIER: ${{ github.ref_name }}
+        UPLOAD: ${{ inputs.upload }}
+        RELEASE_METRICS: ${{ inputs.release_metrics }}
+      run: |
+        chmod +x $GITHUB_ACTION_PATH/../utils/calculate-and-upload/main.sh
+        $GITHUB_ACTION_PATH/../utils/calculate-and-upload/main.sh

.github/actions/artifact-size-metrics/download-and-process/action.yml

@@ -0,0 +1,31 @@
+name: Process artifact size metrics
+description: Compares artifact size metrics, leaves a comment, and fails if a size increase of ≥5% is detected.
+
+inputs:
+  download:
+    description: Whether the artifact size metrics should be downloaded from S3
+    type: boolean
+
+runs:
+  using: composite
+  steps:
+    # Compares artifact size metrics and sets LARGE_DIFF to true if a size increase of ≥5% is detected.
+    - name: Download and process artifact size metrics
+      shell: bash
+      env:
+        DOWNLOAD: ${{ inputs.download }}
+        GITHUB_REPOSITORY: ${{ github.repository }}
+        IDENTIFIER: ${{ github.ref_name }}
+      run: |
+        chmod +x $GITHUB_ACTION_PATH/../utils/download-and-process/main.sh        
+        $GITHUB_ACTION_PATH/../utils/download-and-process/main.sh
+
+    - name: Large size increase?
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'acknowledge-artifact-size-increase') }}
+      shell: bash
+      run: |
+        if [ "$LARGE_DIFF" == "true" ]; then
+            echo "An artifact has increased in size by more than 5%.
+            If this is expected, please add the acknowledge-artifact-size-increase label to this pull request."
+            exit 1
+        fi

.github/actions/artifact-size-metrics/show-results/action.yml

@@ -0,0 +1,29 @@
+# Gets artifact size metrics from staging dir
+calculateArtifactSizes() {
+  # Artifact staging dir
+  input_dir="build/m2"
+
+  # Create output_file
+  output_file="$1"
+  mkdir -p "$(dirname "$output_file")"
+  touch "$output_file"
+
+  # Write CSV header
+  echo "Artifact, Size (Bytes)" > "$output_file"
+
+  # Find all JARs (exclude sources and javadoc)
+  # TODO: Calculate KN artifacts sizes
+  find "$input_dir" -type f -name "*.jar" ! -name "*-sources.jar" ! -name "*-javadoc.jar" | while read -r jar; do
+      size=$(stat -c%s "$jar")
+
+      # remove dir path, version, optional timestamp, and .jar
+      artifact=$(basename "$jar")
+      artifact=$(echo "$artifact" | sed -E 's/-[0-9].*\.jar$//')
+
+      # Add artifact size to CSV
+      echo "$artifact, $size" >> "$output_file"
+  done
+
+  # Print results for debugging
+  cat "$output_file"
+}

.github/actions/artifact-size-metrics/utils/calculate-and-upload/calculate_metrics.sh

@@ -0,0 +1,42 @@
+# Upload the artifact size metrics to cloudwatch
+uploadToCloudwatch() {
+  metrics_file="$1"
+  metrics=()
+
+  # Read CSV
+  while IFS=',' read -r artifactName artifactSize; do
+      # Skip header
+      [[ "$artifactName" == "Artifact" ]] && continue
+
+      # Trim spaces
+      artifactName=$(echo "$artifactName" | xargs)
+      artifactSize=$(echo "$artifactSize" | xargs)
+
+      # Build metric JSON
+      metrics+=$(jq -n \
+          --arg name "$GITHUB_REPOSITORY-$artifactName" \
+          --arg value "$artifactSize" \
+          --arg project "$GITHUB_REPOSITORY" \
+          '{
+              MetricName: $name,
+              Timestamp: (now | todate),
+              Unit: "Bytes",
+              Value: ($value | tonumber),
+              Dimensions: [
+                  { Name: "Project", Value: $project }
+              ]
+          }'
+      )
+  done < "$metrics_file"
+
+  namespace="Artifact Size Metrics"
+  chunk_size=1000
+
+  # Send metrics in chunks
+  for ((i=0; i<${#metrics[@]}; i+=chunk_size)); do
+      chunk=("${metrics[@]:i:chunk_size}")
+      aws cloudwatch put-metric-data \
+          --namespace "$namespace" \
+          --metric-data "$(printf '%s\n' "${chunk[@]}" | jq -s '.')"
+  done
+}

.github/actions/artifact-size-metrics/utils/calculate-and-upload/cloudwatch.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Bash script to calculate, and upload artifact size metrics
+
+source "$(dirname "$0")/calculate_metrics.sh"
+source "$(dirname "$0")/cloudwatch.sh"
+source "$(dirname "$0")/../s3.sh"
+source "$(dirname "$0")/../constants.sh"
+source "$(dirname "$0")/../setup.sh"
+
+setup
+
+# Calculate size for artifacts in staging dir (build/m2) and save them to metrics_file
+calculateArtifactSizes "$metrics_file" # see: constants.sh
+
+# Upload metrics to S3/cloudwatch if required
+if [ "$UPLOAD" == "true" ]; then
+  if [ "$RELEASE_METRICS" == "true" ]; then
+    # For record-keeping
+    uploadToMetricsBucket "$metrics_file" "$GITHUB_REPOSITORY"-v"$IDENTIFIER".csv
+    uploadToMetricsBucket "$metrics_file" "$GITHUB_REPOSITORY"-latest.csv
+
+    # For display in our OPS dashboard
+    uploadToCloudwatch "$metrics_file" "$GITHUB_REPOSITORY"
+  else
+    # For downstream consumption in pull requests
+    uploadToMetricsBucket "$metrics_file" [TEMP]"$GITHUB_REPOSITORY"-pull-request-"$IDENTIFIER".csv
+  fi
+fi

.github/actions/artifact-size-metrics/utils/calculate-and-upload/main.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Where current metrics are stored
+metrics_file="build/reports/metrics/artifact-size-metrics.csv"
+
+# Where metrics from latest release are stored
+latest_release_metrics_file="build/reports/metrics/latest-release-artifact-size-metrics.csv"
+
+# Where the metrics comparison results are stored
+metrics_comparison_file="build/reports/metrics/comparison.md"

.github/actions/artifact-size-metrics/utils/constants.sh

@@ -0,0 +1,50 @@
+# Compares artifact size metrics to ones from the latest available release,
+# stores comparison as a markdown table,
+# and returns "true" if a large diff was found (over 5%)
+compareMetrics() {
+    local metrics_file="$1"
+    local latest_release_metrics_file="$2"
+    local metrics_comparison_file="$3"
+
+    # Title and table headers
+    {
+      echo "Affected Artifacts"
+      echo "="
+      echo "| Artifact | Pull Request (bytes) | Latest Release (bytes) | Delta (bytes) | Delta (percentage) |"
+      echo "|----------|----------------------|------------------------|---------------|--------------------|"
+    } > "$metrics_comparison_file"
+
+    large_diff=false
+
+    # Read CSV
+    while IFS=',' read -r artifact size; do
+      # Skip header
+      [ "$artifact" = "Artifact" ] && continue
+
+      # Trim spaces
+      artifact=$(echo "$artifact" | xargs)
+      size=$(echo "$size" | xargs)
+
+      # Find corresponding artifact size in release file or skip
+      latest_release_size=$(awk -F',' -v art="$artifact" 'NR>1 && $1==art {gsub(/ /,"",$2); print $2}' "$latest_release_metrics_file")
+      [ -z "$latest_release_size" ] && continue
+
+      # Find delta
+      delta=$((size - latest_release_size))
+      abs_delta=${delta#-}
+      percent=$((100 * abs_delta / latest_release_size))
+
+      # Add to file
+      echo "| $artifact | $size | $latest_release_size | $delta | ${percent}% |" >> "$metrics_comparison_file"
+
+      # Check for large diff
+      if [ "$percent" -gt 5 ]; then
+          large_diff=true
+      fi
+    done < "$metrics_file"
+
+    # Print results for debugging
+    cat "$metrics_comparison_file"
+
+    $large_diff && echo "true"
+}

.github/actions/artifact-size-metrics/utils/download-and-process/compare.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Bash script to download, and compare artifact size metrics
+
+source "$(dirname "$0")/compare.sh"
+source "$(dirname "$0")/../s3.sh"
+source "$(dirname "$0")/../constants.sh"
+source "$(dirname "$0")/../setup.sh"
+
+setup
+
+if [ "$DOWNLOAD" == "true" ]; then
+  # Get metrics calculated in codebuild - otherwise metrics will already be here
+  downloadFromMetricsBucket [TEMP]"$GITHUB_REPOSITORY"-pull-request-"$IDENTIFIER".csv "$metrics_file" # see: constants.sh
+fi
+
+# Metrics from the latest release are never calculated here so we need to download them
+downloadFromMetricsBucket "$GITHUB_REPOSITORY"-latest.csv "$latest_release_metrics_file" # see: constants.sh
+
+# Compare metrics
+export LARGE_DIFF=$(compareMetrics "$metrics_file" "$latest_release_metrics_file" "$metrics_comparison_file") # see: constants.sh
+
+if [ "$LARGE_DIFF" == "true" ]; then
+  echo "Large diff found!"
+fi

.github/actions/artifact-size-metrics/utils/download-and-process/main.sh

@@ -0,0 +1,12 @@
+# Owned by: aws-kotlin-sdk+ci
+S3_ARTIFACT_SIZE_METRICS_BUCKET="artifact-size-metrics"
+
+# Uploads metrics to the metrics bucket under the specified file name
+uploadToMetricsBucket() {
+  aws s3 cp "$1" s3://"$S3_ARTIFACT_SIZE_METRICS_BUCKET"/"$2"
+}
+
+# Downloads metrics from the metrics bucket to the specified local file
+downloadFromMetricsBucket() {
+  aws s3 cp s3://"$S3_ARTIFACT_SIZE_METRICS_BUCKET"/"$1" "$2"
+}