diff --git a/hack/tools/catalogs/README.md b/hack/tools/catalogs/README.md new file mode 100644 index 000000000..00360a17e --- /dev/null +++ b/hack/tools/catalogs/README.md @@ -0,0 +1,177 @@ +# Hack Catalog Tools + +This directory contains scripts that automate some of the tasks related to catalog interaction and bundle installation. + +--- +> [!WARNING] +> These scripts are intended to help users navigate the catalog and produce installation RBAC until reliable tooling is available for OLM v1, +> and to document the process in code for contributors. These scripts are not officially supported. +> They are not meant to be used in production environments. +--- + +### Prerequisites + +To execute the scripts, the following tools are required: + + * [jq](https://jqlang.github.io/jq/) to filter catalog data + * [yq](https://mikefarah.gitbook.io/yq/) to parse YAML + * [kubectl](https://kubernetes.io/docs/reference/kubectl/) to interact with the cluster running OLM v1 + * [wget](https://www.gnu.org/software/wget/) to download the catalog data + * A container runtime, such as [podman](https://podman.io/) or [docker](https://www.docker.com/) to interact with bundle images. + +#### Container Runtime + +By default, the scripts use `podman` or `docker` as the container runtime. +If you use another container runtime, set the `CONTAINER_RUNTIME` environment variable to the path of the container runtime binary. + +### Tools + +--- +> [!NOTE] +> All examples assume that the current working directory is the `hack/tools/catalogs` directory. +--- + +#### download-catalog + +Download a catalog from an unpacked ClusterCatalog running on a cluster reachable by `kubectl`. + +Example: + + ```terminal + # Download the catalog from the operatorhubio ClusterCatalog + ./download-catalog operatorhubio + ``` + +The downloaded catalog is saved to -catalog.json in the current directory. + +#### list-compatible-bundles + +List (potential) OLM v1 compatible bundles from the catalog. + +Not all registry+v1 bundles made for OLM v0 are compatible with OLM v1. +Compatible bundles must meet the following criteria: + * Support for the 'AllNamespaces' install mode + * No webhooks + * No dependencies on other packages of GVKs + * The operator does not make use of OLM v0's [`OperatorCondition`](https://olm.operatorframework.io/docs/concepts/crds/operatorcondition/) API + + +For more information, see [OLM v1 limitations](../../../docs/refs/olm-v1-limitations.md). + +For some bundles, some of this criteria can only be determined by inspecting the contents bundle image. The script will return all bundles that are potentially compatible. + +Examples: + + ``` terminal + # List (potentially) OLM v1 compatible bundles from the operatorhubio catalog + ./list-compatible-bundles < operatorhubio-catalog.json + ``` + + ``` terminal + # List (potentially) OLM v1 compatible bundles that contain 'argco' in the package name + # -r can be used with any regex supported by jq + ./list-compatible-bundles -r 'argocd' < operatorhubio-catalog.json + ``` + +#### find-bundle-image + +Find the image for a bundle in the catalog. + +Example: + + ``` terminal + # Get the image for the argocd-operator v0.6.0 bundle from the operatorhubio catalog + ./find-bundle-image argocd-operator 0.6.0 < operatorhubio-catalog.json + ``` + +#### unpack-bundle + +Unpack a bundle image to a directory. + +Example: + + ``` terminal + # Unpack the argocd-operator v0.6.0 bundle image to a temporary directory + ./unpack-bundle quay.io/operatorhubio/argocd-operator@sha256:d538c45a813b38ef0e44f40d279dc2653f97ca901fb660da5d7fe499d51ad3b3 + ``` + + ``` terminal + # Unpack the argocd-operator v0.6.0 bundle image to a specific directory + ./unpack-bundle quay.io/operatorhubio/argocd-operator@sha256:d538c45a813b38ef0e44f40d279dc2653f97ca901fb660da5d7fe499d51ad3b3 -o argocd-manifests + ``` + +#### is-bundle-supported + +Check if a bundle is supported by OLM v1 by inspecting the unpacked bundle manifests. + + +For more information on bundle support, see [OLM v1 limitations](../../../docs/refs/olm-v1-limitations.md). + +Example: + + ``` terminal + # Check if the argocd-operator v0.6.0 bundle from the operatorhubio catalog is supported by OLM v1 + ./is-bundle-supported argocd-manifests + ``` + + ``` terminal + # Find bundle image, unpack, and verify support in one command + ./find-bundle-image argocd-operator 0.6.0 < operatorhubio-catalog.json | ./unpack-bundle | ./is-bundle-supported + ``` + +#### generate-manifests + +Generate RBAC or installation manifests for a bundle. The generated manifests can be templates or fully rendered manifests. + +The following options can be used to override resource naming defaults: + -n Namespace where the extension is installed + -e - Name of the extension + -cr - Name of the cluster role + -r - Name of the role + -s - Name of the service account + --template - Generate template manifests + +Default resource name format: + * Namespace: -system + * Extension name: + * ClusterRole name: -cluster-role + * Role name: -installer-role + * ServiceAccount name: -installer + * ClusterRoleBinding name: -binding + * RoleBinding name: -binding + +Use `--template` to generate templated manifests that can be customized before applying to the cluster. +Template manifests will contain the following template variables: + +Template Variables: +* `${NAMESPACE}` - Namespace where the extension is installed +* `${EXTENSION_NAME}` - Name of the extension +* `${CLUSTER_ROLE_NAME}` - Name of the cluster role +* `${ROLE_NAME}` - Name of the role +* `${SERVICE_ACCOUNT_NAME}` - Name of the service account + +Examples: + + ``` terminal + # Generate installation manifests for the argocd-operator v0.6.0 bundle from the operatorhubio catalog + ./generate-manifests install argocd-operator 0.6.0 < operatorhubio-catalog.json + ``` + + ``` terminal + # Generate templated installation manifests for the argocd-operator v0.6.0 bundle from the operatorhubio catalog + generate-manifests install argocd-operator 0.6.0 --template < operatorhubio-catalog.json + ``` + + ``` terminal + # Generate RBAC manifests for the argocd-operator v0.6.0 bundle from the operatorhubio catalog + generate-manifests rbac argocd-operator 0.6.0 < operatorhubio-catalog.json + ``` + + ``` terminal + # Generate templated RBAC manifests for the argocd-operator v0.6.0 bundle from the operatorhubio catalog + generate-manifests rbac argocd-operator 0.6.0 --template < operatorhubio-catalog.json + ``` diff --git a/hack/tools/catalogs/download-catalog b/hack/tools/catalogs/download-catalog new file mode 100755 index 000000000..cfb7e51a9 --- /dev/null +++ b/hack/tools/catalogs/download-catalog @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +SCRIPT_ROOT=$(dirname "$(realpath "$0")") +source "${SCRIPT_ROOT}/lib/utils.sh" + +# Check required tools are installed +assert-commands kubectl jq wget + +# ClusterCatalog coordinates +: "${CATALOGD_CATALOGD_SERVICE_NAMESPACE:=olmv1-system}" +: "${CATALOGD_SERVICE_NAME:=catalogd-catalogserver}" +: "${CATALOGD_SERVICE_PORT:=443}" # Assumes the service uses HTTPS on port 443 +: "${CATALOGD_LOCAL_SERVICE_PORT:=8001}" + +echo "Namespace: $CATALOGD_CATALOGD_SERVICE_NAMESPACE" +echo "Service Name: $CATALOGD_SERVICE_NAME" +echo "Service Port: $CATALOGD_SERVICE_PORT" +echo "Local Service Port: $CATALOGD_LOCAL_SERVICE_PORT" + +# Display usage +usage() { + print-banner + echo "" + echo "Usage: $0 " + echo "" + echo "Download catalog from a ClusterCatalog in a cluster reachable from KUBECONFIG" + echo "Downloaded catalog will be saved as -catalog.json" + echo "" + echo "Example:" + echo " $0 operatorhubio" +} + +# Check if catalog name is provided +if [ -z "$1" ]; then + usage + exit 1 +fi + +CATALOG_NAME="$1" + +# Check if the clustercatalog resource exists +echo "Checking if ClusterCatalog $CATALOG_NAME exists..." +CLUSTER_CATALOG=$(kubectl get clustercatalog "$CATALOG_NAME" -o json 2>/dev/null) +if [ -z "$CLUSTER_CATALOG" ]; then + echo "ClusterCatalog $CATALOG_NAME does not exist." + exit 1 +fi + +# Check if the Unpacked condition is true +UNPACKED_CONDITION=$(echo "$CLUSTER_CATALOG" | jq -r '.status.conditions[]? // [] | select(.type=="Unpacked") | .status') +if [ "$UNPACKED_CONDITION" != "True" ]; then + echo "ClusterCatalog $CATALOG_NAME is not unpacked yet." + exit 1 +fi + +# Get the contentURL +CONTENT_URL=$(echo "$CLUSTER_CATALOG" | jq -r '.status.contentURL') +if [ -z "$CONTENT_URL" ]; then + echo "Content URL not found for ClusterCatalog $CATALOG_NAME." + exit 1 +fi + +# Start port forwarding +echo "Starting kubectl port-forward to $CATALOGD_SERVICE_NAME on port $CATALOGD_LOCAL_SERVICE_PORT..." +kubectl port-forward -n "$CATALOGD_CATALOGD_SERVICE_NAMESPACE" svc/"$CATALOGD_SERVICE_NAME" "$CATALOGD_LOCAL_SERVICE_PORT:$CATALOGD_SERVICE_PORT" &>/dev/null & +PORT_FORWARD_PID=$! + +# Poll the service until it responds or timeout after 30 seconds +timeout=30 +while ! curl -s "http://localhost:${CATALOGD_LOCAL_SERVICE_PORT}" >/dev/null; do + timeout=$((timeout - 1)) + if [ $timeout -le 0 ]; then + echo "Port forwarding failed to start within 30 seconds." + kill $PORT_FORWARD_PID + exit 1 + fi + sleep 1 +done + +# Modify the contentURL to hit localhost: +LOCAL_CONTENT_URL=${CONTENT_URL//https:\/\/$CATALOGD_SERVICE_NAME.$CATALOGD_CATALOGD_SERVICE_NAMESPACE.svc/https:\/\/localhost:$CATALOGD_LOCAL_SERVICE_PORT} +echo "Found content URL: $CONTENT_URL" +echo "Using local port: $CATALOGD_LOCAL_SERVICE_PORT" +echo "Using local content URL: $LOCAL_CONTENT_URL" + +# shellcheck disable=SC2001 +# Download the catalog using wget +echo "Downloading catalog from $LOCAL_CONTENT_URL..." +wget --no-check-certificate "$LOCAL_CONTENT_URL" -O "${CATALOG_NAME}-catalog.json" + +# Stop the port forwarding +echo "Stopping kubectl port-forward..." +kill $PORT_FORWARD_PID + +echo "Catalog downloaded to ${CATALOG_NAME}-catalog.json" \ No newline at end of file diff --git a/hack/tools/catalogs/find-bundle-image b/hack/tools/catalogs/find-bundle-image new file mode 100755 index 000000000..9394e6b72 --- /dev/null +++ b/hack/tools/catalogs/find-bundle-image @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Get the directory of the current script +SCRIPT_ROOT=$(dirname "$(realpath "$0")") + +source "${SCRIPT_ROOT}/lib/bundle.sh" +source "${SCRIPT_ROOT}/lib/utils.sh" + +# Check required tools are installed +assert-commands jq + +usage() { + print-banner + echo "" + echo "Usage: $0 " + echo "" + echo "Find the bundle image for a package in a catalog in " + echo "" + echo "Example:" + echo " $0 argocd-operator 0.6.0 < operatorhubio-catalog.json" +} + +if [ "$#" -lt 2 ]; then + usage + exit 1 +fi + +package_name="$1" +package_version="$2" + +# Find bundle image +image="$(cat - | get-bundle-image "${package_name}" "${package_version}")" +echo "${image}" diff --git a/hack/tools/catalogs/generate-manifests b/hack/tools/catalogs/generate-manifests new file mode 100755 index 000000000..ee1ee97b0 --- /dev/null +++ b/hack/tools/catalogs/generate-manifests @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +# Get the directory of the current script +SCRIPT_ROOT=$(dirname "$(realpath "$0")") + +source "${SCRIPT_ROOT}/lib/unpack.sh" +source "${SCRIPT_ROOT}/lib/collect-rbac.sh" +source "${SCRIPT_ROOT}/lib/utils.sh" + +# Check there is a container runtime (podman, or docker) +# If neither are found, check CONTAINER_RUNTIME is set and exists in PATH +assert-container-runtime + +# Check required tools are installed +assert-commands jq + +function usage() { + print-banner + echo "" + echo "Usage:" + echo "" + echo "Generate installation manifests" + echo "$0 install [-n namespace] [-e cluster-extension-name] [-cr cluster-role-name] [-r role-name] [-s service-account-name] [--template]" + echo "" + echo "Generate RBAC manifests" + echo "$0 rbac [-n namespace] [-e cluster-extension-name] [-cr cluster-role-name] [-r role-name] [-s service-account-name] [--template]" + echo "" + echo "Generate installation or RBAC manifests for a registry+v1 package given an FBC catalog in stdin" + echo "" + echo "Options:" + echo " -n - Namespace where the extension is installed" + echo " -e - Name of the extension" + echo " -cr - Name of the cluster role" + echo " -r - Name of the role" + echo " -s - Name of the service account" + echo " --template - Generate template manifests" + echo "" + echo "Template Variables:" + echo " * \${NAMESPACE} - Namespace where the extension is installed" + echo " * \${EXTENSION_NAME} - Name of the extension" + echo " * \${CLUSTER_ROLE_NAME} - Name of the cluster role" + echo " * \${ROLE_NAME} - Name of the role" + echo " * \${SERVICE_ACCOUNT_NAME} - Name of the service account" + echo "" + echo "Default Resource Name Format:" + echo " * Namespace: -system" + echo " * Extension name: " + echo " * ClusterRole name: -cluster-role" + echo " * Role name: -installer-role" + echo " * ServiceAccount name: -installer" + echo " * ClusterRoleBinding name: -binding" + echo " * RoleBinding name: -binding" + echo "" + echo "Examples:" + echo " # Generate installation manifests for the argocd-operator package" + echo " $0 install argocd-operator 0.6.0 < operatorhubio-catalog.json" + echo "" + echo " # Generate RBAC manifests for the argocd-operator package" + echo " $0 rbac argocd-operator 0.6.0 < operatorhubio-catalog.json" + echo "" + echo " # Generate templated installation manifests for the argocd-operator package" + echo " $0 install argocd-operator 0.6.0 --template < operatorhubio-catalog.json" + echo "" + echo " # Generate templated RBAC manifests for the argocd-operator package" + echo " $0 rbac argocd-operator 0.6.0 --template < operatorhubio-catalog.json" + echo "" + echo "WARNING: This script is a stopgap solution until proper tools are available in OLMv1 - it is not guaranteed to work with all packages." +} + +# Check for at least 3 arguments +if [ "$#" -lt 3 ]; then + usage + exit 1 +fi + +# Command and package details +COMMAND=$1 +export PACKAGE_NAME=$2 +export PACKAGE_VERSION=$3 + +# Initialize environment variables with template defaults +export NAMESPACE="\${NAMESPACE}" +export EXTENSION_NAME="\${EXTENSION_NAME}" +export CLUSTER_ROLE_NAME="\${CLUSTER_ROLE_NAME}" +export ROLE_NAME="\${ROLE_NAME}" +export SERVICE_ACCOUNT_NAME="\${SERVICE_ACCOUNT_NAME}" +export DEBUG=false +template=false + +# Parse optional arguments +shift 3 +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -n) + export NAMESPACE="$2" + shift 2 + ;; + -e) + export EXTENSION_NAME="$2" + shift 2 + ;; + -cr) + export CLUSTER_ROLE_NAME="$2" + shift 2 + ;; + -r) + export ROLE_NAME="$2" + shift 2 + ;; + -s) + export SERVICE_ACCOUNT_NAME="$2" + shift 2 + ;; + --template) + template=true + shift + ;; + --debug) + DEBUG=1 + shift + ;; + *) + echo "Unknown option $1" + usage + exit 1 + ;; + esac +done + +# Apply default values only to unset parameters if --template is not set +if [ "$template" = false ]; then + [ "$EXTENSION_NAME" == "\${EXTENSION_NAME}" ] && export EXTENSION_NAME="${PACKAGE_NAME}" + [ "$NAMESPACE" == "\${NAMESPACE}" ] && export NAMESPACE="${EXTENSION_NAME}-system" + [ "$SERVICE_ACCOUNT_NAME" == "\${SERVICE_ACCOUNT_NAME}" ] && export SERVICE_ACCOUNT_NAME="${PACKAGE_NAME}-installer" + [ "$CLUSTER_ROLE_NAME" == "\${CLUSTER_ROLE_NAME}" ] && export CLUSTER_ROLE_NAME="${SERVICE_ACCOUNT_NAME}-cluster-role" + [ "$ROLE_NAME" == "\${ROLE_NAME}" ] && export ROLE_NAME="${SERVICE_ACCOUNT_NAME}-installer-role" +fi + +# Output the set environment variables for confirmation +debug "Environment variables set:" +debug "NAMESPACE=${NAMESPACE}" +debug "EXTENSION_NAME=${EXTENSION_NAME}" +debug "CLUSTER_ROLE_NAME=${CLUSTER_ROLE_NAME}" +debug "ROLE_NAME=${ROLE_NAME}" +debug "SERVICE_ACCOUNT_NAME=${SERVICE_ACCOUNT_NAME}" + +# Find bundle image +image="$(cat - | get-bundle-image "${PACKAGE_NAME}" "${PACKAGE_VERSION}")" + +# Unpack and close container +bundle_manifest_dir="$("${SCRIPT_ROOT}/unpack-bundle" "${image}")" + +# Derive rbac from bundle manifests +collect_installer_rbac "${bundle_manifest_dir}" +echo "Done" >&2 + +# Example output or further processing based on command +case "${COMMAND}" in + install) + generate_install_manifests | envsubst + ;; + rbac) + generate_rbac_manifests | envsubst + ;; + *) + echo "Unknown command ${COMMAND}" + usage + exit 1 + ;; +esac + +# Clean up manifest directory +if [ "${DEBUG,,}" != "false" ]; then + debug "Skipping cleanup of manifest directory: ${bundle_manifest_dir}" +else + rm -rf "${bundle_manifest_dir}" +fi \ No newline at end of file diff --git a/hack/tools/catalogs/is-bundle-supported b/hack/tools/catalogs/is-bundle-supported new file mode 100755 index 000000000..b197b5310 --- /dev/null +++ b/hack/tools/catalogs/is-bundle-supported @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Get the directory of the current script +SCRIPT_ROOT=$(dirname "$(realpath "$0")") + +source "${SCRIPT_ROOT}/lib/unpack.sh" +source "${SCRIPT_ROOT}/lib/collect-rbac.sh" +source "${SCRIPT_ROOT}/lib/utils.sh" + +# Check required tools are installed +assert-commands jq + +usage() { + print-banner + echo "" + echo "Usage: $0 " + echo "" + echo "Check if a bundle is supported by OLM v1 given a bundle manifest directory" + echo "" + echo "Example:" + echo " $0 " +} + +# Must have a single argument +if [ -z "$1" ]; then + if [ -t 0 ]; then + echo "Error: Docker image name is required." + usage + exit 1 + else + read -r manifest_dir + fi +else + manifest_dir="$1" +fi + + # Check the bundle is supported +assert-bundle-supported "${manifest_dir}" + +echo "Bundle is supported by OLM v1" >&2 +echo "true" \ No newline at end of file diff --git a/hack/tools/catalogs/lib/bundle.sh b/hack/tools/catalogs/lib/bundle.sh new file mode 100644 index 000000000..2adfc260f --- /dev/null +++ b/hack/tools/catalogs/lib/bundle.sh @@ -0,0 +1,194 @@ +# Library of functions for interacting with bundles and their manifests + +# SCRIPT_ROOT is the root of the script +source "$(dirname "${BASH_SOURCE[0]}")/hash.sh" + +# Given package and version grabs the bundle image from stdin FBC stream +get-bundle-image(){ + local package_name="${1}" + local package_version="${2}" + local image + image=$(jq -r --arg pkg "$package_name" --arg ver "$package_version" \ + 'select(.schema == "olm.bundle" and (.properties[] | select(.type == "olm.package" and .value.packageName == $pkg and .value.version == $ver))) | .image') + + if [ -z "$image" ]; then + echo "ERROR: No matching image found for package '$package_name' with version '$package_version'." >&2 + exit 1 + fi + + echo "${image}" +} + +is-all-namespace-mode-enabled() { + local csv="${1}" + local valid + # Note: The 'and true' is to ensure that the expression evaluates to true or false + # Without it, yq will return the matched value. This is done for succinctness. Without the 'and true', + # the same behavior could be achieved by checking if the output is non-empty. + valid=$(yq eval '(.spec.installModes[] | select(.type == "AllNamespaces" and .supported == true)) and true' "${csv}") + echo "${valid}" +} + +does-not-have-webhooks() { + local csv="${1}" + local valid + valid=$(yq eval '((.spec.webhookdefinitions == null) or (.spec.webhookdefinitions | length == 0))' "${csv}") + echo "${valid}" +} + +does-not-have-dependencies() { + local csv="${1}" + local valid + valid=$(yq eval '((.spec.customresourcedefinitions.required == null) or (.spec.customresourcedefinitions.required | length == 0))' "${csv}") + echo "${valid}" +} + +is-crd-version-supported() { + local manifest_dir="${1}" + local valid=true + while IFS= read -r resource_file; do + version=$(yq eval '.apiVersion' "$resource_file") + if [ "${version}" != "apiextensions.k8s.io/v1" ]; then + valid="${version}" + break + fi + done < <(find "${manifest_dir}" -type f -exec grep -l "^kind: CustomResourceDefinition" {} \;) + echo "$valid" +} + +is-bundle-supported() { + local manifest_dir="${1}" + csv="$(find_csv "${manifest_dir}")" + + crd_version="$(is-crd-version-supported "${manifest_dir}")" + + if [ "$(is-all-namespace-mode-enabled "${csv}")" != "true" ]; then + echo "Bundle not supported: AllNamespaces install mode is disabled" >&2 + echo "false" + elif [ "$(does-not-have-webhooks "${csv}")" != "true" ]; then + echo "Bundle not supported: contains webhooks" >&2 + echo "false" + elif [ "$(does-not-have-dependencies "${csv}")" != "true" ]; then + echo "Bundle not supported: contains dependencies" >&2 + echo "false" + elif [ "${crd_version}" != "true" ]; then + echo "Bundle not supported: unsupported CRD api version (${crd_version})" >&2 + echo "false" + fi + + echo "true" +} + +# Function to validate the bundle is supported +assert-bundle-supported() { + local manifest_dir="${1}" + if [ "$(is-bundle-supported "${manifest_dir}")" != "true" ]; then + exit 1 + fi +} + +# Function to get all resource names for a particular kind +# from the manifest directory +collect_resource_names() { + local manifest_dir="${1}" + local kind="${2}" + local resource_names=() + while IFS= read -r resource_file; do + name=$(yq eval -r '.metadata.name' "$resource_file") + if [ -n "$name" ]; then + resource_names+=("$name") + fi + done < <(find "${manifest_dir}" -type f -exec grep -l "^kind: ${kind}" {} \;) + echo "${resource_names[@]}" +} + +# Function that collects all the rules for all the ClusterRole manifests +# shipped with the bundle +collect_manifest_cluster_role_perms() { + local manifest_dir="${1}" + local kind="ClusterRole" + local all_cr_rules="[]" + + while IFS= read -r resource_file; do + # Extract the entire rules array from the current file and ensure it's treated as a valid JSON array + cr_rules=$(yq eval -o=json -r '.rules // []' "$resource_file") + # Validate and merge the current rules array with the cumulative rules array + if jq -e . >/dev/null 2>&1 <<<"$cr_rules"; then + all_cr_rules=$(jq -c --argjson existing "$all_cr_rules" --argjson new "$cr_rules" '$existing + $new' <<<"$all_cr_rules") + fi + done < <(find "${manifest_dir}" -type f -exec grep -l "^kind: ${kind}" {} \;) + echo "$all_cr_rules" +} + +# Function that collects all the rules for all the Role manifests +# shipped with the bundle +collect_manifest_role_perms() { + local manifest_dir="${1}" + local kind="Role" + local all_cr_rules="[]" + + while IFS= read -r resource_file; do + # Extract the entire rules array from the current file and ensure it's treated as a valid JSON array + cr_rules=$(yq eval -o=json -r '.rules // []' "$resource_file") + # Validate and merge the current rules array with the cumulative rules array + if jq -e . >/dev/null 2>&1 <<<"$cr_rules"; then + all_cr_rules=$(jq -c --argjson existing "$all_cr_rules" --argjson new "$cr_rules" '$existing + $new' <<<"$all_cr_rules") + fi + done < <(find "${manifest_dir}" -type f -exec grep -l "^kind: ${kind}" {} \;) + echo "$all_cr_rules" +} + +# Function to get the apiGroup for a named resource of a given kind +# from the manifests dir +get_api_group() { + local dir_path="$1" + local kind="$2" + local name="$3" + + # Find the file containing the specified kind and name + local file + file=$(grep -rl "kind: $kind" "$dir_path" | xargs grep -l "name: $name") + + # Extract the apiGroup from the found file + if [ -n "$file" ]; then + local api_group + api_group=$(yq eval '.apiVersion' "$file" | awk -F'/' '{print $1}') + echo "$api_group" + fi +} + +# Function to get the generated clusterrole resource names +generated_cluster_role_names() { + local csvFile="${1}" + local generated_cluster_role_names=() + csv_name=$(yq eval -r '.metadata.name' "${csvFile}") + cperms=$(yq eval -o=json -r '.spec.install.spec.clusterPermissions? // []' "$csvFile" | jq -c '.[] | {serviceAccountName, rules: [.rules[] | {verbs, apiGroups, resources, resourceNames, nonResourceURLs} | with_entries(select(.value != null and .value != []))]}') + rbacPerms=$(yq eval -o=json -r '.spec.install.spec.permissions? // []' "$csvFile" | jq -c '.[] | {serviceAccountName, rules: [.rules[] | {verbs, apiGroups, resources, resourceNames, nonResourceURLs} | with_entries(select(.value != null and .value != []))]}') + allPerms=("${cperms[@]}" "${rbacPerms[@]}") + for perm in "${allPerms[@]}"; do + sa=$(echo "$perm" | yq eval -r '.serviceAccountName') + generated_name="$(generate_name "${csv_name}-${sa}" "${perm}")" + generated_cluster_role_names+=("${generated_name}") + done + echo "${generated_cluster_role_names[@]}" +} + +# Get CSV from manifest directory +find_csv() { + local manifest_dir="${1}" + local csv_files + + # Use grep -l to find files containing "kind: ClusterServiceVersion" + csv_files=$(grep -l "kind: ClusterServiceVersion" "$manifest_dir"/*.yaml) + + # Check if multiple CSV files are found + if [ "$(echo "$csv_files" | wc -l)" -gt 1 ]; then + echo "Error: Multiple CSV files found in ${manifest_dir}." + return 1 + elif [ -z "$csv_files" ]; then + echo "Error: No CSV file found in ${manifest_dir}." + return 1 + else + echo "${csv_files}" + fi +} \ No newline at end of file diff --git a/hack/tools/catalogs/lib/collect-rbac.sh b/hack/tools/catalogs/lib/collect-rbac.sh new file mode 100644 index 000000000..33ed54da3 --- /dev/null +++ b/hack/tools/catalogs/lib/collect-rbac.sh @@ -0,0 +1,144 @@ +# Library of functions for collecting RBAC from manifests for the purposes of generating +# The required RBAC for the cluster extension installation +source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/rbac.sh" +source "$(dirname "${BASH_SOURCE[0]}")/manifests.sh" +source "$(dirname "${BASH_SOURCE[0]}")/bundle.sh" + +# Function to add the specified rules +add_required_rules() { + local finalizer_perm + finalizer_perm=$(make_rbac_rule "olm.operatorframework.io" "clusterextensions/finalizers" '"update"' "$EXTENSION_NAME") + aggregate_rules "${finalizer_perm}" "cluster" +} + +collect_crd_rbac() { + debug "Collecting CRD permissions" + local csv="${1}" + crds=() + while IFS=$'\n' read -r crd; do + crds+=("$crd") + done < <(yq eval -o=json -r '.spec.customresourcedefinitions.owned[]?.name' "$csv") + add_rbac_rules "apiextensions.k8s.io" "customresourcedefinitions" "cluster" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${crds[@]}" +} + +collect_cluster_role_rbac() { + local manifest_dir="${1}" + local csv="${2}" + debug "Adding ClusterRole RBAC" + + # Collect shipped ClusterRole names + # And the OLMv1 generated ClusterRole names + read -ra manifest_cluster_role_names <<< "$(collect_resource_names "${manifest_dir}" "ClusterRole")" + read -ra generated_cluster_role_names <<< "$(generated_cluster_role_names "${csv}")" + all_cluster_role_names=("${manifest_cluster_role_names[@]}" "${generated_cluster_role_names[@]}") + add_rbac_rules "rbac.authorization.k8s.io" "clusterroles" "cluster" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${all_cluster_role_names[@]}" + + # Add all rules for defined in shipped ClusterRoles + # This allows the installer service account to grant rbac + manifest_cr_perms="$(collect_manifest_cluster_role_perms "${manifest_dir}")" + for item in $(echo "$manifest_cr_perms" | jq -c -r '.[]'); do + aggregate_rules "${item}" "cluster" + done + + debug "Adding ClusterPermissions" + # Add all cluster scoped rules for defined in the CSV + # This allows the installer service account to grant rbac + cluster_permissions=$(yq eval -o=json '.spec.install.spec.clusterPermissions[].rules?' "$csv" | jq -c '.[]') + for perm in ${cluster_permissions}; do + aggregate_rules "${perm}" "cluster" + done + + # Collect RBAC for cluster scoped manifest objects + collect_cluster_scoped_resource_rbac "${manifest_dir}" + + debug "Adding ClusterRoleBinding RBAC" + # Collect shipped ClusterRoleBinding names + # And the OLMv1 generated ClusterRoleBinding names (same as the generated ClusterRole names) + read -ra manifest_cluster_role_binding_names <<< "$(collect_resource_names "${manifest_dir}" "ClusterRoleBinding")" + all_cluster_role_binding_names=("${manifest_cluster_role_binding_names[@]}" "${generated_cluster_role_names[@]}") + add_rbac_rules "rbac.authorization.k8s.io" "clusterrolebindings" "cluster" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${all_cluster_role_binding_names[@]}" +} + +collect_cluster_scoped_resource_rbac() { + debug "Adding other ClusterScoped Resources" + local manifest_dir="${1}" + for kind in "${CLUSTER_SCOPED_RESOURCES[@]}"; do + read -ra resource_names <<< "$(collect_resource_names "${manifest_dir}" "${kind}")" + if [ ${#resource_names[@]} -eq 0 ]; then + continue + fi + api_group=$(get_api_group "${manifest_dir}" "${kind}" "${resource_names[1]}") + add_rbac_rules "${api_group}" "${kind,,}s" "cluster" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${resource_names[@]}" + done +} + +collect_operator_deployment_rbac() { + local manifest_dir="${1}" + local csv="${2}" + + debug "Adding Deployment RBAC" + read -ra manifest_dep_names <<< "$(collect_resource_names "${manifest_dir}" "Deployment")" + read -ra csv_deployments <<< "$(yq eval -o=json -r '.spec.install.spec.deployments[]?.name' "$csv")" + all_deployments=("${manifest_dep_names[@]}" "${csv_deployments[@]}") + add_rbac_rules "apps" "deployments" "namespace" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${all_deployments[@]}" +} + +collect_service_account_rbac() { + debug "Adding ServiceAccount RBAC" + local manifest_dir="${1}" + local csv="${2}" + read -ra manifest_sas <<< "$(collect_resource_names "${manifest_dir}" "ServiceAccount")" + read -ra csv_sas <<< "$(yq eval '.. | select(has("serviceAccountName")) | .serviceAccountName' "${csv}" | sort -u)" + all_sas=("${manifest_sas[@]}" "${csv_sas[@]}") + add_rbac_rules "" "serviceaccounts" "namespace" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${all_sas[@]}" +} + +collect_role_rbac() { + debug "Collecting Role RBAC" + local manifest_dir="${1}" + local csv="${2}" + + # Shipped Role manifest permissions + manifest_role_perms="$(collect_manifest_role_perms "${manifest_dir}")" + for item in $(echo "$manifest_role_perms" | jq -c -r '.[]'); do + aggregate_rules "${item}" "namespace" + done + + # CSV namespaced permissions + namespace_permissions=$(yq eval -o=json '.spec.install.spec.permissions[].rules?' "$csv" | jq -c '.[]') + for perm in ${namespace_permissions}; do + aggregate_rules "${perm}" "cluster" + done + + # Account for all other shipped namespace scoped resources + for kind in "${NAMESPACE_SCOPED_RESOURCES[@]}"; do + read -ra resource_names <<< "$(collect_resource_names "${manifest_dir}" "${kind}")" + if [ ${#resource_names[@]} -eq 0 ]; then + continue + fi + api_group=$(get_api_group "${manifest_dir}" "${kind}" "${resource_names[@]}") + add_rbac_rules "${api_group}" "${kind,,}s" "namespace" "${ALL_RESOURCE_VERBS}" "${NAMED_RESOURCE_VERBS}" "${resource_names[@]}" + done +} + +# Expects a supported bundle +collect_installer_rbac() { + local manifest_dir="${1}" + csv="$(find_csv "${manifest_dir}")" + + echo "Collecting RBAC from bundle (manifest_dir:${manifest_dir} csv: ${csv})" >&2 + + # Ensure bundle is supported by olmv1 + assert-bundle-supported "${manifest_dir}" + + # Add the required permissions rules + add_required_rules + + # Grab CSV name + collect_crd_rbac "${csv}" + collect_cluster_role_rbac "${manifest_dir}" "${csv}" + collect_operator_deployment_rbac "${manifest_dir}" "${csv}" + collect_service_account_rbac "${manifest_dir}" "${csv}" + collect_role_rbac "${manifest_dir}" "${csv}" +} diff --git a/hack/tools/catalogs/lib/hash.sh b/hack/tools/catalogs/lib/hash.sh new file mode 100644 index 000000000..deccb7e8d --- /dev/null +++ b/hack/tools/catalogs/lib/hash.sh @@ -0,0 +1,63 @@ +# Library of functions that reproduce hashing behavior in internal/rukpak/convert/registryv1.go:generate_name() +# This is used to generate unique names for the ClusterRoles generated by O-C during bundle installation + +base36encode() { + if [[ -z "$1" || ! "$1" =~ ^[0-9a-fA-F]+$ ]]; then + echo "Invalid input. Please provide a hexadecimal number." + return 1 + fi + + # Convert hexadecimal to decimal + local bigint + bigint=$(echo "ibase=16; $1" | bc) + + # Convert decimal to base 36 + local base36="" + while [ "$(echo "$bigint > 0" | bc)" -eq 1 ]; do + remainder=$(echo "$bigint % 36" | bc) + bigint=$(echo "$bigint / 36" | bc) + + if [ "$remainder" -lt 10 ]; then + base36="${remainder}${base36}" + else + # Convert remainder (10-35) to corresponding ASCII letter ('a'-'z') + val=$((remainder + 87)) + char=$(echo "$val" | awk '{printf "%c", $1}') + base36="${char}${base36}" + fi + done + echo "$base36" +} + +deep_hash_object() { + local obj="$1" + + # Compute SHA-224 hash and convert to uppercase for consistency + local hash_hex + hash_hex=$(echo "$obj" | sha224sum | awk '{print toupper($1)}') + + # Encode the hash to base36 + local base36_hash + base36_hash=$(base36encode "${hash_hex}") + echo "${base36_hash}" +} + +# Function to generate a name based on the base string and hashed object +generate_name() { + local base="$1" + local obj="$2" + local max_name_length=63 # Define the maximum name length (similar to DNS limits) + + # Generate a hash from the object using deep_hash_object function + local hash_str + hash_str=$(deep_hash_object "$obj") + + # Check if the combined length exceeds the maximum length + if [ $((${#base} + ${#hash_str})) -gt $max_name_length ]; then + # Truncate the base string + base="${base:0:$((max_name_length - ${#hash_str} - 1))}" + fi + + # Return the concatenated string + echo "${base}-${hash_str}" +} \ No newline at end of file diff --git a/hack/tools/catalogs/lib/manifests.sh b/hack/tools/catalogs/lib/manifests.sh new file mode 100644 index 000000000..55a448f55 --- /dev/null +++ b/hack/tools/catalogs/lib/manifests.sh @@ -0,0 +1,121 @@ +# Library of functions for generating kube manifests + +# Function to generate the target install namespace +generate_namespace() { + cat <&2 + container_id=$($CONTAINER_RUNTIME create --quiet --entrypoint="/bin/bash" "$image") + if [ -z "$container_id" ]; then + echo "Failed to create container from image '$image'." >&2 + exit 1 + fi + + echo "${container_id}" +} + +unpack() { + local container_id="${1}" + local output_dir="${2}" + + # Extract the directory from the "operators.operatorframework.io.bundle.manifests.v1" label + local manifest_dir + echo "Unpacking bundle to ${output_dir}" >&2 + manifest_dir=$($CONTAINER_RUNTIME inspect --format '{{ index .Config.Labels "operators.operatorframework.io.bundle.manifests.v1" }}' "$container_id") + + if [ -z "$manifest_dir" ]; then + echo "No manifest directory label found on the image." + $CONTAINER_RUNTIME rm "$container_id" + exit 1 + fi + + # Copy files from the container to a temporary directory + $CONTAINER_RUNTIME cp "$container_id:$manifest_dir/." "$output_dir" > /dev/null + + # Clean up the container + $CONTAINER_RUNTIME rm "$container_id" > /dev/null +} \ No newline at end of file diff --git a/hack/tools/catalogs/lib/utils.sh b/hack/tools/catalogs/lib/utils.sh new file mode 100644 index 000000000..bebbf27cb --- /dev/null +++ b/hack/tools/catalogs/lib/utils.sh @@ -0,0 +1,49 @@ +# Library of utility functions + +debug() { + if [ "${DEBUG,,}" != "false" ] && [ -n "$DEBUG" ]; then + echo "DEBUG: $1" >&2 + fi +} + +print-banner() { + local red='\033[91m' + local white='\033[97m' + local reset='\033[0m' + local green='\033[92m' + + echo -e "${white}===========================================================================================================================${reset}" + echo -e "${white}‖${red} ____ __ ______ __ ${white}‖${reset}" + echo -e "${white}‖${red} / __ \ ____ ___ _____ ____ _ / /_ ____ _____ / ____/_____ ____ _ ____ ___ ___ _ __ ____ _____ / /__ ${white}‖${reset}" + echo -e "${white}‖${red} / / / // __ \ / _ \ / ___// __ \`// __// __ \ / ___/ / /_ / ___// __ \`// __ \`__ \ / _ \| | /| / // __ \ / ___// //_/ ${white}‖${reset}" + echo -e "${white}‖${red} / /_/ // /_/ // __// / / /_/ // /_ / /_/ // / / __/ / / / /_/ // / / / / // __/| |/ |/ // /_/ // / / ,< ${white}‖${reset}" + echo -e "${white}‖${red} \____// .___/ \___//_/ \__,_/ \__/ \____//_/ /_/ /_/ \__,_//_/ /_/ /_/ \___/ |__/|__/ \____//_/ /_/|_| ${white}‖${reset}" + echo -e "${white}‖${red} /_/${green} OLM v1 ${white}‖${reset}" + echo -e "${white}===========================================================================================================================${reset}" +} + +assert-commands() { + for cmd in "$@"; do + if ! command -v "$cmd" &>/dev/null; then + echo "Required command '$cmd' not found in PATH" >&2 + exit 1 + fi + done +} + +assert-container-runtime() { + if [ -z "$CONTAINER_RUNTIME" ]; then + if command -v podman &>/dev/null; then + export CONTAINER_RUNTIME="podman" + elif command -v docker &>/dev/null; then + export CONTAINER_RUNTIME="docker" + else + echo "No container runtime found in PATH. If not using docker or podman, please set the CONTAINER_RUNTIME environment variable to your container runtime" >&2 + exit 1 + fi + fi + if ! command -v "$CONTAINER_RUNTIME" &>/dev/null; then + echo "Configured container runtime '$CONTAINER_RUNTIME' not found in PATH" >&2 + exit 1 + fi +} \ No newline at end of file diff --git a/hack/tools/catalogs/list-compatible-bundles b/hack/tools/catalogs/list-compatible-bundles new file mode 100755 index 000000000..8e0560414 --- /dev/null +++ b/hack/tools/catalogs/list-compatible-bundles @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Get the directory of the current script +SCRIPT_ROOT=$(dirname "$(realpath "$0")") +source "${SCRIPT_ROOT}/lib/utils.sh" + +# Check required tools are installed +assert-commands jq + +usage() { + print-banner + echo "" + echo "Usage: $0 [-r ] < " + echo "" + echo "Filter Catalog FBC in stdin for OLMv1 (potentially) supported bundles" + echo "" + echo "Examples:" + echo " # Filter for all OLMv1 (potentially) supported bundles" + echo " $0 < operatorhubio-catalog.json" + echo "" + echo " # Filter for all OLMv1 (potentially) supported bundles that contain argocd in the package name" + echo " $0 -r argocd < operatorhubio-catalog.json" + echo "" + echo "NOTE: OLM v1 currently only supports bundles that fulfill the following criteria: " + echo " - Support AllNamespaces install model" + echo " - Don't have dependencies" + echo " - Don't contain webhooks" + echo " - Don't interact with the OperatorConditions API (only verifiable at runtime)" + echo "WARNING: These results may include bundles with webhooks, which are incompatible" +} + +# Parse the optional regex argument +REGEX="" +while getopts "r:" opt; do + case ${opt} in + r ) + REGEX=$OPTARG + ;; + \? ) + usage + exit 1 + ;; + esac +done +shift $((OPTIND -1)) + +# Select bundle documents +function select-bundle-documents() { + jq 'select(.schema == "olm.bundle")' +} + +# Select bundles that declare AllNamespace install mode +# or declare nothing at all (older released bundles sans "olm.csv.metadata" property) +function that-support-allnamespace-install-mode() { + jq 'select( + all(.properties[].type; . != "olm.csv.metadata") or + (.properties[]? | + select(.type == "olm.csv.metadata" and + (.value.installModes[]? | + select(.type == "AllNamespaces" and .supported == true) + ) + ) + ) + )' +} + +# Select bundles without dependencies +function that-dont-have-dependencies() { + jq 'select(all(.properties[].type; . != "olm.package.required" and . != "olm.gvk.required"))' +} + +# Select the "olm.package" property from the bundles +# This contains the packageName and version information +function extract-olm-package-property() { + jq '.properties[] | select(.type == "olm.package") |.value' +} + +# Group packages by name and collect versions into an array +function group-versions-by-package-name() { + jq -s 'group_by(.packageName) | map({packageName: .[0].packageName, versions: map(.version)})' | regex_it +} + +# Apply regex on name +function filter-by-regex-if-necessary() { + jq --arg regex "$REGEX" ' + if $regex != "" then + map(select(.packageName | test($regex))) + else + . + end' +} + +cat - | select-bundle-documents | that-support-allnamespace-install-mode | that-dont-have-dependencies | group-versions-by-package-name | filter-by-regex-if-necessary + +echo "NOTE: OLM v1 currently only supports bundles that support AllNamespaces install model, don't have dependencies, and don't contain webhooks" >&2 +echo "WARNING: These results may include bundles with webhooks, which are incompatible" >&2 \ No newline at end of file diff --git a/hack/tools/catalogs/unpack-bundle b/hack/tools/catalogs/unpack-bundle new file mode 100755 index 000000000..684c90d48 --- /dev/null +++ b/hack/tools/catalogs/unpack-bundle @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Get the directory of the current script +SCRIPT_ROOT=$(dirname "$(realpath "$0")") + +source "${SCRIPT_ROOT}/lib/unpack.sh" +source "${SCRIPT_ROOT}/lib/utils.sh" + +# Check there is a container runtime (podman, or docker) +# If neither are found, check CONTAINER_RUNTIME is set and exists in PATH +assert-container-runtime + +usage() { + print-banner + echo "" + echo "Usage: $0 [-o output_directory] bundle_image" + echo "" + echo "Unpack a bundle image to a directory``" + echo "" + echo "Examples:" + echo " # Unpack argcocd-operator bundle image to a temporary directory" + echo " $0 quay.io/operatorhubio/argocd-operator@sha256:d538c45a813b38ef0e44f40d279dc2653f97ca901fb660da5d7fe499d51ad3b3" + echo "" + echo " # Unpack the bundle image to a specific directory" + echo " $0 -o argocd-manifests quay.io/operatorhubio/argocd-operator@sha256:d538c45a813b38ef0e44f40d279dc2653f97ca901fb660da5d7fe499d51ad3b3" +} + +# Initialize variables +output_directory=$(mktemp -d) +bundle_image="" + +# Parse command line arguments +while getopts "o:" opt; do + case ${opt} in + o) + output_directory=${OPTARG} + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND -1)) + +# Check if bundle image argument is provided or read from stdin +if [ -z "$1" ]; then + if [ -t 0 ]; then + echo "Error: Docker image name is required." + usage + else + read -r bundle_image + fi +else + bundle_image="$1" +fi + +if [ -z "${bundle_image}" ]; then + echo "Error: Docker image name is required." + usage + exit 1 +fi + +# Validate output directory +if [ ! -d "$output_directory" ]; then + mkdir -p "$output_directory" +fi + +# Unpack the bundle image +container_id="$(create_container "$bundle_image")" +unpack "${container_id}" "${output_directory}" + +echo "${output_directory}" \ No newline at end of file