packaging: add options to parameterise server/key for testing (#5444)

* packaging: add options to parameterise server/key for testing

Signed-off-by: Patrick Stephens <[email protected]>

* codeowners: update for install script

Signed-off-by: Patrick Stephens <[email protected]>

* workflows: reduce noise from CI updates

Signed-off-by: Patrick Stephens <[email protected]>

* packaging: update test script to handle new URLs

Signed-off-by: Patrick Stephens <[email protected]>

* packaging: fix linting failure

Signed-off-by: Patrick Stephens <[email protected]>

* packaging: fix typo in test script

Signed-off-by: Patrick Stephens <[email protected]>

* packaging: switch to fluent-bit service

Signed-off-by: Patrick Stephens <[email protected]>

* packaging: fix linting issue

Signed-off-by: Patrick Stephens <[email protected]>

Author: Pat

.github/workflows/docs-generate.yaml

@@ -8,6 +8,7 @@ on:
     # Do not re-trigger when our own PRs are merged
     paths-ignore:
       - codebase-structure.svg
+      - .github/
 jobs:
   update_diagram:
     name: Update the codebase structure diagram

CODEOWNERS

@@ -23,6 +23,7 @@
 /dockerfiles/            @niedbalski @patrick-stephens
 /packaging/              @niedbalski @patrick-stephens
 /codebase-structure.svg  @niedbalski @patrick-stephens
+/install.sh              @niedbalski @patrick-stephens
 
 # Core: Signv4
 # ------------

install.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 set -e
 
+# Provided primarily to simplify testing for staging, etc.
+RELEASE_URL=${FLUENT_BIT_PACKAGES_URL:-https://packages.fluentbit.io}
+RELEASE_KEY=${FLUENT_BIT_PACKAGES_KEY:-https://packages.fluentbit.io/fluentbit.key}
+
 echo "================================"
 echo " Fluent Bit Installation Script "
 echo "================================"
@@ -31,29 +35,29 @@ sudo -k
 case ${OS} in
     amzn|amazonlinux)
         sudo sh <<'SCRIPT'
-rpm --import https://packages.fluentbit.io/fluentbit.key
+rpm --import $RELEASE_KEY
 cat > /etc/yum.repos.d/fluent-bit.repo <<EOF
 [fluent-bit]
 name = Fluent Bit
-baseurl = https://packages.fluentbit.io/amazonlinux/\$releasever/\$basearch/
+baseurl = $RELEASE_URL/amazonlinux/\$releasever/\$basearch/
 gpgcheck=1
 repo_gpgcheck=1
-gpgkey=https://packages.fluentbit.io/fluentbit.key
+gpgkey=$RELEASE_KEY
 enabled=1
 EOF
 yum -y install fluent-bit || yum -y install td-agent-bit
 SCRIPT
     ;;
     centos|centoslinux|rhel|redhatenterpriselinuxserver|fedora|rocky|almalinux)
         sudo sh <<'SCRIPT'
-rpm --import https://packages.fluentbit.io/fluentbit.key
+rpm --import $RELEASE_KEY
 cat > /etc/yum.repos.d/fluent-bit.repo <<EOF
 [fluent-bit]
 name = Fluent Bit
-baseurl = https://packages.fluentbit.io/centos/\$releasever/\$basearch/
+baseurl = $RELEASE_URL/centos/\$releasever/\$basearch/
 gpgcheck=1
 repo_gpgcheck=1
-gpgkey=https://packages.fluentbit.io/fluentbit.key
+gpgkey=$RELEASE_KEY
 enabled=1
 EOF
 yum -y install fluent-bit || yum -y install td-agent-bit
@@ -65,9 +69,9 @@ SCRIPT
         sudo sh <<SCRIPT
 export DEBIAN_FRONTEND=noninteractive
 mkdir -p /usr/share/keyrings/
-curl https://packages.fluentbit.io/fluentbit.key | gpg --dearmor > /usr/share/keyrings/fluentbit-keyring.gpg
+curl $RELEASE_KEY | gpg --dearmor > /usr/share/keyrings/fluentbit-keyring.gpg
 cat > /etc/apt/sources.list.d/fluent-bit.list <<EOF
-deb [signed-by=/usr/share/keyrings/fluentbit-keyring.gpg] https://packages.fluentbit.io/${OS}/${CODENAME} ${CODENAME} main
+deb [signed-by=/usr/share/keyrings/fluentbit-keyring.gpg] $RELEASE_URL/${OS}/${CODENAME} ${CODENAME} main
 EOF
 apt-get -y update
 apt-get -y install fluent-bit || apt-get -y install td-agent-bit

packaging/test-release-packages.sh

@@ -1,10 +1,40 @@
 #!/bin/bash
 set -eux
-# Verify package install for a latest release version
-docker run --rm -it ubuntu:20.04 sh -c "apt-get update && apt-get install -y sudo gpg curl;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
-docker run --rm -it ubuntu:18.04 sh -c "apt-get update && apt-get install -y sudo gpg curl;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
-docker run --rm -it debian:10 sh -c "apt-get update && apt-get install -y sudo gpg curl;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
-docker run --rm -it debian:11 sh -c "apt-get update && apt-get install -y sudo gpg curl;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
-docker run --rm -it centos:7 sh -c "yum install -y curl sudo;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
-docker run --rm -it rockylinux:8 sh -c "yum install -y curl sudo;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
-docker run --rm -it amazonlinux:2 sh -c "yum install -y curl sudo;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# Verify package install for a release version
+
+if [[ -f "$SCRIPT_DIR/.env" ]]; then
+    # shellcheck disable=SC1091
+    source "$SCRIPT_DIR/.env"
+fi
+
+CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker}
+
+APT_TARGETS=("ubuntu:18.04"
+    "ubuntu:20.04"
+    "debian:10"
+    "debian:11")
+
+YUM_TARGETS=("centos:7"
+    "rockylinux:8"
+    "amazonlinux:2")
+
+for IMAGE in "${APT_TARGETS[@]}"
+do
+    echo "Testing $IMAGE"
+    $CONTAINER_RUNTIME run --rm -it \
+        -e FLUENT_BIT_PACKAGES_URL="${FLUENT_BIT_PACKAGES_URL:-https://packages.fluentbit.io}" \
+        -e FLUENT_BIT_PACKAGES_KEY="${FLUENT_BIT_PACKAGES_KEY:-https://packages.fluentbit.io/fluentbit.key}" \
+        "$IMAGE" \
+        sh -c "apt-get update && apt-get install -y sudo gpg curl;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
+done
+
+for IMAGE in "${YUM_TARGETS[@]}"
+do
+    echo "Testing $IMAGE"
+    $CONTAINER_RUNTIME run --rm -it \
+        -e FLUENT_BIT_PACKAGES_URL="${FLUENT_BIT_PACKAGES_URL:-https://packages.fluentbit.io}" \
+        -e FLUENT_BIT_PACKAGES_KEY="${FLUENT_BIT_PACKAGES_KEY:-https://packages.fluentbit.io/fluentbit.key}" \
+        "$IMAGE" \
+        sh -c "yum install -y curl sudo;curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh"
+done

packaging/testing/smoke/packages/Dockerfile.amazonlinux2

@@ -1,8 +1,16 @@
 # For staging upgrade we use the 'staging-upgrade-prep' as the base
 ARG STAGING_BASE=dokken/amazonlinux-2
 
+ARG RELEASE_URL=https://packages.fluentbit.io
+ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key
+
 FROM dokken/amazonlinux-2 as official-install
+
 ARG RELEASE_URL
+ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL}
+
+ARG RELEASE_KEY
+ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
@@ -12,15 +20,21 @@ COPY ./test.sh /test.sh
 RUN chmod a+x /test.sh
 
 FROM official-install as staging-upgrade-prep
-RUN rm -f /etc/yum.repos.d/td-agent-bit.repo
+RUN rm -f /etc/yum.repos.d/*-bit.repo
 
 FROM ${STAGING_BASE} as staging-install
-ARG STAGING_URL
 ARG STAGING_VERSION
 ENV STAGING_VERSION=${STAGING_VERSION}
+
+ARG STAGING_URL
+ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL}
+
+ARG STAGING_KEY=${STAGING_URL}/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY}
+
 # hadolint ignore=DL3032
-RUN rpm --import "$STAGING_URL/fluentbit.key" && \
-    wget -q "$STAGING_URL/amazonlinux-2.repo" -O /etc/yum.repos.d/staging.repo && \
+RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \
+    wget -q "$FLUENT_BIT_PACKAGES_URL/amazonlinux-2.repo" -O /etc/yum.repos.d/staging.repo && \
     yum update -y && yum install -y fluent-bit && \
     systemctl enable fluent-bit
 

packaging/testing/smoke/packages/Dockerfile.centos7

@@ -1,7 +1,17 @@
 # For staging upgrade we use the 'staging-upgrade-prep' as the base
 ARG STAGING_BASE=dokken/centos-7
 
+ARG RELEASE_URL=https://packages.fluentbit.io
+ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key
+
 FROM dokken/centos-7 as official-install
+
+ARG RELEASE_URL
+ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL}
+
+ARG RELEASE_KEY
+ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY}
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # Use the one-line install
 RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
@@ -14,11 +24,17 @@ FROM official-install as staging-upgrade-prep
 RUN rm -f /etc/yum.repos.d/*-bit.repo
 
 FROM ${STAGING_BASE} as staging-install
-ARG STAGING_URL
 ARG STAGING_VERSION
 ENV STAGING_VERSION=${STAGING_VERSION}
-RUN rpm --import "$STAGING_URL/fluentbit.key" && \
-    wget -nv "$STAGING_URL/centos-7.repo" -O /etc/yum.repos.d/staging.repo
+
+ARG STAGING_URL
+ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL}
+
+ARG STAGING_KEY=${STAGING_URL}/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY}
+
+RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \
+    wget -nv "$FLUENT_BIT_PACKAGES_URL/centos-7.repo" -O /etc/yum.repos.d/staging.repo
 # hadolint ignore=DL3032
 RUN yum update -y && yum install -y fluent-bit && \
     systemctl enable fluent-bit

packaging/testing/smoke/packages/Dockerfile.centos8

@@ -1,13 +1,19 @@
 # For staging upgrade we use the 'staging-upgrade-prep' as the base
 ARG STAGING_BASE=dokken/centos-8
 
+ARG RELEASE_URL=https://packages.fluentbit.io
+ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key
+
 FROM dokken/centos-8 as official-install
 # CentOS is now EOL so have to use the vault repos
 RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
     sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
 
 ARG RELEASE_URL
-RUN rpm --import $RELEASE_URL/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL}
+
+ARG RELEASE_KEY
+ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
@@ -17,18 +23,24 @@ COPY ./test.sh /test.sh
 RUN chmod a+x /test.sh
 
 FROM official-install as staging-upgrade-prep
-RUN rm -f /etc/yum.repos.d/td-agent-bit.repo
+RUN rm -f /etc/yum.repos.d/*-bit.repo
 
 FROM ${STAGING_BASE} as staging-install
 # CentOS is now EOL so have to use the vault repos
 RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
     sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
 
-ARG STAGING_URL
 ARG STAGING_VERSION
 ENV STAGING_VERSION=${STAGING_VERSION}
-RUN rpm --import "$STAGING_URL/fluentbit.key" && \
-    wget -nv "$STAGING_URL/centos-8.repo" -O /etc/yum.repos.d/staging.repo
+
+ARG STAGING_URL
+ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL}
+
+ARG STAGING_KEY=${STAGING_URL}/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY}
+
+RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \
+    wget -nv "$FLUENT_BIT_PACKAGES_URL/centos-8.repo" -O /etc/yum.repos.d/staging.repo
 # hadolint ignore=DL3032
 RUN yum update -y && yum install -y fluent-bit && \
     systemctl enable fluent-bit

packaging/testing/smoke/packages/Dockerfile.debian10

@@ -1,28 +1,44 @@
 # For staging upgrade we use the 'official-install' as the base
 ARG STAGING_BASE=dokken/debian-10
 
+ARG RELEASE_URL=https://packages.fluentbit.io
+ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key
+
 FROM dokken/debian-10 as official-install
+
 ARG RELEASE_URL
-RUN wget -qO - $RELEASE_URL/fluentbit.key | apt-key add -
-RUN echo "deb $RELEASE_URL/debian/buster buster main" >> /etc/apt/sources.list
-RUN apt-get update && apt-get install -y td-agent-bit
-RUN systemctl enable td-agent-bit
+ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL}
+
+ARG RELEASE_KEY
+ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY}
+
+# Use the one-line install
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
+RUN systemctl enable fluent-bit || systemctl enable td-agent-bit
 
 COPY ./test.sh /test.sh
 RUN chmod a+x /test.sh
 
 FROM official-install as staging-upgrade-prep
-RUN head -n -1 /etc/apt/sources.list > /tmp/sources.list && mv /tmp/sources.list /etc/apt/sources.list
+RUN rm -f /etc/apt/sources.list.d/fluent-bit.list
 
 FROM ${STAGING_BASE} as staging-install
-ARG STAGING_URL
 ARG STAGING_VERSION
 ENV STAGING_VERSION=${STAGING_VERSION}
+
+ARG STAGING_URL
+ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL}
+
+ARG STAGING_KEY=${STAGING_URL}/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY}
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN wget -qO - $STAGING_URL/fluentbit.key | apt-key add -
-RUN echo "deb $STAGING_URL/debian/buster buster main" >> /etc/apt/sources.list
-RUN apt-get update && apt-get install -y td-agent-bit
-RUN systemctl enable td-agent-bit
+RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add -
+RUN echo "deb $FLUENT_BIT_PACKAGES_URL/debian/buster buster main" >> /etc/apt/sources.list
+# hadolint ignore=DL3015,DL3008,DL3009
+RUN apt-get update && apt-get install -y fluent-bit
+RUN systemctl enable fluent-bit
 
 COPY ./test.sh /test.sh
 RUN chmod a+x /test.sh

packaging/testing/smoke/packages/Dockerfile.debian11

@@ -1,8 +1,17 @@
 # For staging upgrade we use the 'official-install' as the base
 ARG STAGING_BASE=dokken/debian-11
 
+ARG RELEASE_URL=https://packages.fluentbit.io
+ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key
+
 FROM dokken/debian-11 as official-install
+
 ARG RELEASE_URL
+ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL}
+
+ARG RELEASE_KEY
+ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY}
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
 RUN systemctl enable fluent-bit || systemctl enable td-agent-bit
@@ -14,14 +23,21 @@ FROM official-install as staging-upgrade-prep
 RUN rm -f /etc/apt/sources.list.d/fluent-bit.list
 
 FROM ${STAGING_BASE} as staging-install
-ARG STAGING_URL
 ARG STAGING_VERSION
 ENV STAGING_VERSION=${STAGING_VERSION}
+
+ARG STAGING_URL
+ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL}
+
+ARG STAGING_KEY=${STAGING_URL}/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY}
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN wget -qO - $STAGING_URL/fluentbit.key | apt-key add -
-RUN echo "deb $STAGING_URL/debian/bullseye bullseye main" >> /etc/apt/sources.list
-RUN apt-get update && apt-get install -y td-agent-bit
-RUN systemctl enable td-agent-bit
+RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add -
+RUN echo "deb $FLUENT_BIT_PACKAGES_URL/debian/bullseye bullseye main" >> /etc/apt/sources.list
+# hadolint ignore=DL3015,DL3008,DL3009
+RUN apt-get update && apt-get install -y fluent-bit
+RUN systemctl enable fluent-bit
 
 COPY ./test.sh /test.sh
 RUN chmod a+x /test.sh