STF 1.5.6 release

- sg-core is now built on ubi9
- Update to Golang to 1.21.13
- Github Actions now run on ubuntu-latest
- Pin Devstack version to stable/2024.2. New OpenStack versions deprecated the AMQP driver in oslo.messaging and this breaks sg-core testing
- Parse custom labels from ceilometer metadata. Metrics without the user_metadata field are unchanged (current behavior is preserved). When the user_metadata field is present in the metric received from ceilometer it's parsed and each key=value pair is used as a separate metric label

Author: vkmc

.github/workflows/integration.yml

@@ -1,7 +1,7 @@
 name: Integration testing
 env:
   PROJECT_ROOT: /root/go/src/github.com/infrawatch/sg-core
-  OPSTOOLS_REPO: https://raw.githubusercontent.com/infrawatch/sg-core/04dcb34edd2c234b378222d2f9a17e15c0dad936/build/repos/opstools.repo
+  OPSTOOLS_REPO: https://raw.githubusercontent.com/infrawatch/sg-core/0d28a1c2f87bd64f2d67effcda926855af057a82/build/repos/opstools.repo
 
   QDR_IMAGE: quay.io/interconnectedcloud/qdrouterd:1.17.0
   QDR_VOLUME: "--volume=${{ github.workspace }}/ci/service_configs/qdr:/etc/qpid-dispatch:ro"
@@ -10,7 +10,7 @@ env:
   BRIDGE_IMAGE: quay.io/infrawatch/sg-bridge:latest
   BRIDGE_VOLUME: "--volume=${{ github.workspace }}/tmp:/tmp/sg-bridge:z"
 
-  TEST_IMAGE: registry.access.redhat.com/ubi8
+  TEST_IMAGE: registry.access.redhat.com/ubi9
   TEST_PORT: "-p 3000:3000"
 
 on: [push, pull_request]
@@ -22,7 +22,7 @@ jobs:
     env:
       QDR_CHANNEL: collectd/metrics
       BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
-      PROMETHEUS_IMAGE: prom/prometheus:latest
+      PROMETHEUS_IMAGE: quay.io/prometheus/prometheus:latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.3
@@ -48,7 +48,7 @@ jobs:
       - name: Install devstack
         run: |
           SOURCE=$(pwd)
-          git clone http://github.com/openstack/devstack /opt/stack/devstack
+          git clone -b stable/2024.2 http://github.com/openstack/devstack /opt/stack/devstack
           pushd /opt/stack/devstack
           cp $SOURCE/ci/integration/metrics/local.conf .
           sudo apt-get update
@@ -62,8 +62,8 @@ jobs:
         id: bridge_branch
         continue-on-error: true
         run: |
-          echo "${GITHUB_REF#refs/heads/}"
-          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
+          echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}")"
       - name: Start sg-bridge from container image
         if: steps.bridge_branch.outcome != 'success'
         run: |
@@ -74,7 +74,7 @@ jobs:
         if: steps.bridge_branch.outcome == 'success'
         run: |
           docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
-            -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
+            -e GITHUB_HEAD_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
             --workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
             $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
       - name: Install collectd
@@ -118,7 +118,7 @@ jobs:
     env:
       QDR_CHANNEL: anycast/ceilometer/metering.sample
       BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
-      PROMETHEUS_IMAGE: prom/prometheus:latest
+      PROMETHEUS_IMAGE: quay.io/prometheus/prometheus:latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.3
@@ -144,7 +144,7 @@ jobs:
       - name: Install devstack
         run: |
           SOURCE=$(pwd)
-          git clone http://github.com/openstack/devstack /opt/stack/devstack
+          git clone -b stable/2024.2 http://github.com/openstack/devstack /opt/stack/devstack
           pushd /opt/stack/devstack
           cp $SOURCE/ci/integration/metrics/local.conf .
           sudo apt-get update
@@ -158,8 +158,8 @@ jobs:
         id: bridge_branch
         continue-on-error: true
         run: |
-          echo "${GITHUB_REF#refs/heads/}"
-          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
+          echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}")"
       - name: Start sg-bridge from container image
         if: steps.bridge_branch.outcome != 'success'
         run: |
@@ -170,7 +170,7 @@ jobs:
         if: steps.bridge_branch.outcome == 'success'
         run: |
           docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
-            -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
+            -e GITHUB_HEAD_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
             --workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
             $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
       - name: Set Ceilometer pipelines to QDR output and restart notification agent
@@ -218,7 +218,7 @@ jobs:
     name: "[metrics] transport: socket(tcp); handler: ceilometer-metrics; application: prometheus"
     runs-on: ubuntu-22.04
     env:
-      PROMETHEUS_IMAGE: prom/prometheus:latest
+      PROMETHEUS_IMAGE: quay.io/prometheus/prometheus:latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.3
@@ -244,7 +244,7 @@ jobs:
       - name: Install devstack
         run: |
           SOURCE=$(pwd)
-          git clone http://github.com/openstack/devstack /opt/stack/devstack
+          git clone -b stable/2024.2 http://github.com/openstack/devstack /opt/stack/devstack
           pushd /opt/stack/devstack
           cp $SOURCE/ci/integration/metrics/local.conf .
           sudo apt-get update
@@ -323,8 +323,8 @@ jobs:
         id: bridge_branch
         continue-on-error: true
         run: |
-          echo "${GITHUB_REF#refs/heads/}"
-          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
+          echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}")"
       - name: Start sg-bridge from container image
         if: steps.bridge_branch.outcome != 'success'
         run: |
@@ -334,7 +334,7 @@ jobs:
         if: steps.bridge_branch.outcome == 'success'
         run: |
           docker run --name=sgbridge $BRIDGE_VOLUME -d -uroot --network host -e OPSTOOLS_REPO \
-            -e GITHUB_REF -e BRIDGE_SOCKET --workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
+            -e GITHUB_HEAD_REF -e BRIDGE_SOCKET --workdir=$(dirname $BRIDGE_SOCKET) --volume ${{ github.workspace }}:$PROJECT_ROOT:z \
             $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_bridge.sh
       - name: Run rsyslog to produce log messages
         run: |

.github/workflows/tests.yml

@@ -1,7 +1,7 @@
 name: CI
 env:
   PROJECT_ROOT: /root/go/src/github.com/infrawatch/sg-core
-  OPSTOOLS_REPO: https://raw.githubusercontent.com/infrawatch/sg-core/04dcb34edd2c234b378222d2f9a17e15c0dad936/build/repos/opstools.repo
+  OPSTOOLS_REPO: https://raw.githubusercontent.com/infrawatch/sg-core/0d28a1c2f87bd64f2d67effcda926855af057a82/build/repos/opstools.repo
 
   LOKI_IMAGE: quay.io/infrawatch/loki:2.4.2
   LOKI_VOLUME: "--volume=${{ github.workspace }}/ci/service_configs/loki:/etc/loki:ro"
@@ -10,32 +10,32 @@ env:
   ELASTIC_IMAGE: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
   ELASTIC_PORT: "-p 9200:9200 -p 9300:9300"
 
-  TEST_IMAGE: registry.access.redhat.com/ubi8
+  TEST_IMAGE: registry.access.redhat.com/ubi9
   COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
 
 on: [push, pull_request]
 
 jobs:
   golangci:
     name: Linting
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.21.13'
       - uses: actions/checkout@v4.1.3
       #- name: download libraries
       #  run: go mod download
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6.1.0
         with:
           # Caching conflicts happen in GHA, so just disable for now
           skip-cache: true
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.51
+          version: v1.59.1
   unit-tests:
     name: Unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.3
@@ -72,12 +72,12 @@ jobs:
           path-to-profile: ${{ github.workspace }}/profile.cov
   image-build:
     name: Image build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.1.3
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.21.13'
       - name: Verify image builds
         run: |
           docker build --tag infrawatch/sg-core:latest --file build/Dockerfile .

.github/workflows/updates.yml

@@ -16,7 +16,7 @@ jobs:
 #        (github.event.issue.author_association == 'CONTRIBUTOR') ||
 #        (github.event.issue.author_association == 'MEMBER')
 #      )
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: update PR with coveralls badge
         uses: actions/github-script@v7.0.1

.golangci.yaml

@@ -1,11 +1,3 @@
-run:
-  skip-dirs:
-    - plugins/transport/dummy-alertmanager
-    - plugins/transport/dummy-events
-    - plugins/transport/dummy-metrics
-    - plugins/transport/dummy-logs
-    - plugins/application/print
-    - devenv
 issues:
   exclude-rules:
     - linters:
@@ -21,12 +13,26 @@ issues:
         - staticcheck
       # https://staticcheck.io/docs/checks#SA4008 (The variable in the loop condition never changes, are you incrementing the wrong variable?)
       text: "SA4008:"
-
+    # Don't warn on unused parameters.
+    # Parameter names are useful; replacing them with '_' is undesirable.
+    - linters: [revive]
+      text: 'unused-parameter: parameter \S+ seems to be unused, consider removing or renaming it as _'
+    - linters: [revive]
+      text: 'redefines-builtin-id: redefinition of the built-in function new'
+    - linters: [revive]
+      text: 'redefines-builtin-id: redefinition of the built-in function len'
+  exclude-dirs:
+    - plugins/transport/dummy-alertmanager
+    - plugins/transport/dummy-events
+    - plugins/transport/dummy-metrics
+    - plugins/transport/dummy-logs
+    - plugins/application/print
+    - devenv
 linters:
   disable-all: true
   enable:
     - bodyclose
-    - depguard
+    # - depguard
     - dogsled
     - dupl
     - errcheck

build/Dockerfile

@@ -1,19 +1,19 @@
 # --- build smart gateway ---
-FROM registry.access.redhat.com/ubi8:latest AS builder
+FROM registry.access.redhat.com/ubi9:latest AS builder
 ENV GOPATH=/go
 ENV D=/go/src/github.com/infrawatch/sg-core
 
 WORKDIR $D
 COPY . $D/
 
 # dependencies for qpid-proton-c
-COPY build/repos/opstools.repo /etc/yum.repos.d/opstools.repo
+COPY build/repos/opstools.repo /etc/yum.repos.d/CentOS-OpsTools.repo
 
 RUN dnf install golang git qpid-proton-c-devel -y --setopt=tsflags=nodocs
-RUN go install golang.org/dl/go1.20@latest && /go/bin/go1.20 download && PRODUCTION_BUILD=false CONTAINER_BUILD=true GOCMD=/go/bin/go1.20 ./build.sh
+RUN go install golang.org/dl/go1.21.13@latest && /go/bin/go1.21.13 download && PRODUCTION_BUILD=false CONTAINER_BUILD=true GOCMD=/go/bin/go1.21.13 ./build.sh
 
 # --- end build, create smart gateway layer ---
-FROM registry.access.redhat.com/ubi8-minimal:latest
+FROM registry.access.redhat.com/ubi9-minimal:latest
 
 LABEL io.k8s.display-name="Smart Gateway" \
       io.k8s.description="A component of the Service Telemetry Framework on the server side that ingests data from AMQP 1.x and provides a metrics scrape endpoint for Prometheus, and forwards events to ElasticSearch" \

build/repos/opstools.repo

@@ -1,19 +1,8 @@
-# CentOS-OpsTools.repo
-#
-# Please see http://wiki.centos.org/SpecialInterestGroup/OpsTools for more
-# information
+# CentOS 9 Opstools repos
 
-[centos-opstools-testing]
-name=CentOS-OpsTools - testing repo
-baseurl=https://buildlogs.centos.org/centos/$releasever-stream/opstools/$basearch/collectd-5/
-gpgcheck=0
-enabled=0
-
-[centos-opstools]
-name=CentOS-OpsTools - collectd
-#mirrorlist=http://mirrorlist.centos.org/?arch=$basearch&release=$releasever-stream&repo=opstools-collectd-5
-baseurl=http://vault.centos.org/$releasever-stream/opstools/$basearch/collectd-5/
-gpgcheck=0
+[centos9-opstools]
+name=centos9-opstools
+baseurl=http://mirror.stream.centos.org/SIGs/9-stream/opstools/$basearch/collectd-5/
 enabled=1
-skip_if_unavailable=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-OpsTools
+gpgcheck=0
+module_hotfixes=1

ci/integration/logging/run_bridge.sh

@@ -1,17 +1,16 @@
 #!/bin/env bash
-# CI script for UBI8 job
+# CI script for UBI9 job
 # purpose: spawn sg-bridge for message bus connection
 
 set -ex
 
 # enable required repo(s)
 curl -o /etc/yum.repos.d/CentOS-OpsTools.repo $OPSTOOLS_REPO
-sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/CentOS-OpsTools.repo
 
 dnf install -y git gcc make qpid-proton-c-devel redhat-rpm-config
 
 # install and start sg-bridge
-BRANCH="$(echo ${GITHUB_REF#refs/heads/})"
+BRANCH="$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}})"
 git clone https://github.com/infrawatch/sg-bridge.git
 pushd sg-bridge
 git checkout $BRANCH || true

ci/integration/logging/run_rsyslog.sh

@@ -1,5 +1,5 @@
 #!/bin/env bash
-# CI script for UBI8 job
+# CI script for UBI9 job
 # purpose: spawn rsyslog with omamqp1 plugin and simulate log records creation
 
 set -ex

ci/integration/logging/run_sg.sh

@@ -1,26 +1,22 @@
 #!/bin/env bash
-# CI script for UBI8 job
+# CI script for UBI9 job
 # purpose: spawn sg-core to process messages sent by rsyslog
 
 set -ex
 
 # enable required repo(s)
 curl -o /etc/yum.repos.d/CentOS-OpsTools.repo $OPSTOOLS_REPO
-sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/CentOS-OpsTools.repo
-# Update to use the vault mirror since Centos 8s is EOL
-sed -i 's/^#baseurl.*$/baseurl=http:\/\/vault.centos.org\/$releasever-stream\/opstools\/$basearch\/collectd-5/g' /etc/yum.repos.d/CentOS-OpsTools.repo
-sed -i 's/^mirror/#mirror/g' /etc/yum.repos.d/CentOS-OpsTools.repo
 
 dnf install -y git golang gcc make qpid-proton-c-devel
 
 export GOBIN=$GOPATH/bin
 export PATH=$PATH:$GOBIN
 
-go install golang.org/dl/go1.20@latest
-go1.20 download
+go install golang.org/dl/go1.21.13@latest
+go1.21.13 download
 
 # install sg-core and start sg-core
 mkdir -p /usr/lib64/sg-core
-PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.20 BUILD_ARGS=-buildvcs=false ./build.sh
+PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.21.13 BUILD_ARGS=-buildvcs=false ./build.sh
 
 ./sg-core -config ./ci/integration/logging/sg_config.yaml

ci/integration/logging/run_validation.sh

@@ -1,5 +1,5 @@
 #!/bin/env bash
-# CI script for UBI8 job
+# CI script for UBI9 job
 # purpose: verify the expected logging data is saved in supported storage types