Add and move around build scripts

Author: ibmmqmet

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Changelog
 Newest updates are at the top of this file.
 
+### Aug 10 2020
+* Update to use v5.1.1 of the mq-golang repository
+* Moved build shell commands to `scripts` subdirectory and added README there
+* Add a Windows batch file as example of building locally
+
 ### Jul 23 2020
 * Update to use v5.1.0 of the mq-golang repository
 * Update to use MQ 9.2.0

Dockerfile renamed to Dockerfile.build

@@ -61,12 +61,12 @@ RUN cd /opt/mqm \
  && rm -f ./*.tar.gz
 
 # Insert the script that will do the build
-COPY buildInDocker.sh $GOPATH
+COPY scripts/buildInDocker.sh $GOPATH
 RUN chmod 777 $GOPATH/buildInDocker.sh
 
 WORKDIR $GOPATH/src/$ORG/$REPO
-COPY go.mod .          
-COPY go.sum .          
+COPY go.mod .
+COPY go.sum .
 RUN chmod 777 go.*
 
 COPY config.common.yaml .

Dockerfile.run

@@ -0,0 +1,54 @@
+# © Copyright IBM Corporation 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE=ubuntu:19.10
+FROM $BASE_IMAGE
+
+# Create location for the git clone and MQ installation
+RUN mkdir -p /opt/bin \
+  && chmod -R 777 /opt/bin \
+  && mkdir -p /opt/mqm \
+  && chmod a+rx /opt/mqm \
+  && mkdir -p /opt/config \
+  && chmod a+rx /opt/config
+
+# Install curl
+RUN apt-get update \
+    && apt-get install -y curl bash ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Location of the downloadable MQ client package \
+ENV RDURL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist" \
+    RDTAR="IBM-MQC-Redist-LinuxX64.tar.gz" \
+    VRMF=9.2.0.0
+
+# Install the MQ client from the Redistributable package. This also contains the
+# header files we need to compile against.
+RUN cd /opt/mqm \
+ && curl -LO "$RDURL/$VRMF-$RDTAR" \
+ && tar -zxf ./*.tar.gz \
+ && rm -f ./*.tar.gz
+
+ENV LD_LIBRARY_PATH="/opt/mqm/lib64:/usr/lib64" \
+    MQ_CONNECT_TYPE=CLIENT
+
+ARG MONITOR_ARG="mq_prometheus"
+ENV MONITOR=$MONITOR_ARG
+RUN echo "Building container for $MONITOR"
+
+# Copy over the binary file, which is in the same directory as this one
+COPY $MONITOR /opt/bin/$MONITOR
+
+# The configuration file should be mounted from the host into the /opt/config directory.
+CMD /opt/bin/$MONITOR -f /opt/config/$MONITOR.yaml -log.level=info

README.md

@@ -24,8 +24,7 @@ file if you wish to reload all of the dependencies by running `go mod vendor`.
 
 You will require the following programs:
 
-* Go compiler. This should be at least version 12. If you see an error similar to "ld: NULL not defined"
-then it is likely you need to upgrade your compiler.
+* Go compiler. This should be at least version 13.
 
 To build the programs on Linux and MacOS, you may set an environment variable to permit some compile/link flags.
 This is due to security controls in the compiler.
@@ -69,11 +68,16 @@ monitor agent directory also has sample scripts, configuration files etc to help
 with getting the agent running in your specific environment.
 
 ## Using a Docker container to build the programs
-You can use the `buildMonitors.sh` script in this directory to build a Docker container that
+You can use the `buildMonitors.sh` script in the `scripts` subdirectory to build a Docker container that
 in turn will build all the binary programs and copy them to a local directory. That script also
 sets some extra version-related flags that will be shown when the program starts. The container will
 automatically download and install the MQ client runtime files needed for compilation.
 
+## Building on Windows
+There is a `buildMonitors.bat` file that may help with building on Windows. It assumes you have
+the [tdm-gcc-64](https://jmeubank.github.io/tdm-gcc/download/) 64-bit compiler suite installed. It
+builds all the collectors and corresponding YAML configuration files into %GOPATH%/bin
+
 ## Monitor configuration
 The monitors always collect all of the available queue manager-wide metrics.
 They can also be configured to collect statistics for specific sets of queues where
@@ -154,7 +158,7 @@ table combines multiple metrics on the same line now.
 
 Information about channels comes from the DISPLAY CHSTATUS CURRENT command. That
 only shows channels with a known state and does not report on inactive channels.
-To also see the inactive channels, then set the showInactiveChannels 
+To also see the inactive channels, then set the showInactiveChannels
 configuration attribute to true.
 
 ### z/OS Support
@@ -203,7 +207,7 @@ point at the file.
 All of the exporters support
 the same configuration options for how to connect to MQ and which objects are monitored. There is
 then an exporter-specific section for additional configuration such as how to contact the back-end
-database. 
+database.
 The common options are shown in a template in this directory; the exporter-specific options are in individual files in each directory. Combine the two pieces into a single file to get a complete deployable configuration.
 
 Unlike the command line flags, lists are provided in a more natural format instead of comma-separated

go.mod

@@ -4,7 +4,7 @@ go 1.13
 
 require (
 	github.com/aws/aws-sdk-go v1.30.18
-	github.com/ibm-messaging/mq-golang/v5 v5.1.0
+	github.com/ibm-messaging/mq-golang/v5 v5.1.1
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
 	github.com/prometheus/client_golang v1.6.0
 	github.com/sirupsen/logrus v1.5.0

go.sum

@@ -42,6 +42,8 @@ github.com/ibm-messaging/mq-golang/v5 v5.0.0 h1:9J8bsDoCo60rbSgB7ZAURPG3L5Kpr+F8
 github.com/ibm-messaging/mq-golang/v5 v5.0.0/go.mod h1:ywCwmYbJOU/E0rl+z4GiNoxVMty68O+LVO39a1VMXrE=
 github.com/ibm-messaging/mq-golang/v5 v5.1.0 h1:fWTWSV9uBv847rVQqGPRGGCQr61FJR7eSxM5RJMa3NQ=
 github.com/ibm-messaging/mq-golang/v5 v5.1.0/go.mod h1:ywCwmYbJOU/E0rl+z4GiNoxVMty68O+LVO39a1VMXrE=
+github.com/ibm-messaging/mq-golang/v5 v5.1.1 h1:rXlyYWyG4paNImj7cDXJsxJHONcI1nhD6XBiebDsoeQ=
+github.com/ibm-messaging/mq-golang/v5 v5.1.1/go.mod h1:ywCwmYbJOU/E0rl+z4GiNoxVMty68O+LVO39a1VMXrE=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=

scripts/README.md

@@ -0,0 +1,27 @@
+This directory contains scripts as examples of building and running
+the various collector programs. The files are:
+
+* `buildMonitors.sh`: Creates a container and uses it as the build environment
+to build all the collectors. Output goes to a real local directory, `$HOME/tmp/mq-metric-samples`
+which holds the binaries, the default YAML configuration files and MQSC scripts that can be
+used to run the collectors as MQ Services.
+
+* `buildInDocker.sh`: Used inside the container created by the previous script. It downloads the
+MQ Redistributable Client needed for compiling the packages, and then does the `go build`
+operations.
+
+* `buildRuntime.sh`: Create a container that holds the runtime for a particular collector. By
+default, build containers for all collectors, but can give a command line list to explicitly ask
+for a subset. For example, `buildRuntime.sh mq_prometheus mq_json`. The container is tagged, ready to run.
+
+* `runMonitor.sh`: Execute one of the containers build in the previous step. The configuration file
+is mounted from the host image. This is the file you'll most likely want to change, to match your
+own setup. But it's a useful model to show how parameters can be passed.
+
+* `buildMonitors.bat`: Example of building on Windows. It assumes you
+already have the MQ client package installed, and have a suitable `gcc`
+compiler available under c:\TDM-GCC-64. Output of the binaries and
+YAML files go into `%GOPATH%\bin`.
+
+* `docker-compose.yaml`: A fragment of what might go into a larger composition file. In its current
+state it tries to pull an image from a repository even though it has been built and exists locally.

buildInDocker.sh renamed to scripts/buildInDocker.sh

@@ -0,0 +1,34 @@
+@echo off
+
+rem Don't affect parent environment
+setlocal
+
+rem Set a PATH to include a suitable gcc build
+set PATH=c:\tdm-gcc-64\bin;%PATH%
+
+rem Go to the root of my Go tree
+rem Usually something like c:\Gowork
+cd %GOPATH%
+
+rem And set references to the commands that have been git cloned under it
+set R=src\github.com\ibm-messaging\mq-metric-samples
+set D=%R%\cmd
+
+rem Make sure the correct size is going to be used - The invoked gcc may not set it
+set CGO_CFLAGS=-D_WIN64
+
+rem Do the builds - since we can't always use *.go, then name the files explicitly
+rem Some have slightly different sets of files
+for %%M in (mq_prometheus mq_json mq_coll mq_influx) do (
+echo Building %%M
+go build -o bin/%%M.exe %D%\%%M\config.go %D%\%%M\main.go %D%\%%M\exporter.go
+type %R%\config.common.yaml %D%\%%M\config.collector.yaml > bin\%%M.yaml 2>NUL:
+)
+
+for %%M in (mq_aws mq_opentsdb) do (
+echo Building %%M
+go build -o bin/%%M.exe %D%\%%M\config.go %D%\%%M\main.go %D%\%%M\exporter.go %D%\%%M\points.go
+type %R%\config.common.yaml %D%\%%M\config.collector.yaml > bin\%%M.yaml 2>NUL:
+)
+
+endlocal

scripts/buildMonitors.bat

@@ -17,44 +17,25 @@
 # to compile the binary components of the monitoring programs, and then to copy those
 # programs to a local temporary directory.
 
-function latestSemVer {
-  (for x in $*
-  do
-    echo $x | sed "s/^v//g"
-  done) | sort -n | tail -1
-}
+. ./common.sh
 
 GOPATH="/go"
-
 TAG="mq-metric-samples-gobuild"
-# Assume repo tags have been created in a sensible order. Find the mq-golang
-# version in the dep file (it's the only dependency explicitly listed for now)
-# and the current Git tag for this repo. Then pick the latest version to create
-# the Docker tag
-VERDEP=`cat go.mod | awk '/mq-golang/ {print $2}' `
-VERREPO=`git tag -l | sort | tail -1 `
 
-VER=`latestSemVer $VERDEP $VERREPO`
-if [ -z "$VER" ]
-then
-  VER="latest"
-fi
-# echo "VERDEP=$VERDEP VERREPO=$VERREPO"
 echo "Building container $TAG:$VER"
 
 # Set the userid we will run the container as
 uid=`id -u`
 gid=`id -g`
 
 # Build a container that has all the pieces needed to compile the Go programs for MQ
-docker build --build-arg GOPATH_ARG=$GOPATH -t $TAG:$VER .
+docker build --build-arg GOPATH_ARG=$GOPATH -f Dockerfile.build -t $TAG:$VER .
 rc=$?
 
 if [ $rc -eq 0 ]
 then
   # Run the image to do the compilation and extract the files
   # from it into a local directory mounted into the container.
-  OUTDIR=$HOME/tmp/mq-metric-samples/bin
   rm -rf $OUTDIR
   mkdir -p $OUTDIR