HYRAX-1964: Build images from bes base docker image (bes_core) instead of bes and libdap rpms

.travis.yml

@@ -1,7 +1,7 @@
 # paste config here
 #
 # Initial Travis-CI control file. 6.11.15 jhrg
-# Added sonar scan 3.12.19 jhrg 
+# Added sonar scan 3.12.19 jhrg
 #
 #
 #
@@ -42,7 +42,7 @@ env:
     - secure: "SpCKRpuU9nIHayBZCQVoUIEK/2niy1Lw0fub/O/mAvzq33/VQa1K4WZcAJjl95KS/mrFjj3e5Gp35jXRNeViB8D3DKE9oZ3R5WDk1V66qDpmxYCSW/6iR2Zm6rjBB4wV6wWPZ1WYBg3Hg6oVZD1X8c5ABqJk7PE59L1IRF8Ocf3p8K0EUYDGINakXZ/RJHe6HtA4++oMhq54dPUTAa6xUrkCw3KNC1UXkoofzS3r/KBAZ+MPtzZkR8/dK8qPQ8hiOuH4Blj4T+d4oHPzcbmz5Nze10wZAhAm6+KwLLBSCs7Morverb1acdKPyye/t1dhzhT3/wiaP75vxDF/STdBYH6GX9HuV9JWXC8bJZxrkZcxyuVsreq/ClpLOhOk0Yfd6pTEa/QZZ+U6hyxcNbu6rROAAZpgsjPAiH3vVC9OdFo11p7bZMpBGyqUeiS82XlZ/3Svj//tiUFji2xwua1SHiM3wmgrLjII2iw8eSYuCEjqtoV6jPT0LfKzLSzuCt1S77SzaujkwBiQ1mSIJuE9SNvpXyJEQ4BInVFn+jOxCxQf6sT7ZnW7mxfK1MmgSKpzoBp8WlIAQAIewTJeaU59ws1nYbha84b80vEXp839Eesfpsf5C44dlnFSCa7sZFJ2tRjPaHv7Swt0dyJ2AmL0EHCfecX3e5sSKQAtxLN9+1E="
     # DOCKER_HUB_PSWD for our DockerHub account
     - secure: "TwtIaxZhEc+THFGe7qcOTmgeXG37tG9Hqmx0/JQS4UBYmbM+n/RbxV0IK5WjTBsvV/UBPVPdYLb0EOgsZDWjrFR7kyemd1dLc4fgb7EnHEyl17GUK3CR+ZtnRNAn+u10CJ3lq1d+ttda0GrEOpGT9e2nqPo19BxQ4vlhuFq8TAb4lKfBz3y89sY0Py+3htDtqNrIcZ1Ca0cyKmkU90/42Hx5kmBJPcFAIIrxy46cF4iNyQ8cS5TAmY/vQ+ZcoiO5yCA6ACqKWqFH2i7qUZhVsBWI6VWgQTbuRKOJ4RdOv2QtHb3QGO+PdxxitpQvw2aYam4nI7WapqNk84hu+oXoavDSR0kN21vlpfvcyfIi9JuEGGmVb7irbCCVo27kLHrRrntByKJ7GpR5Wf9s8frWd53iri9k0DSwxvKGZBIaFPodnYsJKF1Ck16IrJrr6AgPZvTKN4vBesFiRvI6sIJrZL6gD62cW2s196zKaSzcT0DccUKCaG6/javNazMo+sk74tOgqaeX0r/whXv8Rc/oobdUFI0AkGGOTzp4+QHVKX1swPEVEjP00ewOyurQinG1yhkqOgQU7s6f5dXabepmnFtH5j5W8AjDL6kISk1w/UIfMwGkUDXyxSGZWLaLag2oDxb+0Xe4CC+9m6SxtnRSzUjXDzOL4QBB8ri0VBSMA2E="
-    - BUILD_RECIPE="travis-build-recipe"
+    - BUILD_RECIPE="el8-build-recipe"
     - TARGET_OS=$( grep "TARGET_OS" "$BUILD_RECIPE" | awk '{print $2;}' )
     - RELEASE_DATE="$(date +%s)"
     - S3_BUILD_BUCKET="opendap.travis.build"
@@ -51,6 +51,7 @@ env:
 before_install:
   - echo "# travis.before_install() - BUILD_RECIPE is $BUILD_RECIPE"
   - echo "# travis.before_install() - TARGET_OS is $TARGET_OS"
+  - echo "# travis.before_install() - BES_CORE_IMAGE_TAG is $BES_CORE_IMAGE_TAG"
   - echo "# travis.before_install() - PythonVersion is $(python3 --version)"
   - echo "# travis.before_install() - Using pip3 to install awscli"
   - pip3 install awscli
@@ -78,13 +79,6 @@ stages:
 
 jobs:
   include:
-
-    ############################################################################
-    #
-    # RHEL Builds
-    #
-
-# BUILD_AND_TEST
     - stage: BUILD_AND_TEST
       name: "Build and Test opendap/hyrax"
       script:
@@ -101,22 +95,18 @@ jobs:
         - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION")
         - show_version
         - get_tomcat_distro "$DOCKER_NAME" "$TOMCAT_VERSION"
-        - s3_get_besd_distro
-          "$S3_BUILD_BUCKET"
-          "$DOCKER_NAME"
-          "$TARGET_OS"
-          "$LIBDAP_VERSION"
-          "$BES_VERSION" $ADD_DEBUG 2>&1
         - s3_get_olfs_distro
           "$S3_BUILD_BUCKET"
           "$DOCKER_NAME"
           "$OLFS_VERSION" 2>&1
+        - set -e # Don't keep going if the docker build fails....
         - docker build
           --build-arg TOMCAT_VERSION
           --build-arg RELEASE_DATE
           --build-arg HYRAX_VERSION
           --build-arg LIBDAP_VERSION
           --build-arg BES_VERSION
+          --build-arg BES_CORE_IMAGE_TAG
           --build-arg OLFS_VERSION
           --tag "$OS_SNAPSHOT_IMAGE_TAG"
           --tag "$OS_BUILD_VERSION_TAG"
@@ -126,7 +116,6 @@ jobs:
         - echo "Running Test Script"
         - ./travis/regression_test_script -i "$OS_SNAPSHOT_IMAGE_TAG" -l "$TESTS_LOG_BASE"
 
-# BUILD
     - stage: BUILD
       name: "Build opendap/besd"
       script:
@@ -135,28 +124,20 @@ jobs:
         - cd "$TARGET_OS-builds"
         - source "./build-$TARGET_OS"
         - show_version
-        #- export BES_VERSION_TAG="opendap/$DOCKER_NAME:$BES_VERSION-$TEST_DEPLOYMENT"
-        #- echo "# BES_VERSION_TAG '$BES_VERSION_TAG'"
         - OS_BUILD_VERSION_TAG="opendap/$DOCKER_NAME:$BES_VERSION-$TARGET_OS$TEST_DEPLOYMENT"
         - echo "#  OS_BUILD_VERSION_TAG '$OS_BUILD_VERSION_TAG'"
         - echo "# OS_SNAPSHOT_IMAGE_TAG '$OS_SNAPSHOT_IMAGE_TAG'"
-        - s3_get_besd_distro
-          "$S3_BUILD_BUCKET"
-          "$DOCKER_NAME"
-          "$TARGET_OS"
-          "$LIBDAP_VERSION"
-          "$BES_VERSION" "$ADD_DEBUG_RPMS" 2>&1
         - docker build
           --build-arg RELEASE_DATE
           --build-arg HYRAX_VERSION
           --build-arg LIBDAP_VERSION
           --build-arg BES_VERSION
+          --build-arg BES_CORE_IMAGE_TAG
           --tag "$OS_SNAPSHOT_IMAGE_TAG"
           --tag "$OS_BUILD_VERSION_TAG"
           "$DOCKER_NAME"
         - docker image ls -a
 
-# DISABLE PERMANENTLY??
     - stage: BUILD
       name: "Build opendap/hyrax_ncwms"
       script:
@@ -167,13 +148,6 @@ jobs:
         - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION")
         - show_version
         - get_tomcat_distro "$DOCKER_DIR" "$TOMCAT_VERSION"
-        - s3_get_besd_distro
-          "$S3_BUILD_BUCKET"
-          "$DOCKER_DIR"
-          "$TARGET_OS"
-          "$LIBDAP_VERSION"
-          "$BES_VERSION" "$ADD_DEBUG_RPMS" 2>&1
-        - ls -l "$DOCKER_DIR"/*.rpm
         - s3_get_olfs_distro
           "$S3_BUILD_BUCKET"
           "$DOCKER_DIR"
@@ -185,6 +159,7 @@ jobs:
           --build-arg HYRAX_VERSION
           --build-arg LIBDAP_VERSION
           --build-arg BES_VERSION
+          --build-arg BES_CORE_IMAGE_TAG
           --build-arg OLFS_VERSION
           --build-arg USE_NCWMS=true
           --tag "$OS_SNAPSHOT_IMAGE_TAG"
@@ -193,7 +168,6 @@ jobs:
         - docker image ls -a
 
 
-# BUILD
     - stage: BUILD
       name: "Build opendap/olfs"
       script:
@@ -251,4 +225,4 @@ deploy:
     script: bash "$TRAVIS_BUILD_DIR/travis/deploy_to_docker_hub.sh"
     on:
       all_branches: true
-      condition: ( "$DEPLOY_ME" = "true" )
+      condition: $DEPLOY_ME = "true"

el8-builds/besd/Dockerfile

@@ -13,30 +13,36 @@
 # it for debugging stuff.
 #
 # Set one or more individual labels
-FROM rockylinux:8
+ARG BES_CORE_IMAGE_TAG
+FROM ${BES_CORE_IMAGE_TAG:-"rockylinux:8"}
+
+ARG BES_CORE_IMAGE_TAG
+RUN if [ -z "$BES_CORE_IMAGE_TAG" ]; then \
+        echo "Error: Non-empty BES_CORE_IMAGE_TAG must be specified. Exiting."; \
+        exit 1; \
+    fi
 
 ARG RELEASE_DATE
 ENV RELEASE_DATE ${RELEASE_DATE:-"unknown"}
 RUN echo "RELEASE_DATE: ${RELEASE_DATE}"
 
-ARG LIBDAP_VERSION
-ENV LIBDAP_VERSION=${LIBDAP_VERSION:-"snapshot"}
-RUN echo "LIBDAP_VERSION: ${LIBDAP_VERSION}"
-
-ARG LIBDAP_RPM
-ENV LIBDAP_RPM=${LIBDAP_RPM:-"libdap-${LIBDAP_VERSION}.el8.x86_64.rpm"}
-RUN echo "LIBDAP_RPM: ${LIBDAP_RPM}"
-COPY ${LIBDAP_RPM} /${LIBDAP_RPM}
-
 ARG BES_VERSION
-ENV BES_VERSION=${BES_VERSION:-"snapshot"}
-RUN echo "BES_VERSION: ${BES_VERSION}"
-
-ARG BES_RPM
-ENV BES_RPM=${BES_RPM:-"bes-${BES_VERSION}.static.el8.x86_64.rpm"}
-RUN echo "BES_RPM: ${BES_RPM}"
-COPY ${BES_RPM} /${BES_RPM}
+RUN BES_VERSION_FOUND=$(cat bes_VERSION); \
+    if [ "${BES_VERSION_FOUND}" == "${BES_VERSION:-missing}" ]; then \
+        echo "BES Version: ${BES_VERSION}"; \
+    else \
+        echo "Error: Expected BES_VERSION \"${BES_VERSION}\", found version \"${BES_VERSION_FOUND}\". Exiting."; \
+        exit 1; \
+    fi
 
+ARG LIBDAP_VERSION
+RUN LIBDAP_VERSION_FOUND=$(cat libdap_VERSION); \
+    if [ "${LIBDAP_VERSION_FOUND}" == "${LIBDAP_VERSION:-missing}" ]; then \
+        echo "LIBDAP Version: ${LIBDAP_VERSION}"; \
+    else \
+        echo "Error: Expected LIBDAP_VERSION `${LIBDAP_VERSION}`, found version `${LIBDAP_VERSION_FOUND}`. Exiting."; \
+        exit 1; \
+    fi
 
 LABEL vendor="OPeNDAP Incorporated"
 LABEL org.opendap.besdaemon.version=${BES_VERSION}
@@ -45,12 +51,14 @@ LABEL org.opendap.hyrax.version.is-production="false"
 LABEL org.opencontainers.image.authors="support@opendap.org"
 
 USER root
+WORKDIR /
 
 # Update and install the needful.
 RUN set -e && dnf install -y python3.11 \
-    && python3 --version
-
-RUN set -e && dnf install -y which bc emacs vim \
+    && python3 --version \
+#
+# RUN set -e
+    && dnf install -y which bc emacs vim \
     && dnf install -y procps \
     && dnf update -y \
     && dnf clean all
@@ -59,22 +67,37 @@ RUN set -e && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \
     && python3 -m pip install --upgrade pip \
     && python3 -m pip install --no-cache-dir awscli
 
-###############################################################
-# Retrieve and install the latest libdap snapshot
-RUN set -e \
-    && echo "Installing libdap snapshot rpm. ${LIBDAP_RPM}" \
-    && ls -l ./libdap* \
-    && dnf -y install ./${LIBDAP_RPM} \
-    && rm -f ./${LIBDAP_RPM}
-
-###############################################################
-# Retrieve and install the latest BES snapshot
-RUN set -e \
-    && echo "Install the besd rpm ${BES_RPM} " \
-    && ls -l ./bes* \
-    && dnf -y install ./${BES_RPM} \
-    && rm -f ./${BES_RPM}
-
+# Set up BES
+#
+# BES is already installed on the base image, but
+# we need to do some additional configuration (to match what was
+# previously handled by bes RPM installation)
+
+# Add besd service to start at boot
+RUN cp ${PREFIX}/etc/rc.d/init.d/besd /etc/rc.d/init.d/besd \
+    && chkconfig --add besd \
+    && ldconfig \
+    && chkconfig --list | grep besd \
+    && echo "whoami: "`whoami` \
+    && echo "besdaemon is here: "`which besdaemon`
+
+# Adapted from bes/spec.all_static.in
+# Default macros: https://docs.fedoraproject.org/en-US/packaging-guidelines/RPMMacros/
+RUN sed -i.dist \
+    -e 's:=.*/bes.log:=/var/log/bes/bes.log:' \
+    -e 's:=.*/lib/bes:='"$PREFIX"'/lib/bes:' \
+    -e 's:=.*/share/bes:='"$PREFIX"'/share/bes:' \
+    -e 's:=.*/share/hyrax:='"$PREFIX"'/share/hyrax:' \
+    # -e 's:=/full/path/to/serverside/certificate/file.pem:=/etc/pki/bes/cacerts/file.pem:' \
+    # -e 's:=/full/path/to/serverside/key/file.pem:=/etc/pki/bes/public/file.pem:' \
+    # -e 's:=/full/path/to/clientside/certificate/file.pem:=/etc/pki/bes/cacerts/file.pem:' \
+    # -e 's:=/full/path/to/clientside/key/file.pem:=/etc/pki/bes/public/file.pem:' \
+    -e 's:=user_name:='"$USER"':' \
+    -e 's:=group_name:='"$USER"':' \
+    $PREFIX/etc/bes/bes.conf \
+    && mkdir -p "/var/log/bes/" \
+    && touch "/var/log/bes/bes.log" \
+    && sudo chown -R $USER:$USER "/var/log/bes/"
 
 RUN echo "besdaemon is here: "`which besdaemon`
 
@@ -85,10 +108,5 @@ ENTRYPOINT ["/entrypoint.sh"]
 EXPOSE 10022
 EXPOSE 11002
 
-# can't use USER with entrypoint that needs root
-# use gosu or, as done, enable bes user write so the entrypoint doe snot need root
-RUN  chown -R bes /etc/bes
-USER root
-
 CMD ["-"]
 

el8-builds/besd/entrypoint.sh

@@ -94,24 +94,24 @@ set -e
 #
 if [ $SERVER_HELP_EMAIL != "not_set" ]; then
     echo "Setting Admin Contact To: $SERVER_HELP_EMAIL" >&2
-    sed -i "s/admin.email.address@your.domain.name/$SERVER_HELP_EMAIL/" /etc/bes/bes.conf
+    sed -i "s/admin.email.address@your.domain.name/$SERVER_HELP_EMAIL/" $PREFIX/etc/bes/bes.conf
 fi
 if [ $FOLLOW_SYMLINKS != "not_set" ]; then
     echo "Setting BES FollowSymLinks to YES." >&2
-    sed -i "s/^BES.Catalog.catalog.FollowSymLinks=No/BES.Catalog.catalog.FollowSymLinks=Yes/" /etc/bes/bes.conf
+    sed -i "s/^BES.Catalog.catalog.FollowSymLinks=No/BES.Catalog.catalog.FollowSymLinks=Yes/" $PREFIX/etc/bes/bes.conf
 fi
 
 
 # Start the BES daemon process
 # /usr/bin/besdaemon -i /usr -c /etc/bes/bes.conf -r /var/run/bes.pid
-/usr/bin/besctl start; 
+$PREFIX/usr/bin/besctl start;
 status=$?
 if [ $status -ne 0 ]; then
     echo "Failed to start BES: $status" >&2
     exit $status
 fi
 
-besd_pid=`ps aux | grep /usr/bin/besdaemon | grep -v grep | awk '{print $2;}' - `;
+besd_pid=`ps aux | grep $PREFIX/usr/bin/besdaemon | grep -v grep | awk '{print $2;}' - `;
 echo "The besdaemon is UP! pid: $besd_pid"  >&2
 
 echo "BES Has Arrived..."  >&2