diff --git a/.travis.yml b/.travis.yml index a76af017..ada0d351 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,15 @@ # paste config here - -# Initial Travic-CI control file. 6.11.15 jhrg +# +# Initial Travis-CI control file. 6.11.15 jhrg # Added sonar scan 3.12.19 jhrg - -dist: xenial +# +# +# +dist: jammy language: python python: - - "3.9.7" - -# command to install dependencies -install: - - pip install -r requirements.txt + - "3.12" services: - docker @@ -19,6 +17,7 @@ services: branches: only: - master + - el9 notifications: email: @@ -41,90 +40,75 @@ env: - secure: "SpCKRpuU9nIHayBZCQVoUIEK/2niy1Lw0fub/O/mAvzq33/VQa1K4WZcAJjl95KS/mrFjj3e5Gp35jXRNeViB8D3DKE9oZ3R5WDk1V66qDpmxYCSW/6iR2Zm6rjBB4wV6wWPZ1WYBg3Hg6oVZD1X8c5ABqJk7PE59L1IRF8Ocf3p8K0EUYDGINakXZ/RJHe6HtA4++oMhq54dPUTAa6xUrkCw3KNC1UXkoofzS3r/KBAZ+MPtzZkR8/dK8qPQ8hiOuH4Blj4T+d4oHPzcbmz5Nze10wZAhAm6+KwLLBSCs7Morverb1acdKPyye/t1dhzhT3/wiaP75vxDF/STdBYH6GX9HuV9JWXC8bJZxrkZcxyuVsreq/ClpLOhOk0Yfd6pTEa/QZZ+U6hyxcNbu6rROAAZpgsjPAiH3vVC9OdFo11p7bZMpBGyqUeiS82XlZ/3Svj//tiUFji2xwua1SHiM3wmgrLjII2iw8eSYuCEjqtoV6jPT0LfKzLSzuCt1S77SzaujkwBiQ1mSIJuE9SNvpXyJEQ4BInVFn+jOxCxQf6sT7ZnW7mxfK1MmgSKpzoBp8WlIAQAIewTJeaU59ws1nYbha84b80vEXp839Eesfpsf5C44dlnFSCa7sZFJ2tRjPaHv7Swt0dyJ2AmL0EHCfecX3e5sSKQAtxLN9+1E=" # DOCKER_HUB_PSWD for our DockerHub account - secure: "TwtIaxZhEc+THFGe7qcOTmgeXG37tG9Hqmx0/JQS4UBYmbM+n/RbxV0IK5WjTBsvV/UBPVPdYLb0EOgsZDWjrFR7kyemd1dLc4fgb7EnHEyl17GUK3CR+ZtnRNAn+u10CJ3lq1d+ttda0GrEOpGT9e2nqPo19BxQ4vlhuFq8TAb4lKfBz3y89sY0Py+3htDtqNrIcZ1Ca0cyKmkU90/42Hx5kmBJPcFAIIrxy46cF4iNyQ8cS5TAmY/vQ+ZcoiO5yCA6ACqKWqFH2i7qUZhVsBWI6VWgQTbuRKOJ4RdOv2QtHb3QGO+PdxxitpQvw2aYam4nI7WapqNk84hu+oXoavDSR0kN21vlpfvcyfIi9JuEGGmVb7irbCCVo27kLHrRrntByKJ7GpR5Wf9s8frWd53iri9k0DSwxvKGZBIaFPodnYsJKF1Ck16IrJrr6AgPZvTKN4vBesFiRvI6sIJrZL6gD62cW2s196zKaSzcT0DccUKCaG6/javNazMo+sk74tOgqaeX0r/whXv8Rc/oobdUFI0AkGGOTzp4+QHVKX1swPEVEjP00ewOyurQinG1yhkqOgQU7s6f5dXabepmnFtH5j5W8AjDL6kISk1w/UIfMwGkUDXyxSGZWLaLag2oDxb+0Xe4CC+9m6SxtnRSzUjXDzOL4QBB8ri0VBSMA2E=" - - TARGET_OS="el8" - - RELEASE_DATE=`date +%s` - - VERSION_FILE="${TARGET_OS}-build-recipe" - - LIBDAP_VERSION=$(grep "libdap4-" ${VERSION_FILE} | awk '{print $1;}' - | sed "s/libdap4-//g") - - BES_VERSION=$(grep "bes-" ${VERSION_FILE} | awk '{print $1;}' - | sed "s/bes-//g") - - HYRAX_VERSION=$(grep "hyrax-" ${VERSION_FILE} | awk '{print $1;}' - | sed "s/hyrax-//g") - - OLFS_VERSION=$(grep "olfs-" ${VERSION_FILE} | awk '{print $1;}' - | sed "s/olfs-//g") - - OLFS_DISTRO="olfs-${OLFS_VERSION}-webapp.tgz" - - OLFS_DISTRO_URL="s3://opendap.travis.build/${OLFS_DISTRO}" - - NGAP_DISTRO="ngap-${OLFS_VERSION}-webapp.tgz" - - NGAP_DISTRO_URL="s3://opendap.travis.build/${NGAP_DISTRO}" - - ROBOTS_DISTRO="robots-olfs-${OLFS_VERSION}-webapp.tgz" - - ROBOTS_DISTRO_URL="s3://opendap.travis.build/${ROBOTS_DISTRO}" - - LIBDAP_RH8_RPM="libdap-${LIBDAP_VERSION}.el8.x86_64.rpm" - - LIBDAP_RH8_RPM_URL="s3://opendap.travis.build/${LIBDAP_RH8_RPM}" - - BES_RH8_RPM="bes-${BES_VERSION}.static.el8.x86_64.rpm" - - BES_RH8_RPM_URL="s3://opendap.travis.build/${BES_RH8_RPM}" - - TOMCAT_MAJOR_VERSION=9 + - BUILD_RECIPE="travis-build-recipe" + - TARGET_OS=$( grep "TARGET_OS" "$BUILD_RECIPE" | awk '{print $2;}' ) + - RELEASE_DATE="$(date +%s)" - S3_BUILD_BUCKET="opendap.travis.build" - ADD_DEBUG_RPMS="" before_install: - - python3 --version + - echo "# travis.before_install() - BUILD_RECIPE is $BUILD_RECIPE" + - echo "# travis.before_install() - TARGET_OS is $TARGET_OS" + - echo "# travis.before_install() - PythonVersion is $(python3 --version)" + - echo "# travis.before_install() - Using pip3 to install awscli" - pip3 install awscli - - if test "${TRAVIS_PULL_REQUEST}" = "false"; then export DEPLOY_ME="true"; else export DEPLOY_ME="false"; fi + - echo "# travis.before_install() - TRAVIS_PULL_REQUEST is $TRAVIS_PULL_REQUEST" + - echo "# travis.before_install() - TRAVIS_BRANCH is $TRAVIS_BRANCH" + - if test "$TRAVIS_PULL_REQUEST" = "false" -o "$TRAVIS_BRANCH" = "el9" ; then export DEPLOY_ME="true"; else export DEPLOY_ME="false"; fi + - echo "# travis.before_install() - DEPLOY_ME is $DEPLOY_ME" + install: - echo "print env:" - printenv - echo $DOCKER_HUB_PSWD | docker login -u $DOCKER_HUB_UID --password-stdin - # - echo "Retrieving libdap C7 RPM ${LIBDAP_C7_RPM}" - # - aws --region=us-east-1 s3 cp ${LIBDAP_C7_RPM_URL} ./builds/${LIBDAP_C7_RPM} - # - echo "Retrieving BES C7 RPM ${BES_C7_RPM}" - # - aws --region=us-east-1 s3 cp ${BES_C7_RPM_URL} ./builds/${BES_C7_RPM} - - echo "Retrieving libdap RH8 RPM ${LIBDAP_RH8_RPM}" - - aws --region=us-east-1 s3 cp ${LIBDAP_RH8_RPM_URL} ./builds/${LIBDAP_RH8_RPM} - - echo "Retrieving BES RH8 RPM ${BES_RH8_RPM}" - - aws --region=us-east-1 s3 cp ${BES_RH8_RPM_URL} ./builds/${BES_RH8_RPM} - - echo "Retrieving OLFS Distribution file ${OLFS_DISTRO}" - - aws --region=us-east-1 s3 cp ${OLFS_DISTRO_URL} ./builds/${OLFS_DISTRO} - - echo "Retrieving ROBOTS Distribution file ${ROBOTS_DISTRO}" - - aws --region=us-east-1 s3 cp ${ROBOTS_DISTRO_URL} ./builds/${ROBOTS_DISTRO} - - echo "Retrieving NGAP Distribution file ${NGAP_DISTRO}" - - aws --region=us-east-1 s3 cp ${NGAP_DISTRO_URL} ./builds/${NGAP_DISTRO} - - ls -l ./builds stages: - name: BUILD_AND_TEST - if: branch = master + if: branch = master OR branch = el9 - name: BUILD - if: branch = master + if: branch = master OR branch = el9 - name: TEST_DEPLOYED_IMAGE - if: (branch = master) and (type != pull_request) + if: ((branch = master) and (type != pull_request)) or (branch = el9) - name: disabled if: branch = disabled + jobs: include: ############################################################################ # - # RH8 Builds + # RHEL Builds # + +# BUILD_AND_TEST - stage: BUILD_AND_TEST - name: "Build and Test opendap/hyrax:snapshot (RH8)" + name: "Build and Test opendap/hyrax" script: - - cd "${TARGET_OS}-builds" - export DOCKER_NAME="hyrax" - - source ./build-rh8 - - export SNAPSHOT_IMAGE_TAG="opendap/${DOCKER_NAME}:snapshot$TEST_DEPLOYMENT" - - export BUILD_VERSION_TAG="opendap/${DOCKER_NAME}:${HYRAX_VERSION}$TEST_DEPLOYMENT" - - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + - export DOCKER_DIR="$DOCKER_NAME" + - echo "DOCKER_NAME '$DOCKER_NAME'" + - echo "RELEASE_DATE '$RELEASE_DATE'" + - echo "BUILDING FOR $TARGET_OS" + - cd "$TARGET_OS-builds" + - source "./build-$TARGET_OS" - show_version - - get_tomcat_distro "${DOCKER_NAME}" "${TOMCAT_VERSION}" + - echo "USING TOMCAT MAJOR VERSION '$TOMCAT_MAJOR_VERSION'" + - export TESTS_LOG_BASE="$DOCKER_NAME-regression-tests-$TRAVIS_JOB_NUMBER" + - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION") + - show_version + - get_tomcat_distro "$DOCKER_NAME" "$TOMCAT_VERSION" - s3_get_besd_distro - "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" - "el8" - "${LIBDAP_VERSION}" - "${BES_VERSION}" $ADD_DEBUG 2>&1 + "$S3_BUILD_BUCKET" + "$DOCKER_NAME" + "$TARGET_OS" + "$LIBDAP_VERSION" + "$BES_VERSION" $ADD_DEBUG 2>&1 - s3_get_olfs_distro - "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" - "${OLFS_VERSION}" 2>&1 + "$S3_BUILD_BUCKET" + "$DOCKER_NAME" + "$OLFS_VERSION" 2>&1 - docker build --build-arg TOMCAT_VERSION --build-arg RELEASE_DATE @@ -132,60 +116,67 @@ jobs: --build-arg LIBDAP_VERSION --build-arg BES_VERSION --build-arg OLFS_VERSION - --tag "${SNAPSHOT_IMAGE_TAG}" - --tag "${BUILD_VERSION_TAG}" - "${DOCKER_NAME}" + --tag "$OS_SNAPSHOT_IMAGE_TAG" + --tag "$OS_BUILD_VERSION_TAG" + "$DOCKER_NAME" - docker image ls -a - cd .. - echo "Running Test Script" - - ./travis/regression_test_script -i ${SNAPSHOT_IMAGE_TAG} + - ./travis/regression_test_script -i "$OS_SNAPSHOT_IMAGE_TAG" -l "$TESTS_LOG_BASE" +# BUILD - stage: BUILD - name: "Build opendap/besd:snapshot (RH8)" + name: "Build opendap/besd" script: - - cd "${TARGET_OS}-builds" - export DOCKER_NAME="besd" - - source ./build-rh8 - - export SNAPSHOT_IMAGE_TAG="opendap/${DOCKER_NAME}:snapshot$TEST_DEPLOYMENT" - - export BUILD_VERSION_TAG="opendap/${DOCKER_NAME}:${BES_VERSION}$TEST_DEPLOYMENT" + - echo "# RELEASE_DATE '$RELEASE_DATE'" + - cd "$TARGET_OS-builds" + - source "./build-$TARGET_OS" - show_version + #- export BES_VERSION_TAG="opendap/$DOCKER_NAME:$BES_VERSION-$TEST_DEPLOYMENT" + #- echo "# BES_VERSION_TAG '$BES_VERSION_TAG'" + - OS_BUILD_VERSION_TAG="opendap/$DOCKER_NAME:$BES_VERSION-$TARGET_OS$TEST_DEPLOYMENT" + - echo "# OS_BUILD_VERSION_TAG '$OS_BUILD_VERSION_TAG'" + - echo "# OS_SNAPSHOT_IMAGE_TAG '$OS_SNAPSHOT_IMAGE_TAG'" - s3_get_besd_distro - "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" - "el8" - "${LIBDAP_VERSION}" - "${BES_VERSION}" "${ADD_DEBUG_RPMS}" 2>&1 + "$S3_BUILD_BUCKET" + "$DOCKER_NAME" + "$TARGET_OS" + "$LIBDAP_VERSION" + "$BES_VERSION" "$ADD_DEBUG_RPMS" 2>&1 - docker build --build-arg RELEASE_DATE --build-arg HYRAX_VERSION --build-arg LIBDAP_VERSION --build-arg BES_VERSION - --tag "${SNAPSHOT_IMAGE_TAG}" - --tag "${BUILD_VERSION_TAG}" - "${DOCKER_NAME}" + --tag "$OS_SNAPSHOT_IMAGE_TAG" + --tag "$OS_BUILD_VERSION_TAG" + "$DOCKER_NAME" - docker image ls -a +# DISABLE PERMANENTLY?? - stage: BUILD - name: "Build opendap/hyrax_ncwms:snapshot (RH8)" + name: "Build opendap/hyrax_ncwms" script: - - cd "${TARGET_OS}-builds" - - export DOCKER_NAME="hyrax" - - source ./build-rh8 - - export SNAPSHOT_IMAGE_TAG="opendap/${DOCKER_NAME}_ncwms:snapshot$TEST_DEPLOYMENT" - - export BUILD_VERSION_TAG="opendap/${DOCKER_NAME}_ncwms:${HYRAX_VERSION}$TEST_DEPLOYMENT" - - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + - export DOCKER_NAME="hyrax_ncwms" + - export DOCKER_DIR="hyrax" + - cd "$TARGET_OS-builds" + - source "./build-$TARGET_OS" + - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION") - show_version - - get_tomcat_distro "${DOCKER_NAME}" "${TOMCAT_VERSION}" + - get_tomcat_distro "$DOCKER_DIR" "$TOMCAT_VERSION" - s3_get_besd_distro - "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" - "el8" - "${LIBDAP_VERSION}" - "${BES_VERSION}" "${ADD_DEBUG_RPMS}" 2>&1 + "$S3_BUILD_BUCKET" + "$DOCKER_DIR" + "$TARGET_OS" + "$LIBDAP_VERSION" + "$BES_VERSION" "$ADD_DEBUG_RPMS" 2>&1 + - ls -l "$DOCKER_DIR"/*.rpm - s3_get_olfs_distro - "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" - "${OLFS_VERSION}" 2>&1 + "$S3_BUILD_BUCKET" + "$DOCKER_DIR" + "$OLFS_VERSION" 2>&1 + - ls -l "$DOCKER_DIR"/*.tar.gz - docker build --build-arg TOMCAT_VERSION --build-arg RELEASE_DATE @@ -194,85 +185,76 @@ jobs: --build-arg BES_VERSION --build-arg OLFS_VERSION --build-arg USE_NCWMS=true - --tag "${SNAPSHOT_IMAGE_TAG}" - --tag "${BUILD_VERSION_TAG}" - "${DOCKER_NAME}" + --tag "$OS_SNAPSHOT_IMAGE_TAG" + --tag "$OS_BUILD_VERSION_TAG" + "$DOCKER_DIR" - docker image ls -a + +# BUILD - stage: BUILD - name: "Build opendap/hyrax:ngap-snapshot (RH8)" + name: "Build opendap/olfs" script: - - cd "${TARGET_OS}-builds" - - export DOCKER_NAME="ngap" - - source ./build-rh8 - - export SNAPSHOT_IMAGE_TAG="opendap/hyrax:${DOCKER_NAME}-snapshot$TEST_DEPLOYMENT" - - export BUILD_VERSION_TAG="opendap/hyrax:${DOCKER_NAME}-${HYRAX_VERSION}$TEST_DEPLOYMENT" - - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + - export DOCKER_NAME="olfs" + - export DOCKER_DIR="olfs" + - cd "$TARGET_OS-builds" + - source "./build-$TARGET_OS" + - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION") - show_version - get_tomcat_distro "${DOCKER_NAME}" "${TOMCAT_VERSION}" - - s3_get_besd_distro - "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" - "el8" - "${LIBDAP_VERSION}" - "${BES_VERSION}" "${ADD_DEBUG_RPMS}" 2>&1 - - get_ngap_olfs_distro + - s3_get_olfs_distro "${S3_BUILD_BUCKET}" - "${DOCKER_NAME}" + "olfs" "${OLFS_VERSION}" 2>&1 - docker build --build-arg TOMCAT_VERSION --build-arg RELEASE_DATE --build-arg HYRAX_VERSION - --build-arg LIBDAP_VERSION - --build-arg BES_VERSION --build-arg OLFS_VERSION - --tag "${SNAPSHOT_IMAGE_TAG}" - --tag "${BUILD_VERSION_TAG}" - "${DOCKER_NAME}" + --tag "$OS_SNAPSHOT_IMAGE_TAG" + --tag "$OS_BUILD_VERSION_TAG" olfs - docker image ls -a - stage: BUILD - name: "Build opendap/olfs:snapshot" + name: "Build opendap/hyrax:ngap" script: - - cd "${TARGET_OS}-builds" - - export DOCKER_NAME="olfs" - - source ./build-rh8 - - export SNAPSHOT_IMAGE_TAG="opendap/${DOCKER_NAME}:snapshot$TEST_DEPLOYMENT" - - export BUILD_VERSION_TAG="opendap/${DOCKER_NAME}:${OLFS_VERSION}$TEST_DEPLOYMENT" - - export TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + - cd "$TARGET_OS-builds" + - export DOCKER_NAME="ngap" + - export DOCKER_DIR="ngap" + - source "./build-$TARGET_OS" - show_version - - get_tomcat_distro "${DOCKER_NAME}" "${TOMCAT_VERSION}" - - s3_get_olfs_distro - "${S3_BUILD_BUCKET}" - "olfs" - "${OLFS_VERSION}" 2>&1 - - docker build - --build-arg TOMCAT_VERSION - --build-arg RELEASE_DATE - --build-arg HYRAX_VERSION - --build-arg OLFS_VERSION - --tag "${SNAPSHOT_IMAGE_TAG}" - --tag "${BUILD_VERSION_TAG}" olfs + - ./build-hyrax-ngap.sh - docker image ls -a - stage: TEST_DEPLOYED_IMAGE - name: "Test Deployed Image (RH8)" + name: "Test Deployed Image" script: - - source ./el8-builds/build-rh8 ./el8-build-recipe - - export TEST_IMAGE="opendap/hyrax:${HYRAX_VERSION}$TEST_DEPLOYMENT" - - echo "Running Hyrax Regression Test Script on ${TEST_IMAGE}" - - ./travis/regression_test_script -i "${TEST_IMAGE}" + - cd "$TARGET_OS-builds" + - source "./build-$TARGET_OS" + - cd .. + - echo "Running Hyrax Regression Test Script on $OS_SNAPSHOT_IMAGE_TAG" + - ./travis/regression_test_script -i "$OS_SNAPSHOT_IMAGE_TAG" -l "deployed-image-regression-tests-$TRAVIS_JOB_NUMBER" - export DEPLOY_ME="false" +after_script: + - ./travis/upload-test-results.sh "$DOCKER_NAME" "$TESTS_LOG_BASE.tgz" + before_deploy: - export DOCKER_LOGIN=`aws ecr get-login --region us-east-1 | sed "s/-e none //g"`; - $DOCKER_LOGIN; - - echo "DEPLOY_ME is ${DEPLOY_ME}" + - echo "# DEPLOY_ME is $DEPLOY_ME" deploy: - provider: script - script: bash ${TRAVIS_BUILD_DIR}/travis/deploy_to_docker_hub.sh + script: bash "$TRAVIS_BUILD_DIR/travis/deploy_to_docker_hub.sh" on: branch: master - condition: ${DEPLOY_ME} = "true" + condition: $DEPLOY_ME = "true" + + - provider: script + script: bash "$TRAVIS_BUILD_DIR/travis/deploy_to_docker_hub.sh" + on: + branch: el9 + condition: $DEPLOY_ME = "true" + + diff --git a/build_from_release_recipe.sh b/build_from_release_recipe.sh index 023371bd..fd354a93 100755 --- a/build_from_release_recipe.sh +++ b/build_from_release_recipe.sh @@ -31,9 +31,7 @@ woo_get_besd_distro \ true # Get the OLFS web archive -woo_get_olfs_distro \ - `pwd` \ - "${OLFS_VERSION}" +curl_get_olfs_distro `pwd` "${OLFS_VERSION}" ls -l *.rpm *.gz *.tgz diff --git a/el8-builds/besd/Dockerfile b/el8-builds/besd/Dockerfile index cb6bf753..95232c34 100644 --- a/el8-builds/besd/Dockerfile +++ b/el8-builds/besd/Dockerfile @@ -42,8 +42,7 @@ LABEL vendor="OPeNDAP Incorporated" LABEL org.opendap.besdaemon.version=${BES_VERSION} LABEL org.opendap.besdaemon.release-date=RELEASE_DATE LABEL org.opendap.hyrax.version.is-production="false" - -MAINTAINER support@opendap.org +LABEL org.opencontainers.image.authors="support@opendap.org" USER root diff --git a/el8-builds/build-rh8 b/el8-builds/build-el8 similarity index 91% rename from el8-builds/build-rh8 rename to el8-builds/build-el8 index ecdd2f2d..38bfd03a 100755 --- a/el8-builds/build-rh8 +++ b/el8-builds/build-el8 @@ -32,6 +32,11 @@ function read_build_recipe() { export BES_VERSION="" export BUILD_DMRPP_VERSION="" export LIBDAP_VERSION="" + export SNAPSHOT_IMAGE_TAG="" + export BUILD_VERSION_TAG="" + export OS_SNAPSHOT_IMAGE_TAG="" + export OS_BUILD_VERSION_TAG="" + RELEASE_DATE=${RELEASE_DATE:-$(echo "${build_recipe}" | grep hyrax | awk '{print $2;}')} HYRAX_VERSION=${HYRAX_VERSION:-$(echo "${build_recipe}" | grep hyrax | awk '{print $1;}' | sed "s/hyrax-//g" )} OLFS_VERSION=${OLFS_VERSION:-$(echo "${build_recipe}" | grep olfs | sed "s/olfs-//g" | awk '{print $1;}')} @@ -54,36 +59,82 @@ function read_build_recipe() { TEST_DEPLOYMENT="$td_value" fi fi + + + if test "$DOCKER_NAME" == "ngap" + then + ############################################################################################### + # For the ngap distribution we make special SNAPSHOT_IMAGE_TAG and BUILD_VERSION_TAG values + # because the ngap product gets tagged differently than the products from our other repos. + # Specifically, we use the opendap/hyrax repo since the ngap product is just a specialization + # of hyrax, at least for now. So we use the opendap/hyrax project identifier, and we add the + # $DOCKER_NAME ("ngap" in this case) to the image tag: + # Examples: + # opendap/hyrax:ngap-snapshot + # opendap/hyrax:ngap-snapshot-el8 + # opendap/hyrax:ngap-1.17.1-846 + # opendap/hyrax:ngap-1.17.1-846-el8 + # opendap/hyrax:ngap-snapshot-el8-test-deploy + # opendap/hyrax:ngap--1.17.1-846-el8-test-deploy + # We set (and replace) these values here, in the stage, so that they can be accessed by the + # travis/deploy_to_docker_hub.sh script used by the Travis deploy operation below. + SNAPSHOT_IMAGE_TAG="${SNAPSHOT_IMAGE_TAG:-"opendap/hyrax:$DOCKER_NAME-snapshot$TEST_DEPLOYMENT"}" + BUILD_VERSION_TAG="${BUILD_VERSION_TAG:-"opendap/hyrax:$DOCKER_NAME-$HYRAX_VERSION$TEST_DEPLOYMENT"}" + + OS_SNAPSHOT_IMAGE_TAG="${OS_SNAPSHOT_IMAGE_TAG:-"opendap/hyrax:$DOCKER_NAME-snapshot-$TARGET_OS$TEST_DEPLOYMENT"}" + OS_BUILD_VERSION_TAG="${OS_BUILD_VERSION_TAG:-"opendap/hyrax:$DOCKER_NAME-$HYRAX_VERSION-$TARGET_OS$TEST_DEPLOYMENT"}" + else + ############################################################################################### + # This is the nominal tagging scheme + # + + SNAPSHOT_IMAGE_TAG="${SNAPSHOT_IMAGE_TAG:-"opendap/$DOCKER_NAME:snapshot$TEST_DEPLOYMENT"}" + BUILD_VERSION_TAG="${BUILD_VERSION_TAG:-"opendap/$DOCKER_NAME:$HYRAX_VERSION$TEST_DEPLOYMENT"}" + + OS_SNAPSHOT_IMAGE_TAG="${OS_SNAPSHOT_IMAGE_TAG:-"opendap/$DOCKER_NAME:snapshot-$TARGET_OS$TEST_DEPLOYMENT"}" + OS_BUILD_VERSION_TAG="${OS_BUILD_VERSION_TAG:-"opendap/$DOCKER_NAME:$HYRAX_VERSION-$TARGET_OS$TEST_DEPLOYMENT"}" + fi + } read_build_recipe "${BUILD_RECIPE_FILE}" -export SNAPSHOT_IMAGE_TAG=${SNAPSHOT_IMAGE_TAG:-"opendap/${DOCKER_NAME}:snapshot"} -export BUILD_VERSION_TAG=${BUILD_VERSION_TAG:-"opendap/${DOCKER_NAME}:${HYRAX_VERSION}"} + + + + export RELEASE_HOST=${RELEASE_HOST:-"https://www.opendap.org"} function show_version(){ loggy "${HR}" - loggy " RELEASE_DATE: ${RELEASE_DATE}" - loggy " HYRAX_VERSION: ${HYRAX_VERSION}" - loggy " OLFS_VERSION: ${OLFS_VERSION}" - loggy " BES_VERSION: ${BES_VERSION}" - loggy " BUILD_DMRPP_VERSION: ${BUILD_DMRPP_VERSION}" - loggy " LIBDAP_VERSION: ${LIBDAP_VERSION}" - loggy " DOCKER_NAME: ${DOCKER_NAME}" - loggy " BUILD_VERSION_TAG: ${BUILD_VERSION_TAG}" - loggy " SNAPSHOT_IMAGE_TAG: ${SNAPSHOT_IMAGE_TAG}" - loggy " S3_BUILD_BUCKET: ${S3_BUILD_BUCKET}" - loggy " SLEEP_INTERVAL: ${SLEEP_INTERVAL}" + loggy " RELEASE_DATE: ${RELEASE_DATE}" + loggy " TARGET_OS: $TARGET_OS" + loggy " DOCKER_DIR: $DOCKER_DIR" + loggy " DOCKER_NAME: $DOCKER_NAME" + loggy " HYRAX_VERSION: ${HYRAX_VERSION}" + loggy " OLFS_VERSION: ${OLFS_VERSION}" + loggy " BES_VERSION: ${BES_VERSION}" + loggy " BUILD_DMRPP_VERSION: ${BUILD_DMRPP_VERSION}" + loggy " LIBDAP_VERSION: ${LIBDAP_VERSION}" + loggy " DOCKER_NAME: ${DOCKER_NAME}" + loggy "" + loggy " BUILD_VERSION_TAG: '$BUILD_VERSION_TAG'" + loggy " OS_BUILD_VERSION_TAG: '$OS_BUILD_VERSION_TAG'" + loggy "" + loggy " SNAPSHOT_IMAGE_TAG: '$SNAPSHOT_IMAGE_TAG'" + loggy "OS_SNAPSHOT_IMAGE_TAG: '$OS_SNAPSHOT_IMAGE_TAG'" + loggy "" + loggy " S3_BUILD_BUCKET: ${S3_BUILD_BUCKET}" + loggy " SLEEP_INTERVAL: ${SLEEP_INTERVAL}" loggy "" - loggy "TOMCAT_MAJOR_VERSION: ${TOMCAT_MAJOR_VERSION}" + loggy " TOMCAT_MAJOR_VERSION: ${TOMCAT_MAJOR_VERSION}" loggy " TOMCAT_VERSION: ${TOMCAT_VERSION}" loggy "" - loggy " NO_CACHE: ${NO_CACHE}" + loggy " NO_CACHE: ${NO_CACHE}" loggy "" - loggy " TEST_DEPLOYMENT: '$TEST_DEPLOYMENT'" + loggy " TEST_DEPLOYMENT: '$TEST_DEPLOYMENT'" loggy "" } @@ -615,15 +666,15 @@ function s3_get_build_dmrpp_distro() { ################################################################################ # -# woo_get_olfs_distro() - Get the rpm and war files need for a Hyrax installation +# curl_get_olfs_distro() - Get the rpm and war files need for a Hyrax installation # from the identified S3 bucket. # $1 - The AWS S3 bucket from which to pull the Hyrax components # $2 - The directory into which to place the Hyrax components. # $3 - The olfs version number ##.##.##-## # -function woo_get_olfs_distro() { +function curl_get_olfs_distro() { loggy "${HR}" - loggy "TASK: woo_get_olfs_distro()" + loggy "TASK: curl_get_olfs_distro()" local target_dir="${1}" loggy " target_dir: ${target_dir}" local olfs_version="${2}" @@ -770,7 +821,7 @@ function build_hyrax_ncwms_docker() { loggy "build_hyrax_ncwms_docker()" loggy "" export DOCKER_NAME="hyrax" - source ./build-rh8 + source ./build-el8 export SNAPSHOT_IMAGE_TAG="opendap/${DOCKER_NAME}_ncwms:rh8-snapshot" export BUILD_VERSION_TAG="opendap/${DOCKER_NAME}_ncwms:rh8-${HYRAX_VERSION}" export TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") diff --git a/el8-builds/build-hyrax-ngap.sh b/el8-builds/build-hyrax-ngap.sh new file mode 100755 index 00000000..d10203e4 --- /dev/null +++ b/el8-builds/build-hyrax-ngap.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# +source "./build-$TARGET_OS" +HR0="#######################################################################" +HR1="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" +HR2="--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---" +############################################################################# +# loggy() +prolog="build-hyrax-ngap.sh" +function loggy(){ + echo "$@" | awk -v prolog="$prolog" '{ print "# " prolog " - " $0;}' >&2 +} +loggy "$HR0" +loggy "BEGIN" + +export DOCKER_NAME="${DOCKER_NAME:-"ngap"}" +loggy "DOCKER_NAME: $DOCKER_NAME" + +############################################################################################### +#export SNAPSHOT_IMAGE_TAG="${SNAPSHOT_IMAGE_TAG:-"opendap/hyrax:$DOCKER_NAME-snapshot-$TARGET_OS$TEST_DEPLOYMENT"}" +loggy " SNAPSHOT_IMAGE_TAG: '$SNAPSHOT_IMAGE_TAG'" >&2 +loggy "OS_SNAPSHOT_IMAGE_TAG: '$OS_SNAPSHOT_IMAGE_TAG'" >&2 +# +#export BUILD_VERSION_TAG="${BUILD_VERSION_TAG:-"opendap/hyrax:$DOCKER_NAME-$HYRAX_VERSION-$TARGET_OS$TEST_DEPLOYMENT"}" +loggy " BUILD_VERSION_TAG: '$OS_BUILD_VERSION_TAG'" >&2 +loggy " OS_BUILD_VERSION_TAG: '$OS_BUILD_VERSION_TAG'" >&2 +############################################################################################### + +loggy "TOMCAT_MAJOR_VERSION: $TOMCAT_MAJOR_VERSION" >&2 +export TOMCAT_VERSION= +TOMCAT_VERSION="$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION")" +loggy "TOMCAT_VERSION: $TOMCAT_VERSION" >&2 + +show_version + +get_tomcat_distro "$DOCKER_NAME" "$TOMCAT_VERSION" + +s3_get_besd_distro \ + "$S3_BUILD_BUCKET" \ + "$DOCKER_NAME" \ + "$TARGET_OS" \ + "$LIBDAP_VERSION" \ + "$BES_VERSION" \ + "$ADD_DEBUG_RPMS" 2>&1 + +get_ngap_olfs_distro "$S3_BUILD_BUCKET" "$DOCKER_NAME" "$OLFS_VERSION" 2>&1 + +docker build \ + --build-arg TOMCAT_VERSION \ + --build-arg RELEASE_DATE \ + --build-arg HYRAX_VERSION \ + --build-arg LIBDAP_VERSION \ + --build-arg BES_VERSION \ + --build-arg OLFS_VERSION \ + --tag "$OS_SNAPSHOT_IMAGE_TAG" \ + --tag "$OS_BUILD_VERSION_TAG" \ + "$DOCKER_NAME" + +docker image ls -a + +loggy "END" +loggy "$HR0" diff --git a/el8-builds/ngap/Dockerfile b/el8-builds/ngap/Dockerfile index 7ffc9e7d..7dac1683 100644 --- a/el8-builds/ngap/Dockerfile +++ b/el8-builds/ngap/Dockerfile @@ -25,74 +25,72 @@ ENV HR2="# -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - ENV DEFAULT_NGAP_DEPLOYMENT_CONTEXT="ngap" ARG DEPLOYMENT_CONTEXT -ENV DEPLOYMENT_CONTEXT ${DEPLOYMENT_CONTEXT:-"ROOT"} +ENV DEPLOYMENT_CONTEXT="${DEPLOYMENT_CONTEXT:-"ROOT"}" RUN echo "DEPLOYMENT_CONTEXT: ${DEPLOYMENT_CONTEXT}" -RUN export NOW=`date +%s` - # # Hyrax Components Version Negotiation # ARG RELEASE_DATE -ENV RELEASE_DATE ${RELEASE_DATE:-"${NOW}"} +ENV RELEASE_DATE="${RELEASE_DATE:-$(date +%s)}" RUN echo "RELEASE_DATE: ${RELEASE_DATE}" ARG TOMCAT_VERSION -ENV TOMCAT_VERSION ${TOMCAT_VERSION:-"9.0.64"} +ENV TOMCAT_VERSION="${TOMCAT_VERSION:-"9.0.64"}" RUN echo "TOMCAT_VERSION: ${TOMCAT_VERSION}" -ENV TOMCAT_DISTRO="apache-tomcat-${TOMCAT_VERSION}" -COPY "${TOMCAT_DISTRO}.tar.gz" / +ENV TOMCAT_DISTRO="apache-tomcat-$TOMCAT_VERSION" +COPY "$TOMCAT_DISTRO.tar.gz" / RUN ls -l /apache-tomcat-* >&2 ARG HYRAX_VERSION -ENV HYRAX_VERSION=${HYRAX_VERSION:-"snapshot"} -RUN echo "HYRAX_VERSION: ${HYRAX_VERSION}" +ENV HYRAX_VERSION="${HYRAX_VERSION:-"snapshot"}" +RUN echo "HYRAX_VERSION: $HYRAX_VERSION" ARG LIBDAP_VERSION -ENV LIBDAP_VERSION=${LIBDAP_VERSION:-"snapshot"} -RUN echo "LIBDAP_VERSION: ${LIBDAP_VERSION}" +ENV LIBDAP_VERSION="${LIBDAP_VERSION:-"snapshot"}" +RUN echo "LIBDAP_VERSION: $LIBDAP_VERSION" ARG LIBDAP_RPM -ENV LIBDAP_RPM=${LIBDAP_RPM:-"libdap-${LIBDAP_VERSION}.el8.x86_64.rpm"} -RUN echo "LIBDAP_RPM: ${LIBDAP_RPM}" -COPY ${LIBDAP_RPM} libdap-devel-* libdap-debuginfo-* / +ENV LIBDAP_RPM="${LIBDAP_RPM:-"libdap-$LIBDAP_VERSION.el8.x86_64.rpm"}" +RUN echo "LIBDAP_RPM: $LIBDAP_RPM" +COPY "$LIBDAP_RPM" libdap-devel-* libdap-debuginfo-* / RUN ls -l /libdap-* >&2 ARG BES_VERSION -ENV BES_VERSION=${BES_VERSION:-"snapshot"} -RUN echo "BES_VERSION: ${BES_VERSION}" +ENV BES_VERSION="${BES_VERSION:-"snapshot"}" +RUN echo "BES_VERSION: $BES_VERSION" ARG BES_RPM -ENV BES_RPM=${BES_RPM:-"bes-${BES_VERSION}.static.el8.x86_64.rpm"} -RUN echo "BES_RPM: ${BES_RPM}" -COPY ${BES_RPM} bes-devel-* bes-debuginfo-* / +ENV BES_RPM="${BES_RPM:-"bes-$BES_VERSION.static.el8.x86_64.rpm"}" +RUN echo "BES_RPM: $BES_RPM" +COPY "$BES_RPM" bes-devel-* bes-debuginfo-* / RUN ls -l /bes-* >&2 ARG OLFS_VERSION -ENV OLFS_VERSION=${OLFS_VERSION:-"unknown"} -RUN echo "OLFS_VERSION: ${OLFS_VERSION}" +ENV OLFS_VERSION="${OLFS_VERSION:-"unknown"}" +RUN echo "OLFS_VERSION: $OLFS_VERSION" ARG NGAP_DISTRO -ENV NGAP_DISTRO=${NGAP_DISTRO:-"ngap-${OLFS_VERSION}-webapp"} -RUN echo "NGAP_DISTRO: ${NGAP_DISTRO}" -COPY ${NGAP_DISTRO}.tgz /${NGAP_DISTRO}.tgz -RUN tar -tf /${NGAP_DISTRO}.tgz +ENV NGAP_DISTRO="${NGAP_DISTRO:-"ngap-$OLFS_VERSION-webapp"}" +RUN echo "NGAP_DISTRO: $NGAP_DISTRO" +COPY "$NGAP_DISTRO.tgz" "/$NGAP_DISTRO.tgz" +RUN tar -tf "/$NGAP_DISTRO.tgz" ARG TOMCAT_CONNECTION_TIMEOUT_MS -ENV TOMCAT_CONNECTION_TIMEOUT_MS=${TOMCAT_CONNECTION_TIMEOUT_MS:-"1000000"} -RUN echo "TOMCAT_CONNECTION_TIMEOUT_MS: ${TOMCAT_CONNECTION_TIMEOUT_MS}" +ENV TOMCAT_CONNECTION_TIMEOUT_MS="${TOMCAT_CONNECTION_TIMEOUT_MS:-"1000000"}" +RUN echo "TOMCAT_CONNECTION_TIMEOUT_MS: $TOMCAT_CONNECTION_TIMEOUT_MS" LABEL vendor="OPeNDAP" -LABEL org.opendap.hyrax.version=${HYRAX_VERSION} -LABEL org.opendap.hyrax.release-date=${RELEASE_DATE} +LABEL org.opendap.hyrax.version="$HYRAX_VERSION" +LABEL org.opendap.hyrax.release-date="$RELEASE_DATE" LABEL org.opendap.hyrax.version.is-production="false" -MAINTAINER support@opendap.org +LABEL org.opencontainers.image.authors="support@opendap.org" USER root ARG DEVELOPER_MODE -ENV DEVELOPER_MODE ${DEVELOPER_MODE:-"false"} +ENV DEVELOPER_MODE="${DEVELOPER_MODE:-"false"}" RUN set -e && \ if [ $DEVELOPER_MODE = "true" ];then echo "DEVELOPER_MODE: ENABLED"; else echo "DEVELOPER_MODE: DISABLED"; fi diff --git a/el8-builds/olfs/Dockerfile b/el8-builds/olfs/Dockerfile index 3dc1a0f9..dc543927 100644 --- a/el8-builds/olfs/Dockerfile +++ b/el8-builds/olfs/Dockerfile @@ -11,7 +11,7 @@ ENV python_version="3.11" ENV HR="###########################################################################" ENV HR2="# -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - --" -MAINTAINER support@opendap.org +LABEL org.opencontainers.image.authors="support@opendap.org" USER root RUN export NOW=`date +%s` diff --git a/el9-build-recipe b/el9-build-recipe new file mode 100644 index 00000000..f39ed165 --- /dev/null +++ b/el9-build-recipe @@ -0,0 +1,6 @@ +TARGET_OS: el9 +TOMCAT_MAJOR_VERSION: 11 +libdap4-3.21.1-366 2026-01-27T19:58:24+0000 +bes-3.21.1-1065 2026-02-23T20:47:04+0000 +olfs-1.18.15-704 2026-02-24T18:37:24+0000 +hyrax-1.17.1-704 2026-02-24T18:37:24+0000 diff --git a/el9-builds/besd/Dockerfile b/el9-builds/besd/Dockerfile new file mode 100644 index 00000000..30a61730 --- /dev/null +++ b/el9-builds/besd/Dockerfile @@ -0,0 +1,120 @@ +############################################################################################### +# +# Dockerfile for besdaemon image +# +# +# Some shell state reference: +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. +# +# In general use "set -e" when running commands that matter and don't use +# it for debugging stuff. +# +# Set one or more individual labels +# +FROM rockylinux:9 +ENV HR="###############################################################################################################" +ENV HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +ENV TARGET_OS="el9" +ENV ARCH="x86_64" + +RUN echo "$HR" +RUN echo "# BEGIN - besd docker build" +# rocky9 comes with python 3.9 by default, and replacing it seems problematic at best. +RUN echo "# Python Version: $(python3 --version)" + +ENV dnf_runtime_packages="which unzip procps bc" +RUN echo "# dnf_runtime_packages: $dnf_runtime_packages" + +ENV dnf_developer_tools_packages="emacs vim awscli jq gdb valgrind" +RUN echo "# dnf_developer_tools_packages: $dnf_developer_tools_packages" + +ARG RELEASE_DATE +ENV RELEASE_DATE="${RELEASE_DATE:-$(date +%s)}" +RUN echo "RELEASE_DATE: $RELEASE_DATE" + +ARG LIBDAP_VERSION +ENV LIBDAP_VERSION=${LIBDAP_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "LIBDAP_VERSION: $LIBDAP_VERSION" + +ARG LIBDAP_RPM +ENV LIBDAP_RPM=${LIBDAP_RPM:-"libdap-$LIBDAP_VERSION.$TARGET_OS.x86_64.rpm"} +RUN echo "LIBDAP_RPM: $LIBDAP_RPM" +COPY "$LIBDAP_RPM" "/$LIBDAP_RPM" + +ARG BES_VERSION +ENV BES_VERSION=${BES_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "BES_VERSION: $BES_VERSION" + +ARG BES_RPM +ENV BES_RPM=${BES_RPM:-"bes-$BES_VERSION.static.$TARGET_OS.x86_64.rpm"} +RUN echo "BES_RPM: $BES_RPM" +COPY "$BES_RPM" "/$BES_RPM" + + +LABEL vendor="OPeNDAP Incorporated" +LABEL org.opendap.besdaemon.version="$BES_VERSION" +LABEL org.opendap.besdaemon.release-date="$RELEASE_DATE" +LABEL org.opendap.hyrax.version.is-production="false" +LABEL org.opencontainers.image.authors="support@opendap.org" + +USER root + +###################################################################### +# Update the OS and create a handy executable shell scipt to simplify +# the installation of the (not as yet installed) but super useful +# developer tools packages :) ndp +RUN set -e \ + && dnf update -y \ + && dnf install -y $dnf_runtime_packages \ + && echo "#!/bin/bash" > /install-dev-tools \ + && echo "dnf install -y $dnf_developer_tools_packages" > /install-dev-tools \ + && chmod 755 /install-dev-tools + +################################################################ +# Install the libdap rpm +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing the libdap rpm: $LIBDAP_RPM" >&2 \ + && ls -l ./libdap* \ + && dnf -y install ./libdap-*.rpm \ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && rm -f "./$LIBDAP_RPM" \ + && dnf clean all \ + && echo "# libdap4 is installed." >&2 \ + && echo "$HR" >&2 + +################################################################ +# Install the the BES +# besd, besdaemon, beslistener, besctl, besstandalone, bescmdline +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing the BES rpm: $BES_RPM" >&2 \ + && ls -l ./bes* \ + && dnf -y install ./bes-*.rpm \ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && rm -f "./$BES_RPM" \ + && dnf clean all \ + && echo "# besdaemon is here: $(which besdaemon)" >&2 \ + && echo "# besdaemon -v: '$(besdaemon -v)'" >&2 \ + && echo "$HR" >&2 + + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 10022 +EXPOSE 11002 + +# can't use USER with entrypoint that needs root +# use gosu or, as done, enable bes user write so the entrypoint doe snot need root +RUN chown -R bes /etc/bes +USER root + +CMD ["-"] + diff --git a/el9-builds/besd/entrypoint.sh b/el9-builds/besd/entrypoint.sh new file mode 100755 index 00000000..8dbc4289 --- /dev/null +++ b/el9-builds/besd/entrypoint.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# This is the entrypoint.sh file for the besd container. +# +# +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. +# +export debug=false +export BANNER="################################# BES #############################################" +export HR0="###################################################################################" +export HR1="-----------------------------------------------------------------------------------" +export HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +export prolog="entrypoint.sh -" +function loggy(){ + echo "$@" | awk -v prolog="$prolog" '{ print "# " prolog " " $0;}' >&2 +} + +loggy "$HR0" +loggy "$BANNER" +loggy "Greetings, I am $(whoami) (uid: $UID)" +loggy "PythonVersion: $(python3 --version)" + +if [ "$SERVER_HELP_EMAIL" ] && [ -n "$SERVER_HELP_EMAIL" ] ; then + loggy "Found existing SERVER_HELP_EMAIL: $SERVER_HELP_EMAIL" +else + SERVER_HELP_EMAIL="not_set" + loggy "SERVER_HELP_EMAIL is $SERVER_HELP_EMAIL" +fi +if [ "$FOLLOW_SYMLINKS" ] && [ -n "$FOLLOW_SYMLINKS" ] ; then + loggy "Found existing FOLLOW_SYMLINKS: $FOLLOW_SYMLINKS" +else + FOLLOW_SYMLINKS="not_set"; + loggy "FOLLOW_SYMLINKS is $FOLLOW_SYMLINKS" +fi + +export SLEEP_INTERVAL="${SLEEP_INTERVAL:-60}" +loggy "SLEEP_INTERVAL: $SLEEP_INTERVAL seconds." + +#AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}" +#loggy "AWS_SECRET_ACCESS_KEY is ${AWS_SECRET_ACCESS_KEY}" +# +#AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}" +#loggy "AWS_ACCESS_KEY_ID is ${AWS_ACCESS_KEY_ID}" +# +#AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-}" +#loggy "AWS_DEFAULT_REGION is ${AWS_DEFAULT_REGION}" + + +while getopts "e:sdi:k:r:" opt; do + case $opt in + e) + SERVER_HELP_EMAIL=$OPTARG + loggy "SERVER_HELP_EMAIL: $SERVER_HELP_EMAIL"; + ;; + s) + FOLLOW_SYMLINKS="Yes" + loggy "FOLLOW_SYMLINKS: $FOLLOW_SYMLINKS"; + ;; + d) + debug=true; + loggy "Debug is enabled"; + ;; + k) + AWS_SECRET_ACCESS_KEY="${OPTARG}" + loggy "Found command line value for AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}"; + ;; + i) + AWS_ACCESS_KEY_ID="${OPTARG}" + loggy "Found command line value for AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}"; + ;; + r) + AWS_DEFAULT_REGION="${OPTARG}" + loggy "Found command line value for AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION}"; + ;; + \?) + loggy "Invalid option: -$OPTARG" + loggy "options: [-e xxx] [-s] [-d] [-i xxx] [-k xxx] [-r xxx]" + loggy " -e xxx where xxx is the email address of the admin contact for the server." + loggy " -s When present causes the BES to follow symbolic links." + loggy " -d Enables debugging output for this script." + loggy " -i xxx Where xxx is an AWS CLI AWS_ACCESS_KEY_ID." + loggy " -k xxx Where xxx is an AWS CLI AWS_SECRET_ACCESS_KEY." + loggy " -r xxx Where xxx is an AWS CLI AWS_DEFAULT_REGION." + exit 2; + ;; + esac +done + +################################################################################ +loggy "Checking AWS CLI: " +set +e +aws_bin="$(which aws 2>&1)" +ab_status=$? +set -e +if test $ab_status -ne 0; then + loggy "WARNING: It appears that the AWS CLI is not installed. Not found on '$PATH' ('which' status: $ab_status, msg: $aws_bin)" +else + acl="$(aws configure list 2>&1)" + acl_status=$? + loggy "$acl" + if test $acl_status -ne 0; then + loggy "WARNING: Problem with AWS CLI! ('aws' status: $acl_status msg: $acl)" + fi +fi +# loggy "$@" + + +# modify bes.conf based on environment variables before startup. These are set in +# the Docker file to "not_set" and are overriden by the commandline here +# +if [ "$SERVER_HELP_EMAIL" != "not_set" ]; then + loggy "Setting Admin Contact To: $SERVER_HELP_EMAIL" + sed -i "s/admin.email.address@your.domain.name/$SERVER_HELP_EMAIL/" /etc/bes/bes.conf +fi +if [ "$FOLLOW_SYMLINKS" != "not_set" ]; then + loggy "Setting BES FollowSymLinks to YES." + sed -i "s/^BES.Catalog.catalog.FollowSymLinks=No/BES.Catalog.catalog.FollowSymLinks=Yes/" /etc/bes/bes.conf +fi + +#------------------------------------------------------------------------------- +# We use 'echo' in the following because downstream code is expecting this +# output to be a key value pair, so none of that loggy() stuff +bes_uid="$(id -u bes)" +echo "bes_uid: $bes_uid" +bes_gid="$(id -g bes)" +echo "bes_gid: $bes_gid" + +# Start the BES daemon process +# /usr/bin/besdaemon -i /usr -c /etc/bes/bes.conf -r /var/run/bes.pid +loggy "Calling 'besctl start'" +/usr/bin/besctl start +status=$? +if [ $status -ne 0 ]; then + loggy "ERROR: Failed to start BES: $status" + exit $status +fi + +besd_pid="$(ps aux | grep /usr/bin/besdaemon | grep -v grep | awk '{print $2;}' - )" +loggy "The besdaemon is UP! pid: $besd_pid" +start_time= +start_time="$(date "+%s")" +loggy "The BES Has Arrived...(time: $start_time, SLEEP_INTERVAL: $SLEEP_INTERVAL)" +loggy "$HR0" + +while /bin/true; do + sleep $SLEEP_INTERVAL + besd_ps="$(ps -f "$besd_pid")"; + BESD_STATUS=$? + if test $BESD_STATUS -ne 0 + then + loggy "BESD_STATUS: $BESD_STATUS bes_pid: $bes_pid" + loggy "The BES daemon appears to have died! Exiting." + exit 1; + fi + + if test "$debug" = "true"; + then + loggy "$(date) BESD_STATUS: $BESD_STATUS besd_pid: $besd_pid" + fi +done diff --git a/el9-builds/build-el9 b/el9-builds/build-el9 new file mode 100755 index 00000000..70fd92b9 --- /dev/null +++ b/el9-builds/build-el9 @@ -0,0 +1,1382 @@ +#!/bin/bash +HR="##########################################################################" +HR2="== == == == == == == == == == == == == == == == == == == == == == == == ==" +HR3="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" +prolog="build_el9 -" +########################################################################### +function loggy(){ + if test -n "${log_file}" + then + echo "$@" | awk '{ print "# "$0;}' | tee -a "${log_file}" >&2 + else + echo "$@" | awk '{ print "# "$0;}' >&2 + fi +} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +loggy "$HR" +loggy "$prolog BEGIN" + +export BUILD_RECIPE="${1:-"../el9-build-recipe"}" +loggy "$prolog BUILD_RECIPE: '$BUILD_RECIPE'" + +export S3_BUILD_BUCKET="${S3_BUILD_BUCKET:-"opendap.travis.build"}" +loggy "$prolog S3_BUILD_BUCKET: '$S3_BUILD_BUCKET'" + +export DOCKER_NAME="${DOCKER_NAME:-"hyrax"}" +loggy "$prolog DOCKER_NAME: '$DOCKER_NAME'" + +#export TOMCAT_MAJOR_VERSION=${TOMCAT_MAJOR_VERSION:-"11"} +#loggy "TOMCAT_MAJOR_VERSION: $TOMCAT_MAJOR_VERSION" + +export SLEEP_INTERVAL="${SLEEP_INTERVAL:-60}" +loggy "$prolog SLEEP_INTERVAL: $SLEEP_INTERVAL" + +export NO_CACHE="${NO_CACHE:-""}" +loggy "$prolog NO_CACHE: $NO_CACHE" + +export ARCHIVE_HOST=${ARCHIVE_HOST:-"https://www.opendap.org"} +loggy "$prolog ARCHIVE_HOST: $ARCHIVE_HOST" + +export aws_quiet="${aws_quiet:-"shhhh"}" + +################################################################################ +# +# get_test_deployment_tag() +# Returns a value of "-test-deploy" if either the current branch or +# the build-recipe file contains the string "test-deploy" +# +function get_test_deployment_tag(){ + local build_recipe_file="$1" + + local td_value="-test-deploy" + local td + if [[ "$TRAVIS_BRANCH" == *"-test-deploy" ]] + then + td="$td_value" + fi + + if test -z "$td" + then + grep "test-deploy" "$build_recipe_file" > /dev/null + if test $? -eq 0 + then + td="$td_value" + fi + fi + echo "$td" + +} + +function read_build_recipe() { + local prolog="read_build_recipe() -" + local build_recipe_file="${1}" + local build_recipe="" + build_recipe=$(cat "${build_recipe_file}") + + export TARGET_OS + export TOMCAT_MAJOR_VERSION + export LIBDAP_VERSION + export BES_VERSION + export OLFS_VERSION + export HYRAX_VERSION + export RELEASE_DATE + # export APR_VERSION + export OS_SNAPSHOT_IMAGE_TAG + export OS_BUILD_VERSION_TAG + export TEST_DEPLOYMENT + + + TARGET_OS="${TARGET_OS:-$(echo "$build_recipe" | grep "TARGET_OS" | awk '{print $2;}')}" + TOMCAT_MAJOR_VERSION="${TOMCAT_MAJOR_VERSION:-$(echo "$build_recipe" | grep "TOMCAT_MAJOR_VERSION" | awk '{print $2;}')}" + LIBDAP_VERSION=${LIBDAP_VERSION:-$(echo "$build_recipe" | grep libdap4 | sed "s/libdap4-//g"| awk '{print $1;}')} + BES_VERSION=${BES_VERSION:-$(echo "$build_recipe" | grep bes | sed "s/bes-//g" | awk '{print $1;}')} + OLFS_VERSION=${OLFS_VERSION:-"$(echo "$build_recipe" | grep olfs | sed "s/olfs-//g" | awk '{print $1;}')-$TARGET_OS"} + HYRAX_VERSION=${HYRAX_VERSION:-$(echo "$build_recipe" | grep hyrax | awk '{print $1;}' | sed "s/hyrax-//g" )} + RELEASE_DATE=${RELEASE_DATE:-$(echo "$build_recipe" | grep hyrax | awk '{print $2;}')} + #APR_VERSION="1.7.6-1" + #APR_VERSION=${APR_VERSION:-$(echo "${build_recipe}" | grep apr | sed "s/apr-//g" | awk '{print $1;}')} + TEST_DEPLOYMENT="${TEST_DEPLOYMENT:-"$(get_test_deployment_tag "$build_recipe_file")"}" + + if test "$DOCKER_NAME" == "ngap" + then + ############################################################################################### + # For the ngap distribution we make special SNAPSHOT_IMAGE_TAG and BUILD_VERSION_TAG values + # because the ngap product gets tagged differently than the products from our other repos. + # Specifically, we use the opendap/hyrax repo since the ngap product is just a specialization + # of hyrax, at least for now. So we use the opendap/hyrax project identifier, and we add the + # $DOCKER_NAME ("ngap" in this case) to the image tag: + # Examples: + # opendap/hyrax:ngap-snapshot-el9 + # opendap/hyrax:ngap-1.17.1-846-el9 + # opendap/hyrax:ngap-snapshot-el9-test-deploy + # opendap/hyrax:ngap--1.17.1-846-el9-test-deploy + # We set (and replace) these values here, in the stage, so that they can be accessed by the + # travis/deploy_to_docker_hub.sh script used by the Travis deploy operation below. + OS_SNAPSHOT_IMAGE_TAG="${OS_SNAPSHOT_IMAGE_TAG:-"opendap/hyrax:$DOCKER_NAME-snapshot-$TARGET_OS$TEST_DEPLOYMENT"}" + OS_BUILD_VERSION_TAG="${OS_BUILD_VERSION_TAG:-"opendap/hyrax:$DOCKER_NAME-$HYRAX_VERSION-$TARGET_OS$TEST_DEPLOYMENT"}" + else + OS_SNAPSHOT_IMAGE_TAG="${OS_SNAPSHOT_IMAGE_TAG:-"opendap/$DOCKER_NAME:snapshot-$TARGET_OS$TEST_DEPLOYMENT"}" + OS_BUILD_VERSION_TAG="${OS_BUILD_VERSION_TAG:-"opendap/$DOCKER_NAME:$HYRAX_VERSION-$TARGET_OS$TEST_DEPLOYMENT"}" + fi + + # We unset these so that they are surely NOT set when we deploy + # we don't use these tags in the el9 food chain, at least not yet + unset SNAPSHOT_IMAGE_TAG + unset BUILD_VERSION_TAG +} +read_build_recipe "${BUILD_RECIPE}" + + + + +function show_version(){ + local prolog="show_version() -" + loggy "${HR}" + loggy "$prolog BEGIN" + loggy "$prolog RELEASE_DATE: $RELEASE_DATE" + loggy "$prolog TARGET_OS: $TARGET_OS" + loggy "$prolog TEST_DEPLOYMENT: $TEST_DEPLOYMENT" + loggy "$prolog LIBDAP_VERSION: $LIBDAP_VERSION" + loggy "$prolog BES_VERSION: $BES_VERSION" + loggy "$prolog OLFS_VERSION: $OLFS_VERSION" + loggy "$prolog HYRAX_VERSION: $HYRAX_VERSION" + loggy "$prolog DOCKER_NAME: $DOCKER_NAME" + loggy "$prolog S3_BUILD_BUCKET: $S3_BUILD_BUCKET" + loggy "$prolog SLEEP_INTERVAL: $SLEEP_INTERVAL" + loggy "$prolog" + loggy "$prolog BUILD_VERSION_TAG: '$BUILD_VERSION_TAG'" + loggy "$prolog OS_BUILD_VERSION_TAG: '$OS_BUILD_VERSION_TAG'" + loggy "$prolog" + loggy "$prolog SNAPSHOT_IMAGE_TAG: '$SNAPSHOT_IMAGE_TAG'" + loggy "$prolog OS_SNAPSHOT_IMAGE_TAG: '$OS_SNAPSHOT_IMAGE_TAG'" + loggy "$prolog" + loggy "$prolog TOMCAT_MAJOR_VERSION: $TOMCAT_MAJOR_VERSION" + loggy "$prolog TOMCAT_VERSION: $TOMCAT_VERSION" + loggy "$prolog APACHE_APR_VERSION: $APACHE_APR_VERSION" + loggy "$prolog" + loggy "$prolog NO_CACHE: '$NO_CACHE'" + loggy "$prolog END" + loggy "$HR" +} + + + +################################################################################ +# +# get_tomcat_tag_names() +# +# Goes to the Tomcat GitHub and retrieves the number of tag names, t_count using +# the api.github endpoint. The function checks for the value of t_count in $1 +# and if it's not found defaults to 120. If t_count is greater than 100 +# then this code will make multiple requests to api.github in order to gather +# the values from multiple pages. +# +function get_tomcat_tag_names(){ + local t_count="${1}" + if test -z "$t_count"; then t_count=120; fi + loggy "Retrieving $t_count tags." + + local tomcat_gh_repo="https://api.github.com/repos/apache/tomcat/tags" + loggy "Tomcat tomcat_gh_repo: $tomcat_gh_repo" + + local remaining=$t_count + local tag_count=0 + local tag_names="" + local ntc + local page=0 + while test $remaining -gt 0 + do + if test $remaining -gt 100 + then + ntc=100; + else + ntc=$remaining; + fi + let "page++" + let "remaining-=ntc" + + new_tags=$(curl -s "${tomcat_gh_repo}?per_page=${ntc}&page=${page}" | \ + jq -r '.[] | .name') + #loggy "new_tags: " + #loggy "$new_tags" + if test -n "$tag_names" + then + tag_names=$(echo "$tag_names"; echo "" ; echo "$new_tags" ) + else + tag_names="$new_tags" + fi + #loggy "tag_names: " + #loggy "$tag_names" + done + + tag_count=$( echo "$tag_names" | wc -l ) + loggy "Retrieved $tag_count tags from $tomcat_gh_repo" + echo "$tag_names" | sort -V +} + +################################################################################ +# +# get_latest_tomcat_version_number() +# +# Using the passed tomcat_major_version this function uses the Tomcat GitHub +# repo to find the most recent release for that major version, and then checks +# to make sure that the release is available from the Tomcat download server. +# If the version is not available on the download server, the release is +# discarded and the next most recent release is chosen. +# +# https://github.com/apache/tomcat +# +function get_latest_tomcat_version_number(){ + local prolog="get_latest_tomcat_version_number() -" + loggy "$prolog BEGIN" + + local tomcat_major_version="${1}" + local release_endpoint_base="https://dlcdn.apache.org/tomcat/tomcat" + loggy "$prolog Tomcat release_endpoint_base: $release_endpoint_base" + + local tomcat_gh_repo="https://api.github.com/repos/apache/tomcat/tags?per_page=100" + loggy "$prolog Tomcat tomcat_gh_repo: $tomcat_gh_repo" + + local retry=true + local ignore_list= + local tomcat_tags= + + tomcat_tags=$(get_tomcat_tag_names 200) + #loggy "Tomcat tomcat_tags: $tomcat_tags" + while $retry ; do + tomcat_major_version_list=$(echo "${tomcat_tags}" | grep "^${tomcat_major_version}" ) + + loggy "$prolog tomcat_major_version_list: " + loggy "${tomcat_major_version_list}" + + local tomcat_version="" + if test -n "${ignore_list}" ; then + loggy "$prolog Using ignore_list: '${ignore_list}'" + tomcat_version=$(echo "${tomcat_major_version_list}" | \ + grep -v ${ignore_list} | \ + tail -n 1 ) + else + loggy "$prolog The ignore_list is empty." + tomcat_version=$(echo "${tomcat_major_version_list}" | \ + tail -n 1 ) + fi + loggy "$prolog Candidate tomcat_version: ${tomcat_version}" + + local release_endpoint_url="${release_endpoint_base}-${tomcat_major_version}/v${tomcat_version}" + loggy "$prolog release_endpoint_url=${release_endpoint_url}" + + stat=$(curl -s -o /dev/null -w "%{response_code}" "${release_endpoint_url}") + if test $stat -eq 404 ; then + loggy "$prolog Candidate tomcat_version: ${tomcat_version} not found on download site: ${release_endpoint_url} " + if test -n "${tomcat_version}"; then + ignore_list+="${ignore_list} -e ${tomcat_version} " + else + loggy "$prolog ERROR: The tomcat_version is inexplicably empty. Exiting." + loggy "$prolog FAILED_TO_LOCATE_TOMCAT_VERSION" + return 1; + fi + retry=true + else + retry=false + ignore_list= + fi + done + loggy "$prolog Using tomcat_version: ${tomcat_version} " + loggy "$prolog END" + echo "${tomcat_version}" +} + + +################################################################################ +# +# get_tomcat_distro() +# $1 - the directory into which the Tomcat distribution files +# will be placed +# $2 - The version number of the Tomcat distribution to retrieve. +# Example: 9.0.64 +# +function get_tomcat_distro(){ + local prolog="get_tomcat_distro() -" + loggy "${HR}" + loggy "$prolog BEGIN" + + # $1 - The directory into which the Tomcat distribution files + # will be placed + local target_dir="${1}" + loggy "$prolog target_dir: ${target_dir}" + + # $2 - The version number of the Tomcat distribution to retrieve. + # Example: 9.0.64 + local tomcat_version="${2}" + loggy "$prolog tomcat_version: ${tomcat_version}" + + local tomcat_major_version="" + tomcat_major_version=$(echo "${tomcat_version}" | awk 'BEGIN{FS="."}{print $1;}') + loggy "$prolog tomcat_major_version: ${tomcat_major_version}" + + + local silent="-s" + + local tomcat_distro="apache-tomcat-${tomcat_version}.tar.gz" + loggy "$prolog tomcat_distro: ${tomcat_distro}" + + local tomcat_major_url="https://archive.apache.org/dist/tomcat/tomcat-${tomcat_major_version}" + loggy "$prolog tomcat_major_url: ${tomcat_major_url}" + + local tomcat_distro_url="${tomcat_major_url}/v${tomcat_version}/bin/${tomcat_distro}" + loggy "$prolog tomcat_distro_url: ${tomcat_distro_url}" + local tomcat_distro_file="${target_dir}/${tomcat_distro}" + loggy "$prolog tomcat_distro_file: ${tomcat_distro_file}" + + local tomcat_sig_url="${tomcat_distro_url}.asc" + loggy "$prolog tomcat_sig_url: ${tomcat_sig_url}" + local tomcat_sig_file="${tomcat_distro_file}.asc" + loggy "$prolog tomcat_sig_file: ${tomcat_sig_file}" + + local tomcat_keys_url="${tomcat_major_url}/KEYS" + loggy "$prolog tomcat_keys_url: ${tomcat_keys_url}" + local tomcat_keys_file="${target_dir}/apache_tomcat_keys" + loggy "$prolog tomcat_keys_file: ${tomcat_keys_file}" + + if test -f "$tomcat_distro_file" + then + loggy "$prolog The tomcat distribution file $tomcat_distro_file already exists." + else + loggy "" + loggy "$prolog cURL retrieving: ${tomcat_distro_url}" + curl ${silent} -o "${tomcat_distro_file}" "${tomcat_distro_url}" + if test $? -ne 0; then + echo "ERROR! Failed to retrieve Tomcat distribution from ${tomcat_distro_url}" >&2 + return 2 + fi + fi + + loggy "" + loggy "$prolog cURL retrieving: ${tomcat_sig_url}" + curl ${silent} -o "${tomcat_sig_file}" "${tomcat_sig_url}" + if test $? -ne 0; then + echo "ERROR! Failed to retrieve Tomcat distribution signature from ${tomcat_sig_url}" >&2 + return 2 + fi + + loggy "" + loggy "$prolog cURL retrieving: ${tomcat_keys_url}" + curl ${silent} -o "${tomcat_keys_file}" "${tomcat_keys_url}" + if test $? -ne 0; then + echo "ERROR! Failed to retrieve Tomcat public keys from ${tomcat_keys_url}" >&2 + return 2 + fi + + loggy "" + loggy "$prolog Importing Tomcat public keys:" + loggy "$prolog Local keys file: ${tomcat_keys_file}" + loggy "$prolog Source: ${tomcat_keys_url}" + gpg --import "${tomcat_keys_file}" 2>&1 | awk '{print "# ", $0;}' + if test $? -ne 0 ; then + echo "ERROR! Failed to import Tomcat public keys!" >&2 + return 2 + fi + + loggy "" + loggy "$prolog Verifying Tomcat distribution:" + loggy "$prolog Tomcat public keys: ${tomcat_keys_file}" + loggy "$prolog distribution file: ${tomcat_distro_file}" + loggy "$prolog signature: ${tomcat_sig_file}" + loggy "" + + gpg --verify "${tomcat_sig_file}" "${tomcat_distro_file}" 2>&1 | awk '{print "# ", $0;}' + if test $? -ne 0 ; then + echo "$prolog ERROR! Failed to verify Tomcat distribution!" >&2 + return 2 + fi + echo "$prolog SUCCESS - Retrieved Tomcat distribution: ${tomcat_distro_file}" >&2 + echo "$prolog $(ls -l ${tomcat_distro_file})" >&2 + loggy "" + return 0 +} + +################################################################################ +# +# check_aws() - Get a thing from an S3 bucket +# +# [-bash: ~] aws configure list +# Name Value Type Location +# ---- ----- ---- -------- +# profile None None +# access_key None None +# secret_key None None +# region us-west-2 config-file ~/.aws/config +# +# export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +# export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +# export AWS_DEFAULT_REGION=us-west-2 +# +# +function check_aws(){ + loggy "${HR}" + loggy "" + loggy "check_aws()" + loggy "" + + local aws_config_list= + aws_config_list=$(aws configure list | sed -e "s/not set/not_set/g") + local access_key + access_key=$(echo "${aws_config_list}" | grep -i access_key | awk '{print $2;}') + local secret_key + secret_key=$(echo "${aws_config_list}" | grep -i secret_key | awk '{print $2;}') + local region + region=$(echo "${aws_config_list}" | grep -i region | awk '{print $2;}') + local partial= + partial=$(echo "${aws_config_list}" | grep -i "Partial credentials found in env") + + ns="" + status=0; + if test "${access_key}" = "${ns}"; then + status=1 + loggy "The AWS_ACCESS_KEY_ID has not been set." + fi + if test "${secret_key}" = "${ns}"; then + status=1 + loggy "The AWS_SECRET_ACCESS_KEY has not been set." + fi + if test "${region}" = "${ns}"; then + status=1 + loggy "The AWS_DEFAULT_REGION has not been set." + fi + + if test -n "${partial}"; then + status=1 + loggy "${partial}" + fi + + if test $status -ne 0 + then + loggy "AWS CLI IS NOT CONFIGURED" + else + loggy "AWS CLI APPEARS TO BE CONFIGURED" + fi + loggy "${HR}" + + return $status +} + +################################################################################ +# +# s3_pull() - Get a thing from an S3 bucket +# +function s3_pull(){ + loggy "$HR3" + local prolog="s3_pull() -" + loggy "$prolog BEGIN" + + local bucket_name="${1}" + loggy "$prolog bucket_name: $bucket_name" + + local object_name="${2}" + loggy "$prolog object_name: $object_name" + + local target_dir="${3}" + loggy "$prolog target_dir: $target_dir" + + local s3_url="s3://${bucket_name}/${object_name}" + loggy "$prolog s3_url: $s3_url" + loggy "$prolog aws s3 ls: $(aws s3 ls "$s3_url")" + + local target_file="${target_dir}/${object_name}" + loggy "$prolog target_file: $target_file" + + local be_quiet + if test -n "${aws_quiet}"; then be_quiet="--quiet"; fi + loggy "$prolog be_quiet: '$be_quiet'" + + loggy "$prolog Retrieving: $s3_url" + aws s3 cp $be_quiet "$s3_url" "$target_file" + loggy "$prolog END" +} + +################################################################################ +# +# s3_get_besd_distro() - Get the rpm and war files need for a Hyrax installation +# from the identified S3 bucket. +# $1 - The AWS S3 bucket from which to pull the Hyrax components +# $2 - The directory into which to place the Hyrax components. +# $3 - The linux architecture (aka el7, el8, etc.) +# $4 - The libap version number ##.##.##-## +# $5 - The bes version number ##.##.##-## +# $6 - A debug flag, if present (i.e. not empty string) then the devel and +# debuginfo rpm files for libdap and bes will be retrieved as well. +# +function s3_get_besd_distro() { + local prolog="s3_get_besd_distro() -" + loggy "${HR2}" + loggy "$prolog BEGIN" + local s3_bucket="${1}" + loggy "$prolog s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy "$prolog target_dir: ${target_dir}" + local larch="${3}" + loggy "$prolog linux_architecture: ${larch}" + local libdap_version="${4}" + loggy "$prolog libdap_version: ${libdap_version}" + local bes_version="${5}" + loggy "$prolog bes_version: ${bes_version}" + local debug="${6}" + loggy "$prolog debug: ${debug}" + + set -e + + loggy "$prolog Cleaning up..." + rm -fv "${target_dir}/*.rpm" + + loggy "" + s3_pull "${s3_bucket}" \ + "libdap-${libdap_version}.${larch}.x86_64.rpm" \ + "${target_dir}" + + if test -n "${debug}"; then # Only get the debug libs when invoked in debug mode + s3_pull "${s3_bucket}" \ + "libdap-devel-${libdap_version}.${larch}.x86_64.rpm" \ + "${target_dir}" + s3_pull "${s3_bucket}" \ + "libdap-debuginfo-${libdap_version}.${larch}.x86_64.rpm" \ + "${target_dir}" + fi + + s3_pull "${s3_bucket}" \ + "bes-${bes_version}.static.${larch}.x86_64.rpm" \ + "${target_dir}" + + if test -n "${debug}"; then # Only get the debug libs when invoked in debug mode + s3_pull "${s3_bucket}" \ + "bes-devel-${bes_version}.static.${larch}.x86_64.rpm" \ + "${target_dir}" + s3_pull "${s3_bucket}" \ + "bes-debuginfo-${bes_version}.static.${larch}.x86_64.rpm" \ + "${target_dir}" + fi + + set +e + loggy "" + return 0 +} + +################################################################################ +# +# s3_get_apache_apr_distro() - Get the rpm and war files need for a Hyrax installation +# from the identified S3 bucket. +# $1 - The AWS S3 bucket from which to pull the Hyrax components +# $2 - The directory into which to place the Hyrax components. +# $3 - The APR version number ##.##.##-## +# $4 - A debug flag, if present (i.e. not empty string) then the devel and +# debuginfo rpm files for Apache APR will be retrieved as well. +# +function s3_get_apache_apr_distro() { + loggy "${HR2}" + loggy "TASK: s3_get_apache_apr_distro()" + local s3_bucket="${1}" + loggy " s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy " target_dir: ${target_dir}" + local apr_version="${3}" + loggy " apr_version: ${apr_version}" + local debug="${4}" + loggy " debug: ${debug}" + + set -e + + loggy "Cleaning up..." + rm -fv "${target_dir}/*.rpm" + + loggy "" + s3_pull "${s3_bucket}" \ + "apr-${apr_version}.x86_64.rpm" \ + "${target_dir}" + + #if test -n "${debug}"; then # Only get the debug libs when invoked in debug mode + s3_pull "${s3_bucket}" \ + "apr-devel-${apr_version}.x86_64.rpm" \ + "${target_dir}" + s3_pull "${s3_bucket}" \ + "apr-debuginfo-${apr_version}.x86_64.rpm" \ + "${target_dir}" + #fi + set +e + loggy "" + return 0 +} + +################################################################################ +# +# s3_get_apache_apr_distro() - Get the rpm and war files need for a Hyrax installation +# from the identified S3 bucket. +# $1 - The AWS S3 bucket from which to pull the Hyrax components +# $2 - The directory into which to place the Hyrax components. +# $3 - The APR version number ##.##.##-## +# $4 - A debug flag, if present (i.e. not empty string) then the devel and +# debuginfo rpm files for Apache APR will be retrieved as well. +# +function s3_get_openssl_distro() { + loggy "${HR2}" + loggy "TASK: s3_get_openssl_distro()" + local s3_bucket="${1}" + loggy " s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy " target_dir: ${target_dir}" + local openssl_version="${3}" + loggy " openssl_version: ${openssl_version}" + local larch="${4}" + loggy "linux_architecture: ${larch}" + local debug="${5}" + loggy " debug: ${debug}" + + set -e + + loggy "Cleaning up..." + rm -fv "${target_dir}/*.rpm" + + loggy "" + s3_pull "${s3_bucket}" \ + "openssl-${openssl_version}.${larch}.x86_64.rpm" \ + "${target_dir}" + + s3_pull "${s3_bucket}" \ + "openssl-libs-${openssl_version}.${larch}.x86_64.rpm" \ + "${target_dir}" + + s3_pull "${s3_bucket}" \ + "openssl-devel-${openssl_version}.${larch}.x86_64.rpm" \ + "${target_dir}" + + set +e + loggy "" + return 0 +} + +function curl_pull(){ + local release_url="${1}" + local target_file="${2}" + local target_dir="${3}" + + local target_url="${release_url}/${target_file}" + loggy "$HR3" + loggy "curl_pull(), retrieving:" + loggy " target_file: ${target_file}" + loggy " release_url: ${release_url}" + loggy " target_url: ${target_url}" + curl -s -L -o "${target_dir}/${target_file}" "${target_url}" + loggy "$(ls -l "${target_dir}/${target_file}")" + loggy "" +} + +################################################################################ +# +# woo_get_besd_distro() - Get the rpm and war files need for a Hyrax release +# from https://www.opendap.org/pub/binaries +# $1 - The directory into which to place the Hyrax components. +# $2 - The linux architecture (aka el7, el8, etc.) +# $3 - The libap version number ##.##.##-## +# $4 - The bes version number ##.##.##-## +# $5 - A debug flag, if present (i.e. not empty string) then the devel and +# debuginfo rpm files for libdap and bes will be retrieved as well. +# +function woo_get_besd_distro() { + loggy "${HR}" + loggy "TASK: woo_get_besd_distro()" + local target_dir="${1}" + loggy " target_dir: ${target_dir}" + local larch="${2}" + loggy "linux_architecture: ${larch}" + local libdap_version="${3}" + loggy " libdap_version: ${libdap_version}" + local bes_version="${4}" + loggy " bes_version: ${bes_version}" + local hyrax_version="${5}" + loggy " hyrax_version: ${hyrax_version}" + local debug="${6}" + loggy " debug: ${debug}" + + set -e + loggy "" + loggy "Cleaning up..." + rm -fv "${target_dir}/*.rpm" + loggy "" + + local release_dir="pub/binary" + release_dir="${release_dir}/hyrax-"$(echo "${hyrax_version}" | awk '{split($0,a,"."); print a[1]"."a[2];}') + ######### FIXME ######### + # FIXME - rocky-8 or rhel8? wtf + release_dir="${release_dir}/rocky-8" + ######## FIXME ######### + loggy "release_dir: ${release_dir}" + + release_url="${ARCHIVE_HOST}/${release_dir}" + loggy "release_url: ${release_url}" + + target_file="libdap-${libdap_version}.${larch}.x86_64.rpm" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + if test -n "${debug}"; then # Only get the debug libs when invoked in debug mode + target_file="libdap-devel-${libdap_version}.${larch}.x86_64.rpm" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + target_file="libdap-debuginfo-${libdap_version}.${larch}.x86_64.rpm" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + fi + + + target_file="bes-${bes_version}.static.${larch}.x86_64.rpm" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + if test -n "${debug}"; then # Only get the debug libs when invoked in debug mode + target_file="bes-devel-${bes_version}.static.${larch}.x86_64.rpm" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + target_file="bes-debuginfo-${bes_version}.static.${larch}.x86_64.rpm" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + fi + + set +e + loggy "" + return 0 +} + +function s3_get_olfs_ngap_distro() { + local prolog="s3_get_olfs_ngap_distro() -" + loggy "${HR}" + loggy "$prolog BEGIN" + local s3_bucket="${1}" + loggy "$prolog s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy "$prolog target_dir: ${target_dir}" + local ngap_version="${3}" + loggy "$prolog ngap_version: ${ngap_version}" + + set -e + loggy "$prolog Cleaning up..." + rm -fv "${target_dir}/nap*.tgz" + + loggy "" + s3_pull "${s3_bucket}" \ + "ngap-${ngap_version}-webapp.tgz" \ + "${target_dir}" + + set +e + loggy "$prolog END" + return 0 + +} +################################################################################ +# +# s3_get_olfs_distro() - Get the rpm and war files need for a Hyrax installation +# from the identified S3 bucket. +# $1 - The AWS S3 bucket from which to pull the Hyrax components +# $2 - The directory into which to place the Hyrax components. +# $3 - The olfs version number ##.##.##-## +# +function s3_get_olfs_distro() { + local prolog="s3_get_olfs_distro() -" + loggy "${HR}" + loggy "$prolog BEGIN" + local s3_bucket="${1}" + loggy "$prolog s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy "$prolog target_dir: ${target_dir}" + local olfs_version="${3}" + loggy "$prolog olfs_version: ${olfs_version}" + + set -e + loggy "$prolog Cleaning up..." + rm -fv "${target_dir}/olfs*.tgz" + + loggy "" + s3_pull "${s3_bucket}" \ + "olfs-${olfs_version}-webapp.tgz" \ + "${target_dir}" + + s3_pull "${s3_bucket}" \ + "robots-olfs-${olfs_version}-webapp.tgz" \ + "${target_dir}" + set +e + loggy "$prolog END" + return 0 +} + +################################################################################ +# +# s3_get_build_dmrpp_distro() - Get the rpm and war files need for a Hyrax +# installation from the identified S3 bucket. +# $1 - The AWS S3 bucket from which to pull the Hyrax components +# $2 - The directory into which to place the Hyrax components. +# $3 - The olfs version number ##.##.##-## +# +function s3_get_build_dmrpp_distro() { + local prolog="s3_get_build_dmrpp_distro() -" + loggy "${HR}" + loggy "$prolog BEGIN" + local s3_bucket="${1}" + loggy "$prolog s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy "$prolog target_dir: ${target_dir}" + local build_dmrpp_version="${3}" + loggy "$prolog build_dmrpp_version: ${build_dmrpp_version}" + + set -e + loggy "$prolog Cleaning up..." + rm -fv "${target_dir}/build_dmrpp*.tgz" + + loggy "" + s3_pull "${s3_bucket}" \ + "build_dmrpp-${build_dmrpp_version}-webapp.tgz" \ + "${target_dir}" + + set +e + loggy "" + loggy "$prolog END" + return 0 +} + + +################################################################################ +# +# curl_get_olfs_distro() - Get the rpm and war files need for a Hyrax installation +# from the identified S3 bucket. +# $2 - The directory into which to place the Hyrax components. +# $3 - The olfs version number ##.##.##-## +# +function curl_get_olfs_distro() { + local prolog="curl_get_olfs_distro() -" + loggy "${HR}" + loggy "$prolog BEGIN" + local target_dir="${1}" + loggy "$prolog target_dir: ${target_dir}" + local olfs_version="${2}" + loggy "$prolog olfs_version: ${olfs_version}" + + release_url="${ARCHIVE_HOST}/pub/olfs" + loggy "$prolog release_url: ${release_url}" + + set -e + loggy "$prolog Cleaning up..." + rm -fv "${target_dir}/olfs*.tgz" + rm -fv "${target_dir}/robots-olfs*.tgz" + + target_file="olfs-${olfs_version}-webapp.tgz" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + + target_file="robots-olfs-${olfs_version}-webapp.tgz" + curl_pull "${release_url}" "${target_file}" "${target_dir}" + + set +e + loggy "" + loggy "$prolog END" + return 0 +} + + +################################################################################ +# +# get_ngap_olfs_distro() - Get the rpm and war files need for a Hyrax NGAP OLFS +# installation from the identified S3 bucket. +# $1 - The AWS S3 bucket from which to pull the Hyrax components +# $2 - The directory into which to place the Hyrax components. +# $3 - The olfs version number ##.##.##-## +# +function get_ngap_olfs_distro() { + local prolog="get_ngap_olfs_distro() -" + loggy "${HR}" + loggy "$prolog BEGIN" + local s3_bucket="${1}" + loggy "$prolog s3_bucket: ${s3_bucket}" + local target_dir="${2}" + loggy "$prolog target_dir: ${target_dir}" + local olfs_version="${3}" + loggy "$prolog olfs_version: ${olfs_version}" + + set -e + loggy "$prolog Cleaning up..." + rm -fv "${target_dir}/ngap*.tgz" + + loggy "" + s3_pull "${s3_bucket}" \ + "ngap-${olfs_version}-webapp.tgz" \ + "${target_dir}" + + set +e + loggy "" + loggy "$prolog END" + + return 0 +} + + + + + +################################################################################ +# +# +function build_hyrax_docker() { + local docker_name="${1}" + + show_version + + loggy "${HR}" + loggy "TASK: build_hyrax_docker()" +# --no-cache \ + docker build ${NO_CACHE} \ + --platform linux/amd64 \ + --build-arg TOMCAT_VERSION \ + --build-arg RELEASE_DATE \ + --build-arg HYRAX_VERSION \ + --build-arg LIBDAP_VERSION \ + --build-arg BES_VERSION \ + --build-arg OLFS_VERSION \ + --tag "${OS_BUILD_VERSION_TAG}" \ + --tag "${OS_SNAPSHOT_IMAGE_TAG}" \ + ${docker_name} + loggy "" + loggy "" +} + + + +################################################################################ +# +# +function get_platform(){ + local platform="" + if test "$(uname -m)" = "arm64" ; then platform="--platform linux/amd64"; fi + echo "${platform}" +} + + +################################################################################ +# +# +function start_hyrax(){ + local image_tag="${1}" + + loggy "${HR}" + loggy "TASK: start_hyrax()" + loggy " Starting docker image: ${image_tag}" + local platform="" + platform=$(get_platform) + loggy " platform: ${platform}" + + docker run \ + -d \ + ${platform} \ + -h hyrax \ + -p 8080:8080 \ + --name=hyrax \ + --env SLEEP_INTERVAL \ + "${image_tag}" + + loggy "" +} + +################################################################################ +# +# +function debug_hyrax(){ + local image_tag="${1}" + + loggy "${HR}" + loggy "TASK: debug_hyrax()" + loggy " Starting docker image: ${image_tag}" + local platform="" + platform=$(get_platform) + loggy " platform: ${platform}" + + # docker run -d -h hyrax -p 8080:8080 --name=hyrax "${IMAGE_TAG}" + docker run \ + ${platform} \ + -h hyrax \ + -p 8080:8080 \ + --name=hyrax \ + --env SLEEP_INTERVAL \ + "${image_tag}" +} + +################################################################################ +# +# +function cleanup_build_files(){ + local docker_name="${1}" + + loggy "${HR}" + loggy "TASK: cleanup_build_files()" + loggy " Cleaning ${docker_name}" + local build_files="" + build_files="${build_files} ${docker_name}/*.rpm" + build_files="${build_files} ${docker_name}/olfs*.tgz" + build_files="${build_files} ${docker_name}/ngap*.tgz" + build_files="${build_files} ${docker_name}/robots*.tgz" + build_files="${build_files} ${DOCKER_NAME}/apache*" + # ls -Gl ${build_files} + rm -fv ${build_files} + loggy "" + loggy "DONE: cleanup_build_files()" + loggy "" +} + +################################################################################ +# +# +function build_hyrax() { + local debug="${1}" + + local log_file="build_hyrax.log" + show_version 2>&1 | tee "${log_file}" + + export TOMCAT_VERSION + TOMCAT_VERSION="$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION")" + + show_version + + get_tomcat_distro \ + "${DOCKER_NAME}" \ + "${TOMCAT_VERSION}" 2>&1 | tee -a "${log_file}" + + s3_get_besd_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "$TARGET_OS" \ + "${LIBDAP_VERSION}" \ + "${BES_VERSION}" $debug 2>&1 | tee -a "${log_file}" + + s3_get_olfs_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "${OLFS_VERSION}" 2>&1 | tee -a "${log_file}" + + get_ngap_olfs_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "${OLFS_VERSION}" 2>&1 | tee -a "${log_file}" + + build_hyrax_docker "${DOCKER_NAME}" 2>&1 | tee -a "${log_file}" + cleanup_build_files "${DOCKER_NAME}" 2>&1 | tee -a "${log_file}" +} + + + + + +################################################################################ +# +# +function docker_build_ngap() { + loggy "${HR}" + loggy "TASK: build_ngap_rh9()" +# --no-cache \ + docker build ${NO_CACHE} \ + --platform linux/amd64 \ + --build-arg TOMCAT_VERSION \ + --build-arg RELEASE_DATE \ + --build-arg HYRAX_VERSION \ + --build-arg LIBDAP_VERSION \ + --build-arg BES_VERSION \ + --build-arg OLFS_VERSION \ + --tag "${OS_BUILD_VERSION_TAG}" \ + --tag "${OS_SNAPSHOT_IMAGE_TAG}" \ + ${DOCKER_NAME} + loggy "" + loggy "" +} +################################################################################ +# +# +function build_ngap() { + local log_file="./build_ngap.log" + + export DOCKER_NAME="ngap" + show_version 2>&1 | tee "${log_file}" + + local tomcat_version="unset" + tomcat_version=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + get_tomcat_distro \ + "${DOCKER_NAME}" \ + "${tomcat_version}" 2>&1 | tee -a "${log_file}" + + s3_get_besd_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "el8" \ + "${LIBDAP_VERSION}" \ + "${BES_VERSION}" 2>&1 | tee -a "${log_file}" + + get_ngap_olfs_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "${OLFS_VERSION}" 2>&1 | tee -a "${log_file}" + + build_hyrax_docker "${DOCKER_NAME}" 2>&1 | tee -a "${log_file}" + cleanup_build_files "${DOCKER_NAME}" 2>&1 | tee -a "${log_file}" +} + + + + + + +################################################################################ +# +# +function build_olfs_docker() { + loggy "${HR}" + loggy "TASK: build_olfs_docker()" +# --no-cache \ + docker build ${NO_CACHE} \ + --platform linux/amd64 \ + --build-arg TOMCAT_VERSION \ + --build-arg RELEASE_DATE \ + --build-arg HYRAX_VERSION \ + --build-arg OLFS_VERSION \ + --tag "${OS_BUILD_VERSION_TAG}" \ + --tag "${OS_SNAPSHOT_IMAGE_TAG}" \ + ${DOCKER_NAME} + loggy "" + loggy "" +} + +################################################################################ +# +# +function build_olfs() { + local log_file="build_olfs.log" + + export DOCKER_NAME="olfs" + show_version 2>&1 | tee "${log_file}" + + local tomcat_version="unset" + tomcat_version=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + get_tomcat_distro \ + "${DOCKER_NAME}" \ + "${tomcat_version}" 2>&1 | tee -a "${log_file}" + + s3_get_olfs_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "${OLFS_VERSION}" 2>&1 | tee -a "${log_file}" + + build_olfs_docker 2>&1 | tee -a "${log_file}" +# cleanup_build_files "${DOCKER_NAME}" 2>&1 | tee -a "${log_file}" +} + + +################################################################################ +# +# +function build_besd_docker() { + loggy "${HR}" + loggy "TASK: build_besd_docker()" +# --no-cache \ + docker build ${NO_CACHE} \ + --platform linux/amd64 \ + --build-arg RELEASE_DATE \ + --build-arg LIBDAP_VERSION \ + --build-arg BES_VERSION \ + --tag "${OS_BUILD_VERSION_TAG}" \ + --tag "${OS_SNAPSHOT_IMAGE_TAG}" \ + ${DOCKER_NAME} + loggy "" + loggy "" +} + +################################################################################ +# +# +function build_besd() { + local log_file="build_besd.log" + + export DOCKER_NAME="besd" + show_version 2>&1 | tee "${log_file}" + + s3_get_besd_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "$TARGET_OS" \ + "${LIBDAP_VERSION}" \ + "${BES_VERSION}" 2>&1 | tee -a "${log_file}" + + build_besd_docker 2>&1 | tee -a "${log_file}" +# cleanup_build_files "${DOCKER_NAME}" 2>&1 | tee -a "${log_file}" +} + +################################################################################ +# pull_then_tag_and_push_new_tag() +# +function pull_then_tag_and_push_new_tag(){ + set -e + loggy "${HR2}" + loggy "" + loggy "pull_tag_push_image()" + loggy "" + local target_image_tag="${1}" + loggy " target_image_tag: ${target_image_tag}" + local new_tag="${2}" + loggy " new_tag: ${new_tag}" + + loggy "" + loggy "PULLING: ${target_image_tag}" + loggy "" + docker pull ${target_image_tag} + loggy "" + loggy "TAGGING AS: ${new_tag}" + loggy "" + set -x + docker tag ${target_image_tag} ${new_tag} + set +x + loggy "" + loggy "PUSHING: ${new_tag}" + loggy "" + set -x + docker push ${new_tag} + set +x + set +e +} + +################################################################################ +# +# mk_easy_docker_release() +# +# $1 - You must pass a build recipe (aka snapshot.time) file name containing the +# versions and build times. +# +# Reads a snapshot.time file (aka a build recipe) which contains a list of the +# Hyrax Docker files that made up the build. The idea is that is the release +# process is followed then recipe file will look something like: +# +# libdap4-3.20.11-0 2022-07-21T23:48:51+0000 +# bes-3.20.13-0 2022-07-22T14:41:58+0000 +# olfs-1.18.13-0 2022-07-22T16:36:38+0000 +# hyrax-1.16.8-0 2022-07-22T16:36:38+0000 +# +# Note that the build numbers at the end of the version numbers are all 0. +# +# The Docker images for: +# opendap/hyrax:HYRAX_BUILD_VERSION +# opendap/hyrax_ncwms:HYRAX_BUILD_VERSION +# opendap/besd:BES_BUILD_VERSION +# opendap/olfs:OLFS_BUILD_VERSION +# +# Are pulled from Docker Hub, then they are tagged with the release number. For +# our release that means the the trailing build number (should be "-0") is +# removed from the build version. The image is also tagged with the "latest" +# tag (meaning the most recent release) and then the reults are pushed to Docker +# Hub. +# +# Depends on the functions: +# - read_build_recipe() +# - pull_tag_and_push_image() +# - The HR variable +# +function mk_easy_docker_release(){ + local build_recipe_file="${1}" + loggy "${HR}" + loggy "" + loggy "mk_easy_docker_release() - BEGIN" + loggy "" + read_build_recipe "${build_recipe_file}" + + loggy " RELEASE_DATE: ${RELEASE_DATE}" + loggy " HYRAX_VERSION: ${HYRAX_VERSION}" + local hyrax_major_version= + hyrax_major_version=$(echo "${HYRAX_VERSION}" | awk '{split($0,a,"."); print a[1]"."a[2];}') + loggy " hyrax_major_version: ${hyrax_major_version}" + local hyrax_release_version= + hyrax_release_version=$(echo "${HYRAX_VERSION}" | awk '{split($0,a,"-"); print a[1];}') + loggy " hyrax_release_version: ${hyrax_release_version}" + + loggy " OLFS_VERSION: ${OLFS_VERSION}" + local olfs_release_version= + olfs_release_version=$(echo "${OLFS_VERSION}" | awk '{split($0,a,"-"); print a[1];}') + loggy " olfs_release_version: ${olfs_release_version}" + + local bes_version="${4}" + loggy " BES_VERSION: ${BES_VERSION}" + local bes_release_version= + bes_release_version=$(echo "${BES_VERSION}" | awk '{split($0,a,"-"); print a[1];}') + loggy " bes_release_version: ${bes_release_version}" + + pull_then_tag_and_push_new_tag "opendap/hyrax:${HYRAX_VERSION}" "opendap/hyrax:${hyrax_release_version}" + pull_then_tag_and_push_new_tag "opendap/hyrax:${HYRAX_VERSION}" "opendap/hyrax:latest" + + pull_then_tag_and_push_new_tag "opendap/hyrax_ncwms:${HYRAX_VERSION}" "opendap/hyrax_ncwms:${hyrax_release_version}" + pull_then_tag_and_push_new_tag "opendap/hyrax_ncwms:${HYRAX_VERSION}" "opendap/hyrax_ncwms:latest" + + pull_then_tag_and_push_new_tag "opendap/besd:${BES_VERSION}" "opendap/besd:${bes_release_version}" + pull_then_tag_and_push_new_tag "opendap/besd:${BES_VERSION}" "opendap/besd:latest" + + pull_then_tag_and_push_new_tag "opendap/olfs:${OLFS_VERSION}" "opendap/olfs:${olfs_release_version}" + pull_then_tag_and_push_new_tag "opendap/olfs:${OLFS_VERSION}" "opendap/olfs:latest" + + loggy "" + loggy "mk_easy_docker_release() - END" + loggy "${HR}" +} + +# make_hyrax_release "${RELEASE_DATE}" "${HYRAX_VERSION}" "${OLFS_VERSION}" "${BES_VERSION}" "${LIBDAP_VERSION}" + +################################################################################ +# +# +function travis_hyrax() { + check_aws + if test $? -ne 0; then + loggy "There is a problem with the AWS configuration. Fix it and try again" + return $status + fi + export OS_SNAPSHOT_IMAGE_TAG="opendap/${DOCKER_NAME}:snapshot-$TARGET_OS$TEST_DEPLOYMENT" + export OS_BUILD_VERSION_TAG="opendap/${DOCKER_NAME}:${HYRAX_VERSION}-$TARGET_OS$TEST_DEPLOYMENT" + export TOMCAT_VERSION + TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") + + show_version + get_tomcat_distro "${DOCKER_NAME}" "${TOMCAT_VERSION}" + s3_get_besd_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "$TARGET_OS" \ + "${LIBDAP_VERSION}" \ + "${BES_VERSION}" \ + "$ADD_DEBUG" 2>&1 + s3_get_olfs_distro \ + "${S3_BUILD_BUCKET}" \ + "${DOCKER_NAME}" \ + "${OLFS_VERSION}-${TARGET_OS}" 2>&1 + + OLFS_VERSION="$OLFS_VERSION-${TARGET_OS}" + loggy "OLFS_VERSION: $OLFS_VERSION" + docker build \ + --build-arg TOMCAT_VERSION \ + --build-arg RELEASE_DATE \ + --build-arg HYRAX_VERSION \ + --build-arg LIBDAP_VERSION \ + --build-arg BES_VERSION \ + --build-arg OLFS_VERSION \ + --tag "${OS_SNAPSHOT_IMAGE_TAG}" \ + --tag "${OS_BUILD_VERSION_TAG}" \ + "${DOCKER_NAME}" + + docker image ls -a + +} + +################################################################################ +# Stop all running containers +# +function dhalt() { + docker stop $(docker ps -aq); +} + +################################################################################ +# Remove all containers +# +function drmac(){ + docker rm -f $(docker ps -aq); +} + +################################################################################ +# Remove all images +# +function drmai(){ + docker rmi -f $(docker images -q); +} diff --git a/el9-builds/build-hyrax-ngap.sh b/el9-builds/build-hyrax-ngap.sh new file mode 100755 index 00000000..8d94e55d --- /dev/null +++ b/el9-builds/build-hyrax-ngap.sh @@ -0,0 +1,127 @@ +#!/bin/bash +# +# +# We must assume that the shell has sourced ./build-el9 prior (in Travis) so that +# downstream Travis activities (like deployment) will have all the ENV vars they +# need to run. +source "./build-$TARGET_OS" +HR0="#######################################################################" +HR1="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" +HR2="--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---" +############################################################################# +# loggy() +prolog="build-hyrax-ngap.sh" +function loggy(){ + echo "$@" | awk -v prolog="$prolog" '{ print "# " prolog " - " $0;}' >&2 +} +loggy "$HR0" +loggy "BEGIN" + +export DOCKER_NAME="${DOCKER_NAME:-"ngap"}" +loggy "DOCKER_NAME: $DOCKER_NAME" + +############################################################################################### +#export SNAPSHOT_IMAGE_TAG="${SNAPSHOT_IMAGE_TAG:-"opendap/hyrax:$DOCKER_NAME-snapshot-$TARGET_OS$TEST_DEPLOYMENT"}" +loggy " SNAPSHOT_IMAGE_TAG: '$SNAPSHOT_IMAGE_TAG'" >&2 +loggy "OS_SNAPSHOT_IMAGE_TAG: '$OS_SNAPSHOT_IMAGE_TAG'" >&2 +# +#export BUILD_VERSION_TAG="${BUILD_VERSION_TAG:-"opendap/hyrax:$DOCKER_NAME-$HYRAX_VERSION-$TARGET_OS$TEST_DEPLOYMENT"}" +loggy " BUILD_VERSION_TAG: '$OS_BUILD_VERSION_TAG'" >&2 +loggy " OS_BUILD_VERSION_TAG: '$OS_BUILD_VERSION_TAG'" >&2 +############################################################################################### + + +loggy "TOMCAT_MAJOR_VERSION: $TOMCAT_MAJOR_VERSION" >&2 +export TOMCAT_VERSION= +TOMCAT_VERSION="$(get_latest_tomcat_version_number "$TOMCAT_MAJOR_VERSION")" +loggy "TOMCAT_VERSION: $TOMCAT_VERSION" >&2 +# +export APACHE_APR_VERSION="${APACHE_APR_VERSION:-"1.7.6-1"}" +loggy "APACHE_APR_VERSION: $APACHE_APR_VERSION" +# +#export OPENSSL_VERSION="3.5.0-4" +#loggy "OPENSSL_VERSION: $OPENSSL_VERSION" +# +show_version +# +get_tomcat_distro "$DOCKER_NAME" "$TOMCAT_VERSION" +# + +s3_get_besd_distro \ + "$S3_BUILD_BUCKET" \ + "$DOCKER_DIR" \ + "$TARGET_OS" \ + "$LIBDAP_VERSION" \ + "$BES_VERSION" "$ADD_DEBUG_RPMS" +# +s3_get_apache_apr_distro \ + "$S3_BUILD_BUCKET" \ + "$DOCKER_DIR" \ + "$APACHE_APR_VERSION" \ + "$ADD_DEBUG_RPMS" + +s3_get_olfs_ngap_distro \ + "$S3_BUILD_BUCKET" \ + "$DOCKER_DIR" \ + "$OLFS_VERSION" 2>&1 + +#s3_get_openssl_distro \ +# "$S3_BUILD_BUCKET" \ +# "$DOCKER_DIR" \ +# "$OPENSSL_VERSION" \ +# "$TARGET_OS" \ +# "$ADD_DEBUG_RPMS" +# + +set -e +docker build \ + --build-arg TOMCAT_VERSION \ + --build-arg RELEASE_DATE \ + --build-arg HYRAX_VERSION \ + --build-arg LIBDAP_VERSION \ + --build-arg BES_VERSION \ + --build-arg OLFS_VERSION \ + --build-arg OPENSSL_VERSION \ + --tag "${OS_SNAPSHOT_IMAGE_TAG}" \ + --tag "${OS_BUILD_VERSION_TAG}" \ + "${DOCKER_NAME}" +# +set +e + +loggy "docker image ls -a: " +loggy "$(docker image ls -a)" + +function ngap_el9_dnf_sanity_check() { + local prolog="ngap_el9_dnf_sanity_check() -" + set -e + loggy "$HR0" + if test -n "$DEBUG_BUILD" + then + loggy "$prolog Sanity checking installed packages..." + loggy "$prolog openssl is located here: $(which openssl)" + loggy "$prolog openssl version: $(openssl version)" + loggy "$(dnf -y info openssl)" + loggy "$prolog openssl is installed." + loggy "$HR1" + loggy "$prolog libtirpc info:" + loggy "$(dnf -y info libtirpc)" + loggy "$HR2" + loggy "$prolog rpm -ql libtirpc: " + rpm -ql libtirpc + loggy "$HR1" + loggy "$prolog libtirpc-devel info:" + dnf -y info libtirpc-devel + loggy "$HR2" + loggy "$prolog rpm -ql libtirpc-devel: " + rpm -ql libtirpc-devel + loggy "$HR1" + loggy "$prolog libuuid info:" + dnf -y info libuuid + loggy "$prolog libuuid-devel info:" + dnf -y info libuuid-devel + dnf clean all + else + loggy "$prolog Skipping Sanity Checks..." + fi + loggy "$HR0" +} \ No newline at end of file diff --git a/el9-builds/developer.yml b/el9-builds/developer.yml new file mode 100644 index 00000000..f244b3ad --- /dev/null +++ b/el9-builds/developer.yml @@ -0,0 +1,43 @@ +--- +version: '3' +services: + olfs: + build: ./olfs + build: + context: ./olfs + args: + DEVELOPER_MODE: "true" + USE_NCWMS: "true" + env_file: ${PWD}/local.env + image: olfs:snapshot + ports: + - "80:8080" + tty: true + volumes: + - ./logs/olfs_tomcat:/usr/local/tomcat/logs + - ./logs:/usr/local/tomcat/webapps/opendap/WEB-INF/conf/logs + # command: -n http://localhost:8080 # This is the publicly accessible host for ncWMS + besd: + build: ./besd + env_file: ${PWD}/local.env + image: besd:snapshot + ports: + - "10022:10022" + volumes: + - ./logs:/var/log/bes/ + # - ./cache:/var/cache/bes # maps the BES cache directory to the local filesystem + # - ./conf:/etc/bes # Maps a local BES configuration directory onto the bes in docker. + # command: -e admin_contact@email.org -s + ncwms: + build: + context: ./ncWMS + args: + DEVELOPER_MODE: "true" + env_file: ${PWD}/local.env + image: ncwms:latest + ports: + - "8080:8080" + tty: true + volumes: + - ./logs/ncwms_tomcat:/usr/local/tomcat/logs + - ./logs/ncwms:/root/.ncWMS2/logs diff --git a/el9-builds/get_snaphots.sh b/el9-builds/get_snaphots.sh new file mode 100644 index 00000000..69c026e6 --- /dev/null +++ b/el9-builds/get_snaphots.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +docker-compose -f ./hyrax.yml pull olfs besd \ No newline at end of file diff --git a/el9-builds/hyrax.yml b/el9-builds/hyrax.yml new file mode 100644 index 00000000..0dc15ee8 --- /dev/null +++ b/el9-builds/hyrax.yml @@ -0,0 +1,24 @@ +--- +version: '3' +services: + olfs: + # build: ./olfs + env_file: ${PWD}/local.env + image: opendap/olfs:snapshot + ports: + - "8080:8080" + tty: true + volumes: + - ./logs/olfs_tomcat:/usr/local/tomcat/logs + - ./logs:/usr/local/tomcat/webapps/opendap/WEB-INF/conf/logs + besd: + # build: ./besd + env_file: ${PWD}/local.env + image: opendap/besd:snapshot + ports: + - "10022:10022" + volumes: + - ./logs:/var/log/bes/ + # - ./cache:/var/cache/bes # maps the BES cache directory to the local filesystem + # - ./conf:/etc/bes # Maps a local BES configuration directory onto the bes in docker. + # command: -e admin_contact@email.org -s diff --git a/el9-builds/hyrax/Dockerfile b/el9-builds/hyrax/Dockerfile new file mode 100644 index 00000000..f01ab730 --- /dev/null +++ b/el9-builds/hyrax/Dockerfile @@ -0,0 +1,395 @@ +############################################################################################### +# +# Dockerfile for single container Hyrax +# +# +# Some shell state reference: +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. +# +# In general use "set -e" when running commands that matter and don't use +# it for debugging stuff. +# +# Set one or more individual labels +FROM rockylinux:9 +ENV HR="###############################################################################################################" +ENV HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +ENV TARGET_OS="el9" +ENV ARCH="x86_64" +ENV java_version="java-21-openjdk" +# rocky9 comes with python 3.9 by default. +RUN echo "$HR" +RUN echo "# BEGIN - hyrax docker build" +RUN echo "# Python Version: $(python3 --version)" + +################################################################################ +# Stuff we need to run the service: +# which - Because this build likes to say where installed things be livin. +# unzip - to unpack stuff we install +# procps - so we can use 'ps' to track the bes and tomcat processes. +# bc - is used by the cleanup_files.sh script to compute time stuff for +# orphaned files. +ENV dnf_runtime_packages="which unzip procps bc" +RUN echo "# dnf_runtime_packages: $dnf_runtime_packages" + +# Stuff we need to build the hyrax_regression_tests +ENV dnf_regression_test_packages="git make autoconf automake diffutils" +RUN echo "# dnf_regression_test_packages: $dnf_regression_test_packages" + +ENV dnf_developer_tools_packages="diffutils curl emacs vim jq git valgrind gdb awscli" +RUN echo "# dnf_developer_tools_packages: $dnf_developer_tools_packages" + + +ARG RELEASE_DATE +ENV RELEASE_DATE="${RELEASE_DATE:-$(date +%s)}" +RUN echo "# RELEASE_DATE: $RELEASE_DATE" + +ARG TOMCAT_VERSION +ENV TOMCAT_VERSION=${TOMCAT_VERSION:-"11.0.7"} +RUN echo "# TOMCAT_VERSION: $TOMCAT_VERSION" +ENV TOMCAT_DISTRO="apache-tomcat-$TOMCAT_VERSION" +RUN echo "# Retrieving TOMCAT_DISTRO: $TOMCAT_DISTRO" +COPY "$TOMCAT_DISTRO.tar.gz" / +RUN ls -l /apache-tomcat-* >&2 + +ARG HYRAX_VERSION +ENV HYRAX_VERSION=${HYRAX_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "# HYRAX_VERSION: $HYRAX_VERSION" + +ARG LIBDAP_VERSION +ENV LIBDAP_VERSION=${LIBDAP_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "# LIBDAP_VERSION: $LIBDAP_VERSION" + +ARG LIBDAP_RPM +ENV LIBDAP_RPM=${LIBDAP_RPM:-"libdap-$LIBDAP_VERSION.$TARGET_OS.x86_64.rpm"} +RUN echo "# LIBDAP_RPM: $LIBDAP_RPM" +COPY "$LIBDAP_RPM" libdap-devel-* libdap-debuginfo-* / +RUN ls -l /libdap-* >&2 + +ARG BES_VERSION +ENV BES_VERSION=${BES_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "# BES_VERSION: $BES_VERSION" + +ARG BES_RPM +ENV BES_RPM=${BES_RPM:-"bes-$BES_VERSION.static.$TARGET_OS.x86_64.rpm"} +RUN echo "# BES_RPM: $BES_RPM" +COPY "$BES_RPM" bes-devel-* bes-debuginfo-* / +RUN ls -l /bes-* >&2 + +ARG OLFS_VERSION +ENV OLFS_VERSION=${OLFS_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "# OLFS_VERSION: $OLFS_VERSION" + +ARG OLFS_DISTRO +ENV OLFS_DISTRO=${OLFS_DISTRO:-"olfs-$OLFS_VERSION-webapp"} +RUN echo "# OLFS_DISTRO: $OLFS_DISTRO" +COPY "$OLFS_DISTRO.tgz" "/$OLFS_DISTRO.tgz" + +ARG ROBOTS_DISTRO +ENV ROBOTS_DISTRO=${ROBOTS_DISTRO:-"robots-olfs-$OLFS_VERSION-webapp"} +RUN echo "# ROBOTS_DISTRO: $ROBOTS_DISTRO" +COPY "$ROBOTS_DISTRO.tgz" "/$ROBOTS_DISTRO.tgz" + +ARG DEPLOYMENT_CONTEXT +ENV DEPLOYMENT_CONTEXT=${DEPLOYMENT_CONTEXT:-"opendap"} +RUN echo "# DEPLOYMENT_CONTEXT: $DEPLOYMENT_CONTEXT" + +LABEL vendor="OPeNDAP" +LABEL org.opendap.hyrax.version="$HYRAX_VERSION" +LABEL org.opendap.hyrax.release-date="$RELEASE_DATE" +LABEL org.opendap.hyrax.version.is-production="false" +LABEL org.opencontainers.image.authors="support@opendap.org" + +USER root + +# +# The --build-arg USE_NCWMS can be set to "true" in order to +# add the ncWMS application to the build. +ARG USE_NCWMS +ENV USE_NCWMS=${USE_NCWMS:-"false"} +RUN if [ "$USE_NCWMS" = "true" ];then echo "# NCWMS: ENABLED"; else echo "# NCWMS: DISABLED"; fi + +# +# The --build-arg NCWMS_BASE can be set to the base URL for ncWMS. +# The entrypoint.sh code defaults it to +# the URL: https://localhost:8080 if the environment variable NCWMS_BASE +# is not in the shell from which the entrypoint.sh script is called. +ARG NCWMS_BASE +ENV NCWMS_BASE=${NCWMS_BASE:-"https://localhost:8080"} +RUN if [ "$USE_NCWMS" = "true" ];then echo "# NCWMS_BASE: $NCWMS_BASE"; fi + + +ARG DEVELOPER_MODE +ENV DEVELOPER_MODE=${DEVELOPER_MODE:-"false"} +RUN if [ "$DEVELOPER_MODE" = "true" ];then echo "# DEVELOPER_MODE: ENABLED"; else echo "# DEVELOPER_MODE: DISABLED"; fi + +#Get mirrorlist before running yum +#RUN cd /etc/yum.repos.d/ +#RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +#RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + +###################################################################### +# Update the OS and create a handy executable shell scipt to simplify +# the installation of the (not as yet installed) but super useful +# developer tools packages :) ndp +RUN set -e \ + && dnf update -y \ + && echo "#!/bin/bash" > /install-dev-tools \ + && echo "dnf install -y $dnf_developer_tools_packages" > /install-dev-tools \ + && chmod 755 /install-dev-tools + +###################################################################### +# Rocky9 comes with built in curl-minimal. For our puposes we want +# want the full package (I think, probably need to verify). +# The '--allowerasing' parameter enables dnf to erase curl-minimal +# and then install the full package. +# +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing cURL for developers (not minimal cURL)" >&2 \ + && dnf install -y curl curl-devel --allowerasing \ + && echo "# cURL version: $(curl --version)" >&2 \ + && dnf clean all \ + && echo "$HR" >&2 + +###################################################################### +# We want java ac ertain way ($java_version) and +# we want to be sure that +# - We enable the one we want: +# update-alternatives --set java "$java_version.$ARCH +# - We want to see if there are others installed java packages +# (we hope there are not): +# update-alternatives --list +# - We're using the one we wanted: +# echo "# java version: $( java -version 2>&1 )" +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# It's Java Time Witches!" >&2 \ + && echo "# Installing $java_version" >&2 \ + && dnf install -y "$java_version" "$java_version-devel" \ + && echo "$HR2" >&2 \ + && echo "# Using $java_version" >&2 \ + && update-alternatives --set java "$java_version.$ARCH" \ + && echo "# Enabled $java_version" >&2 \ + && echo "# java version: $( java -version 2>&1 )" \ + && update-alternatives --list \ + && dnf clean all \ + && echo "$HR" >&2 + +RUN set -e \ + && dnf install -y $dnf_runtime_packages $dnf_regression_test_packages \ + && dnf clean all +# +# RUN set -e && which diff +# + +################################################################ +# Install the libdap rpm +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing the libdap rpm: $LIBDAP_RPM" >&2 \ + && ls -l ./libdap* \ + && dnf -y install ./libdap-*.rpm \ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && rm -f "./$LIBDAP_RPM" \ + && dnf clean all \ + && echo "# libdap4 is installed." >&2 \ + && echo "$HR" >&2 + +################################################################ +# Install the the BES +# besd, besdaemon, beslistener, besctl, besstandalone, bescmdline +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing the BES rpm: $BES_RPM" >&2 \ + && ls -l ./bes* \ + && dnf -y install ./bes-*.rpm \ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && rm -f "./$BES_RPM" \ + && dnf clean all \ + && echo "# besdaemon is here: $(which besdaemon)" >&2 \ + && echo "# besdaemon -v: '$(besdaemon -v)'" >&2 \ + && echo "$HR" >&2 + +################################################################ +# Install and Setup Tomcat +# +# Make tomcat user and group +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e && useradd -m -U -d /home/tomcat -s /bin/false tomcat +# +# Install the Tomcat package that should have been copied into +# the root directory prior to this layer. +# +RUN set -e \ + && tar -xvf "/$TOMCAT_DISTRO.tar.gz" -C "/home/tomcat" >&2 \ + && ln -s "/home/tomcat/$TOMCAT_DISTRO" "/usr/share/tomcat" >&2 \ + && ln -s "/home/tomcat/$TOMCAT_DISTRO" "/tomcat" >&2 \ + && rm -rvf /usr/share/tomcat/webapps/* >&2 \ + && mkdir -p /var/log/tomcat \ + && chown -R tomcat:tomcat /var/log/tomcat /home/tomcat \ + && echo "# Cleaning up Tomcat distribution files..." >&2 \ + && rm -fv "/$TOMCAT_DISTRO.tar.gz" \ + && echo "# Tomcat is unpacked and ready. >&2 " + +# @TODO REMOVE THIS COMMENTED OUT CODE BEFORE MERGE +#COPY tomcat.service /etc/systemd/system/tomcat.service +#RUN set -e \ +# && which systemctl \ +# && systemctl enable tomcat +#RUN firewall-cmd --add-port 8080/tcp --permanent +#RUN firewall-cmd --reload + +ENV CATALINA_HOME=/usr/share/tomcat +ENV PATH=$CATALINA_HOME/bin:$PATH +RUN set -e && echo "# CATALINA_HOME: $CATALINA_HOME" >&2 + +# Install our modified server.xml so that the server compresses responses. +COPY tomcat11-server.xml / +RUN set -e \ + && mv /tomcat11-server.xml "$CATALINA_HOME/conf/server.xml" \ + && chown -R tomcat:tomcat "$CATALINA_HOME/conf/server.xml" + + +################################################################ +# Install the OLFS web application +# - - - - - - - - - - - - - - - - - - - - - +ENV ROOT=/dev/shm + +RUN echo "# Installing the latest olfs distribution ($OLFS_VERSION)." >&2 +RUN set -e \ + && echo "# working_dir: "`pwd` >&2 \ + && ls -l . \ + && tar -C "$ROOT" -xzvf "./$OLFS_DISTRO.tgz" \ + && ls -l "$ROOT" \ + && echo "# Unpacking war file..." >&2 \ + && mkdir -p "$(readlink -f "$CATALINA_HOME")/webapps/$DEPLOYMENT_CONTEXT/" \ + && unzip -o "$ROOT/$OLFS_DISTRO/opendap.war" -d "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/" \ + && echo "# Cleaning up OLFS distribution files." >&2 \ + && rm -rvf "$ROOT/$OLFS_DISTRO" "./$OLFS_DISTRO.tgz" + +RUN set -e \ + && echo "# Installing the latest dynamic robots.txt distribution ($OLFS_VERSION)." >&2 \ + && ls -l ./robots* \ + && tar -C "$ROOT" -xzf "./$ROBOTS_DISTRO.tgz" \ + && ls -l "$ROOT/$ROBOTS_DISTRO" \ + && echo "# Unpacking dynamic robots.txt war file..." >&2 \ + && unzip -o "$ROOT/$ROBOTS_DISTRO/ROOT.war" -d "$CATALINA_HOME/webapps/ROOT/" \ + && echo "# Cleaning up dynamic robots.txt distribution files..." >&2 \ + && rm -rvf "$ROOT/$ROBOTS_DISTRO" "$ROBOTS_DISTRO.tgz" + +# Fix ownership and access permissions +RUN set -e \ + && mkdir -p "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/logs" \ + && chown -R tomcat:tomcat "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/logs" \ + && chmod 700 "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/logs" + +################################################################ +# Clean up the Hyrax distribution binaries. +# +# +################################################################ + +################################################################ +# retrieve and install the ncWMS web application +# - - - - - - - - - - - - - - - - - - - - - + +ARG NCWMS_VERSION +ENV NCWMS_VERSION=${NCWMS_VERSION:-"2.5.2"} +RUN echo "# NCWMS_VERSION: $NCWMS_VERSION" +ENV NCWMS_WAR_URL="https://github.com/Reading-eScience-Centre/ncwms/releases/download/ncwms-$NCWMS_VERSION/ncWMS2.war" +RUN echo "# NCWMS_WAR_URL: $NCWMS_WAR_URL" + +RUN if [ "$USE_NCWMS" = "true" ]; then \ + echo "# Installing ncWMS." \ + curl -sfSL "$NCWMS_WAR_URL" -o "/dev/shm/ncWMS.war" \ + unzip -o "/dev/shm/ncWMS.war" -d "$CATALINA_HOME/webapps/ncWMS2/" \ + rm -rf /dev/shm/*; \ + else \ + echo "# ncWMS will NOT be installed."; \ + fi + +# set a default ncWMS admin if DEVELOPER_MODE is enabled. +RUN if [ "$DEVELOPER_MODE" = "true" ] && [ "$USE_NCWMS" = "true" ]; then \ + echo "# DEVELOPER MODE: Adding ncWMS admin credentials"; \ + sed -i 'sXX X' $CATALINA_HOME/conf/tomcat-users.xml; \ + else \ + echo "# No ncWMS admin credentials installed."; \ + fi + +# +# make ncWMS work without further configuration +# We will need to adjust this target if we +# decide to run as a different (not root) user. +COPY ncWMS_config.xml /root/.ncWMS2/config.xml +RUN chmod +r /root/.ncWMS2/config.xml + +COPY olfs_viewers.xml /tmp/olfs_viewers.xml +RUN if [ "$USE_NCWMS" = "true" ]; then \ + # If we're installing ncWMS then we copy + # to the server a viewers.xml which has + # been templated so that the ncWMS host can be installed + # at startup. + mv /tmp/olfs_viewers.xml "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/viewers.xml"; \ + else \ + echo "# Skipping OLFS/ncWMS configuration installation."; \ + fi + +################################################################ +# +# Retrieve, verify, and install the hyrax_regression_tests +# project +# +# - - - - - - - - - - - - - - - - - - - - - +ENV TEST_INSTALL_DIR="/hyrax_regression_tests" +RUN set -e \ + && echo "# Retrieving, and building hyrax regression tests." \ + && echo "# h_r_t will be in: $TEST_INSTALL_DIR" \ + && mkdir -p "$TEST_INSTALL_DIR" \ + && git clone -v https://github.com/opendap/hyrax_regression_tests "$TEST_INSTALL_DIR" \ + && cd "$TEST_INSTALL_DIR" \ + && autoreconf -vif \ + && ./configure + +# In order to get the executable "testsuite" to build we have to run +# "make check" But that will fail at build time so the "make check" call +# ends up pretty tortured to avoid a build error. +RUN set -e \ + && cd "$TEST_INSTALL_DIR" \ + && make testsuite > mk.log 2>&1 \ + && echo "# SUCCESS: hyrax_regression_tests ready!" +################################################################ + +RUN ls -l /usr/libexec/ + +################################################################ +# TEMPORARY +# Grab the cleanup files script for managing orphaned files +# from fileout_netcdf +COPY cleanup_files.sh / +RUN chmod +x /cleanup_files.sh +# TEMPORARY +################################################################ + + +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh +ENTRYPOINT [ "/entrypoint.sh" ] + +EXPOSE 8009 +EXPOSE 8080 +EXPOSE 8443 +EXPOSE 10022 +EXPOSE 11002 + +# can't use USER with entrypoint that needs root +# use gosu or, as done, enable bes user write so the entrypoint does not need root +RUN chown -R bes /etc/bes +USER root + +CMD ["-"] + diff --git a/el9-builds/hyrax/cleanup_files.sh b/el9-builds/hyrax/cleanup_files.sh new file mode 100755 index 00000000..b4c59107 --- /dev/null +++ b/el9-builds/hyrax/cleanup_files.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# This script is meant as a temporary patch to the problem of the +# fileout_netcdf handler leaving behind orphaned temporary files +# that clog the deployments disk. When we understand and repair +# the issue with fileout_netcdf this script can be removed +# from the distributions. +# +# +verbose= +default_dir="/tmp/hyrax_fonc" +default_interval="60" # minutes +default_log="/var/log/bes/nc_cleanup.log" + +if test -n "${verbose}"; then echo "##########################################################" >&2 ; fi +if test -n "${verbose}"; then echo "# $0" >&2; echo "#" >&2; fi + +TARGET_DIR="${1:-$default_dir}" +if test -n "${verbose}"; then echo "# TARGET_DIR: ${TARGET_DIR}" >&2; fi + +INTERVAL="${2:-$default_interval}" # minutes +if test -n "${verbose}"; then echo "# INTERVAL: ${INTERVAL}m" >&2; fi + +CLEANUP_LOG="${3:-$default_log}" +if test -n "${verbose}"; then echo "# CLEANUP_LOG: ${CLEANUP_LOG}" >&2; fi + +SLEEP_TIME=$(echo "${INTERVAL}*60/2" | bc) +if test -n "${verbose}"; then echo "# SLEEP_TIME: ${SLEEP_TIME}s" >&2; fi +if test -n "${verbose}"; then echo "#" >&2; fi + + +while true +do + if test -d ${TARGET_DIR}; + then + orphaned_files=$(find ${TARGET_DIR} -type f -mmin +${INTERVAL}) + if test -n "${verbose}"; then echo "orphaned_files: "${orphaned_files} >&2; fi + + if test -n "${orphaned_files}"; + then + echo "########################################################################################" >> "${CLEANUP_LOG}" + echo "# "$(date)" BEGIN File Cleanup of ${TARGET_DIR} " >> "${CLEANUP_LOG}" + for file in ${orphaned_files} + do + target=$(ls -l "${file}") + rm -f "${file}" + echo $(date)" removed: ${target}" 2>&1 >> "${CLEANUP_LOG}" + done + echo "#" >> "${CLEANUP_LOG}" + fi + fi + sleep "${SLEEP_TIME}" + +done + diff --git a/el9-builds/hyrax/entrypoint.sh b/el9-builds/hyrax/entrypoint.sh new file mode 100755 index 00000000..97d1a1f3 --- /dev/null +++ b/el9-builds/hyrax/entrypoint.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# This is the entrypoint.sh file for the single container Hyrax. + +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. + +#echo "entrypoint.sh command line: '$@' " +export debug="false" +export BANNER="################################ HYRAX ############################################" +export HR0="###################################################################################" +export HR1="-----------------------------------------------------------------------------------" +export HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +export prolog="entrypoint.sh" +function loggy(){ + echo "$@" | awk -v prolog="$prolog" '{ print "# " prolog " - " $0;}' >&2 +} +function debug_loggy(){ + if test "$debug" = "true"; then loggy "$@"; fi +} + +loggy "$HR0" +loggy "$BANNER" +loggy "Greetings, I am $(whoami) (uid: $UID)" +# set -e +# set -x + +loggy " PythonVersion: $(python3 --version)" + +export JAVA_HOME=${JAVA_HOME:-"/etc/alternatives/jre"} +loggy " JAVA_HOME: $JAVA_HOME" + +export SLEEP_INTERVAL=${SLEEP_INTERVAL:-60} +loggy " SLEEP_INTERVAL: $SLEEP_INTERVAL seconds." + +export SERVER_HELP_EMAIL=${SERVER_HELP_EMAIL:-"not_set"} +loggy " SERVER_HELP_EMAIL: $SERVER_HELP_EMAIL" + +export FOLLOW_SYMLINKS=${FOLLOW_SYMLINKS:-"false"} +loggy " FOLLOW_SYMLINKS: $FOLLOW_SYMLINKS" + +export NCWMS_BASE=${NCWMS_BASE:-"https://localhost:8080"} +loggy " NCWMS_BASE: $NCWMS_BASE" + +# AWS ########################################################################## +loggy "$HR2" +if test -n "$AWS_ACCESS_KEY_ID" +then + loggy " AWS_ACCESS_KEY_ID: HAS BEEN SET" +else + loggy " AWS_ACCESS_KEY_ID: HAS NOT BEEN SET" +fi + +if test -n "$AWS_SECRET_ACCESS_KEY" +then + loggy "AWS_SECRET_ACCESS_KEY: HAS BEEN SET" +else + loggy "AWS_SECRET_ACCESS_KEY: HAS NOT BEEN SET" +fi + +export AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-"us-west-2"} +loggy " AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION" + + +loggy "Processing Command Line Options." +while getopts "de:sn:i:k:r:" opt; do + loggy "Processing command line opt: $opt" + case $opt in + e) + export SERVER_HELP_EMAIL=$OPTARG + loggy "Set server admin contact email to: $SERVER_HELP_EMAIL" + ;; + s) + export FOLLOW_SYMLINKS="Yes" + loggy "Set FollowSymLinks to: $FOLLOW_SYMLINKS" + ;; + n) + export NCWMS_BASE=$OPTARG + loggy "Set ncWMS public facing service base to : $NCWMS_BASE" + ;; + d) + debug=true + loggy "Debug is enabled" + ;; + i) + export AWS_ACCESS_KEY_ID="$OPTARG" + loggy "Found command line value for AWS_ACCESS_KEY_ID."; + ;; + k) + export AWS_SECRET_ACCESS_KEY="$OPTARG" + loggy "Found command line value for AWS_SECRET_ACCESS_KEY."; + ;; + r) + export AWS_DEFAULT_REGION="$OPTARG" + loggy "Found command line value for AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION"; + ;; + + \?) + loggy "Invalid option: -$OPTARG" + loggy "options: [-e xxx] [-n yyy] [-s] [-d] [-i xxx] [-k xxx] [-r xxx]" + loggy " -e xxx where xxx is the email address of the admin contact for the server." + loggy " -s When present causes the BES to follow symbolic links." + loggy " -n yyy where yyy is the protocol, server and port part " + loggy " of the ncWMS service (for example http://foo.com:8090)." + loggy " -d Enables debugging output for this script." + loggy " -i xxx Where xxx is an AWS CLI AWS_ACCESS_KEY_ID." + loggy " -k xxx Where xxx is an AWS CLI AWS_SECRET_ACCESS_KEY." + loggy " -r xxx Where xxx is an AWS CLI AWS_DEFAULT_REGION." + loggy "EXITING NOW" + exit 2 + ;; + esac +done +loggy "Command Line Options Have Been Processed." +loggy "$HR0" + +if test "$debug" = "true" ; then + loggy "CATALINA_HOME: $CATALINA_HOME" + loggy "$(ls -l "$CATALINA_HOME")" + loggy "$(ls -l "$CATALINA_HOME/bin")" +fi + +export VIEWERS_XML="$CATALINA_HOME/webapps/opendap/WEB-INF/conf/viewers.xml" +if test "$debug" = "true" ; then + loggy "NCWMS: Using NCWMS_BASE: $NCWMS_BASE" + loggy "NCWMS: Setting ncWMS access URLs in viewers.xml (if needed)." + loggy "$(ls -l "$VIEWERS_XML")" +fi + +sed -i "s+@NCWMS_BASE@+$NCWMS_BASE+g" "$VIEWERS_XML" +if test "$debug" = "true" ; then + loggy "$VIEWERS_XML" + loggy "$(cat "$VIEWERS_XML")" +fi + +#------------------------------------------------------------------------------- +# modify bes.conf based on environment variables before startup. +# +if test "$SERVER_HELP_EMAIL" != "not_set" ; then + loggy "Setting Admin Contact To: $SERVER_HELP_EMAIL" + sed -i "s/admin.email.address@your.domain.name/$SERVER_HELP_EMAIL/" "/etc/bes/bes.conf" +fi +if test "$FOLLOW_SYMLINKS" != "not_set" ; then + loggy "Setting BES FollowSymLinks to YES." + sed -i "s/^BES.Catalog.catalog.FollowSymLinks=No/BES.Catalog.catalog.FollowSymLinks=Yes/" "/etc/bes/bes.conf" +fi + +loggy "JAVA VERSION: $( java -version 2>&1 )" + +loggy "Checking for awscli" +set +e +which_output="$(eval "which aws" 2>&1)" +status=$? +set -e +loggy "$which_output" +if test $status -ne 0 +then + loggy "WARNING: AWS CLI not detected on PATH, may not be installed." +else + aws configure list + status=$? + if test $status -ne 0 ; then + loggy "WARNING: Problem with AWS CLI configuration! (status: $status)" + fi +fi + +loggy "PythonVersion (again): $( python3 --version 2>&1 )" + +#------------------------------------------------------------------------------- +# We use 'echo' in the following because downstream code is expecting this +# output to be a key value pair, so none of that loggy() stuff +bes_uid="$(id -u bes)" +echo "bes_uid: $bes_uid" +bes_gid="$(id -g bes)" +echo "bes_gid: $bes_gid" + +#------------------------------------------------------------------------------- +# Start the BES daemon process +# /usr/bin/besdaemon -i /usr -c /etc/bes/bes.conf -r /var/run/bes.pid (old way) +loggy "Launching besd..." +/usr/bin/besctl start +status=$? +if test $status -ne 0 ; then + loggy "ERROR: Failed to start BES: $status" + exit $status +fi +besd_pid="$(ps aux | grep "/usr/bin/besdaemon" | grep -v grep | awk '{print $2;}' - )" +loggy "The besd is UP! [pid: $besd_pid]" + +#------------------------------------------------------------------------------- +# Start Tomcat process +# +export OLFS_CONF="${CATALINA_HOME}/webapps/opendap/WEB-INF/conf" +# mv ${OLFS_CONF}/logback.xml ${OLFS_CONF}/logback.xml.OFF +loggy "Starting Tomcat..." +"$CATALINA_HOME"/bin/startup.sh > /var/log/tomcat/console.log 2>&1 & +status=$? +tomcat_pid=$! # The $! is the PID of the most recently executed background command. +if test $status -ne 0 ; then + loggy "ERROR: Failed to start Tomcat: $status" + exit $status +fi +# When we launch tomcat the initial pid gets "retired" because it spawns a +# secondary processes. +initial_pid="$tomcat_pid" +loggy "Tomcat started, initial pid: $initial_pid" +while test "$initial_pid" -eq "$tomcat_pid" +do + sleep 1 + tomcat_ps="$(ps aux | grep tomcat | grep -v grep)" + loggy "tomcat_ps: $tomcat_ps" + tomcat_pid="$(echo "$tomcat_ps" | awk '{print $2}')" + loggy "tomcat_pid: $tomcat_pid" +done +# New pid and we should be good to go. +loggy "Tomcat is UP! pid: $tomcat_pid" + +# TEMPORARY +/cleanup_files.sh >&2 & +# TEMPORARY + +# sleep --help + +#------------------------------------------------------------------------------- +loggy "$HR1" +loggy "Hyrax Has Arrived. SLEEP_INTERVAL: $SLEEP_INTERVAL" +loggy "$HR0" +while /bin/true; do + sleep $SLEEP_INTERVAL + # debug_loggy "Checking Hyrax Operational State." + besd_ps="$(ps -f "$besd_pid")" + BESD_STATUS=$? + # debug_loggy "BESD_STATUS: $BESD_STATUS" + + tomcat_ps="$(ps -f "${tomcat_pid}")" + TOMCAT_STATUS=$? + # debug_loggy "TOMCAT_STATUS: $TOMCAT_STATUS" + + if test $BESD_STATUS -ne 0 ; then + loggy "BESD_STATUS: $BESD_STATUS besd_ps:$besd_ps" + loggy "The BES daemon appears to have died! Exiting." + exit 1 + fi + if test $TOMCAT_STATUS -ne 0 ; then + loggy "TOMCAT_STATUS: $TOMCAT_STATUS tomcat_pid:$tomcat_pid" + loggy "Tomcat appears to have died! Exiting." + loggy "Tomcat Console Log [BEGIN]" + loggy "$(cat /var/log/tomcat/console.log)" + loggy "Tomcat Console Log [END]" + loggy "catalina.out [BEGIN]" + loggy "$(cat /usr/share/tomcat/logs/catalina.out)" + loggy "catalina.out [END]" + loggy "localhost.log [BEGIN]" + loggy "$(cat /usr/share/tomcat/logs/localhost*)" + loggy "localhost.log [END]" + exit 2 + fi + # debug_loggy "$HR2" + debug_loggy "$(date) SLEEP_INTERVAL: $SLEEP_INTERVAL, BESD_STATUS: $BESD_STATUS, besd_pid: $besd_pid, TOMCAT_STATUS: $TOMCAT_STATUS, tomcat_pid: $tomcat_pid" +done +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + diff --git a/el9-builds/hyrax/ncWMS_config.xml b/el9-builds/hyrax/ncWMS_config.xml new file mode 100644 index 00000000..eb2c9913 --- /dev/null +++ b/el9-builds/hyrax/ncWMS_config.xml @@ -0,0 +1,31 @@ + + + + + + + 256 + 10.0 + + + + + + OPeNDAP Docker + OPeNDAP + + support@opendap.og + + + ncWMS Server + true + 1024 + 1024 + + + http://@COMMAND_LINE_STUFF@ + true + + diff --git a/el9-builds/hyrax/olfs_viewers.xml b/el9-builds/hyrax/olfs_viewers.xml new file mode 100644 index 00000000..49b20ac6 --- /dev/null +++ b/el9-builds/hyrax/olfs_viewers.xml @@ -0,0 +1,65 @@ + + + + + + + idv.jnlp + + + + idv.jnlp + + + + + + + + + Web Mapping Service + + + + + Godiva WMS GUI + + + + + + + diff --git a/el9-builds/hyrax/tomcat.service b/el9-builds/hyrax/tomcat.service new file mode 100644 index 00000000..1675ff65 --- /dev/null +++ b/el9-builds/hyrax/tomcat.service @@ -0,0 +1,25 @@ +[Unit] +Description=Apache Tomcat Servlet container +Wants=network.target +After=network.target + +[Service] +Type=forking + +User=tomcat +Group=tomcat + +Environment="JAVA_HOME=/usr/lib/jvm/jre" +Environment="JAVA_OPTS=-Djava.security.egd=file:///dev/urandom" + +Environment="CATALINA_BASE=/usr/share/tomcat/latest" +Environment="CATALINA_HOME=/usr/share/tomcat/latest" +Environment="CATALINA_PID=/usr/share/tomcat/latest/temp/tomcat.pid" +Environment="CATALINA_OPTS=-Xms512M -Xmx4096M -server -XX:+UseParallelGC" + +ExecStart=/usr/share/tomcat/latest/bin/startup.sh +ExecStop=/usr/share/tomcat/latest/bin/shutdown.sh +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/el9-builds/hyrax/tomcat11-server.xml b/el9-builds/hyrax/tomcat11-server.xml new file mode 100644 index 00000000..6794c91d --- /dev/null +++ b/el9-builds/hyrax/tomcat11-server.xml @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/el9-builds/hyrax_wms.yml b/el9-builds/hyrax_wms.yml new file mode 100644 index 00000000..b31d1549 --- /dev/null +++ b/el9-builds/hyrax_wms.yml @@ -0,0 +1,41 @@ +--- +version: '3' +services: + olfs: + build: + context: ./olfs + args: + USE_NCWMS: "true" + env_file: ${PWD}/local.env + image: olfs:snapshot + ports: + - "80:8080" + tty: true + volumes: + - ./logs/olfs_tomcat:/usr/local/tomcat/logs + - ./logs:/usr/local/tomcat/webapps/opendap/WEB-INF/conf/logs + # command: -n http://localhost:8080 # This will be the publicly accessible host for ncWMS + besd: + build: ./besd + env_file: ${PWD}/local.env + image: besd:snapshot + ports: + - "10022:10022" + volumes: + - ./logs:/var/log/bes/ + # - ./cache:/var/cache/bes # maps the BES cache directory to the local filesystem + # - ./conf:/etc/bes # Maps a local BES configuration directory onto the bes in docker. + # command: -e admin_contact@email.org -s + ncwms: + build: + context: ./ncWMS + args: + DEVELOPER_MODE: "true" + env_file: ${PWD}/local.env + image: ncwms:latest + ports: + - "8080:8080" + tty: true + volumes: + - ./logs/ncwms_tomcat:/usr/local/tomcat/logs + - ./logs/ncwms:/root/.ncWMS2/logs diff --git a/el9-builds/local.env.orig b/el9-builds/local.env.orig new file mode 100644 index 00000000..85114f43 --- /dev/null +++ b/el9-builds/local.env.orig @@ -0,0 +1,9 @@ +# +# Template for runtime setable Hyrax configuration properties +# +# Copy this, and it local.env, and set these values. The 'local.env' +# file is used by the YAML files (e.g., hyrax.yml). +# +#SERVER_HELP_EMAIL=myhelp@myorg +#FOLLOW_SYMLINKS=Yes +#NCWMS_BASE=http://test.opendap.org/ \ No newline at end of file diff --git a/el9-builds/ngap/Dockerfile b/el9-builds/ngap/Dockerfile new file mode 100644 index 00000000..efc4897d --- /dev/null +++ b/el9-builds/ngap/Dockerfile @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +############################################################################################### +# +# Dockerfile for the NGAP deployment Hyrax container +# +# Some shell state reference: +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. +# +# In general use "set -e" when running commands that matter and don't use +# it for debugging stuff. +# +# Set one or more individual labels +FROM rockylinux:9 +LABEL org.opencontainers.image.authors="support@opendap.org" +USER root +ENV DEBUG_BUILD="" +ENV HR="###############################################################################################################" +ENV HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +ENV TARGET_OS="el9" +ENV ARCH="x86_64" +ENV java_version="java-21-openjdk" + +##################################################### +# Runtime packages needed by the service. +# 'jq' is used by enterypoint.sh +# 'awscli' is used by entrypoint.sh +# 'ps' (aka procps) is used by besctl and entrypoint.sh +# 'bc' is used entrypoint.sh and cleanup_files.sh +ENV dnf_runtime_packages="which unzip jq procps bc" +RUN echo "# dnf_runtime_packages: $dnf_runtime_packages" + + +##################################################### +# Packages needed to compile the Tomcat Native APR +# that we locate in the tomcat distribution after +# we have unpacked/installed it. +# @TODO Build the Tomcat Native APR outside of this Dockerfile and bring in the compiled result. +# The CRB (CodeReady Builder) repository has to be +# enabled to get the libtirpc-devel package. +ENV dnf_build_the_things_packages="libtirpc libtirpc-devel libuuid libuuid-devel openssl openssl-devel" +RUN echo "# dnf_build_the_things_packages: $dnf_build_the_things_packages" + +##################################################### +# +ENV dnf_developer_tools_packages="diffutils emacs vim git valgrind gdb awscli" +RUN echo "# dnf_developer_tools_packages: $dnf_developer_tools_packages" + + +RUN echo "$HR" +RUN echo "# BEGIN - ngap docker build" +# rocky9 comes with python 3.9 by default, and replacing it seems problematic at best. +RUN echo "# Python Version: $(python3 --version)" + + +# ENV DEPLOYMENT_CONTEXT="ROOT" +ENV DEFAULT_NGAP_DEPLOYMENT_CONTEXT="ngap" + +ARG DEPLOYMENT_CONTEXT +ENV DEPLOYMENT_CONTEXT="${DEPLOYMENT_CONTEXT:-"ROOT"}" +RUN echo "# DEPLOYMENT_CONTEXT: $DEPLOYMENT_CONTEXT" + +ARG RELEASE_DATE +ENV RELEASE_DATE="${RELEASE_DATE:-$(date +%s)}" +RUN echo "# RELEASE_DATE: $RELEASE_DATE" + +# +# Hyrax Components Version Negotiation +# +ARG TOMCAT_VERSION +ENV TOMCAT_VERSION="${TOMCAT_VERSION:-"11.0.7"}" +RUN echo "# TOMCAT_VERSION: $TOMCAT_VERSION" + +ENV TOMCAT_DISTRO="apache-tomcat-${TOMCAT_VERSION}" +COPY "${TOMCAT_DISTRO}.tar.gz" / +RUN ls -l /apache-tomcat-* >&2 + +ARG HYRAX_VERSION +ENV HYRAX_VERSION=${HYRAX_VERSION:-"snapshot-$TARGET_OS"} +RUN echo "# HYRAX_VERSION: ${HYRAX_VERSION}" + +ARG LIBDAP_VERSION +ENV LIBDAP_VERSION="${LIBDAP_VERSION:-"snapshot"}" +RUN echo "# LIBDAP_VERSION: $LIBDAP_VERSION" + +ARG LIBDAP_RPM +ENV LIBDAP_RPM="${LIBDAP_RPM:-"libdap-$LIBDAP_VERSION.$TARGET_OS.x86_64.rpm"}" +RUN echo "# LIBDAP_RPM: $LIBDAP_RPM" +COPY ${LIBDAP_RPM} libdap-devel-* libdap-debuginfo-* / +RUN ls -l /libdap-* >&2 + +ARG BES_VERSION +ENV BES_VERSION="${BES_VERSION:-"snapshot-$TARGET_OS"}" +RUN echo "# BES_VERSION: $BES_VERSION" + +ARG BES_RPM +ENV BES_RPM="${BES_RPM:-"bes-$BES_VERSION.static.$TARGET_OS.x86_64.rpm"}" +RUN echo "# BES_RPM: $BES_RPM" +COPY "$BES_RPM" bes-devel-* bes-debuginfo-* / +RUN ls -l /bes-* >&2 + +ARG OLFS_VERSION +ENV OLFS_VERSION="${OLFS_VERSION:-"unknown"}" +RUN echo "# OLFS_VERSION: $OLFS_VERSION" + +ARG REDIS_VERSION +ENV REDIS_VERSION="redisson-3.50.0" +RUN echo "# REDIS_VERSION: $REDIS_VERSION" + +ARG NGAP_DISTRO +ENV NGAP_DISTRO=${NGAP_DISTRO:-"ngap-${OLFS_VERSION}-webapp"} +RUN echo "# NGAP_DISTRO: ${NGAP_DISTRO}" +COPY ${NGAP_DISTRO}.tgz /${NGAP_DISTRO}.tgz + +############################################################ +# @TODO Do we need this? +# @TODO Is this duplicating the compile Tomcat Native APR?? +ARG APACHE_APR_VERSION +ENV APACHE_APR_VERSION="${APACHE_APR_VERSION:-"1.7.6-1"}" +RUN echo "# Apache APR version: $APACHE_APR_VERSION" + +ARG APACHE_APR_RPM +ENV APACHE_APR_RPM="${APACHE_APR_RPM:-"apr-$APACHE_APR_VERSION.x86_64.rpm"}" +RUN echo "# APACHE_APR_RPM: $APACHE_APR_RPM" +COPY "$APACHE_APR_RPM" apr-devel-* apr-debuginfo-* / +RUN ls -l /apr-* >&2 + +ARG TOMCAT_CONNECTION_TIMEOUT_MS +ENV TOMCAT_CONNECTION_TIMEOUT_MS="${TOMCAT_CONNECTION_TIMEOUT_MS:-"1000000"}" +RUN echo "# TOMCAT_CONNECTION_TIMEOUT_MS: $TOMCAT_CONNECTION_TIMEOUT_MS" + +LABEL vendor="OPeNDAP" +LABEL org.opendap.hyrax.ngap.version=${HYRAX_VERSION} +LABEL org.opendap.hyrax.ngap.release-date=${RELEASE_DATE} +LABEL org.opendap.hyrax.version.is-production="false" +LABEL org.opencontainers.image.authors="support@opendap.org" + +USER root + +###################################################################### +# Update the OS and create a handy executable shell scipt to simplify +# the installation of the (not as yet installed) but super useful +# developer tools packages :) ndp +RUN set -e \ + && dnf update -y \ + && echo "#!/bin/bash" > /install-dev-tools \ + && echo "dnf install -y $dnf_developer_tools_packages" > /install-dev-tools \ + && chmod 755 /install-dev-tools + + +###################################################################### +# Rocky9 comes with built in curl-minimal. For our puposes we want +# want the full package (I think, probably need to verify). +# The '--allowerasing' parameter enables dnf to erase curl-minimal +# and then install the full package. +# +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing cURL for developers (not minimal cURL)" >&2 \ + && dnf install -y curl curl-devel --allowerasing \ + && echo "# cURL version: $(curl --version)" >&2 \ + && dnf clean all \ + && echo "$HR" >&2 + +###################################################################### +# We want java ac ertain way ($java_version) and +# we want to be sure that +# - We enable the one we want: +# update-alternatives --set java "$java_version.$ARCH +# - We want to see if there are others installed java packages +# (we hope there are not): +# update-alternatives --list +# - We're using the one we wanted: +# echo "# java version: $( java -version 2>&1 )" +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# It's Java Time Witches!" >&2 \ + && echo "# Installing $java_version" >&2 \ + && dnf install -y "$java_version" "$java_version-devel" \ + && echo "$HR2" >&2 \ + && echo "# Using $java_version" >&2 \ + && update-alternatives --set java "$java_version.$ARCH" \ + && echo "# Enabled $java_version" >&2 \ + && echo "# java version: $( java -version 2>&1 )" \ + && update-alternatives --list \ + && dnf clean all \ + && echo "$HR" >&2 + +###################################################################### +# This Dockerfile brings in the Tomcat Native APR by using the one +# that ships in the Tomcat package that we install. The Tomcat Native +# APR requires the regular C/C++ development sweet (make, gcc, etc) +# to build so we install those here. +# We also enable the CodeReady Builder repo so we can get the +# libtirpc-devel package. +# +RUN set -e \ + && echo "$HR" >&2 \ + # The CRB (CodeReady Builder) repository has to be enabled to get the libtirpc-devel package. + && echo "# Installing 'dnf-plugins-core', enabling CRB (CodeReady Builder) repository and installing 'Development Tools'" >&2 \ + && dnf install -y dnf-plugins-core \ + && dnf config-manager --set-enabled crb \ + && dnf groupinstall "Development Tools" -y \ + && echo "$HR2" >&2 \ + +###################################################################### +# Install our runtime and build time packages. +# +RUN set -e \ + && echo "$HR2" >&2 \ + && echo "# Installing NGAP package lists..." >&2 \ + && dnf install -y $dnf_runtime_packages $dnf_build_the_things_packages \ + && dnf clean all \ + && echo "$HR" >&2 + +###################################################################### +# The follow is a sanity check so that we can see what we got. +RUN set -e \ + && echo "$HR" >&2 \ + && if test -n "$DEBUG_BUILD"; then \ + echo "# Sanity checking installed packages..." >&2 \ + && echo "# openssl is located here: $(which openssl)" >&2 \ + && echo "# openssl version: $(openssl version)" >&2 \ + && dnf -y info openssl >&2 \ + && echo "# openssl is installed." >&2 \ + && echo "$HR" >&2 \ + && echo "# libtirpc info:" >&2 \ + && dnf -y info libtirpc >&2 \ + && echo "$HR2" >&2 \ + && echo "# rpm -ql libtirpc: " >&2 \ + && rpm -ql libtirpc >&2 \ + && echo "$HR" >&2 \ + && echo "# libtirpc-devel info:" >&2 \ + && dnf -y info libtirpc-devel >&2 \ + && echo "$HR2" >&2 \ + && echo "# rpm -ql libtirpc-devel: " >&2 \ + && rpm -ql libtirpc-devel >&2 \ + && echo "$HR" >&2 \ + && echo "# libuuid info:" >&2 \ + && dnf -y info libuuid >&2 \ + && echo "# libuuid-devel info:" >&2 \ + && dnf -y info libuuid-devel >&2 \ + && dnf clean all; \ + else echo "# Skipping Sanity Checks." >&2 ; fi \ + && echo "$HR" >&2 + + +################################################################ +# Install the libdap rpm +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing the libdap rpm: $LIBDAP_RPM" >&2 \ + && ls -l ./libdap* \ + && dnf -y install ./libdap-*.rpm \ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && rm -f "./$LIBDAP_RPM" \ + && dnf clean all \ + && echo "# libdap4 is installed." >&2 \ + && echo "$HR" >&2 + +################################################################ +# Install the the BES +# besd, besdaemon, beslistener, besctl, besstandalone, bescmdline +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing the BES rpm: $BES_RPM" >&2 \ + && ls -l ./bes* \ + && dnf -y install ./bes-*.rpm \ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && rm -f "./$BES_RPM" \ + && dnf clean all \ + && echo "# besdaemon is here: $(which besdaemon)" >&2 \ + && echo "# besdaemon -v: '$(besdaemon -v)'" >&2 \ + && echo "$HR" >&2 + + +################################################################ +# Install the Apache-Portable-Runtime (Apache APR) +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e \ + && echo "$HR" >&2 \ + && echo "Installing the Apache APR rpm: $APACHE_APR_RPM" >&2 \ + && ls -l ./apr* >&2\ + && dnf -y install ./apr-*.rpm >&2\ + && echo "# The 'dnf install' command exited with status: $?" >&2 \ + && dnf list installed | grep apr >&2 \ + && rm -f "./$APACHE_APR_RPM" \ + && dnf clean all \ + && echo "Apache APR is installed." >&2 \ + && echo "$HR" >&2 +################################################################ +# Install and Setup Tomcat +# +# Make tomcat user and group +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e && useradd -m -U -d /home/tomcat -s /bin/false tomcat +# +# Install the Tomcat package that should have been copied into +# the root directory prior to this layer. +# +RUN set -e \ + && echo "$HR" >&2 \ + && df -h >&2 \ + && echo "$HR" >&2 \ + && echo "# Unpacking Tomcat distribution: $TOMCAT_DISTRO.tar.gz" >&2 \ + && tar -xvf "/$TOMCAT_DISTRO.tar.gz" -C "/home/tomcat" >&2 \ + && echo "# Distribution unpacked. Linking to /usr/share/tomcat" >&2 \ + && echo "# $(ln -vs "/home/tomcat/$TOMCAT_DISTRO" "/usr/share/tomcat")" >&2 \ + && echo "# Cleaning webapps dir" >&2 \ + && rm -rf /usr/share/tomcat/webapps/* >&2 \ + && echo "# Setting up logs" >&2 \ + && mkdir -p /var/log/tomcat \ + && chown -R tomcat:tomcat /var/log/tomcat /home/tomcat \ + && echo "# Tomcat is unpacked and ready. >&2 " + +# @TODO REMOVE THIS FOLLOWING COMMENTED OUT CODE BEFORE MERGE +#COPY tomcat.service /etc/systemd/system/tomcat.service +#RUN set -e \ +# && echo "$HR2" >&2 \ +# && echo "# Calling: 'systemctl enable tomcat'" >&2 \ +# && echo "# $(systemctl enable tomcat)" >&2 \ +# && echo "# systemctl status: $?" >&2 \ +# && echo "$HR" >&2 +#RUN firewall-cmd --add-port 8080/tcp --permanent +#RUN firewall-cmd --reload + +ENV CATALINA_HOME="/usr/share/tomcat" +ENV PATH="$CATALINA_HOME/bin:$PATH" +RUN set -e && echo "# CATALINA_HOME: $CATALINA_HOME" >&2 + +# Install our modified server.xml so that the server compresses responses. +COPY tomcat11-server.xml / +RUN set -e \ + && mv "/tomcat11-server.xml" "$CATALINA_HOME/conf/server.xml" \ + && chown -R tomcat:tomcat "$CATALINA_HOME/conf/server.xml" + +RUN set -e \ + && echo "Cleaning up Tomcat distribution files..." >&2 \ + && rm -fv "/$TOMCAT_DISTRO.tar.gz" >&2 + +########################################################################################### +# +# Build and Install the Tomcat Native-APR library. +# @TODO This building should be done elsewhere and brought in +# +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# Installing Tomcat Native APR" >&2 \ + && export JAVA_HOME="$(dirname $(dirname $(readlink -f $(which javac))))" >&2 \ + && echo "# JAVA_HOME: $JAVA_HOME" >&2 \ + && echo "$HR2" >&2 \ + && ls -l "$CATALINA_HOME/bin" >&2 \ + && echo "$HR2" >&2 \ + && tar -xf "/$CATALINA_HOME/bin/tomcat-native.tar.gz" -C "$CATALINA_HOME/bin" >&2 \ + && echo "$HR2" >&2 \ +# && ls -l ${CATALINA_HOME}/bin/tomcat-native-*-src >&2 \ + && cd $CATALINA_HOME/bin/tomcat-native-*-src/native >&2 \ + && ./configure --with-apr=/usr/bin >&2 \ + && echo "$HR2" >&2 \ + && make >&2 \ + && echo "$HR2" >&2 \ + && make install >&2 \ + && echo "$HR2" >&2 \ + && echo "# SUCCESS: Tomcat Native-APR Library installed." >&2 \ + && echo "$HR" >&2 + +ENV CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib" + +RUN set -e \ + && echo "$HR" >&2 \ + && echo "# The java executable is here: $(which java)" >&2 \ + && echo "# Using java version: $( java -version 2>&1 )" >&2 \ + && echo "$HR" >&2 + +################################################################ +# Retrieve, verify, and install the NGAP/OLFS web application +# - - - - - - - - - - - - - - - - - - - - - +ENV ROOT=/dev/shm + +# NGAP package uses the OLFS_VERSION number. jhrg 4/2/21 +RUN set -e \ + && echo "$HR" \ + && echo "Installing the latest NGAP distribution ($NGAP_DISTRO)." >&2 \ + && echo "working dir: "$(pwd)" ROOT: $ROOT" >&2 \ + && tar -C "$ROOT" -xzvf "/$NGAP_DISTRO.tgz" >&2 \ + && ls -l "$ROOT" >&2 \ + && echo "Unpacking war file..." >&2 \ + && unzip -o "$ROOT/$NGAP_DISTRO/$DEFAULT_NGAP_DEPLOYMENT_CONTEXT.war" -d "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/" \ + && echo "Cleaning up." >&2 \ + && rm -rf "$ROOT/$NGAP_DISTRO" "/$NGAP_DISTRO.tgz" \ + && echo "$HR" >&2 + +# Fix ownership and access permissions +RUN set -e \ + && mkdir -p "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/logs" \ + && chown -R tomcat:tomcat "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/logs" \ + && chmod 700 "$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf/logs" +################################################################ + +################################################################ +# Install our modified tomcat.conf so that the server use the memory +# cached session state and not require sticky sessions to operate +# behind a load balancer. +# +COPY tomcat.conf / +RUN set -e \ + && mkdir -p /etc/tomcat \ + && mv /tomcat.conf /etc/tomcat/tomcat.conf \ + && chown -R tomcat:tomcat /etc/tomcat + +# +# Request Redisson and Apache Session Level logging in the logging.properties +RUN set -e \ + && echo "org.redisson.level = FINE" >> "$CATALINA_HOME/conf/logging.properties" \ + && echo "org.apache.catalina.core.level = FINE" >> "$CATALINA_HOME/conf/logging.properties" \ + && echo "org.apache.catalina.session.level = FINE" >> "$CATALINA_HOME/conf/logging.properties" +# +# Install Memcached Session Manager jars into the Tomcat: ${CATALINA_HOME}/lib +RUN set -e \ + && echo "$HR" >&2 \ + && echo "Installing Session Manager jar files. Cloning OLFS..." >&2 \ + && git clone https://github.com/OPENDAP/olfs \ + #&& git clone --depth 1 https://github.com/OPENDAP/olfs \ + && cd olfs \ + && git checkout tomcat-11 \ + && cd .. \ + && echo "$HR2" >&2 \ + && echo "Copying elasticache client" >&2 \ + && cp -v "olfs/resources/ngap/lib/elasticache-java-cluster-client-1.1.2.jar" "$CATALINA_HOME/lib" \ + && echo "Copying redisson all" >&2 \ + && cp -v "olfs/resources/ngap/lib/redisson-all-3.50.0.jar" "$CATALINA_HOME/lib" \ + && echo "Copying redisson Session Manager TC11" >&2 \ + && cp -v "olfs/resources/ngap/lib/redisson-tomcat-11-3.50.0.jar" "$CATALINA_HOME/lib" \ + && chown -R tomcat:tomcat "$CATALINA_HOME/lib" \ + && rm -rfv olfs \ + && echo "$HR" >&2 + +################################################################ +# +# Retrieve, verify, and install the hyrax_regression_tests +# project +# +# - - - - - - - - - - - - - - - - - - - - - +ENV TEST_INSTALL_DIR="/hyrax_regression_tests" +RUN set -e \ + && echo "$HR" >&2 \ + && echo "Retrieving, and building hyrax regression tests." >&2 \ + && echo "h_r_t will be in: $TEST_INSTALL_DIR" >&2 \ + && mkdir -p "$TEST_INSTALL_DIR" \ + && git clone -v "https://github.com/opendap/hyrax_regression_tests" "$TEST_INSTALL_DIR" \ + && cd "$TEST_INSTALL_DIR" \ + && git status >&2 \ + && autoreconf -vif >&2 \ + && ./configure >&2 \ + && make testsuite > mk.log 2>&1 \ + && echo "SUCCESS: hyrax_regression_tests ready!" >&2 \ + && echo "$HR" >&2 + +################################################################ +# TEMPORARY +# Grab the cleanup files script for managing orphaned files +# from fileout_netcdf +COPY cleanup_files.sh / +RUN set -e && chmod +x /cleanup_files.sh +# TEMPORARY +################################################################ + +COPY entrypoint.sh / +RUN set -e && chmod +x /entrypoint.sh +ENTRYPOINT [ "/entrypoint.sh" ] + +EXPOSE 8080 +EXPOSE 8443 +EXPOSE 10022 +EXPOSE 11002 + +# can't use USER with entrypoint that needs root +# use gosu or, as done, enable bes user write so the entrypoint does not need root +RUN set -e && chown -R bes /etc/bes +USER root + +CMD ["-"] + diff --git a/el9-builds/ngap/cleanup_files.sh b/el9-builds/ngap/cleanup_files.sh new file mode 100755 index 00000000..78245269 --- /dev/null +++ b/el9-builds/ngap/cleanup_files.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# This script is meant as a temporary patch to the problem of the +# fileout_netcdf handler leaving behind orphaned temporary files +# that clog the deployments disk. When we understand and repair +# the issue with fileout_netcdf this script can be removed +# from the distributions. +# +# +verbose= +default_dir="/tmp/hyrax_fonc" +default_interval="60" # minutes +default_log="/var/log/bes/cleanup.log" +BES_TMP_FILES_DIR="/tmp/bes_rr_tmp" + +if test -n "${verbose}"; then echo "##########################################################" >&2 ; fi +if test -n "${verbose}"; then echo "# $0" >&2; echo "#" >&2; fi + +TARGET_DIR="${1:-$default_dir}" +if test -n "${verbose}"; then echo "# TARGET_DIR: ${TARGET_DIR}" >&2; fi + +INTERVAL="${2:-$default_interval}" # minutes +if test -n "${verbose}"; then echo "# INTERVAL: ${INTERVAL}m" >&2; fi + +CLEANUP_LOG="${3:-$default_log}" +if test -n "${verbose}"; then echo "# CLEANUP_LOG: ${CLEANUP_LOG}" >&2; fi + +SLEEP_TIME=$(echo "${INTERVAL}*60/2" | bc) +if test -n "${verbose}"; then echo "# SLEEP_TIME: ${SLEEP_TIME}s" >&2; fi +if test -n "${verbose}"; then echo "#" >&2; fi + +TARGET_DIRS="${TARGET_DIR} ${BES_TMP_FILES_DIR}" + +let pass=0 +while true +do + let pass+=1 + echo "########################################################################################" >> "${CLEANUP_LOG}" + echo "# Begin Pass: ${pass} ("$(date)")" >> "${CLEANUP_LOG}" + for target_dir in ${TARGET_DIRS} + do + if test -d ${target_dir}; + then + orphaned_files=$(find ${target_dir} -type f -mmin +${INTERVAL}) + if test -n "${verbose}"; then echo "orphaned_files: "${orphaned_files} >&2; fi + + if test -n "${orphaned_files}"; + then + echo "# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" >> "${CLEANUP_LOG}" + echo "# Begin cleanup of ${target_dir} ("$(date)")" >> "${CLEANUP_LOG}" + echo "#" >> "${CLEANUP_LOG}" + for file in ${orphaned_files} + do + target=$(ls -l "${file}") + rm -f "${file}" + echo $(date)" removed: ${target}" 2>&1 >> "${CLEANUP_LOG}" + done + echo "#" >> "${CLEANUP_LOG}" + echo "# End cleanup of ${target_dir} ("$(date)")" >> "${CLEANUP_LOG}" + fi + fi + done + sleep "${SLEEP_TIME}" + +done + diff --git a/el9-builds/ngap/cloudwatch_metrics_transmitter.sh b/el9-builds/ngap/cloudwatch_metrics_transmitter.sh new file mode 100644 index 00000000..cb940534 --- /dev/null +++ b/el9-builds/ngap/cloudwatch_metrics_transmitter.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# Here are the fields/contents of /proc/meminfo +# +# MemTotal: 1817080 kB +# MemFree: 604092 kB +# MemAvailable: 984796 kB +# Buffers: 0 kB +# Cached: 580776 kB +# SwapCached: 0 kB +# Active: 282772 kB +# Inactive: 759916 kB +# Active(anon): 46812 kB +# Inactive(anon): 513800 kB +# Active(file): 235960 kB +# Inactive(file): 246116 kB +# Unevictable: 0 kB +# Mlocked: 0 kB +# SwapTotal: 0 kB +# SwapFree: 0 kB +# Dirty: 48 kB +# Writeback: 0 kB +# AnonPages: 461936 kB +# Mapped: 87612 kB +# Shmem: 98700 kB +# KReclaimable: 58508 kB +# Slab: 116820 kB +# SReclaimable: 58508 kB +# SUnreclaim: 58312 kB +# KernelStack: 2576 kB +# PageTables: 9100 kB +# NFS_Unstable: 0 kB +# Bounce: 0 kB +# WritebackTmp: 0 kB +# CommitLimit: 908540 kB +# Committed_AS: 974356 kB +# VmallocTotal: 34359738367 kB +# VmallocUsed: 0 kB +# VmallocChunk: 0 kB +# Percpu: 1064 kB +# HardwareCorrupted: 0 kB +# AnonHugePages: 348160 kB +# ShmemHugePages: 0 kB +# ShmemPmdMapped: 0 kB +# FileHugePages: 0 kB +# FilePmdMapped: 0 kB +# HugePages_Total: 0 +# HugePages_Free: 0 +# HugePages_Rsvd: 0 +# HugePages_Surp: 0 +# Hugepagesize: 2048 kB +# Hugetlb: 0 kB +# DirectMap4k: 141224 kB +# DirectMap2M: 1916928 kB +# DirectMap1G: 0 kB + + +function aws_units(){ + local memproc_units="${1}" + local aws_units="Bytes" +# An error occurred (InvalidParameterValue) when calling the PutMetricData operation: The parameter MetricData.member.1.Unit must be a value in the set [ Megabits, Terabits, Gigabits, Count, Bytes, Gigabytes, Gigabytes/Second, Kilobytes, Kilobits/Second, Terabytes, Terabits/Second, Bytes/Second, Percent, Megabytes, Megabits/Second, Milliseconds, Microseconds, Kilobytes/Second, Gigabits/Second, Megabytes/Second, Bits, Bits/Second, Count/Second, Seconds, Kilobits, Terabytes/Second, None ]. + + if test "${memproc_units}" == "kB"; then aws_units="Kilobytes"; + elif test "${memproc_units}" == "MB"; then aws_units="Megabytes"; + elif test "${memproc_units}" == "GB"; then aws_units="Gigabytes"; + elif test "${memproc_units}" == "TB"; then aws_units="Terabytes"; fi + echo "${aws_units}" +} + +function mem_watch() { + + # Acquire instance metadata, doesn't change so outside loop. + export instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) + if test -n "${verbose}"; then echo "instance_id: ${instance_id}" >&2; fi + + export instance_type=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) + if test -n "${verbose}"; then echo "instance_type: ${instance_type}" >&2; fi + + while(true) + do + # Set sample_time + sample_time=$(date '+%Y-%m-%dT%H:%M:%S.000%Z') + if test -n "${verbose}"; then echo "sample_time: ${sample_time}" >&2; fi + # Sample /proc/memproc + memproc_sample=$(cat /proc/meminfo) + + # Get the goodies... + mem_avail=$(echo "${memproc_sample}" | grep MemAvailable /proc/meminfo | awk '{print $2}') + mem_units=$(echo "${memproc_sample}" | grep MemAvailable /proc/meminfo | awk '{print $3}') + units=$(aws_units ${mem_units}) + + if test -n "${verbose}"; then echo "mem_avail: ${mem_avail} ${units}" >&2; fi + + + aws cloudwatch put-metric-data \ + --metric-name MemAvailable \ + --namespace hyrax \ + --unit "${units}" \ + --value "${mem_avail}" \ + --timestamp "${sample_time}"\ + --dimensions InstanceId="${instance_id}",InstanceType="${instance_type}" + + status=$? + if test ${status} -ne 0 ; then echo "ERROR - The call to 'aws cloudwatch put-metric-data' failed. status: ${status}" >&2; fi + + sleep 1 + done +} + +mem_watch + diff --git a/el9-builds/ngap/entrypoint.sh b/el9-builds/ngap/entrypoint.sh new file mode 100755 index 00000000..407f33e3 --- /dev/null +++ b/el9-builds/ngap/entrypoint.sh @@ -0,0 +1,579 @@ +#!/bin/bash +# This is the entrypoint.sh file for the single container Hyrax. + +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. +export debug="false" + +export SYSTEM_ID="${INSTANCE_ID:-"undetermined"}" +export LOG_KEY_PREFIX="${LOG_KEY_PREFIX:-"hyrax-"}" +if [[ "$LOG_KEY_PREFIX" != *"-" ]]; then + LOG_KEY_PREFIX="$LOG_KEY_PREFIX-" +fi +########################################################################## +# +# Functions +# + +########################################################################## +# set_log_key_names() - Adds the LOG_KEY_PREFIX to the log key names +# +function set_log_key_names() { + export TIME_KEY + TIME_KEY="${LOG_KEY_PREFIX}time" + + export INSTANCE_ID_KEY + INSTANCE_ID_KEY="${LOG_KEY_PREFIX}instance-id" + + export PID_KEY + PID_KEY="${LOG_KEY_PREFIX}pid" + + export TYPE_KEY + TYPE_KEY="${LOG_KEY_PREFIX}type" + + export MESSAGE_KEY + MESSAGE_KEY="${LOG_KEY_PREFIX}message" +} + +########################################################################## +# loggy() +# Makes json log records +# +function startup_log() { + echo "{ \"$TIME_KEY\": $(date "+%s")," \ + "\"$INSTANCE_ID_KEY\": \"$SYSTEM_ID\"," \ + "\"$PID_KEY\": $(echo $$)," \ + "\"$TYPE_KEY\": \"start-up\"," \ + "\"$MESSAGE_KEY\": $(echo -n "$*" | jq -Rsa '.')" \ + "}" >&2 +} + +function heartbeat_log() { + echo "{ \"$TIME_KEY\": $(date "+%s"), " \ + "\"$INSTANCE_ID_KEY\": \"$SYSTEM_ID\", " \ + "\"$PID_KEY\": $$, " \ + "\"$TYPE_KEY\": \"heartbeat\", " \ + "\"$MESSAGE_KEY\": $(echo -n "$*" | jq -Rsa '.')" \ + "}" >&2 +} + +function error_log() { + echo "{ \"$TIME_KEY\": $(date "+%s"),"\ + "\"$INSTANCE_ID_KEY\": \"$SYSTEM_ID\"," \ + "\"$PID_KEY\": $$," \ + "\"$TYPE_KEY\": \"error\"," \ + "\"$MESSAGE_KEY\": $(echo -n "$*" | jq -Rsa '.')" \ + "}" >&2 +} + +########################################################################## +# get_instance_id() +# Try to get the AWS instance-id and if that fails make up a unique one. +# +function get_aws_instance_id() { + local aws_instance_id_url + local id_file + local http_status + local curl_status + local instance_id + + aws_instance_id_url="http://169.254.169.254/latest/meta-data/instance-id" + id_file="./instance-id.txt" + startup_log "Checking for AWS instance-id by requesting: $aws_instance_id_url" + + set +e # This cURL command may fail, and that's ok. + http_status"=$(curl -s -w "%{http_code}" --max-time 5 -o "$id_file" -L "$aws_instance_id_url")" + curl_status=$? + set -e + + startup_log "curl_status: $curl_status" + startup_log "http_status: $http_status" + if test $curl_status -ne 0 || test "$http_status" -gt 400; then + startup_log "WARNING! Failed to determine the AWS instance-d by requesting: $aws_instance_id_url (curl_status: $curl_status http_status: $http_status)" + startup_log "Inventing a random instance-id value." + instance_id="h-$(python3 -c 'import uuid; print(str(uuid.uuid4()))')" + else + instance_id="$(cat $id_file)" + fi + startup_log "Using instance_id: $instance_id" + echo "$instance_id" +} +########################################################################## +# write_tomcat_logs() +# Writes the last log_lines of the Tomcat logs console.log, catalina.out, +# and the most recent localhost*.log files as error log messages. +# +function write_tomcat_logs() { + local log_lines="$1" + local wait_time="$2" + + error_log "Tomcat Console Log [BEGIN]" + error_log "$(tail --lines "$log_lines" /var/log/tomcat/console.log)" + error_log "Tomcat Console Log [END]" + error_log "catalina.out [BEGIN]" + error_log "$(tail --lines "$log_lines" /usr/share/tomcat/logs/catalina.out)" + error_log "catalina.out [END]" + error_log "localhost.log [BEGIN]" + error_log "$(tail --lines "$log_lines" "$(ls -t /usr/share/tomcat/logs/localhost* | head -1)")" + error_log "localhost.log [END]" + sleep $wait_time +} + +########################################################################## +########################################################################## +########################################################################## +# +# Execution Begins Here. +# + +# Make sure the startup log keys have the correct prefix. +set_log_key_names + +#loggy "entrypoint.sh command line: \"$@\"" +startup_log "########################### HYRAX #################################" +startup_log "Greetings, I am $(whoami)." +set -e +#set -x +startup_log "PythonVersion: $(python3 --version)" + +################################################################################ +SYSTEM_ID="$(get_aws_instance_id)" + +################################################################################ +startup_log "Checking AWS CLI: " +set +e +aws_bin="$(which aws 2>&1)" +ab_status=$? +set -e +if test $ab_status -ne 0; then + startup_log "WARNING: It appears that the AWS CLI is not installed. Not found on '$PATH' ('which' status: $ab_status, msg: $aws_bin)" +else + acl="$(aws configure list 2>&1)" + acl_status=$? + startup_log "$acl" + if test $acl_status -ne 0; then + startup_log "WARNING: Problem with AWS CLI! ('aws' status: $acl_status msg: $acl)" + fi +fi + +################################################################################ +startup_log "JAVA VERSION: $( java -version 2>&1 | sed -e "s/\"//g"; )" # Java version has undesired double quote chars +export JAVA_HOME="${JAVA_HOME:-"/etc/alternatives/jre"}" +startup_log "JAVA_HOME: $JAVA_HOME" + +export CATALINA_HOME="${CATALINA_HOME:-"NOT_SET"}" +startup_log "CATALINA_HOME: $CATALINA_HOME" + +export DEPLOYMENT_CONTEXT="${DEPLOYMENT_CONTEXT:-"ROOT"}" +startup_log "DEPLOYMENT_CONTEXT: $DEPLOYMENT_CONTEXT" + +export OLFS_CONF_DIR="$CATALINA_HOME/webapps/$DEPLOYMENT_CONTEXT/WEB-INF/conf" +startup_log "OLFS_CONF_DIR: $OLFS_CONF_DIR" + +export TOMCAT_CONTEXT_FILE="/usr/share/tomcat/conf/context.xml" +startup_log "TOMCAT_CONTEXT_FILE: $TOMCAT_CONTEXT_FILE" + +export TOMCAT_REDISSON_FILE="/usr/share/tomcat/conf/redisson.json" +startup_log "TOMCAT_REDISSON_FILE: $TOMCAT_REDISSON_FILE" + +export NCWMS_BASE="${NCWMS_BASE:-"https://localhost:8080"}" +startup_log "NCWMS_BASE: $NCWMS_BASE" + +################################################################################ +if test -n "$AWS_ACCESS_KEY_ID"; then + startup_log "AWS_ACCESS_KEY_ID: HAS BEEN SET" +else + startup_log "AWS_ACCESS_KEY_ID: HAS NOT BEEN SET" +fi + +if test -n "$AWS_SECRET_ACCESS_KEY"; then + startup_log "AWS_SECRET_ACCESS_KEY: HAS BEEN SET" +else + startup_log "AWS_SECRET_ACCESS_KEY: HAS NOT BEEN SET" +fi + +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-"us-west-2"}" +startup_log "AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION" + +################################################################################ +export NGAP_CERTIFICATE_FILE="/usr/share/tomcat/conf/NGAP-CA-certificate.crt" +startup_log "NGAP_CERTIFICATE_FILE: $NGAP_CERTIFICATE_FILE" + +export NGAP_CERTIFICATE_CHAIN_FILE="/usr/share/tomcat/conf/NGAP-CA-certificate-chain.crt" +startup_log "NGAP_CERTIFICATE_CHAIN_FILE: $NGAP_CERTIFICATE_CHAIN_FILE" + +export NGAP_CERTIFICATE_KEY_FILE="/usr/share/tomcat/conf/NGAP-CA-certificate.key" +startup_log "NGAP_CERTIFICATE_KEY_FILE: $NGAP_CERTIFICATE_KEY_FILE" + +################################################################################ +export NETRC_FILE="/etc/bes/ngap_netrc" +startup_log "NETRC_FILE: $NETRC_FILE" + +export BES_SITE_CONF_FILE="/etc/bes/site.conf" +startup_log "BES_SITE_CONF_FILE: $BES_SITE_CONF_FILE" + +export BES_LOG_FILE="/var/log/bes/bes.log" +startup_log "BES_LOG_FILE: $BES_LOG_FILE" + +export SLEEP_INTERVAL="${SLEEP_INTERVAL:-10}" +startup_log "SLEEP_INTERVAL: $SLEEP_INTERVAL seconds." + +export SERVER_HELP_EMAIL="${SERVER_HELP_EMAIL:-"not_set"}" +startup_log "SERVER_HELP_EMAIL: $SERVER_HELP_EMAIL" + +export FOLLOW_SYMLINKS="${FOLLOW_SYMLINKS:-"not_set"}" +startup_log "FOLLOW_SYMLINKS: $FOLLOW_SYMLINKS" + +################################################################################ +# Inject one set of credentials into .netrc +# Only modify the .netrc file if all three environment variables are defined +# +if test -n "$HOST" && test -n "$USERNAME" && test -n "$PASSWORD"; then + startup_log "Updating netrc file: $NETRC_FILE" + # machine is a domain name or an ip address, not a URL. + echo "machine $HOST" | sed -e "s_https:__g" -e "s_http:__g" -e "s+/++g" >> "$NETRC_FILE" + echo " login $USERNAME" >> "$NETRC_FILE" + echo " password $PASSWORD" >> "$NETRC_FILE" + chown bes:bes "$NETRC_FILE" + chmod 400 "$NETRC_FILE" + startup_log "$(ls -l "$NETRC_FILE")" + # startup_log "$( cat "$NETRC_FILE" )" +fi +################################################################################ + +################################################################################ +# Inject olfs.xml configuration document. +# +# Test if the olfs.xml env variable is set (by way of not unset) and +# not empty and use its value if present and non-empty.olfs +# +if test -n "$OLFS_XML"; then + OLFS_XML_FILE="$OLFS_CONF_DIR/olfs.xml" + startup_log "Updating OLFS configuration file: $OLFS_XML_FILE" + echo "$OLFS_XML" > "$OLFS_XML_FILE" + startup_log "$( ls -l "$OLFS_XML_FILE" )" + startup_log "$( cat "$OLFS_XML_FILE" )" +fi +################################################################################ + +################################################################################ +# Inject user-access.xml document to define the servers relationship to +# EDL and the user access rules. +# +# Test if the user-access.xml env variable is set (by way of not unset) and +# not empty +# +if test -n "$USER_ACCESS_XML"; then + USER_ACCESS_XML_FILE="$OLFS_CONF_DIR/user-access.xml" + startup_log "Updating OLFS user access controls: $USER_ACCESS_XML_FILE" + echo "$USER_ACCESS_XML" > "$USER_ACCESS_XML_FILE" + startup_log "$(ls -l "$USER_ACCESS_XML_FILE")" + # startup_log ( cat "$USER_ACCESS_XML_FILE" ) +fi +################################################################################ + +################################################################################ +# Inject BES configuration site.conf document to configure the BES to operate +# in the NGAP environment. +# +# Test if the bes.conf env variable is set (by way of not unset) and not empty +if test -n "$BES_SITE_CONF"; then + startup_log "Updating BES site.conf: $BES_SITE_CONF_FILE" + # echo "$BES_SITE_CONF" > "$BES_SITE_CONF_FILE" + # @TODO THis seems like a crappy hack, we should just change the source file in BitBucket to be correct + echo "$BES_SITE_CONF" | sed -e "s+BES.LogName=stdout+BES.LogName=$BES_LOG_FILE+g" > "$BES_SITE_CONF_FILE" + startup_log "$( ls -l "$BES_SITE_CONF_FILE" )" + # startup_log $( cat "$BES_SITE_CONF_FILE" ) +fi +# +# Update site.conf with the instance-id of this system. +echo "AWS.instance-id=$SYSTEM_ID" >> "$BES_SITE_CONF_FILE" +startup_log "instance-id in site.conf: $( tail -1 "$BES_SITE_CONF_FILE" )" +################################################################################ + +################################################################################ +# Inject Tomcat's context.xml configuration document to configure the Tomcat to +# utilize Session Management in the NGAP environment. +# +# Test if the bes.conf env variable is set (by way of not unset) and not empty +if test -n "$TOMCAT_CONTEXT_XML"; then + startup_log "Writing Tomcat context.xml file: $TOMCAT_CONTEXT_FILE" + echo "$TOMCAT_CONTEXT_XML" > "$TOMCAT_CONTEXT_FILE" + startup_log "$( ls -l "$TOMCAT_CONTEXT_FILE" )" + # startup_log "$(cat "$TOMCAT_CONTEXT_FILE" )" +fi +################################################################################ + +################################################################################ +# Inject Tomcat's NGAP[CA] certificate document to configure the Tomcat to +# utilize SSL/TLS Data-In-Transit in the NGAP environment. +# +# Test if the bes.conf env variable is set (by way of not unset) and not empty +if test -n "$NGAP_CERTIFICATE"; then + startup_log "Writing certificate file: $NGAP_CERTIFICATE_FILE" + echo "$NGAP_CERTIFICATE" > "$NGAP_CERTIFICATE_FILE" + startup_log "$( ls -l "$NGAP_CERTIFICATE_FILE" )" + # startup_log "$(cat "$NGAP_CERTIFICATE_FILE" )" +fi +################################################################################ + +################################################################################ +# Inject Tomcat's NGAP[CA] certificate-chain to configure the Tomcat to +# utilize SSL/TLS Data-In-Transit in the NGAP environment. +# +# Test if the bes.conf env variable is set (by way of not unset) and not empty +if test -n "$NGAP_CERTIFICATE_CHAIN"; then + startup_log "Writing credentials chain file: $NGAP_CREDENTIALS_CHAIN_FILE" + echo "$NGAP_CERTIFICATE_CHAIN" > "$NGAP_CERTIFICATE_CHAIN_FILE" + startup_log "$( ls -l "$NGAP_CERTIFICATE_CHAIN_FILE" )" + # startup_log "$(cat "$NGAP_CERTIFICATE_CHAIN_FILE" )" +fi +################################################################################ + +################################################################################ +# Inject Tomcat's NGAP[CA] certificate key to configure the Tomcat to +# utilize SSL/TLS Data-In-Transit in the NGAP environment. +# +# Test if the bes.conf env variable is set (by way of not unset) and not empty +if test -n "$NGAP_CERTIFICATE_KEY"; then + startup_log "Writing key file: $NGAP_CERTIFICATE_KEY_FILE" + echo "$NGAP_CERTIFICATE_KEY" > "$NGAP_CERTIFICATE_KEY_FILE" + startup_log "$( ls -l "$NGAP_CERTIFICATE_KEY_FILE" )" + # startup_log "$(cat "$NGAP_CERTIFICATE_KEY_FILE" )" +fi +################################################################################ + +################################################################################ +# Inject Tomcat's redisson.json configuration document to configure the Tomcat to +# utilize Redisson Session Management in the NGAP environment. +# +# Test if the bes.conf env variable is set (by way of not unset) and not empty +if test -n "$TOMCAT_REDISSON_XML"; then + startup_log "Writing Tomcat redisson.json file: $TOMCAT_REDISSON_FILE" + echo "$TOMCAT_REDISSON_XML" > "$TOMCAT_REDISSON_FILE" + chown tomcat:tomcat "$TOMCAT_REDISSON_FILE" + startup_log "$( ls -l "$TOMCAT_REDISSON_FILE" )" + startup_log "$(cat "$TOMCAT_REDISSON_FILE" )" +fi +################################################################################ + +################################################################################ +# +# Process commandline arguments +# +# + +while getopts "e:sn:dk:i:r:" opt; do + startup_log "Processing command line opt: $opt" + case "$opt" in + d) + export debug="true" + startup_log "Set debug: $debug" + ;; + e) + export SERVER_HELP_EMAIL=$OPTARG + startup_log "Set SERVER_HELP_EMAIL: $SERVER_HELP_EMAIL" + ;; + s) + export FOLLOW_SYMLINKS="Yes" + startup_log "Set FOLLOW_SYMLINKS: $FOLLOW_SYMLINKS" + ;; + n) + export NCWMS_BASE=$OPTARG + startup_log "Set NCWMS_BASE: $NCWMS_BASE" + ;; + k) + export AWS_SECRET_ACCESS_KEY="$OPTARG" + startup_log "Set AWS_SECRET_ACCESS_KEY" + ;; + i) + export AWS_ACCESS_KEY_ID="$OPTARG" + startup_log "Set AWS_ACCESS_KEY_ID" + ;; + r) + export AWS_DEFAULT_REGION="$OPTARG" + startup_log "Set AWS_DEFAULT_REGION" + ;; + + \?) + echo "Invalid option: -$OPTARG" >&2 + echo "options: [-e xxx] [-n yyy] [-s] [-d] [-i xxx] [-k xxx] [-r xxx]" >&2 + echo " -e xxx where xxx is the email address of the admin contact for the server." >&2 + echo " -s When present causes the BES to follow symbolic links." >&2 + echo " -n yyy where yyy is the protocol, server and port part " >&2 + echo " of the ncWMS service (for example https://foo.com:8090)." >&2 + echo " -d Enables debugging output for this script." >&2 + echo " -i xxx Where xxx is an AWS CLI AWS_ACCESS_KEY_ID." >&2 + echo " -k xxx Where xxx is an AWS CLI AWS_SECRET_ACCESS_KEY." >&2 + echo " -r xxx Where xxx is an AWS CLI AWS_DEFAULT_REGION." >&2 + echo "EXITING NOW" >&2 + exit 2 + ;; + esac +done +# +# END Command Line Processing +################################################################################ + +if test "$debug" = "true"; then + startup_log "CATALINA_HOME: $CATALINA_HOME" + startup_log "$(ls -l "$CATALINA_HOME")" + startup_log "CATALINA_HOME/bin: $CATALINA_HOME/bin" + startup_log "$(ls -l "$CATALINA_HOME/bin")" +fi + +################################################################################ +# +# Configuring NcWMS +# +VIEWERS_XML="$OLFS_CONF_DIR/viewers.xml" +if test "$debug" = "true"; then + startup_log "NCWMS: Using NCWMS_BASE: $NCWMS_BASE" + startup_log "NCWMS: Setting ncWMS access URLs in viewers.xml (if needed)." + startup_log "$(ls -l "$VIEWERS_XML")" +fi + +if test -f "$VIEWERS_XML"; then + sed -i "s+@NCWMS_BASE@+$NCWMS_BASE+g" "$VIEWERS_XML" +fi + +if test "$debug" = "true"; then + startup_log "$VIEWERS_XML: " + startup_log "$(cat "$VIEWERS_XML")" +fi +################################################################################ + +################################################################################ +# +# Configure OLFS debug logging if debug is enabled. +if test "$debug" = "true"; then + startup_log "Configuring OLFS to debug logging..." + logback_xml="$OLFS_CONF_DIR/logback.xml" + ngap_logback_xml="$OLFS_CONF_DIR/logback-ngap.xml" + if test -f "$ngap_logback_xml" + then + cp "$ngap_logback_xml" "$logback_xml" + startup_log "Enabled Logback (slf4j) debug logging for NGAP." + else + startup_log "Unable to locate the debug logging configuration file $ngap_logback_xml. No changes were made to logging configuration." + fi + startup_log "$( cat "$logback_xml" )" +fi +################################################################################ + +################################################################################ +# +# modify bes.conf based on environment variables before startup. +# +if test "$SERVER_HELP_EMAIL" != "not_set"; then + startup_log "Setting Admin Contact To: $SERVER_HELP_EMAIL" + sed -i "s/admin.email.address@your.domain.name/$SERVER_HELP_EMAIL/" /etc/bes/bes.conf +fi +if test "$FOLLOW_SYMLINKS" != "not_set"; then + startup_log "Setting BES FollowSymLinks to YES." + sed -i "s/^BES.Catalog.catalog.FollowSymLinks=No/BES.Catalog.catalog.FollowSymLinks=Yes/" /etc/bes/bes.conf +fi +################################################################################ + +#------------------------------------------------------------------------------- +# Start the BES daemon process +# /usr/bin/besdaemon -i /usr -c /etc/bes/bes.conf -r /var/run/bes.pid +bes_uid="$(id -u bes)" +bes_gid="$(id -g bes)" +startup_log "Launching besd [uid: $bes_uid gid: $bes_gid]" +/usr/bin/besctl start > ./besctl.log 2>&1 # dropped debug control -d "/dev/null,timing" - ndp 10/12/2023 +status=$? +startup_log "$(cat ./besctl.log)" +if test $status -ne 0; then + error_log "ERROR: Failed to start BES: $status" + #exit $status +fi +besd_pid="$(ps aux | grep /usr/bin/besdaemon | grep -v grep | awk '{print $2;}' -)" +startup_log "The besd is UP! [pid: $besd_pid]" + +#------------------------------------------------------------------------------- +# Start Tomcat process +# +startup_log "Starting tomcat/olfs..." + +# mv "$OLFS_CONF_DIR/logback.xml" "$OLFS_CONF_DIR/logback.xml.OFF" + +"$CATALINA_HOME"/bin/startup.sh > "/var/log/tomcat/console.log" 2>&1 & +status=$? +tomcat_pid=$! +if test $status -ne 0; then + error_log "ERROR: Failed to start Tomcat: $status" + #exit $status +fi +# When we launch tomcat the initial pid gets "retired" because it spawns a +# secondary processes. +initial_pid="$tomcat_pid" +startup_log "Tomcat started, initial pid: $initial_pid" +while test $initial_pid -eq "$tomcat_pid"; do + sleep 1 + tomcat_ps="$(ps aux | grep tomcat | grep -v grep)" + startup_log "tomcat_ps: $tomcat_ps" + tomcat_pid=$(echo "$tomcat_ps" | awk '{print $2}') + startup_log "tomcat_pid: $tomcat_pid" +done +# New pid and we should be good to go. +startup_log "Tomcat is UP! pid: $tomcat_pid" + +# TEMPORARY ################################################################### +/cleanup_files.sh >&2 & +# TEMPORARY ################################################################### + +#------------------------------------------------------------------------------- +# Get the bes log, make it json, and send it to stdout +# +tail -f "$BES_LOG_FILE" | beslog2json.py --prefix "$LOG_KEY_PREFIX" & + +start_time= +now= +suptime= +start_time="$(date "+%s")" +#------------------------------------------------------------------------------- +startup_log "Hyrax Has Arrived...(time: $start_time SLEEP_INTERVAL: $SLEEP_INTERVAL)" +#------------------------------------------------------------------------------- +while /bin/true; do + sleep $SLEEP_INTERVAL + + # Compute service_uptime in hours + now="$(date "+%s")" + suptime="$(echo "scale=4; ($now - $start_time)/60/60" | bc)" + + if test "$debug" = "true"; then + heartbeat_log "Checking Hyrax Operational State. service_uptime: $suptime hours"; + fi + + besd_ps="$(ps -f "$besd_pid")" + BESD_STATUS=$? + + if test "$debug" = "true"; then + heartbeat_log "besd_ps: $besd_ps"; + fi + if test "$debug" = "true"; then + heartbeat_log "BESD_STATUS: $BESD_STATUS"; + fi + + if test $BESD_STATUS -ne 0; then + error_log "BESD_STATUS: $BESD_STATUS besd_pid:$besd_pid" + error_log "The BES daemon appears to have died! Exiting. (service_uptime: $suptime hours)" + #exit $BESD_STATUS + fi + + tomcat_ps="$(ps -f "$tomcat_pid")" + TOMCAT_STATUS=$? + if test "$debug" = "true"; then heartbeat_log "TOMCAT_STATUS: $TOMCAT_STATUS"; fi + if test $TOMCAT_STATUS -ne 0; then + error_log "TOMCAT_STATUS: $TOMCAT_STATUS tomcat_pid:$tomcat_pid" + error_log "Tomcat appears to have died! Exiting. (service_uptime: $suptime hours)" + # write_tomcat_logs 100 5 # [number of log lines to grab from each file] [time to sleep after sending] + #exit $TOMCAT_STATUS + fi + +done +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- diff --git a/el9-builds/ngap/tomcat.conf b/el9-builds/ngap/tomcat.conf new file mode 100644 index 00000000..f64edf6a --- /dev/null +++ b/el9-builds/ngap/tomcat.conf @@ -0,0 +1,62 @@ +# System-wide configuration file for tomcat services +# This will be loaded by systemd as an environment file, +# so please keep the syntax. For shell expansion support +# place your custom files as /etc/tomcat/conf.d/*.conf +# +# There are 2 "classes" of startup behavior in this package. +# The old one, the default service named tomcat.service. +# The new named instances are called tomcat@instance.service. +# +# Use this file to change default values for all services. +# Change the service specific ones to affect only one service. +# For tomcat.service it's /etc/sysconfig/tomcat, for +# tomcat@instance it's /etc/sysconfig/tomcat@instance. + +# This variable is used to figure out if config is loaded or not. +TOMCAT_CFG_LOADED="1" + +# In new-style instances, if CATALINA_BASE isn't specified, it will +# be constructed by joining TOMCATS_BASE and NAME. +TOMCATS_BASE="/var/lib/tomcats/" + +# Where your java installation lives +JAVA_HOME="/usr/lib/jvm/jre" + +# Where your tomcat installation lives +CATALINA_HOME="/usr/share/tomcat" + +# System-wide tmp +CATALINA_TMPDIR="/var/cache/tomcat/temp" + +# You can pass some parameters to java here if you wish to +#JAVA_OPTS="-Xminf0.1 -Xmaxf0.3" + +# Use JAVA_OPTS to set java.library.path for libtcnative.so +#JAVA_OPTS="-Djava.library.path=/usr/lib" + +# You can change your tomcat locale here +#LANG="en_US" + +# Run tomcat under the Java Security Manager +SECURITY_MANAGER="false" + +# SHUTDOWN_WAIT has been deprecated. To change the shutdown wait time, set +# TimeoutStopSec in tomcat.service. + +# If you wish to further customize your tomcat environment, +# put your own definitions here +# (i.e. LD_LIBRARY_PATH for some jdbc drivers) + +################################################################################ +# NGAP Configuration +# +# Memory +# +JAVA_OPTS="-Xms512M -Xmx2048M" +# +# Allow URL encoded slash and backslash in URL paths +# +JAVA_OPTS="${JAVA_OPTS} -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true" +JAVA_OPTS="${JAVA_OPTS} -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true" +# +# \ No newline at end of file diff --git a/el9-builds/ngap/tomcat.service b/el9-builds/ngap/tomcat.service new file mode 100644 index 00000000..1675ff65 --- /dev/null +++ b/el9-builds/ngap/tomcat.service @@ -0,0 +1,25 @@ +[Unit] +Description=Apache Tomcat Servlet container +Wants=network.target +After=network.target + +[Service] +Type=forking + +User=tomcat +Group=tomcat + +Environment="JAVA_HOME=/usr/lib/jvm/jre" +Environment="JAVA_OPTS=-Djava.security.egd=file:///dev/urandom" + +Environment="CATALINA_BASE=/usr/share/tomcat/latest" +Environment="CATALINA_HOME=/usr/share/tomcat/latest" +Environment="CATALINA_PID=/usr/share/tomcat/latest/temp/tomcat.pid" +Environment="CATALINA_OPTS=-Xms512M -Xmx4096M -server -XX:+UseParallelGC" + +ExecStart=/usr/share/tomcat/latest/bin/startup.sh +ExecStop=/usr/share/tomcat/latest/bin/shutdown.sh +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/el9-builds/ngap/tomcat11-server.xml b/el9-builds/ngap/tomcat11-server.xml new file mode 100644 index 00000000..80c71f77 --- /dev/null +++ b/el9-builds/ngap/tomcat11-server.xml @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/el9-builds/olfs/Dockerfile b/el9-builds/olfs/Dockerfile new file mode 100644 index 00000000..d06f19ae --- /dev/null +++ b/el9-builds/olfs/Dockerfile @@ -0,0 +1,231 @@ +### +# Dockerfile for Hyrax OLFS +### + +# +FROM rockylinux:9 +LABEL org.opencontainers.image.authors="support@opendap.org" +USER root +ENV HR="######################################################" +ENV HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +ENV TARGET_OS="el9" +ENV ARCH="x86_64" +ENV java_version="java-21-openjdk" +# rocky9 comes with python 3.9 by default. +ENV python_version="3.9" +RUN echo "$HR" && echo "# Python Version: $(python3 --version) (config says we want: $python_version)" && echo "$HR" + +ENV dnf_runtime_packages="which unzip procps bc" +RUN echo "# dnf_runtime_packages: $dnf_runtime_packages" + +ENV dnf_developer_tools_packages="diffutils emacs vim jq git bc valgrind gdb awscli" +RUN echo "# dnf_developer_tools_packages: $dnf_developer_tools_packages" + + + +ARG RELEASE_DATE +ENV RELEASE_DATE=${RELEASE_DATE:-"$(date +%s)"} +RUN echo "# RELEASE_DATE: ${RELEASE_DATE}" + +ARG TOMCAT_VERSION +ENV TOMCAT_VERSION="${TOMCAT_VERSION:-"11.0.7"}" +RUN echo "# TOMCAT_VERSION: ${TOMCAT_VERSION}" +ENV TOMCAT_DISTRO="apache-tomcat-${TOMCAT_VERSION}" +COPY "${TOMCAT_DISTRO}.tar.gz" / +RUN ls -l /apache-tomcat-* >&2 + +ARG HYRAX_VERSION +ENV HYRAX_VERSION=${HYRAX_VERSION:-"snapshot"} +RUN echo "# HYRAX_VERSION: ${HYRAX_VERSION}" + +ARG OLFS_VERSION +ENV OLFS_VERSION=${OLFS_VERSION:-"unknown"} +RUN echo "# OLFS_VERSION: ${OLFS_VERSION}" + +ARG OLFS_DISTRO +ENV OLFS_DISTRO=${OLFS_DISTRO:-"olfs-${OLFS_VERSION}-webapp"} +RUN echo "# OLFS_DISTRO: ${OLFS_DISTRO}" +COPY ${OLFS_DISTRO}.tgz /${OLFS_DISTRO}.tgz + +ARG ROBOTS_DISTRO +ENV ROBOTS_DISTRO=${ROBOTS_DISTRO:-"robots-olfs-${OLFS_VERSION}-webapp"} +RUN echo "# ROBOTS_DISTRO: ${ROBOTS_DISTRO}" +COPY ${ROBOTS_DISTRO}.tgz /${ROBOTS_DISTRO}.tgz + +LABEL vendor="OPeNDAP" +LABEL org.opendap.olfs.version=${OLFS_VERSION} +LABEL org.opendap.olfs.release-date=RELEASE_DATE +LABEL org.opendap.hyrax.version.is-production="false" + + +# The --build-arg USE_NCWMS can be set to "true" in order to +# add the ncWMS application to the build. +ARG USE_NCWMS +ENV USE_NCWMS="${USE_NCWMS:-"false"}" +RUN set -e && \ + if [ $USE_NCWMS = "true" ];then echo "NCWMS: ENABLED"; else echo "NCWMS: DISABLED"; fi + +# +# The --build-arg NCWMS_BASE can be set to the base URL for ncWMS. +# The entrypoint.sh code defaults it to +# the URL: https://localhost:8080 if the environment variable NCWMS_BASE +# is not in the shell from which the entrypoint.sh script is called. +ARG NCWMS_BASE +ENV NCWMS_BASE="${NCWMS_BASE:-"https://localhost:8080"}" +RUN set -e && \ + if [ $USE_NCWMS = "true" ];then echo "NCWMS_BASE: {$NCWMS_BASE}"; fi + + +################################################################ +# Update and install the needful. +# +# Installing utils for debugging convenience. +# +RUN set -e \ + && dnf update -y \ + && echo "#!/bin/bash" > /install-dev-tools \ + && echo "dnf install -y $dnf_developer_tools_packages" > /install-dev-tools \ + && chmod 755 /install-dev-tools \ + && dnf clean all + + +RUN set -e \ + && echo "$HR" \ + && echo "# Installing cURL" \ + && yum install -y curl curl-devel --allowerasing \ + && echo "# cURL version: $(curl --version)" \ + && dnf clean all + +RUN set -e \ + && echo "$HR" \ + && echo "# It's Java Time Witches!" \ + && echo "# Installing $java_version" \ + && yum install -y "$java_version" "$java_version-devel" \ + && echo "$HR2" \ + && echo "# Using $java_version" \ + && update-alternatives --set java "$java_version.$ARCH" \ + && echo "# Enabled $java_version" \ + && echo "# java version: $( java -version 2>&1 )" \ + && update-alternatives --list \ + && dnf install -y tzdata-java \ + && dnf clean all \ + && echo "$HR" + +RUN set -e \ + && dnf install -y $dnf_runtime_packages \ + && dnf clean all + + + + +################################################################ +# Install and Setup Tomcat +# +# Make tomcat user and group +# - - - - - - - - - - - - - - - - - - - - - +RUN set -e && useradd -m -U -d /home/tomcat -s /bin/false tomcat +# +# Install the Tomcat package that should have been copied into +# the root directory prior to this layer. +# +RUN set -e \ + && tar -xvf "/${TOMCAT_DISTRO}.tar.gz" -C /home/tomcat >&2 \ + && ln -s /home/tomcat/${TOMCAT_DISTRO} /usr/share/tomcat >&2 \ + && rm -rf /usr/share/tomcat/webapps/* >&2 \ + && mkdir -p /var/log/tomcat \ + && chown -R tomcat:tomcat /var/log/tomcat /home/tomcat +#COPY tomcat.service /etc/systemd/system/tomcat.service +#RUN set -e \ +# && which systemctl \ +# && systemctl enable tomcat +#RUN firewall-cmd --add-port 8080/tcp --permanent +#RUN firewall-cmd --reload + +ENV CATALINA_HOME="/usr/share/tomcat" +ENV PATH="$CATALINA_HOME/bin:$PATH" +RUN set -e && echo "# CATALINA_HOME: $CATALINA_HOME" >&2 + +# Install our modified server.xml so that the server compresses responses. +COPY tomcat11-server.xml / +RUN set -e \ + && mv /tomcat11-server.xml ${CATALINA_HOME}/conf/server.xml \ + && chown -R tomcat:tomcat ${CATALINA_HOME}/conf/server.xml + +RUN set -e \ + && echo "# Cleaning up Tomcat distribution files..." >&2 \ + && rm -fv "/${TOMCAT_DISTRO}.tar.gz" + + +################################################################ +# Install the OLFS web application +# - - - - - - - - - - - - - - - - - - - - - +ENV ROOT="/dev/shm" + +RUN echo "# Installing the latest olfs distribution ($OLFS_VERSION)." >&2 +RUN set -e \ + && echo "# Working_dir: "`pwd` >&2 \ + && ls -l . \ + && tar -C "$ROOT" -xzvf "./$OLFS_DISTRO.tgz" \ + && ls -l "$ROOT" \ + && echo "# Unpacking war file..." >&2 \ + && mkdir -p "$(readlink -f "$CATALINA_HOME")/webapps/opendap/"\ + && unzip -o "$ROOT/$OLFS_DISTRO/opendap.war" -d "$CATALINA_HOME/webapps/opendap/" \ + && echo "# Cleaning up." >&2 \ + && rm -rf "$ROOT/$OLFS_DISTRO" "./$OLFS_DISTRO.tgz" + +RUN set -e \ + && echo "# Installing the latest dynamic robots.txt distribution ($OLFS_VERSION)." >&2 \ + && ls -l ./robots* \ + && tar -C "$ROOT" -xzf "./$ROBOTS_DISTRO.tgz" \ + && ls -l "$ROOT/$ROBOTS_DISTRO" \ + && echo "# Unpacking dynamic robots.txt war file..." >&2 \ + && unzip -o "$ROOT/$ROBOTS_DISTRO/ROOT.war" -d "$CATALINA_HOME/webapps/ROOT/" \ + && echo "# Cleaning up dynamic robots.txt snapshot..." >&2 \ + && rm -rvf "$ROOT/$ROBOTS_DISTRO" "$ROBOTS_DISTRO.tgz" + +# Fix ownership and access permissions +RUN set -e \ + && mkdir -p "$CATALINA_HOME/webapps/opendap/WEB-INF/conf/logs" \ + && chown -R tomcat:tomcat "$CATALINA_HOME/webapps/opendap/WEB-INF/conf/logs" \ + && chmod 700 "$CATALINA_HOME/webapps/opendap/WEB-INF/conf/logs" + +################################################################ +# Clean up the distribution binaries. +# +RUN rm -vf /olfs*.tgz /robots*.tgz /apache* +# +################################################################ + + + +# set host for bes that olfs will contact - this is expected to be over docker's internal network +ARG BES_HOST +ENV BES_HOST="${BES_HOST:-besd}" +RUN sed -i "s/localhost/$BES_HOST/" "$CATALINA_HOME/webapps/opendap/WEB-INF/conf/olfs.xml" + +# +# Setting NCWMS_HOST to the protocol, host, and port +# section of the publicly acessible URL of the +# ncWMS service. Using localhost is all well and good +# for testing but this needs to be settable at +# build time for sure and maybe even docker runtime? +# +COPY olfs_viewers.xml /tmp/olfs_viewers.xml +RUN set -e \ + && if [ $USE_NCWMS = "true" ]; then \ + mv /tmp/olfs_viewers.xml ${CATALINA_HOME}/webapps/opendap/WEB-INF/conf/viewers.xml; \ + else \ + echo "Skipping OLFS/ncWMS configuration installation."; \ + fi + +### +# Expose ports +### + +EXPOSE 8080 8443 + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh && cat /entrypoint.sh +ENTRYPOINT [ "/entrypoint.sh" ] + +CMD ["-"] diff --git a/el9-builds/olfs/entrypoint.sh b/el9-builds/olfs/entrypoint.sh new file mode 100755 index 00000000..b309ab45 --- /dev/null +++ b/el9-builds/olfs/entrypoint.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# This is the entrypoint.sh file for the olfs container. +# +# +# set -f # "set -o noglob" Disable file name generation using metacharacters (globbing). +# set -v # "set -o verbose" Prints shell input lines as they are read. +# set -x # "set -o xtrace" Print command traces before executing command. +# set -e # Exit on error. +export debug="false" +export BANNER="################################ OLFS #############################################" +export HR0="###################################################################################" +export HR1="-----------------------------------------------------------------------------------" +export HR2="-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --" +export prolog="entrypoint.sh" +function loggy(){ + echo "$@" | awk -v prolog="$prolog" '{ print "# " prolog " - " $0;}' >&2 +} + +loggy "$HR0" +loggy "$BANNER" +loggy "Greetings, I am $(whoami)." +set -e +#set -x +loggy "PythonVersion: $(python3 --version)" + +export SLEEP_INTERVAL="${SLEEP_INTERVAL:-60}" +loggy "SLEEP_INTERVAL: $SLEEP_INTERVAL seconds." + +export NCWMS_BASE="${NCWMS_BASE:-"https://localhost:8080"}" +loggy "NCWMS_BASE: $NCWMS_BASE" + +while getopts "n:d" opt; do + case $opt in + n) + NCWMS_BASE=$OPTARG + loggy "Using commandline NCWMS_BASE: $NCWMS_BASE" + ;; + d) + debug="true"; + loggy "Debug is enabled"; + ;; + \?) + loggy "Invalid option: -$OPTARG" + loggy "options: [-n ncwms_base_url] [-d] " + loggy " -n xxx where xxx is the protocol, server and port part " + loggy " of the ncWMS service (for example http://foo.com:8090/)." + loggy " -d Enables debugging output for this script." + loggy "EXITING NOW" + exit 2; + ;; + esac +done + +if test "$debug" = true ; then + loggy "CATALINA_HOME: ${CATALINA_HOME}" + loggy "$(ls -l "$CATALINA_HOME" "$CATALINA_HOME/bin")" + loggy "NCWMS_BASE: ${NCWMS_BASE}" + loggy "Setting ncWMS access URLs in viewers.xml (if needed)." +fi + +sed -i "s+@NCWMS_BASE@+$NCWMS_BASE+g" ${CATALINA_HOME}/webapps/opendap/WEB-INF/conf/viewers.xml; +if test "$debug" = true ; then + loggy "${CATALINA_HOME}/webapps/opendap/WEB-INF/conf/viewers.xml" + loggy "$(cat "$CATALINA_HOME/webapps/opendap/WEB-INF/conf/viewers.xml")" +fi + +export OLFS_CONF="$CATALINA_HOME/webapps/opendap/WEB-INF/conf" +# mv ${OLFS_CONF}/logback.xml ${OLFS_CONF}/logback.xml.OFF +loggy "Starting Tomcat..." +"$CATALINA_HOME"/bin/startup.sh > /var/log/tomcat/console.log 2>&1 & +status=$? +tomcat_pid=$! +if test $status -ne 0 ; then + loggy "ERROR: Failed to start Tomcat: $status" + exit $status +fi +# When we launch tomcat the initial pid gets "retired" because it spawns a +# secondary processes. +initial_pid="$tomcat_pid" +loggy "Tomcat started, initial pid: $initial_pid" +while test $initial_pid -eq $tomcat_pid +do + sleep 1 + tomcat_ps="$(ps aux | grep tomcat | grep -v grep)" + loggy "tomcat_ps: $tomcat_ps" + tomcat_pid="$(echo $tomcat_ps | awk '{print $2}')" + loggy "tomcat_pid: $tomcat_pid" +done +# New pid and we should be good to go. +loggy "$HR2" +loggy "Tomcat is UP! pid: $tomcat_pid" +loggy "The OLFS Has Arrived. SLEEP_INTERVAL: $SLEEP_INTERVAL" +loggy "$HR0" +while /bin/true; do + sleep $SLEEP_INTERVAL + tomcat_ps="$(ps -f "$tomcat_pid")" + TOMCAT_STATUS=$? + if test $TOMCAT_STATUS -ne 0 ; then + loggy "TOMCAT_STATUS: $TOMCAT_STATUS tomcat_pid:$tomcat_pid" + loggy "Tomcat appears to have died! Exiting." + loggy "Tomcat Console Log [BEGIN]" + cat /usr/local/tomcat/logs/catalina.out + loggy "Tomcat Console Log [END]" + exit 2 + fi + if test "$debug" = "true" ; then + loggy "$(date) SLEEP_INTERVAL: $SLEEP_INTERVAL TOMCAT_STATUS: $TOMCAT_STATUS tomcat_pid:$tomcat_pid " + fi +done + diff --git a/el9-builds/olfs/olfs_viewers.xml b/el9-builds/olfs/olfs_viewers.xml new file mode 100644 index 00000000..49b20ac6 --- /dev/null +++ b/el9-builds/olfs/olfs_viewers.xml @@ -0,0 +1,65 @@ + + + + + + + idv.jnlp + + + + idv.jnlp + + + + + + + + + Web Mapping Service + + + + + Godiva WMS GUI + + + + + + + diff --git a/el9-builds/olfs/tomcat.service b/el9-builds/olfs/tomcat.service new file mode 100644 index 00000000..1675ff65 --- /dev/null +++ b/el9-builds/olfs/tomcat.service @@ -0,0 +1,25 @@ +[Unit] +Description=Apache Tomcat Servlet container +Wants=network.target +After=network.target + +[Service] +Type=forking + +User=tomcat +Group=tomcat + +Environment="JAVA_HOME=/usr/lib/jvm/jre" +Environment="JAVA_OPTS=-Djava.security.egd=file:///dev/urandom" + +Environment="CATALINA_BASE=/usr/share/tomcat/latest" +Environment="CATALINA_HOME=/usr/share/tomcat/latest" +Environment="CATALINA_PID=/usr/share/tomcat/latest/temp/tomcat.pid" +Environment="CATALINA_OPTS=-Xms512M -Xmx4096M -server -XX:+UseParallelGC" + +ExecStart=/usr/share/tomcat/latest/bin/startup.sh +ExecStop=/usr/share/tomcat/latest/bin/shutdown.sh +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/el9-builds/olfs/tomcat11-server.xml b/el9-builds/olfs/tomcat11-server.xml new file mode 100644 index 00000000..80c71f77 --- /dev/null +++ b/el9-builds/olfs/tomcat11-server.xml @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/el9-builds/playbook.yml b/el9-builds/playbook.yml new file mode 100644 index 00000000..6f9a5591 --- /dev/null +++ b/el9-builds/playbook.yml @@ -0,0 +1,34 @@ +--- +- hosts: localhost + tasks: + - name: build olfs + docker_image: + name: olfs + tag: snapshot + path: olfs + state: present + force: yes + - name: build besd + docker_image: + name: besd + tag: snapshot + path: besd + state: present + force: yes + - name: run hyrax + docker_service: + project_name: hyrax + definition: + version: '2' + services: + besd: + image: besd + env_file: ${PWD}/local.env + olfs: + image: olfs + env_file: ${PWD}/local.env + ports: + - "8080:8080" + depends_on: + - besd + diff --git a/el9-builds/query-redis-cache.sh b/el9-builds/query-redis-cache.sh new file mode 100644 index 00000000..b6839f7d --- /dev/null +++ b/el9-builds/query-redis-cache.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +REDIS="redis-cli -h sit-redis.hmhtzc.0001.usw2.cache.amazonaws.com -p 6379" + +mapfile -t keys < <($REDIS --scan --pattern "hyrax_session:redisson:tomcat_session:*") + +echo "Total keys: ${#keys[@]}" + +for key in "${keys[@]}"; do + echo "Processing $key" + # e.g., fetch metadata + $REDIS HGETALL "$key" +done diff --git a/el9-builds/query-redis-sessions.sh b/el9-builds/query-redis-sessions.sh new file mode 100644 index 00000000..97335345 --- /dev/null +++ b/el9-builds/query-redis-sessions.sh @@ -0,0 +1,149 @@ +#!/bin/bash +set -euo pipefail + +############################################# +# Config (edit or pass as env/args) +############################################# +REDIS_HOST="${REDIS_HOST:-sit-redis.hmhtzc.0001.usw2.cache.amazonaws.com}" +REDIS_PORT="${REDIS_PORT:-6379}" +REDIS_AUTH="${REDIS_AUTH:-}" # leave empty if no auth +REDIS_TLS="${REDIS_TLS:-0}" # 1 to enable --tls +REDIS_CLUSTER="${REDIS_CLUSTER:-0}" # 1 to enable -c (cluster) +PATTERN="${PATTERN:-hyrax_session:redisson:tomcat_session:*}" +OUT_CSV="${OUT_CSV:-sessions.csv}" + +# Allow simple CLI args: ./dump_sessions.sh host port auth +if [[ $# -ge 1 ]]; then REDIS_HOST="$1"; fi +if [[ $# -ge 2 ]]; then REDIS_PORT="$2"; fi +if [[ $# -ge 3 ]]; then REDIS_AUTH="$3"; fi + +############################################# +# Build redis-cli base command +############################################# +REDIS=( + redis-cli + -h "$REDIS_HOST" + -p "$REDIS_PORT" +) + +if [[ "$REDIS_TLS" == "1" ]]; then + REDIS+=(--tls) +fi +if [[ -n "$REDIS_AUTH" ]]; then + REDIS+=(-a "$REDIS_AUTH") +fi +if [[ "$REDIS_CLUSTER" == "1" ]]; then + REDIS+=(-c) +fi + +############################################# +# Helpers +############################################# + +have_cmd() { command -v "$1" >/dev/null 2>&1; } + +# Portable millis->ISO8601 (works on GNU date and macOS/BSD date) +ms_to_iso() { + local ms="$1" + [[ -z "$ms" ]] && { echo ""; return; } + # expect integer millis; if it's JSON like ["java.lang.Long",123], caller strips it + local sec=$(( ms / 1000 )) + if have_cmd gdate; then + gdate -u -d "@$sec" +"%Y-%m-%dT%H:%M:%SZ" + else + # Try GNU date first; if it fails, try BSD `date -r` + if date -u -d "@$sec" +"%Y-%m-%dT%H:%M:%SZ" >/dev/null 2>&1; then + date -u -d "@$sec" +"%Y-%m-%dT%H:%M:%SZ" + else + date -u -r "$sec" +"%Y-%m-%dT%H:%M:%SZ" + fi + fi +} + +# Extract millis from Redisson JsonJacksonCodec format: +# ["java.lang.Long", 1757011521010] -> 1757011521010 +# 1757011521010 -> 1757011521010 +extract_millis() { + local raw="$1" + [[ -z "$raw" || "$raw" == "(nil)" ]] && { echo ""; return; } + # If jq present, try JSON path; else strip non-digits + if have_cmd jq; then + echo "$raw" | jq -r ' + if type=="array" and length>1 and (.[1]|type=="number") then .[1] + elif type=="number" then . + elif type=="string" then + try (fromjson | if type=="array" and length>1 then .[1] else . end) catch empty + else empty end + ' 2>/dev/null || echo "" + else + # Fallback: grab longest digit run + echo "$raw" | tr -cd '0-9' + fi +} + +############################################# +# Fetch keys into an array (safe) +############################################# +echo "Scanning keys with pattern: $PATTERN (host=$REDIS_HOST port=$REDIS_PORT tls=$REDIS_TLS cluster=$REDIS_CLUSTER)" +mapfile -t keys < <("${REDIS[@]}" --scan --pattern "$PATTERN") + +# De-duplicate (SCAN may repeat) +declare -A seen=() +unique_keys=() +for k in "${keys[@]}"; do + [[ -z "${k:-}" ]] && continue + if [[ -z "${seen[$k]:-}" ]]; then + seen["$k"]=1 + unique_keys+=("$k") + fi +done +echo "Found ${#unique_keys[@]} unique keys" + +############################################# +# Emit CSV header +############################################# +echo "sid,creationTime,lastAccessedTime,thisAccessedTime,ttl_seconds,isValid,maxInactiveInterval" > "$OUT_CSV" + +############################################# +# Iterate and print +############################################# +count=0 +for key in "${unique_keys[@]}"; do + ((count++)) + # Extract SID as last colon-delimited token + sid="${key##*:}" + + + # Pull fields + ct_raw=$("${REDIS[@]}" HGET "$key" session:creationTime) + lat_raw=$("${REDIS[@]}" HGET "$key" session:lastAccessedTime) + tat_raw=$("${REDIS[@]}" HGET "$key" session:thisAccessedTime) + mii=$("${REDIS[@]}" HGET "$key" session:maxInactiveInterval) + valid=$("${REDIS[@]}" HGET "$key" session:isValid) + return_url=$("${REDIS[@]}" HGET "$key" session:return_to_url) + ttl=$("${REDIS[@]}" TTL "$key") + + # Parse millis + ct_ms="$(extract_millis "$ct_raw")" + lat_ms="$(extract_millis "$lat_raw")" + tat_ms="$(extract_millis "$tat_raw")" + + # Human ISO8601 + ct_iso="$(ms_to_iso "$ct_ms")" + lat_iso="$(ms_to_iso "$lat_ms")" + tat_iso="$(ms_to_iso "$tat_ms")" + + # CSV line + printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$sid" "$ct_iso" "$lat_iso" "$tat_iso" "${ttl:-}" "${valid:-}" "${mii:-}" "${return_url:-}" \ + >> "$OUT_CSV" + + # Pretty stdout (optional) + printf '(%d/%d) %s\n created: %s\n lastAccessed: %s\n thisAccessed: %s\n ttl: %s sec valid: %s maxInactive: %s sec\n\n' \ + "$count" "${#unique_keys[@]}" "$key" \ + "${ct_iso:-N/A}" "${lat_iso:-N/A}" "${tat_iso:-N/A}" \ + "${ttl:-N/A}" "${valid:-N/A}" "${mii:-N/A}" +done + +echo "Wrote $OUT_CSV" +CSV diff --git a/build_from_s3_binaries.sh b/retired/build_from_s3_binaries.sh similarity index 72% rename from build_from_s3_binaries.sh rename to retired/build_from_s3_binaries.sh index b522fbcb..62a853c0 100755 --- a/build_from_s3_binaries.sh +++ b/retired/build_from_s3_binaries.sh @@ -8,13 +8,15 @@ echo "# build_from_s3_binaries ################################################# start_dir=$(pwd) echo "# BUILD_RECIPE: ${BUILD_RECIPE}" >&2 -cd "${BUILD_DIR}" || exit +TARGET_OS=$( grep "TARGET_OS" "${BUILD_RECIPE}" | awk '{print $2;}' ) + +cd "$TARGET_OS-builds" || exit # -# The build-rh8 script contains a library of bash functions +# The build-el(8|9) scripts contains a library of bash functions # to do the many things, like retrieving binaries from s3 or www.opendap.org # building docker images from said files, etc.build # -source ./build-rh8 "${BUILD_RECIPE}" +source ./build-$TARGET_OS "${BUILD_RECIPE}" export TOMCAT_VERSION=$(get_latest_tomcat_version_number "${TOMCAT_MAJOR_VERSION}") show_version diff --git a/travis-build-recipe b/travis-build-recipe new file mode 100644 index 00000000..6431a1c1 --- /dev/null +++ b/travis-build-recipe @@ -0,0 +1,6 @@ +TARGET_OS: el8 +TOMCAT_MAJOR_VERSION: 9 +libdap4-3.21.1-366 2026-01-27T19:58:24+0000 +bes-3.21.1-1065 2026-02-23T20:47:04+0000 +olfs-1.18.15-703 2026-02-24T18:24:36+0000 +hyrax-1.17.1-703 2026-02-24T18:24:36+0000 diff --git a/travis/deploy_to_docker_hub.sh b/travis/deploy_to_docker_hub.sh index 17dba846..563b5cb4 100755 --- a/travis/deploy_to_docker_hub.sh +++ b/travis/deploy_to_docker_hub.sh @@ -2,25 +2,70 @@ # # Called from the travis.yml. This depends on env vars set by the # travis yaml. jhrg 3/31/21 +HR0="#######################################################################" +HR1="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" +HR2="--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---" +############################################################################# +# loggy() +prolog="deploy_to_docker_hub.sh" +function loggy(){ + echo "$@" | awk -v prolog="$prolog" '{ print "# " prolog " - " $0;}' >&2 +} +loggy "$HR0" +loggy "BEGIN" + set -e -OPENDAP_AWS_ACCOUNT=747931985039 +loggy "Logging into Docker Hub" +echo "$DOCKER_HUB_PSWD" | docker login -u "$DOCKER_HUB_UID" --password-stdin +loggy "$HR1" + +loggy "Running 'docker image ls -a'" +loggy "$(docker image ls -a)" +loggy "$HR2" + + +loggy "Deploying '$OS_SNAPSHOT_IMAGE_TAG' to Docker Hub" +docker push "$OS_SNAPSHOT_IMAGE_TAG" +loggy "$HR2" + +if test -n "$SNAPSHOT_IMAGE_TAG" +then + loggy "Tagging $OS_SNAPSHOT_IMAGE_TAG as $SNAPSHOT_IMAGE_TAG" + docker tag "$OS_SNAPSHOT_IMAGE_TAG" "$SNAPSHOT_IMAGE_TAG" + loggy "Deploying '$SNAPSHOT_IMAGE_TAG' to Docker Hub" + docker push "$SNAPSHOT_IMAGE_TAG" + loggy "$HR2" +fi + +loggy "Deploying '$OS_BUILD_VERSION_TAG' to Docker Hub" +docker push "$OS_BUILD_VERSION_TAG" +loggy "$HR2" -echo "Logging into Docker Hub" -echo $DOCKER_HUB_PSWD | docker login -u $DOCKER_HUB_UID --password-stdin +if test -n "$BUILD_VERSION_TAG" +then + loggy "Tagging $OS_BUILD_VERSION_TAG as $BUILD_VERSION_TAG" + docker tag "$OS_BUILD_VERSION_TAG" "$BUILD_VERSION_TAG" + loggy "Deploying '$BUILD_VERSION_TAG' to Docker Hub" + docker push "$BUILD_VERSION_TAG" + loggy "$HR2" +fi -echo "Deploying ${SNAPSHOT_IMAGE_TAG} to Docker Hub" -docker push ${SNAPSHOT_IMAGE_TAG} -echo "Deploying ${BUILD_VERSION_TAG} to Docker Hub" -docker push ${BUILD_VERSION_TAG} +loggy "Docker Hub deployment complete." +loggy "$HR1" -echo "Docker Hub deployment complete." -echo "AWS configuration: " -aws configure list -echo "Deploying ${SNAPSHOT_IMAGE_TAG} to AWS ECR" -docker tag ${SNAPSHOT_IMAGE_TAG} ${OPENDAP_AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${SNAPSHOT_IMAGE_TAG} -docker push ${OPENDAP_AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${SNAPSHOT_IMAGE_TAG} +if test -n "$SNAPSHOT_IMAGE_TAG" +then + loggy "AWS configuration: " + loggy "$(aws configure list)" + loggy "OPENDAP_AWS_ACCOUNT: $OPENDAP_AWS_ACCOUNT" + loggy "Deploying ${SNAPSHOT_IMAGE_TAG} to AWS ECR" + docker tag "$SNAPSHOT_IMAGE_TAG" "$OPENDAP_AWS_ACCOUNT.dkr.ecr.us-east-1.amazonaws.com/$SNAPSHOT_IMAGE_TAG" + docker push "$OPENDAP_AWS_ACCOUNT.dkr.ecr.us-east-1.amazonaws.com/$SNAPSHOT_IMAGE_TAG" + loggy "AWS ECR deployment complete." +fi -echo "AWS ECR deployment complete." \ No newline at end of file +loggy "END" +loggy "$HR0" diff --git a/travis/get_target_os.sh b/travis/get_target_os.sh new file mode 100755 index 00000000..84f95735 --- /dev/null +++ b/travis/get_target_os.sh @@ -0,0 +1,12 @@ +#!/bin/bash +HR="=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" +prolog="get_target_os.sh -" +########################################################################### +# loggy() +function loggy() { + echo "$@" | awk '{ print "# "$0;}' >&2 +} + +export TARGET_OS=$(grep "TARGET_OS: " "${VERSION_FILE}" | awk '{print $2;}') +loggy "$prolog TARGET_OS: $TARGET_OS" +echo "$TARGET_OS" \ No newline at end of file diff --git a/travis/regression_test_script b/travis/regression_test_script index 9dcdff65..35e94c65 100755 --- a/travis/regression_test_script +++ b/travis/regression_test_script @@ -1,6 +1,20 @@ #!/bin/bash set -e -echo "Starting up ..." + +# +# Travis invocation: ./travis/regression_test_script -i "$SNAPSHOT_IMAGE_TAG" +# +HR0="#######################################################################" +HR1="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" +HR2="--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---" +############################################################################# +# loggy() +function loggy(){ + echo "$@" | awk '{ print "# "$0;}' >&2 +} +prolog="$0() -" +loggy "$HR0" +loggy "$prolog BEGIN" default_time_out=30 default_image_name="opendap/hyrax:snapshot" @@ -11,126 +25,204 @@ test_image_name="${default_image_name}" end_point="${default_endpoint}" netrc_file="" tests="" +tests_log_base= -while getopts "ht:i:e:n:" OPTION; do +while getopts "ht:i:e:n:l:" OPTION; do case $OPTION in - h) - echo "-h help -t time_out -i image_name -e end_point_url -n netrc_file test1 test2 ..." - exit 0 - ;; - t) - time_out=$OPTARG - ;; - i) - test_image_name=$OPTARG - ;; - e) - end_point=$OPTARG - ;; - n) - netrc_file="--netrc ${OPTARG}" - ;; - *) - echo "Incorrect options provided" - exit 1 - ;; + h) + echo "-h help -t time_out -i image_name -e end_point_url -n netrc_file test1 test2 ..." + exit 0 + ;; + t) + time_out=$OPTARG + ;; + i) + test_image_name=$OPTARG + ;; + e) + end_point=$OPTARG + ;; + l) + tests_log_base=$OPTARG + ;; + n) + netrc_file="--netrc ${OPTARG}" + ;; + *) + loggy "Incorrect options provided" + exit 1 + ;; esac done shift $(($OPTIND - 1)) tests="$@" -echo "Using following configurations:" >&2 -echo "Time out duration: ${time_out}" >&2 -echo "Image name: ${test_image_name}" >&2 -echo "End point: ${end_point}" >&2 -echo "Tests: ${tests}" >&2 +loggy "$prolog $HR1" +loggy "$prolog Using the following configuration:" +loggy "$prolog time_out: ${time_out}" +loggy "$prolog test_image_name: ${test_image_name}" +loggy "$prolog end_point: ${end_point}" +loggy "$prolog tests: ${tests}" +loggy "$prolog tests_log_base: ${tests_log_base}" +############################################################################# +# waitForHyrax() +# function waitForHyrax() { + local prolog="waitForHyrax() -" + local docker_id="$1" # Don't fail on errors, they're expected... set +e local start - let start=`date "+%s"` - local done_flag=""; + let start=$(date "+%s") + local done_flag="" local curl_response - while test -z "${done_flag}" - do - echo -n "." - curl -s -c cookies -b cookies -n --netrc-file ${netrc_file} -L ${end_point} > /dev/null + echo -n "$prolog " >&2 + while test -z "${done_flag}"; do + echo -n "." >&2 + curl -s -c cookies -b cookies -n --netrc-file ${netrc_file} -L ${end_point} >/dev/null 2>&1 status=$? # echo "cURL status: ${status}"; - if [ ${status} -eq 0 ] - then - echo "" - echo "Endpoint may have been retrieved. Checking again..." + if [ ${status} -eq 0 ]; then + echo "" >&2 + loggy "$prolog " + loggy "$prolog Endpoint may have been retrieved. Checking again..." # What happens next: We get the response headers "-I" and look # for the string "HTTP/" because we know there will be redirects. # We use tail to look at the last HTTP/ header and if that status is # 200 we're off to the races. - curl_response=$(set -x; curl -s -c cookies -b cookies -n ${netrc_file} -I -L ${end_point}; set +x) - echo "curl_response: $curl_response" - echo "$curl_response" | grep "HTTP/" | tail -1 | grep 200 # > /dev/null; - found=$?; - set +x; # Stop echoing the stuff... + curl_response=$(curl -s -c cookies -b cookies -n "$netrc_file" -I -L "$end_point") + loggy "$prolog curl_response: $curl_response" + echo "$curl_response" | grep "HTTP/" | tail -1 | grep 200 # > /dev/null; + found=$? # echo "cURL found: ${found}"; if [ ${found} -eq 0 ]; then - echo "The Test Endpoint URL: ${end_point} returned a status of 200" - done_flag="yup"; + loggy "$prolog The Test Endpoint URL: ${end_point} returned a status of 200" + done_flag="yup" fi fi let now=$(date "+%s") - let elapsed=$now-$start; + let elapsed=$now-$start # echo "elapsed: $elapsed" - if [ ${elapsed} -gt ${time_out} ] - then - echo "" - echo "ERROR: Hyrax FAILED To Start! Waited for ${elapsed} seconds" - exit 1; - fi + if [ ${elapsed} -gt ${time_out} ]; then + loggy "$prolog " + loggy "$prolog ERROR: Hyrax FAILED To Start! Waited for ${elapsed} seconds" + loggy "$prolog docker logs: " + loggy "$(docker logs "$docker_id")" + loggy "$prolog $HR2" + loggy "$prolog Hyrax failed to arrive...!" + exit 1 + fi done + loggy "" # Now we can fail on errors again. set -e - echo "Hyrax has arrived!" + loggy "$prolog Hyrax has arrived!" + sleep 10 + loggy "$prolog $HR2" + loggy "$prolog Start-up Docker logs: " + loggy "$(docker logs "$docker_id")" + loggy "$prolog $HR2" + return 0 } +function clean_and_start(){ + local prolog="clean_and_start_docker() -" + loggy "$prolog $HR1" + loggy "$prolog BEGIN" -# Dump all other containers -containers=$(docker ps -aq); -if test -n "${containers}" -then - echo "Removing containers: ${containers}"; - docker rm -f ${containers}; -fi + local d_id="$1" + loggy "$prolog d_id: $d_id" -#start Hyrax -container_id=$(docker run -d -h hyrax -p 8080:8080 --name=hyrax ${test_image_name}) + local test_image="$2" + loggy "$prolog test_image: $test_image" + + # Dump all other containers + containers=$(docker ps -aq) + if test -n "${containers}"; then + loggy "$prolog Removing containers: ${containers}" + loggy "$(docker rm -f ${containers})" + fi + + loggy "$prolog Starting: $d_id" + container_id=$(docker run -d -h "$d_id" -p 8080:8080 --name="$d_id" "$test_image") + loggy "$prolog Started container: $container_id" + loggy "$prolog END" +} -waitForHyrax -echo "############################################################################" -echo "# Testing the following image: ${test_image_name}" + +d_id="hyrax" + +clean_and_start "$d_id" "$test_image_name" +waitForHyrax "$d_id" + +loggy "$HR1" +loggy "$prolog Testing image: ${test_image_name}" set -x -docker exec -it hyrax /bin/bash -c "cd hyrax_regression_tests; ./testsuite --hyraxurl=\"${end_point}\" ${netrc_file} ${tests}; " -#./show_fails" -echo "############################################################################" -echo "# Checking beslog2json.awk" -echo "# Python Version: '$(python3 --version)'" -echo "# Acquiring bes.log from docker image." -docker cp hyrax:/var/log/bes/bes.log . -echo "# Acquiring beslog2json.py from docker image." -docker cp hyrax:/usr/bin/beslog2json.py $(pwd)/beslog2json.py + +# If the regression tests fail we want the logs, so don't exit on fail right here +set +e +docker exec -it $d_id /bin/bash -c "cd hyrax_regression_tests; ./testsuite --hyraxurl=\"${end_point}\" ${netrc_file} ${tests};" +test_status=$? +set -e +# http_url https_url stdout testsuite.log +log_file="$tests_log_base.tgz" +loggy "$prolog log_file: $log_file" + +ret_dir=$PWD +cd "/tmp" + +rt_dir="$tests_log_base" +loggy "$prolog rt_dir: $rt_dir" + +loggy "$prolog Creating host directory '$PWD/$rt_dir' for docker run reggression test results." +mkdir -vp "$rt_dir" + +for name in testsuite.log testsuite.dir +do + source_name="$d_id:/hyrax_regression_tests/$name" + target_name="$rt_dir/$name" + loggy "$prolog AWS S3: Copying '$source_name' to '$target_name'" + docker cp "$source_name" "$target_name" +done +loggy "$prolog TarballBeGzipping': $rt_dir)" +tar -cvzf "$log_file" "$rt_dir" +loggy "$prolog log_file: $(ls -l "$log_file")" + +cd "$ret_dir" + +loggy "$HR1" +loggy "$prolog Checking beslog2json.awk" +loggy "$prolog Python Version: '$(python3 --version)'" +loggy "$prolog Acquiring bes.log from docker image." +docker cp $d_id:/var/log/bes/bes.log . + +loggy "$prolog Acquiring beslog2json.py from docker image." +docker cp $d_id:/usr/bin/beslog2json.py $(pwd)/beslog2json.py ls -l ./beslog2json.py -echo "############################################################################" -echo "# head ./bes.log" -head ./bes.log -cat ./bes.log | python3 ./beslog2json.py --all > ./bes.json -echo "############################################################################" -echo "# head -20 ./bes.json" + +loggy "$HR2" +loggy "$prolog head -n 20 ./bes.log" +head -n 20 ./bes.log + +loggy "$HR2" +loggy "Running beslog2json.py: bes.log -> bes.json " +cat ./bes.log | python3 ./beslog2json.py --all >./bes.json + +loggy "$HR2" +loggy "$prolog head -n 10 ./bes.json" head -n 10 ./bes.json -echo "# tail -20 ./bes.json" -tail -n 20 ./bes.json -echo "############################################################################" -jq '.' ./bes.json > /dev/null -echo "JQ STATUS: $?" +loggy "$HR2" +loggy "$prolog tail -n 10 ./bes.json" +tail -n 10 ./bes.json + +loggy "$HR1" +loggy "Checking bes.json parsability jq" +jq '.' ./bes.json >/dev/null +loggy "$prolog JQ STATUS: $?" + +exit $test_status \ No newline at end of file diff --git a/travis/upload-test-results.sh b/travis/upload-test-results.sh new file mode 100755 index 00000000..54c30176 --- /dev/null +++ b/travis/upload-test-results.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Upload the results of tests after running a build on Travis +HR0="#######################################################################" +HR1="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" +HR2="--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---" +############################################################################# +# loggy() +function loggy(){ + echo "$@" | awk '{ print "# "$0;}' >&2 +} + +function upload_test_result() { + local prolog="upload_test_result() -" + loggy "$HR0" + loggy "$prolog BEGIN" + + local docker_name="$1" + loggy "$prolog docker_name: $docker_name" + + local log_file_name="$2" + loggy "$prolog log_file_name: $log_file_name" + + if test -n "$docker_name" -o -n "$log_file_name" + then + if test "$docker_name" = "hyrax"; then + # using: 'test -z "$AWS_ACCESS_KEY_ID" || ...' keeps after_script from running + # the aws cli for forked PRs (where secure env vars are null). I could've used + # an 'if' to block out the whole script, but I didn't... jhrg 3/21/18 + + test -z "$AWS_ACCESS_KEY_ID" || aws s3 cp "/tmp/$log_file_name" s3://opendap.travis.tests/ + fi + else + echo "$prolog Missing docker_name or log_file_name name, SKIPPING. (docker_name: '$docker_name' log_file_name: '$log_file_name')" + fi + loggy "$prolog END" +} + +upload_test_result "$1" "$2" \ No newline at end of file diff --git a/utils/test-rh8 b/utils/test-rh8 index b4f9279c..b701c30e 100755 --- a/utils/test-rh8 +++ b/utils/test-rh8 @@ -163,8 +163,8 @@ function build_hyrax_dependencies_for_rh8() { local docker_result_dir="/root/install" loggy "$prolog Using docker_result_dir: $docker_result_dir" - # export image="opendap/rocky9_hyrax_builder:snapshot" - local image="opendap/rocky8_hyrax_builder:snapshot" + # export image="opendap/rocky9_hyrax_builder:latest" + local image="opendap/rocky8_hyrax_builder:latest" loggy "$prolog Using Docker Image: $image" local build_script="" @@ -235,7 +235,7 @@ function build_hyrax_dependencies_for_rh8() { # --env BES_BUILD_NUMBER=$BES_BUILD_NUMBER # --env AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID # --env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -# opendap/rocky8_hyrax_builder:snapshot /root/travis/travis/build-rh8-rpm.sh +# opendap/rocky8_hyrax_builder:latest /root/travis/travis/build-rh8-rpm.sh # function build_bes_rpms_for_rhel8() { local prolog="build_bes_for_ubuntu() -" @@ -290,7 +290,7 @@ function build_bes_rpms_for_rhel8() { local docker_result_dir="/root/install" loggy "$prolog Using docker_result_dir: $docker_result_dir" - local image="opendap/rocky8_hyrax_builder:snapshot" + local image="opendap/rocky8_hyrax_builder:latest" loggy "$prolog Using Docker Image: $image" # /root/bes/travis/build-rh8-rpm.sh