run_tests_simple.sh

#!/usr/bin/env bash

# This file is the end-to-end test execution orchestrator used by the CI
# system.  This test script is designed for "leaf-node" jedi components whose
# dependencies are primarily upstream. This test script runs a single build
# and then runs package-specific unit tests, followed by integration tests.
# This file can be thought of as being split into several sections.
#
#   1) Environment validation and variable definitions. Several shell vars are
#      used or expected for this script to execute. This section validates the
#      environment has these values. It also sets several other values that are
#      reused throughout this script.
#   2) Function definitions. Any reused logic is defined as functions here.
#   3) Activate spack-stack environment. For now this assumes the use of a
#      docker image, but this could be substituted if needed.
#   4) Install any missing packages. Some applications and Python libraries are
#      missing from our standard environment. These can be set up here, although
#      the build can be sped up in the future by pre-loading these packages in
#      the test images.
#   5) Setup and run tests. Most of the actual test setup is done here, although
#      it depends on the environment and inputs prepared in steps 1-4.

#
# Environment validation and variable definitions.
#

# Load common function definitions.
source $WORKDIR/bundle/jedi_ci_resources/environment.sh
source $WORKDIR/bundle/jedi_ci_resources/util.sh

# Return code for any line of shell code containing a pipe or redirect will
# come from the inner-most executable command.
set -o pipefail

# Validate environment.
valid_environment_found="yes"

if [ -z $GITHUB_APP_PRIVATE_KEY ]; then
    echo "Var GITHUB_APP_PRIVATE_KEY must be set and must contain the text of the GitHub App private key or a file path of the key"
    valid_environment_found="no"
fi
if [ -z $GITHUB_APP_ID ]; then
    echo "Var GITHUB_APP_ID must be set and must contain the GitHub App ID"
    valid_environment_found="no"
fi
if [ -z $GITHUB_INSTALL_ID ]; then
    echo "Var GITHUB_INSTALL_ID must be set and must contain the GitHub App install ID used for API access with target repositories."
    valid_environment_found="no"
fi
if [ -z "${JEDI_COMPILER}" ]; then
    echo "Var JEDI_COMPILER must be set. This variable must be the name of the build environment toolchain."
    valid_environment_found="no"
fi
if [ -z $AWS_BATCH_JOB_ID ]; then
    # This variable is set by AWS Batch.
    echo "Var AWS_BATCH_JOB_ID must be set.."
    valid_environment_found="no"
fi
if [ -z "${ECS_CONTAINER_METADATA_URI_V4}" ]; then
    # This variable is set by AWS Batch.
    echo "Var ECS_CONTAINER_METADATA_URI_V4 must be set."
    valid_environment_found="no"
fi
if [ -z "${TRIGGER_REPO_FULL}" ]; then
    # This variable is set by AWS Batch.
    echo "Var TRIGGER_REPO_FULL must be set."
    valid_environment_found="no"
fi


if [ $valid_environment_found == "no" ]; then
    util.evaluate_debug_timer_then_cleanup
    exit 1
fi

cat << EOF
Configuration:
GITHUB_APP_PRIVATE_KEY_FILE=${GITHUB_APP_PRIVATE_KEY_FILE}
GITHUB_APP_ID=${GITHUB_APP_ID}
GITHUB_INSTALL_ID=${GITHUB_INSTALL_ID}
BUILD_PARALLELISM=${BUILD_PARALLELISM}
WORKDIR=${WORKDIR}
CDASH_URL=${CDASH_URL}
jedi_cmake_ROOT=${jedi_cmake_ROOT}
OMPI_ALLOW_RUN_AS_ROOT=${OMPI_ALLOW_RUN_AS_ROOT}
OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=${OMPI_ALLOW_RUN_AS_ROOT_CONFIRM}
OMPI_MCA_rmaps_base_oversubscribe=${OMPI_MCA_rmaps_base_oversubscribe}
CI_SCRIPTS_DIR=${CI_SCRIPTS_DIR}
CC="${CC}"
CXX="${CXX}"
UNIT_DEPENDENCIES=${UNIT_DEPENDENCIES}
EOF


echo "--------------------------------------------------------------"
echo "Platform debug info"
echo "--------------------------------------------------------------"
echo "aws sts get-caller-identity"
aws sts get-caller-identity
echo "df -h"
df -h
echo "lscpu"
lscpu
echo "ulimit -a"
ulimit -a

# From this point forward we are executing the test and sending debug to stderr.
set -x


#
# Setup and run tests.
#

#REFRESH_CACHE_ON_FETCH="$(jq -r ".skip_cache" $BUILD_JSON)"
#REFRESH_CACHE_ON_WRITE="$(jq -r ".rebuild_cache" $BUILD_JSON)"
#JEDI_BUNDLE_BRANCH="$(jq -r ".jedi_bundle_branch" $BUILD_JSON)"

# Extract just the repo name from the full repository path
TRIGGER_REPO=$(echo "$TRIGGER_REPO_FULL" | cut -d'/' -f2)


# Generate the version ref flag value used later for build config. Ignore
# entries with null version_ref.commit values they are branch references already
# configured in the bundle.
UNIT_DEPENDENCIES=$(jq -r '.dependencies | join(" ")' $BUILD_JSON)


if [ "${CREATE_CHECK_RUNS}" == "yes" ]; then
    UNIT_RUN_ID=$(util.check_run_new $TRIGGER_REPO_FULL "unit" $TRIGGER_SHA)
    INTEGRATION_RUN_ID=$(util.check_run_new $TRIGGER_REPO_FULL "integration" $TRIGGER_SHA)
fi

# Update check-runs to include the batch job URL is included.
util.check_run_runner_allocated $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID
util.check_run_start_build $TRIGGER_REPO_FULL $UNIT_RUN_ID

# Get all GitLFS repositories from s3.
pushd ${JEDI_BUNDLE_DIR}
echo "showing git config"
git config --list
echo "Fetching GitLFS repositories via tarball."
git config --global --add safe.directory '*'
for repo in ioda-data ufo-data fv3-jedi-data mpas-jedi-data ; do
    echo "repo == ${repo}"
    aws s3 cp "s3://jcsda-public-rpays/JCSDA-internal/${repo}.tar.gz" "${repo}.tar.gz" --no-progress
    tar -xf "${repo}.tar.gz"
    cd ${repo}
    # Update refs
    git fetch --all
    cd ..
    rm "${repo}.tar.gz"
done
popd


# Configure cdash integration.
mkdir "${JEDI_BUNDLE_DIR}/cmake"
cp "${SCRIPT_DIR}/ctest_assets/CTestConfig.cmake"       "${JEDI_BUNDLE_DIR}/"
cp "${SCRIPT_DIR}/ctest_assets/CTestCustom.ctest.in"    "${JEDI_BUNDLE_DIR}/cmake/"
cp "${SCRIPT_DIR}/ctest_assets/cdash-integration.cmake" "${JEDI_BUNDLE_DIR}/cmake/"
sed -i "s#CDASH_URL#${CDASH_URL}#g"           "${JEDI_BUNDLE_DIR}/CTestConfig.cmake"
sed -i "s#CDASH_URL#${CDASH_URL}#g"           "${JEDI_BUNDLE_DIR}/CTestConfig.cmake"
sed -i "s#TEST_TARGET_NAME#${TRIGGER_REPO}#g" "${JEDI_BUNDLE_DIR}/CTestConfig.cmake"

# Update the CMakeLists.txt files to include cdash integration. Note that the test
# uses two different CMakeLists.txt files for unit and integration tests (starting with
# unit tests) and each file needs the cdash integration. Both of these cmake files
# were prepared by the GitHub action test launcher.
echo "include(cmake/cdash-integration.cmake)" >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt.integration"
echo ""                                       >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt.integration"
echo "include(CTest)"                         >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt.integration"
echo ""                                       >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt.integration"
# Update the unittest CMakeLists.txt file to include cdash integration.
echo "include(cmake/cdash-integration.cmake)" >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt"
echo ""                                       >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt"
echo "include(CTest)"                         >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt"
echo ""                                       >> "${JEDI_BUNDLE_DIR}/CMakeLists.txt"


if [ $? -ne 0 ]; then
    if grep -qi "remote: Invalid username or password." configure_1.log; then
        util.check_run_fail $TRIGGER_REPO_FULL $UNIT_RUN_ID "Failure: see jcsda-internal/CI/issues/137"
    else
        util.check_run_fail $TRIGGER_REPO_FULL $UNIT_RUN_ID "Bundle configuration failed"
    fi
    util.check_run_skip $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi

# Add the oasim, ropp, and rttov compiler flags if the unittests have one
# of these dependencies. Note that the same compiler flags are used for
# integration tests, so a package without these unittest dependencies will
# not build them for integration testing.
if grep -q -e "oasim" <<< $UNIT_DEPENDENCIES; then
    COMPILER_FLAGS+=( -DBUILD_OASIM=ON )
fi
if  grep -q -e "rttov" <<< $UNIT_DEPENDENCIES; then
    COMPILER_FLAGS+=( -DBUILD_RTTOV=ON )
fi
if  grep -q -e "ropp-ufo" <<< $UNIT_DEPENDENCIES; then
    COMPILER_FLAGS+=( -DBUILD_ROPP=ON )
fi

echo "---- JEDI Bundle CMakeLists.txt - unit tests -----"
cat $JEDI_BUNDLE_DIR/CMakeLists.txt
echo "-------------------------------------------------------"

#
# Build and run unit tests.
#
cd "${BUILD_DIR}"

ecbuild \
      -Wno-dev \
      -DCMAKE_BUILD_TYPE=RelWithDebInfo \
      -DCDASH_OVERRIDE_SYSTEM_NAME="${JEDI_COMPILER}-Container" \
      -DCDASH_OVERRIDE_SITE=AWSBatch \
      -DCDASH_OVERRIDE_GIT_BRANCH=${TRIGGER_PR} \
      -DCTEST_UPDATE_VERSION_ONLY=FALSE \
      -DBUILD_IODA_CONVERTERS=ON \
      -DBUILD_PYIRI=ON \
      ${COMPILER_FLAGS[@]} "${JEDI_BUNDLE_DIR}" | tee configure_2.log

if [ $? -ne 0 ]; then
    if grep -qi "remote: Invalid username or password." configure_2.log; then
        util.check_run_fail $TRIGGER_REPO_FULL $UNIT_RUN_ID "Failure: see jcsda-internal/CI/issues/137"
    else
        util.check_run_fail $TRIGGER_REPO_FULL $UNIT_RUN_ID "Bundle configuration failed"
    fi
    util.check_run_skip $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi

# Back-date source files (search "back-date" in this file for an explanation).
find $JEDI_BUNDLE_DIR -type f -exec touch -d "$SOURCE_BACKDATE_TIMESTAMP" {} \;

make -j $BUILD_PARALLELISM
if [ $? -ne 0 ]; then
    util.check_run_fail $TRIGGER_REPO_FULL $UNIT_RUN_ID "compilation failed"
    util.check_run_skip $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi

util.check_run_start_test $TRIGGER_REPO_FULL $UNIT_RUN_ID

# Run unit tests.
ctest -L $UNITTEST_TAG --timeout 500 -C RelWithDebInfo -D ExperimentalTest

# Upload ctests.
ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous

echo "CDash URL: $(util.create_cdash_url "${BUILD_DIR}/Testing")"

# This is a temporary hack to allow UFO tests to pass until we resolve the
# flakes and/or persistent failures. Once UFO testing failures are resolved
# we can hard-code this failure rate to zero and remove this logic.
ALLOWED_UNIT_FAIL_RATE=0
if [ $UNITTEST_TAG = 'ufo' ]; then
    ALLOWED_UNIT_FAIL_RATE=1
fi

# Close out the check run for unit tests and mark success or failure.
util.check_run_end $TRIGGER_REPO_FULL $UNIT_RUN_ID $ALLOWED_UNIT_FAIL_RATE

# Decision point: if the unit tests failed then we should mark the integration
# tests as skipped and end test execution.
if ! util.check_run_eval_test_xml $ALLOWED_UNIT_FAIL_RATE ; then
    util.check_run_skip $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi


#
# Build and run integration tests. This section will not be run if we detect
# a failure above (implemented)

# Swap out the unittest cmake file for the integration test version. These
# files were created by the GitHub action test launcher.
mv ${JEDI_BUNDLE_DIR}/CMakeLists.txt $JEDI_BUNDLE_DIR/CMakeLists.txt.unittest
cp ${JEDI_BUNDLE_DIR}/CMakeLists.txt.integration $JEDI_BUNDLE_DIR/CMakeLists.txt

echo "---- JEDI Bundle CMakeLists.txt - integration tests -----"
cat $JEDI_BUNDLE_DIR/CMakeLists.txt
echo "-------------------------------------------------------"

# Delete test output to force re-generation of BuildID
TEST_TAG=$(head -1 "${BUILD_DIR}/Testing/TAG")

# Start the integration test run.
util.check_run_start_build $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID

if [ $? -ne 0 ]; then
    util.check_run_fail $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID "Bundle configuration failed"
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi

ecbuild \
      -Wno-dev \
      -DCMAKE_BUILD_TYPE=RelWithDebInfo \
      -DCDASH_OVERRIDE_SYSTEM_NAME="${JEDI_COMPILER}-Container" \
      -DCDASH_OVERRIDE_SITE=AWSBatch \
      -DCDASH_OVERRIDE_GIT_BRANCH=${TRIGGER_PR} \
      -DCTEST_UPDATE_VERSION_ONLY=FALSE \
      -DBUILD_IODA_CONVERTERS=ON \
      -DBUILD_PYIRI=ON \
      ${COMPILER_FLAGS[@]} "${JEDI_BUNDLE_DIR}"
if [ $? -ne 0 ]; then
    util.check_run_fail $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID "ecbuild failed"
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi

# Back-date source files (search "back-date" in this file for an explanation).
find $JEDI_BUNDLE_DIR -type f -exec touch -d "$SOURCE_BACKDATE_TIMESTAMP" {} \;

make -j $BUILD_PARALLELISM
if [ $? -ne 0 ]; then
    util.check_run_fail $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID "compilation failed"
    util.evaluate_debug_timer_then_cleanup
    exit 0
fi

util.check_run_start_test $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID

# Run tests.
ctest -LE "${UNITTEST_TAG}|gsibec|rttov|oasim|ropp-ufo" --timeout 180 -C RelWithDebInfo -D ExperimentalTest

# Upload ctests.
ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous

find ${BUILD_DIR}/Testing -type f
find ${BUILD_DIR}/Testing -type f -exec head -n5 {} \;

echo "CDash URL: $(util.create_cdash_url "${BUILD_DIR}/Testing")"
TEST_TAG=$(head -1 "${BUILD_DIR}/Testing/TAG")
ls -al "${BUILD_DIR}/Testing/${TEST_TAG}/"

# Complete integration tests and allow a failure rate up to 3%
util.check_run_end $TRIGGER_REPO_FULL $INTEGRATION_RUN_ID 3

# Upload codecov data if gcc compiler is used.
if [ "$JEDI_COMPILER" = "gcc" ] && [ -f "${JEDI_BUNDLE_DIR}/${TRIGGER_REPO}/.codecov.yml" ]; then
    bash <(curl -s https://codecov.io/bash) -t 53f87271-b490-453c-b891-afd39cb658af -R "${JEDI_BUNDLE_DIR}/${TRIGGER_REPO}"
fi

util.evaluate_debug_timer_then_cleanup
echo "test complete"