Merge pull request #99 from axiom-data-science/use-official-image

Use official ERDDAP image as base image, substitue env vars in datasets.xml

Author: srstsavage

.github/actions/publish_docker_image/action.yml

@@ -16,8 +16,6 @@ runs:
       with:
         images: |
           axiom/docker-erddap
-        flavor: |
-          suffix=-jdk21-openjdk,onlatest=true
         tags: |
           type=raw,value=latest,enable={{is_default_branch}}
           type=ref,event=tag

Dockerfile

@@ -1,100 +1,58 @@
-ARG BASE_IMAGE=tomcat:10.1.26-jdk21-temurin-jammy
-#referencing a specific image digest pins our unidata tomcat-docker image to platform amd64 (good)
-ARG UNIDATA_TOMCAT_IMAGE=unidata/tomcat-docker:10-jdk17@sha256:af7d3fecec753cbd438f25881deeaf48b40ac1f105971d6f300252e104e39fb2
-FROM ${UNIDATA_TOMCAT_IMAGE} AS unidata-tomcat-image
-FROM ${BASE_IMAGE}
+ARG ERDDAP_VERSION=v2.27.0
+ARG BASE_IMAGE=erddap/erddap:$ERDDAP_VERSION
+FROM $BASE_IMAGE
 
-#use approaches and hardened files from https://github.com/Unidata/tomcat-docker
-#note: we don't inherit directly from Unidata/tomcat-docker to allow more
-#flexibility in building images using different tomcat base images, architectures, etc
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends  \
-        gosu \
-        zip \
-        unzip \
-        && \
-    # Cleanup
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    # Eliminate default web applications
-    rm -rf ${CATALINA_HOME}/webapps/* && \
-    rm -rf ${CATALINA_HOME}/webapps.dist && \
-    # Obscuring server info
-    cd ${CATALINA_HOME}/lib && \
-    mkdir -p org/apache/catalina/util/ && \
-    unzip -j catalina.jar org/apache/catalina/util/ServerInfo.properties \
-        -d org/apache/catalina/util/ && \
-    sed -i 's/server.info=.*/server.info=Apache Tomcat/g' \
-        org/apache/catalina/util/ServerInfo.properties && \
-    zip -ur catalina.jar \
-        org/apache/catalina/util/ServerInfo.properties && \
-    rm -rf org && cd ${CATALINA_HOME} && \
-    # Setting restrictive umask container-wide
-    echo "session optional pam_umask.so" >> /etc/pam.d/common-session && \
-    sed -i 's/UMASK.*022/UMASK           007/g' /etc/login.defs
-
-# Security enhanced web.xml
-COPY --from=unidata-tomcat-image ${CATALINA_HOME}/conf/web.xml ${CATALINA_HOME}/conf/
-
-# Security enhanced server.xml
-COPY --from=unidata-tomcat-image ${CATALINA_HOME}/conf/server.xml ${CATALINA_HOME}/conf/
-
-ARG ERDDAP_VERSION=2.25.1
-ARG ERDDAP_CONTENT_VERSION=1.0.0
-ARG ERDDAP_WAR_URL="https://github.com/ERDDAP/erddap/releases/download/v${ERDDAP_VERSION}/erddap.war"
-ARG ERDDAP_CONTENT_URL="https://github.com/ERDDAP/erddapContent/archive/refs/tags/content${ERDDAP_CONTENT_VERSION}.zip"
-ENV ERDDAP_bigParentDirectory=/erddapData
-
-RUN apt-get update && apt-get install -y unzip xmlstarlet \
-    && if ! command -v gosu &> /dev/null; then apt-get install -y gosu; fi \
+RUN apt-get update && apt-get install -y gettext-base xmlstarlet \
     && rm -rf /var/lib/apt/lists/*
 
-ARG BUST_CACHE=1
-RUN \
-    mkdir -p /tmp/dl && \
-    curl -fSL "${ERDDAP_WAR_URL}" -o /tmp/dl/erddap.war && \
-    unzip /tmp/dl/erddap.war -d ${CATALINA_HOME}/webapps/erddap/ && \
-    curl -fSL "${ERDDAP_CONTENT_URL}" -o /tmp/dl/erddapContent.zip && \
-    unzip /tmp/dl/erddapContent.zip -d /tmp/dl/erddapContent && \
-    find /tmp/dl/erddapContent -type d -name content -exec cp -r "{}" ${CATALINA_HOME} \; && \
-    rm -rf /tmp/dl && \
-    sed -i 's#</Context>#<Resources cachingAllowed="true" cacheMaxSize="100000" />\n&#' ${CATALINA_HOME}/conf/context.xml && \
-    rm -rf /tmp/* /var/tmp/* && \
-    mkdir -p ${ERDDAP_bigParentDirectory}
-
-# Java options
-COPY files/setenv.sh ${CATALINA_HOME}/bin/setenv.sh
-
-# server.xml fixup
-COPY update-server-xml.sh /opt/update-server-xml.sh
-RUN /opt/update-server-xml.sh
-
-# Default configuration
-# Note: Make sure ERDDAP_flagKeyKey is set either in a runtime environment variable or in setup.xml
-#       If a value is not set, a random value for ERDDAP_flagKeyKey will be generated at runtime.
-ENV ERDDAP_baseHttpsUrl="https://localhost:8443" \
-    ERDDAP_emailEverythingTo="[email protected]" \
-    ERDDAP_emailDailyReportsTo="[email protected]" \
-    ERDDAP_emailFromAddress="[email protected]" \
-    ERDDAP_emailUserName="" \
-    ERDDAP_emailPassword="" \
-    ERDDAP_emailProperties="" \
-    ERDDAP_emailSmtpHost="" \
-    ERDDAP_emailSmtpPort="" \
-    ERDDAP_adminInstitution="Axiom Docker Install" \
-    ERDDAP_adminInstitutionUrl="https://github.com/axiom-data-science/docker-erddap" \
-    ERDDAP_adminIndividualName="Axiom Docker Install" \
-    ERDDAP_adminPosition="Software Engineer" \
-    ERDDAP_adminPhone="555-555-5555" \
-    ERDDAP_adminAddress="123 Irrelevant St." \
-    ERDDAP_adminCity="Nowhere" \
-    ERDDAP_adminStateOrProvince="AK" \
-    ERDDAP_adminPostalCode="99504" \
-    ERDDAP_adminCountry="USA" \
-    ERDDAP_adminEmail="[email protected]"
-
-COPY entrypoint.sh datasets.d.sh /
-ENTRYPOINT ["/entrypoint.sh"]
-
-EXPOSE 8080
-CMD ["catalina.sh", "run"]
+COPY datasets.d.sh /
+
+# advise users to use upstream offical ERDDAP docker image
+# if they aren't using experimental features in this image
+COPY --chmod=755 <<EOF /init.d/00-advise-upstream.sh
+#/bin/sh
+cat <<EOF2
+
+███████ ██████  ██████  ██████   █████  ██████  
+██      ██   ██ ██   ██ ██   ██ ██   ██ ██   ██ 
+█████   ██████  ██   ██ ██   ██ ███████ ██████  
+██      ██   ██ ██   ██ ██   ██ ██   ██ ██      
+███████ ██   ██ ██████  ██████  ██   ██ ██      
+
+NOTE: As of version v2.27.0 this image (axiom/docker-erddap)
+is derived from the official ERDDAP Docker image (erddap/erddap).
+
+If you are not using any experimental functionality offered
+by the axiom image (notably datasets.d), you are recommended
+to use the official ERDDAP Docker image instead.
+
+See https://hub.docker.com/r/erddap/erddap for more details.                                                
+
+EOF2
+EOF
+
+COPY --chmod=755 <<'EOF' /init.d/50-datasets.d.sh
+#/bin/sh
+###
+# Add datasets in /datasets.d to datasets.xml
+###
+if [ -d "/datasets.d" ]; then
+  echo "Creating datasets.xml from /datasets.d"
+  ERDDAP_CONTENT_DIR="/usr/local/tomcat/content/erddap"
+  DATASETS_XML="${ERDDAP_CONTENT_DIR}/datasets.xml"
+  if [ -f "$DATASETS_XML" ]; then
+    #datasets.xml exists, make sure we have a backup of it
+    DATASETS_XML_MD5SUM=$(md5sum "$DATASETS_XML" | awk '{print $1}')
+    if ! md5sum "${ERDDAP_CONTENT_DIR}/datasets.xml.*.bak" 2>/dev/null | grep -q "$DATASETS_XML_MD5SUM"; then
+      #we don't have a backup of this version of datasets.xml yet, make one
+      DATASETS_XML_BACKUP="${ERDDAP_CONTENT_DIR}"/datasets.xml.$(date -u +"%Y%m%dT%H%M%SZ").bak
+      echo "Backing up "${DATASETS_XML}" to ${DATASETS_XML_BACKUP}"
+      cp "$DATASETS_XML" "${DATASETS_XML_BACKUP}"
+    fi
+  fi
+  /datasets.d.sh -o "$DATASETS_XML" -w
+fi
+EOF
+
+ENV ERDDAP_useHeadersForUrl=true \
+    ERDDAP_useSaxParser=true

README.md

@@ -1,19 +1,37 @@
 # ERDDAP on Docker
 
-A feature full Tomcat (SSL over APR, etc.) running [ERDDAP](http://coastwatch.pfeg.noaa.gov/erddap/index.html)
+[ERDDAP](http://coastwatch.pfeg.noaa.gov/erddap/index.html) docker image with experimental features (datasets.d, see below).
+
+## Upstream Official ERDDAP Docker Image
+
+As of version v2.27.0 this image (axiom/docker-erddap)
+is derived from the official ERDDAP Docker image (erddap/erddap).
+
+If you are not using any experimental functionality offered
+by the axiom image (notably datasets.d), you are recommended
+to use the official ERDDAP Docker image instead.
+
+See https://hub.docker.com/r/erddap/erddap for more details.
+
+## Versions
 
 Most recent versions:
 
-* `axiom/docker-erddap:latest-jdk21-openjdk` (2.25.1)
+* `axiom/docker-erddap:latest`
+* `axiom/docker-erddap:v2.27.0`
 * `axiom/docker-erddap:2.25.1-jdk21-openjdk`
-* `axiom/docker-erddap:2.24-jdk21-openjdk`
-* `axiom/docker-erddap:2.23-jdk17-openjdk`
 
-See all versions available [here](https://hub.docker.com/r/axiom/docker-erddap/tags). As always, consult the [ERDDAP Changes](https://coastwatch.pfeg.noaa.gov/erddap/download/changes.html) documentation before upgrading your server.
+See all versions available [here](https://hub.docker.com/r/axiom/docker-erddap/tags).
+As always, consult the [ERDDAP Changes](https://coastwatch.pfeg.noaa.gov/erddap/download/changes.html)
+documentation before upgrading your server.
+
+Use any of the `latest-*` images with caution as they follow the upstream image,
+and is not as thoroughly tested as tagged images.
 
-Use any of the `latest-*` images with caution as they follow the upstream image, and is not as thoroughly tested as tagged images.
+## Breaking changes
 
-[Dependabot](https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/keeping-your-dependencies-updated-automatically) is used to automatically make PRs to update the upstream image ([`.github/dependabot.yml`](.github/dependabot.yml)).
+* v2.27.0
+** CORS is now off by default. To enable it, set `ERDDAP_enableCors=true`.
 
 ## Quickstart
 
@@ -37,7 +55,7 @@ or, generate a basic dataset configuration without input for
 later customization
 
 ```bash
-./make-dataset.xml /path/to/your.csv EDDTableFromAsciiFiles > /path/to/your-dataset.xml
+./make-dataset-xml.sh /path/to/your.csv EDDTableFromAsciiFiles > /path/to/your-dataset.xml
 ```
 
 ## Configuration
@@ -48,10 +66,7 @@ See [these instructions for configuring Tomcat](https://github.com/unidata/tomca
 
 ### CORS
 
-The [Tomcat configuration](https://github.com/unidata/tomcat-docker) used by this image enables the
-[Apache Tomcat CORS filter](https://tomcat.apache.org/tomcat-8.5-doc/config/filter.html#CORS_Filter) by
-default. To disable it (maybe you want to handle CORS uniformly in a proxying webserver?), set environment
-variable `DISABLE_CORS` to `1`.
+CORS is off by default. To enable it, set `ERDDAP_enableCors=true`.
 
 ### ERDDAP
 
@@ -241,6 +256,8 @@ Example:
 ERDDAP_DATASETS_cacheMinutes=20 ./datasets.d.sh -d examples/datasets.d
 ```
 
+As of version v2.27.0 environment variables in the generated datasets.xml file will also be replaced using envsubst.
+
 ##### Elegantly removing datasets in datasets.d mode
 
 ERDDAP has a [specific process](https://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#active)

datasets.d.sh

@@ -12,6 +12,8 @@ DATASETSD_OUTPUT_PATH="${DATASETSD_OUTPUT_PATH:-/usr/local/tomcat/content/erddap
 DATASETSD_MARK_REMOVED_DATASETS_INACTIVE="${DATASETSD_MARK_REMOVED_DATASETS_INACTIVE:-0}"
 DATASETSD_WRITE_TO_OUTPUT_PATH="${DATASETSD_WRITE_TO_OUTPUT_PATH:-0}"
 DATASETSD_REFRESH_MISSING_DATASETS="${DATASETSD_REFRESH_MISSING_DATASETS:-0}"
+ERDDAP_bigParentDirectory="${ERDDAP_bigParentDirectory:-/erddapData}"
+
 while getopts ":d:io:rw" opt; do
   case ${opt} in
     d )
@@ -71,18 +73,25 @@ if [ "$DATASETSD_MARK_REMOVED_DATASETS_INACTIVE" == "1" ] && [ -n "$DATASETSD_OU
   true
 fi
 
-#empty edit for formatting
-DXML=$(echo "$DXML" | xmlstarlet edit --inplace)
+#empty edit for formatting, substituting env vars with envsubst
+DXML=$(echo "$DXML" | xmlstarlet edit --inplace | envsubst)
 
 #write output to target file if one was provided and write to stdout flag was not provided
 if [ -n "$DATASETSD_OUTPUT_PATH" ] && [ "$DATASETSD_WRITE_TO_OUTPUT_PATH" == "1" ]; then
   echo "$DXML" > "$DATASETSD_OUTPUT_PATH"
   #set refresh flags for any datasetIDs in datasets.xml that are not in the running ERDDAP config if the refresh option was set
-  if [ -n "$DATASETSD_REFRESH_MISSING_DATASETS" ] && [ "$DATASETSD_REFRESH_MISSING_DATASETS" == "1" ]; then
-    comm -23 \
-      <(xmlstarlet select -t -v "/erddapDatasets/dataset/@datasetID" "$DATASETSD_OUTPUT_PATH" | sort) \
-      <(curl -sS "http://localhost:8080/erddap/tabledap/allDatasets.csv0?datasetID" | grep -v "^allDatasets$" | sort) \
-      | xargs -I{} touch /erddapData/flag/{}
+  #check if erddap is running and /erddapData/flag exists first
+  if [ -n "$DATASETSD_REFRESH_MISSING_DATASETS" ] && [ "$DATASETSD_REFRESH_MISSING_DATASETS" == "1" ] ; then
+    if ! curl -fsS "http://localhost:8080/erddap/index.html" &> /dev/null; then
+      echo "Skipping refresh of missing datasets because ERDDAP isn't running"
+    elif ! [ -d "${ERDDAP_bigParentDirectory}/flag" ]; then
+      echo "Skipping refresh of missing datasets because ${ERDDAP_bigParentDirectory}/flag directory doesn't exist"
+    else
+      comm -23 \
+        <(xmlstarlet select -t -v "/erddapDatasets/dataset/@datasetID" "$DATASETSD_OUTPUT_PATH" | sort) \
+        <(curl -sS "http://localhost:8080/erddap/tabledap/allDatasets.csv0?datasetID" | grep -v "^allDatasets$" | sort) \
+        | xargs -I{} touch "${ERDDAP_bigParentDirectory}"/flag/{}
+    fi
   fi
 else
   echo "$DXML"

entrypoint.sh

@@ -1,4 +1,4 @@
-<dataset type="EDDTableFromAsciiFiles" datasetID="trees" active="true">
+<dataset type="EDDTableFromAsciiFiles" datasetID="${TREES_DATASET_NAME}" active="true">
   <reloadEveryNMinutes>10080</reloadEveryNMinutes>
   <updateEveryNMillis>10000</updateEveryNMillis>
   <fileDir>/data/</fileDir>

examples/datasets.d/trees.csv.xml

@@ -8,7 +8,7 @@ services:
       ERDDAP_DATASETS_standardPrivacyPolicy: "<h1>Any and all usage of this data is permitted.</h1>"
       DATASETSD_MARK_REMOVED_DATASETS_INACTIVE: "1"
       DATASETSD_REFRESH_MISSING_DATASETS: "1"
-      DISABLE_CORS: "1"
+      TREES_DATASET_NAME: "asdf-trees"
     volumes:
       - ./data:/data:ro
       - ./datasets.d:/datasets.d:ro