[hack] move hive & hadoop image to docker-hub and its Dockerfile

Author: dungdm93

hack/Dockerfile.hadoop

@@ -0,0 +1,57 @@
+FROM alpine:3.10 AS downloader
+
+WORKDIR /build
+RUN apk add -U curl gnupg tar
+
+# Main Apache distributions:
+#   * https://apache.org/dist
+#   * https://archive.apache.org/dist
+#   * https://dist.apache.org/repos/dist/release
+# List all Apache mirrors:
+#   * https://apache.org/mirrors
+ARG APACHE_DIST=https://archive.apache.org/dist
+ARG APACHE_MIRROR=${APACHE_DIST}
+ARG HADOOP_VERSION=3.3.0
+
+RUN set -eux; \
+    curl -L  "${APACHE_DIST}/hadoop/common/KEYS" | gpg --batch --import -; \
+    curl -LO "${APACHE_MIRROR}/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz"; \
+    curl -L  "${APACHE_DIST}/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz.asc" \
+           | gpg --batch --verify - "hadoop-${HADOOP_VERSION}.tar.gz";
+RUN tar -xf  "hadoop-${HADOOP_VERSION}.tar.gz" --no-same-owner \
+        --exclude="hadoop-*/share/doc"; \
+    mv       "hadoop-${HADOOP_VERSION}" "hadoop";
+
+
+
+FROM  ubuntu:focal
+LABEL maintainer="Dũng Đặng Minh <[email protected]>"
+SHELL [ "/bin/bash", "-c" ]
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        openjdk-8-jre-headless ca-certificates libc6 \
+        libbz2-1.0 liblz4-1 libsnappy1v5 zlib1g libzstd1 \
+        libssl1.1 libisal2 libnss3 libpam-modules krb5-user procps; \
+    ln -s libcrypto.so.1.1 /usr/lib/x86_64-linux-gnu/libcrypto.so; \
+    ln -s libssl.so.1.1    /usr/lib/x86_64-linux-gnu/libssl.so;    \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*
+
+# TODO: Native Hadoop Library
+# > hadoop checknative -a
+# * libbz2-1.0 liblz4-1 libsnappy1v5 zlib1g libzstd1
+# * libssl1.1
+#   ln -s libcrypto.so.1.1 /usr/lib/x86_64-linux-gnu/libcrypto.so
+#   ln -s libssl.so.1.1    /usr/lib/x86_64-linux-gnu/libssl.so
+# * libisal2
+#   ln -s libisal.so.2     /usr/lib/x86_64-linux-gnu/libisal.so
+
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" \
+    HADOOP_HOME="/opt/hadoop"
+
+COPY --from=downloader "/build/hadoop" "${HADOOP_HOME}"
+
+ENV PATH="${HADOOP_HOME}/bin:${PATH}" \
+    LD_LIBRARY_PATH="${HADOOP_HOME}/lib/native:${LD_LIBRARY_PATH}"

hack/Dockerfile.hive

@@ -0,0 +1,40 @@
+ARG HADOOP_VERSION=3.1.0
+
+FROM alpine:3.10 AS downloader
+
+WORKDIR /build
+RUN apk add -U curl gnupg tar
+
+# Main Apache distributions:
+#   * https://apache.org/dist
+#   * https://archive.apache.org/dist
+#   * https://dist.apache.org/repos/dist/release
+# List all Apache mirrors:
+#   * https://apache.org/mirrors
+ARG APACHE_DIST=https://archive.apache.org/dist
+ARG APACHE_MIRROR=${APACHE_DIST}
+ARG HIVE_VERSION=3.1.2
+
+RUN set -eux; \
+    curl -L  "${APACHE_DIST}/hive/KEYS" | gpg --batch --import -; \
+    curl -LO "${APACHE_MIRROR}/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz"; \
+    curl -L  "${APACHE_DIST}/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz.asc" \
+           | gpg --batch --verify - "apache-hive-${HIVE_VERSION}-bin.tar.gz";
+RUN tar -xf  "apache-hive-${HIVE_VERSION}-bin.tar.gz" --no-same-owner; \
+    mv       "apache-hive-${HIVE_VERSION}-bin" "hive";
+
+
+
+FROM dungdm93/hadoop:${HADOOP_VERSION}
+
+# Tools
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y netcat; \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV HIVE_HOME="/opt/hive" \
+    PATH="/opt/hive/bin:${PATH}"
+
+COPY --from=downloader "/build/hive" "${HIVE_HOME}"

hack/docker-compose.yml

@@ -32,7 +32,7 @@ services:
     - postgres_data:/var/lib/postgresql/data
 
   hive-metastore:
-    image: hub.teko.vn/data/hive:3.1.2-hadoop-3.1.0
+    image: dungdm93/hive:3.1.2
     # HOTFIX: Illegal character in hostname at index 13: thrift://hive_hive-metastore_1.bookshelf:9083
     # Caused at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.resolveUris(HiveMetaStoreClient.java:267)
     container_name: hive-metastore
@@ -49,7 +49,7 @@ services:
     - warehouse_data:/data/warehouse
 
   hive-server2:
-    image: hub.teko.vn/data/hive:3.1.2-hadoop-3.1.0
+    image: dungdm93/hive:3.1.2
     entrypoint: ["/usr/local/scripts/hiveserver2-entrypoint.sh"]
     depends_on:
     - hive-metastore