chores: add Dockerfile.superset

Author: dungdm93

hack/Dockerfile.superset

@@ -0,0 +1,86 @@
+FROM python:3.8-slim as base
+LABEL maintainer="Teko's DataOps Team <[email protected]>"
+SHELL ["/bin/bash", "-c"]
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        libbz2-1.0 liblz4-1 libsnappy1v5 zlib1g libzstd1 \
+        libev4 libssl1.1 libisal2 libnss3 \
+        libpq5 libmariadb3 \
+        curl locales; \
+    \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*
+
+# Firefox + Gecko driver. For Ubuntu, using `apt install firefox firefox-geckodriver`
+ARG GECKO_DRIVER_VERSION=v0.29.0
+RUN set -eux; cd /tmp/; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        xvfb libxi6 libgconf-2-4 \
+        firefox-esr; \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*; \
+    curl -sSL "https://github.com/mozilla/geckodriver/releases/download/${GECKO_DRIVER_VERSION}/geckodriver-${GECKO_DRIVER_VERSION}-linux64.tar.gz" \
+        | tar -xzf - -C /usr/local/bin --no-same-owner;
+
+RUN set -eux; \
+    sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen; \
+    locale-gen; \
+    update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8;
+
+ENV SUPERSET_HOME="/opt/superset"
+RUN set -eux; \
+    useradd -ms "/bin/bash" --uid=1000 superset; \
+    mkdir -p "${SUPERSET_HOME}"; \
+    chown -R superset: "${SUPERSET_HOME}";
+
+WORKDIR ${SUPERSET_HOME}
+
+FROM base AS builder
+
+ARG SUPERSET_VERSION=1.0.1
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        # pip install pyodbc
+        unixodbc-dev \
+        # pip install mysqlclient
+        default-libmysqlclient-dev \
+        # pip install pylibmc
+        libmemcached-dev \
+        # pip install sasl
+        libsasl2-dev;
+
+RUN set -eux; \
+    function join { local IFS="$1"; echo "${*:2}"; }; \
+    SUPERSET_PACKAGES=( \
+        # Cloud
+        athena bigquery redshift \
+        dremio snowflake teradata vertica exasol \
+        # Database
+        mysql postgres mmsql oracle db2 hana \
+        clickhouse cockroachdb elasticsearch \
+        excel gsheets \
+        # Big Data
+        drill druid hive impala kylin pinot presto \
+        # Others
+        cors thumbnails \
+    ); \
+    pip install "apache-superset[$(join ',' ${SUPERSET_PACKAGES[@]})]==${SUPERSET_VERSION}" \
+        "gunicorn[gevent,eventlet]" "flower~=0.9" "authlib~=0.15" "redis~=3.5" "pylibmc~=1.6"; \
+    \
+    rm -rf /usr/local/cx_Oracle-doc;
+
+FROM base
+
+COPY --from=builder /usr/local /usr/local
+COPY ./scripts/superset/* /usr/local/bin/
+RUN mv /usr/local/bin/trino.py /usr/local/lib/python3.8/site-packages/superset/db_engine_specs/
+
+USER superset
+EXPOSE 8088 5555
+ENTRYPOINT [ "/usr/local/bin/docker-entrypoint.sh" ]

hack/scripts/superset/docker-entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -eo pipefail
+
+case "$1" in
+    webserver)
+        superset-tools wait database
+        superset db upgrade
+        # https://docs.gunicorn.org/en/stable/settings.html
+        if [ -n "$GUNICORN_CONFIG_PATH" ]; then
+            GUNICORN_CMD_ARGS="--config=$GUNICORN_CONFIG_PATH ${GUNICORN_CMD_ARGS}"
+        fi
+        exec gunicorn --bind=0.0.0.0:8088 \
+            ${GUNICORN_CMD_ARGS} \
+            "superset.app:create_app()"
+        ;;
+    worker|beat|flower)
+        superset-tools wait broker
+        # https://superset.apache.org/docs/installation/async-queries-celery
+        # https://docs.celeryproject.org/en/stable/userguide/configuration.html
+        # https://docs.celeryproject.org/en/stable/reference/cli.html#celery-worker
+        if [ -n "$CELERY_CONFIG_PATH" ]; then
+            CELERY_CMD_ARGS="--config=$CELERY_CONFIG_PATH ${CELERY_CMD_ARGS}"
+        fi
+        exec celery "$1" ${CELERY_CMD_ARGS} \
+            "--app=superset.tasks.celery_app:app"
+        ;;
+    version)
+        superset version
+        ;;
+    *)
+        exec "$@"
+        ;;
+esac

hack/scripts/superset/superset-tools

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+import time
+from datetime import datetime
+
+from superset import config
+
+
+def wait_for_database(retries: int, interval: float):
+    from sqlalchemy import create_engine, sql
+    from sqlalchemy.engine.base import Engine
+    from sqlalchemy.exc import OperationalError
+
+    engine: Engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
+    uri = repr(engine.url)
+    for i in range(retries):
+        try:
+            engine.execute(sql.text('SELECT 1')).scalar()
+        except OperationalError:
+            print(f'[{datetime.now()}] waiting for database at {uri}... Try {i + 1}/{retries}', file=sys.stderr)
+            time.sleep(interval)
+        else:
+            print(f'[{datetime.now()}] Connect to database at {uri} successfully')
+            return
+
+    sys.exit('SQLAlchemy database still unreachable, giving up')
+
+
+def wait_for_broker(retries: int, interval: float):
+    from kombu.connection import Connection
+    from kombu.exceptions import OperationalError
+
+    class errback:
+        def __init__(self, uri: str, max_retries: int):
+            self.uri = uri
+            self.count = 0
+            self.max_retries = max_retries
+
+        def __call__(self, exc, interval):
+            self.count += 1
+            msg = f'[{datetime.now()}] waiting for broker at {self.uri}... Try {self.count}/{self.max_retries}'
+            print(msg, file=sys.stderr)
+
+    try:
+        conn = Connection(config.CELERY_CONFIG.BROKER_URL)
+        uri = conn.as_uri()
+        conn.ensure_connection(errback=errback(uri, max_retries=retries),
+                               max_retries=retries, interval_start=interval, interval_max=interval)
+        print(f'[{datetime.now()}] Connect to broker at {uri} successfully')
+    except OperationalError:
+        sys.exit('Celery broker still unreachable, giving up')
+
+
+def build_argparse():
+    def wait_service(args: argparse.Namespace):
+        if args.service == 'database':
+            wait_for_database(args.retries, args.interval)
+        elif args.service == 'broker':
+            wait_for_broker(args.retries, args.interval)
+        else:
+            raise ValueError(f'Unknown service {args.service}')
+
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(title='sub-commands')
+
+    # wait sub-command
+    wait = subparsers.add_parser('wait', help='Waiting for external service start-up')
+    wait.add_argument('service', type=str, choices=['database', 'broker'],
+                      help='Service that waiting for')
+    wait.add_argument('--retries', type=int, default=30, help='Maximum number of times to retry')
+    wait.add_argument('--interval', type=float, default=3, help='The number of seconds sleeping for')
+    wait.set_defaults(func=wait_service)
+
+    return parser
+
+
+def main():
+    parser = build_argparse()
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

hack/scripts/superset/trino.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+from typing import Optional
+
+from superset.db_engine_specs import BaseEngineSpec
+from superset.utils import core as utils
+
+
+class TrinoEngineSpec(BaseEngineSpec):
+    engine = "trino"
+    engine_name = "Trino"
+
+    # pylint: disable=line-too-long
+    _time_grain_expressions = {
+        None: "{col}",
+        "PT1S": "date_trunc('second', CAST({col} AS TIMESTAMP))",
+        "PT1M": "date_trunc('minute', CAST({col} AS TIMESTAMP))",
+        "PT1H": "date_trunc('hour', CAST({col} AS TIMESTAMP))",
+        "P1D": "date_trunc('day', CAST({col} AS TIMESTAMP))",
+        "P1W": "date_trunc('week', CAST({col} AS TIMESTAMP))",
+        "P1M": "date_trunc('month', CAST({col} AS TIMESTAMP))",
+        "P0.25Y": "date_trunc('quarter', CAST({col} AS TIMESTAMP))",
+        "P1Y": "date_trunc('year', CAST({col} AS TIMESTAMP))",
+        # "1969-12-28T00:00:00Z/P1W",  # Week starting Sunday
+        # "1969-12-29T00:00:00Z/P1W",  # Week starting Monday
+        # "P1W/1970-01-03T00:00:00Z",  # Week ending Saturday
+        # "P1W/1970-01-04T00:00:00Z",  # Week ending Sunday
+    }
+
+    @classmethod
+    def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
+        tt = target_type.upper()
+        if tt == utils.TemporalType.DATE:
+            value = dttm.date().isoformat()
+            return f"from_iso8601_date('{value}')"
+        if tt == utils.TemporalType.TIMESTAMP:
+            value = dttm.isoformat(timespec='microseconds')
+            return f"from_iso8601_timestamp('{value}')"
+        return None
+
+    @classmethod
+    def epoch_to_dttm(cls) -> str:
+        return "from_unixtime({col})"