Harmony demo

demo.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+set -euo pipefail
+
+DRUID_SHARED_FOLDER=~/druid/home/share
+DATA_OUTPUT_FOLDER=~/druid/data/output
+
+#######
+# MINIO
+#######
+
+# # Create an environment file if it doesn't exist.
+# if [ ! -f ".env.demo.minio" ]; then
+#     password=$(cat /dev/urandom | tr -dc 'A-Za-z0-9!' | head -c 13)
+
+#     echo "MINIO_DATA_FOLDER=./
+# MINIO_ROOT_USER=minio_demo
+# MINIO_ROOT_PASSWORD=$password" > .env.demo.minio
+# fi
+# # Load the environment file
+source .env.demo.minio
+# # Start the minio server.
+# docker compose --env-file .env.demo.minio -f docker-compose.minio.yaml up --detach
+# # Create an alias for the minio server.
+# docker exec -it harmony-minio-1 /bin/bash -c "mc alias set local http://localhost:9000 minio_demo $MINIO_ROOT_PASSWORD"
+# # Create a bucket.
+# docker exec -it harmony-minio-1 /bin/bash -c "mc mb /local/zenysis-harmony-demo"
+# # Create a self serve folder in the bucket
+# docker exec -it harmony-minio-1 /bin/bash -c "touch /tmp/delete_me && mc cp /tmp/delete_me /local/zenysis-harmony-demo/self_serve/delete_me && rm /tmp/delete_me"
+
+#######
+# DRUID
+#######
+
+# mkdir -p ~/druid/home/share
+# mkdir -p ~/druid/data/output
+
+# if [ ! -f "druid_setup/.env" ]; then
+#     echo "SINGLE_SERVER_DOCKER_HOST=
+# DRUID_SHARED_FOLDER=$DRUID_SHARED_FOLDER
+# DATA_OUTPUT_FOLDER=$DATA_OUTPUT_FOLDER" > druid_setup/.env
+# fi
+
+# cd druid_setup
+# make single_server_up
+# cd ..
+
+###############################
+# Prepare Harmony Configuration
+###############################
+
+mkdir -p ./.mc
+if [ ! -f ".mc/config.json" ]; then
+    echo "{
+    \"version\": \"10\",
+    \"hosts\": {
+        \"s3\": {
+            \"url\": \"http://host.docker.internal:9000\",
+            \"accessKey\": \"$MINIO_ROOT_USER\",
+            \"secretKey\": \"$MINIO_ROOT_PASSWORD\",
+            \"api\": \"S3v4\",
+            \"lookup\": \"auto\"
+        }
+    }
+}" > ./.mc/config.json
+fi
+
+if [ ! -f ".env.demo" ]; then
+
+    POSTGRES_PASSWORD=$(cat /dev/urandom | tr -dc 'A-Za-z0-9!' | head -c 13)
+
+    echo "DEFAULT_SECRET_KEY=somesecret
+
+ZEN_ENV=harmony_demo
+
+DRUID_HOST=http://host.docker.internal
+HASURA_HOST=http://hasura:8080
+
+DATABASE_URL='postgresql://postgres:zenysis@postgres:5432/harmony_demo-local'
+
+POSTGRES_HOST=postgres
+
+# You can go to https://www.mapbox.com and create an API token.
+MAPBOX_ACCESS_TOKEN=some_mapbox_access_token
+
+NOREPLY_EMAIL=noreply@zenysis.com
+SUPPORT_EMAIL=suppport@zenysis.com
+
+MC_CONFIG_PATH=./.mc
+
+POSTGRES_PASSWORD=$POSTGRES_PASSWORD
+
+DRUID_SHARED_FOLDER=$DRUID_SHARED_FOLDER
+DATA_OUTPUT_FOLDER=$DATA_OUTPUT_FOLDER
+
+DOCKER_NAMESPACE=zengineering
+DOCKER_TAG=latest
+
+# Assuming you've created a minio alias called "local":
+OBJECT_STORAGE_ALIAS=local" > .env.demo
+fi
+
+source .env.demo
+
+##########################
+# Prepare Harmony Database
+##########################
+
+# docker compose --env-file .env.demo -f docker-compose.yaml -f docker-compose.dev.yaml convert
+# docker compose --env-file .env.demo -f docker-compose.yaml -f docker-compose.dev.yaml run --rm web /bin/bash -c "source venv/bin/activate && yarn init-db harmony_demo"
+
+##################
+# Run the pipeline
+##################
+
+# zengineering
+# latest
+# $(DOCKER_NAMESPACE)/harmony-etl-pipeline:$(DOCKER_TAG)
+# zengineering/harmony-etl-pipeline:latest
+
+# For the Harmony Demo, we skip the generate step.
+
+COMMAND="./pipeline/harmony_demo/process/process_all" \
+    docker compose --project-name harmony-etl-process --env-file .env.demo \
+    -f docker-compose.pipeline.yaml up
+
+COMMAND="./pipeline/harmony_demo/index/index_all" \
+    docker compose --project-name harmony-etl-index --env-file .env.demo \
+    -f docker-compose.pipeline.yaml up
+
+COMMAND="./pipeline/harmony_demo/validate/validate_all" \
+    docker compose --project-name harmony-etl-validate --env-file .env.demo \
+    -f docker-compose.pipeline.yaml up
+
+
+
+
+
+

docker-compose.pipeline.yaml

@@ -5,7 +5,7 @@ services:
     pull_policy: always
     volumes:
       - ${DRUID_SHARED_FOLDER:-/home/share}:/home/share
-      - ${OUTPUT_PATH:-/data/output}:/zenysis/pipeline/out
+      - ${DATA_OUTPUT_FOLDER:-/data/output}:/zenysis/pipeline/out
       # Map minio config folder
       # When pipeline runs, mc should pick up $USER and point to this:
       - ${MC_CONFIG_PATH:-/home/zenysis/.mc}:/home/zenysis/.mc
@@ -26,10 +26,3 @@ services:
           memory: 50g
     user: ${PIPELINE_USER:-1000}:${PIPELINE_GROUP:-1000}
     command: [ "/bin/bash", "-c", "./docker/entrypoint_pipeline.sh" ]
-volumes:
-  mc_config:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: ${MC_CONFIG_PATH:-/home/zenysis/.mc}

docker/dev/Dockerfile

@@ -122,30 +122,25 @@ COPY --chmod=755 --from=downloader /usr/local/bin/mc /usr/local/bin/mc
 WORKDIR /app
 
 # CPython
-# Update setup and create venv
-RUN /opt/python/3.9.16/bin/python3.9 -m pip install --upgrade pip setuptools \
-    && /opt/python/3.9.16/bin/python3.9 -m venv venv \
-    && . venv/bin/activate && python -m pip install --upgrade pip setuptools
+# Create venv
+RUN /opt/python/3.9.16/bin/python3.9 -m venv venv
+
 # Install dependencies
 COPY requirements.txt requirements-web.txt requirements-dev.txt requirements-pipeline.txt ./
 RUN . venv/bin/activate \
-    && python -m pip install -r requirements.txt \
-    && python -m pip install -r requirements-web.txt \
-    && python -m pip install -r requirements-dev.txt \
-    && python -m pip install -r requirements-pipeline.txt
+    && python -m pip install wheel==0.43.0 \
+    && python -m pip install --no-build-isolation -r requirements.txt -r requirements-web.txt -r requirements-dev.txt -r requirements-pipeline.txt
 
 # PyPy
 # Install pypy
 COPY --from=downloader /opt/pypy3.9-v7.3.11 /opt/pypy3.9-v7.3.11
 # Update setup and create venv
 RUN /opt/pypy3.9-v7.3.11/bin/pypy3 -m ensurepip \
-    && /opt/pypy3.9-v7.3.11/bin/pypy3 -m pip install --upgrade pip setuptools \
     && /opt/pypy3.9-v7.3.11/bin/pypy3 -m venv venv_pypy3
 # Install pypy dependencies
 RUN . venv_pypy3/bin/activate \
-    && pypy3 -m pip install --upgrade pip setuptools \
-    && pypy3 -m pip install -r requirements.txt \
-    && pypy3 -m pip install -r requirements-pipeline.txt
+    && pypy3 -m pip install wheel==0.43.0 \
+    && pypy3 -m pip install --no-build-isolation -r requirements.txt -r requirements-pipeline.txt
 
 # Install flow.
 COPY --from=downloader /usr/local/bin/flow /usr/local/bin/flow

pipeline/harmony_demo/index/run/00_druid/00_index

@@ -5,5 +5,6 @@ set -o pipefail
   --data_files="${DRUID_SHARED_FOLDER:-/home/share}/data/harmony_demo/*/current/processed_rows.*" \
   --task_id_file="${PIPELINE_TMP_DIR}/task_id" \
   --task_hash_dir "${DRUID_SHARED_FOLDER:-/home/share}/data/logs/druid_indexing/hash" \
+  --druid_server_shared_folder="${DRUID_SHARED_FOLDER:-/home/share}" \
   --local_server_shared_folder "${DRUID_SHARED_FOLDER:-/home/share}" \
   --min_data_date='1970-01-01'

scripts/clean_published_pipeline.py

@@ -9,8 +9,10 @@
 
 from util.file.directory_util import compute_dir_hash, equal_dir_content
 
+DRUID_SHARED_FOLDER = os.environ.get('DRUID_SHARED_FOLDER', '/home/share')
+
 # Base directory of where to begin search.
-MAIN_DIR = '/home/share/data'
+MAIN_DIR = f'{DRUID_SHARED_FOLDER}/data'
 
 # Directories to whitelist
 WHITELIST_DIRS = set(['current'])

scripts/db/hasura/dev/start_hasura.sh

@@ -32,10 +32,11 @@ if (( CONTAINER_COUNT == 0 )) ; then
   docker run \
       -d \
       -it \
+      --add-host=host.docker.internal:host-gateway \
       --name "${CONTAINER_NAME}" \
       -v "${HASURA_METADATA_DIR}:/hasura-metadata" \
       -v "${HASURA_CONTAINER_SCRIPTS_DIR}:/zenysis:ro" \
-      -p '8088:8080' \
+      -p '5003:8080' \
       --entrypoint 'sh' \
     "${IMAGE_NAME}"
 fi

scripts/local_setup.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -euo pipefail
+
+# This script is used to run the harmony backend locally.
+# It consist of 5 arguments which are:
+# 1. The environment file to be used
+# 2. The email of the user to be created
+# 3. The first name of the user to be created
+# 4. The last name of the user to be created
+# 5. The password of the user to be created
+
+# Function to check if all required arguments are provided
+check_arguments() {
+    if [ $# -ne 5 ]; then
+        echo "Error: Missing arguments."
+        echo "Usage: $0 <env_file> <email> <first_name> <last_name> <password>"
+        exit 1
+    fi
+}
+
+# Function to check if druid is running or not using curl
+check_druid() {
+    if curl localhost:8888 >/dev/null 2>&1; then
+	echo "Druid running"
+    else 
+        echo "Error: Druid not running"
+        exit 1
+    fi
+}
+
+# Function to install and configure the virtual environment
+setup_virtualenv() {
+    if [ ! -d "venv" ]; then
+        echo "Virtual environment not found, creating one..."
+        python3.9 -m venv venv
+    fi
+    echo "Activating virtual environment..."
+    source ./venv/bin/activate
+    echo "Upgrading pip and installing dependencies..."
+    python -m pip install --upgrade pip
+    python -m pip install wheel==0.43.0
+    python -m pip install --no-build-isolation -r requirements.txt -r requirements-dev.txt -r requirements-web.txt -r requirements-pipeline.txt
+}
+
+# Function to load environment variables
+load_env_vars() {
+    echo "Loading environment variables..."
+    set -o allexport
+    source $ENV_FILE
+    set +o allexport
+}
+
+# Function to run pipeline processes
+run_pipeline() {
+    echo "Running pipeline processes..."
+    ./pipeline/harmony_demo/process/process_all
+    ./pipeline/harmony_demo/index/index_all
+    ./pipeline/harmony_demo/validate/validate_all
+}
+
+# Function to initialize the database and populate indicators
+init_db() {
+    echo "Initializing the database..."
+    yarn init-db harmony_demo --populate_indicators_from_config
+}
+
+# Function to create a user
+create_user() {
+    echo "Creating user $EMAIL..."
+    ./scripts/create_user.py --username "$EMAIL" --first_name "$FIRST_NAME" --last_name "$LAST_NAME" --site_admin  -o --password "$PASSWORD"
+}
+
+
+check_arguments "$@"
+
+ENV_FILE=$1
+EMAIL=$2
+FIRST_NAME=$3
+LAST_NAME=$4
+PASSWORD=$5
+
+echo "Using environment file: $ENV_FILE"
+
+# Main execution starts here
+check_druid
+setup_virtualenv
+load_env_vars
+init_db
+run_pipeline
+create_user $EMAIL $FIRST_NAME $LAST_NAME $PASSWORD
+
+echo "All tasks completed successfully!"
+

web/server/routes/webpack_dev_proxy.py

@@ -22,11 +22,11 @@ def _proxy_request(url):
 @webpack_dev_proxy.route('/build/<path:asset>')
 def route_to_webpack_build(asset):
     # Built files will live in webpack-dev-server's virtual `build/` directory.
-    return _proxy_request(f'http://localhost:8080/build/{asset}')
+    return _proxy_request(f'http://localhost:5001/build/{asset}')
 
 
 @webpack_dev_proxy.route('/static/<path:asset>')
 def route_to_webpack_static_asset(asset):
     # Static assets will not be copied into webpack-dev-server's virtual
     # directories but will exist in the same path on the filesystem.
-    return _proxy_request(f'http://localhost:8080/{asset}')
+    return _proxy_request(f'http://localhost:5001/{asset}')

web/webpack.config.js

@@ -36,6 +36,7 @@ module.exports = {
 
     // Needed for friendly errors plugin.
     quiet: true,
+    port: 5001
   },
   devtool: 'source-map',
   entry: {
@@ -184,5 +185,5 @@ module.exports = {
     // You can now require('file') instead of require('file.jsx').
     extensions: ['.js', '.json', '.jsx'],
     modules: [absPath('web/client'), absPath('node_modules')],
-  },
+  }
 };