diff --git a/demo.sh b/demo.sh new file mode 100755 index 00000000..b4f0bd95 --- /dev/null +++ b/demo.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -euo pipefail + +DRUID_SHARED_FOLDER=~/druid/home/share +DATA_OUTPUT_FOLDER=~/druid/data/output + +####### +# MINIO +####### + +# # Create an environment file if it doesn't exist. +# if [ ! -f ".env.demo.minio" ]; then +# password=$(cat /dev/urandom | tr -dc 'A-Za-z0-9!' | head -c 13) + +# echo "MINIO_DATA_FOLDER=./ +# MINIO_ROOT_USER=minio_demo +# MINIO_ROOT_PASSWORD=$password" > .env.demo.minio +# fi +# # Load the environment file +source .env.demo.minio +# # Start the minio server. +# docker compose --env-file .env.demo.minio -f docker-compose.minio.yaml up --detach +# # Create an alias for the minio server. +# docker exec -it harmony-minio-1 /bin/bash -c "mc alias set local http://localhost:9000 minio_demo $MINIO_ROOT_PASSWORD" +# # Create a bucket. +# docker exec -it harmony-minio-1 /bin/bash -c "mc mb /local/zenysis-harmony-demo" +# # Create a self serve folder in the bucket +# docker exec -it harmony-minio-1 /bin/bash -c "touch /tmp/delete_me && mc cp /tmp/delete_me /local/zenysis-harmony-demo/self_serve/delete_me && rm /tmp/delete_me" + +####### +# DRUID +####### + +# mkdir -p ~/druid/home/share +# mkdir -p ~/druid/data/output + +# if [ ! -f "druid_setup/.env" ]; then +# echo "SINGLE_SERVER_DOCKER_HOST= +# DRUID_SHARED_FOLDER=$DRUID_SHARED_FOLDER +# DATA_OUTPUT_FOLDER=$DATA_OUTPUT_FOLDER" > druid_setup/.env +# fi + +# cd druid_setup +# make single_server_up +# cd .. + +############################### +# Prepare Harmony Configuration +############################### + +mkdir -p ./.mc +if [ ! -f ".mc/config.json" ]; then + echo "{ + \"version\": \"10\", + \"hosts\": { + \"s3\": { + \"url\": \"http://host.docker.internal:9000\", + \"accessKey\": \"$MINIO_ROOT_USER\", + \"secretKey\": \"$MINIO_ROOT_PASSWORD\", + \"api\": \"S3v4\", + \"lookup\": \"auto\" + } + } +}" > ./.mc/config.json +fi + +if [ ! -f ".env.demo" ]; then + + POSTGRES_PASSWORD=$(cat /dev/urandom | tr -dc 'A-Za-z0-9!' | head -c 13) + + echo "DEFAULT_SECRET_KEY=somesecret + +ZEN_ENV=harmony_demo + +DRUID_HOST=http://host.docker.internal +HASURA_HOST=http://hasura:8080 + +DATABASE_URL='postgresql://postgres:zenysis@postgres:5432/harmony_demo-local' + +POSTGRES_HOST=postgres + +# You can go to https://www.mapbox.com and create an API token. +MAPBOX_ACCESS_TOKEN=some_mapbox_access_token + +NOREPLY_EMAIL=noreply@zenysis.com +SUPPORT_EMAIL=suppport@zenysis.com + +MC_CONFIG_PATH=./.mc + +POSTGRES_PASSWORD=$POSTGRES_PASSWORD + +DRUID_SHARED_FOLDER=$DRUID_SHARED_FOLDER +DATA_OUTPUT_FOLDER=$DATA_OUTPUT_FOLDER + +DOCKER_NAMESPACE=zengineering +DOCKER_TAG=latest + +# Assuming you've created a minio alias called "local": +OBJECT_STORAGE_ALIAS=local" > .env.demo +fi + +source .env.demo + +########################## +# Prepare Harmony Database +########################## + +# docker compose --env-file .env.demo -f docker-compose.yaml -f docker-compose.dev.yaml convert +# docker compose --env-file .env.demo -f docker-compose.yaml -f docker-compose.dev.yaml run --rm web /bin/bash -c "source venv/bin/activate && yarn init-db harmony_demo" + +################## +# Run the pipeline +################## + +# zengineering +# latest +# $(DOCKER_NAMESPACE)/harmony-etl-pipeline:$(DOCKER_TAG) +# zengineering/harmony-etl-pipeline:latest + +# For the Harmony Demo, we skip the generate step. + +COMMAND="./pipeline/harmony_demo/process/process_all" \ + docker compose --project-name harmony-etl-process --env-file .env.demo \ + -f docker-compose.pipeline.yaml up + +COMMAND="./pipeline/harmony_demo/index/index_all" \ + docker compose --project-name harmony-etl-index --env-file .env.demo \ + -f docker-compose.pipeline.yaml up + +COMMAND="./pipeline/harmony_demo/validate/validate_all" \ + docker compose --project-name harmony-etl-validate --env-file .env.demo \ + -f docker-compose.pipeline.yaml up + + + + + + diff --git a/docker-compose.pipeline.yaml b/docker-compose.pipeline.yaml index addc3ec6..3eab8a25 100644 --- a/docker-compose.pipeline.yaml +++ b/docker-compose.pipeline.yaml @@ -5,7 +5,7 @@ services: pull_policy: always volumes: - ${DRUID_SHARED_FOLDER:-/home/share}:/home/share - - ${OUTPUT_PATH:-/data/output}:/zenysis/pipeline/out + - ${DATA_OUTPUT_FOLDER:-/data/output}:/zenysis/pipeline/out # Map minio config folder # When pipeline runs, mc should pick up $USER and point to this: - ${MC_CONFIG_PATH:-/home/zenysis/.mc}:/home/zenysis/.mc @@ -26,10 +26,3 @@ services: memory: 50g user: ${PIPELINE_USER:-1000}:${PIPELINE_GROUP:-1000} command: [ "/bin/bash", "-c", "./docker/entrypoint_pipeline.sh" ] -volumes: - mc_config: - driver: local - driver_opts: - type: none - o: bind - device: ${MC_CONFIG_PATH:-/home/zenysis/.mc} diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 33720000..9006df06 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -122,30 +122,25 @@ COPY --chmod=755 --from=downloader /usr/local/bin/mc /usr/local/bin/mc WORKDIR /app # CPython -# Update setup and create venv -RUN /opt/python/3.9.16/bin/python3.9 -m pip install --upgrade pip setuptools \ - && /opt/python/3.9.16/bin/python3.9 -m venv venv \ - && . venv/bin/activate && python -m pip install --upgrade pip setuptools +# Create venv +RUN /opt/python/3.9.16/bin/python3.9 -m venv venv + # Install dependencies COPY requirements.txt requirements-web.txt requirements-dev.txt requirements-pipeline.txt ./ RUN . venv/bin/activate \ - && python -m pip install -r requirements.txt \ - && python -m pip install -r requirements-web.txt \ - && python -m pip install -r requirements-dev.txt \ - && python -m pip install -r requirements-pipeline.txt + && python -m pip install wheel==0.43.0 \ + && python -m pip install --no-build-isolation -r requirements.txt -r requirements-web.txt -r requirements-dev.txt -r requirements-pipeline.txt # PyPy # Install pypy COPY --from=downloader /opt/pypy3.9-v7.3.11 /opt/pypy3.9-v7.3.11 # Update setup and create venv RUN /opt/pypy3.9-v7.3.11/bin/pypy3 -m ensurepip \ - && /opt/pypy3.9-v7.3.11/bin/pypy3 -m pip install --upgrade pip setuptools \ && /opt/pypy3.9-v7.3.11/bin/pypy3 -m venv venv_pypy3 # Install pypy dependencies RUN . venv_pypy3/bin/activate \ - && pypy3 -m pip install --upgrade pip setuptools \ - && pypy3 -m pip install -r requirements.txt \ - && pypy3 -m pip install -r requirements-pipeline.txt + && pypy3 -m pip install wheel==0.43.0 \ + && pypy3 -m pip install --no-build-isolation -r requirements.txt -r requirements-pipeline.txt # Install flow. COPY --from=downloader /usr/local/bin/flow /usr/local/bin/flow diff --git a/pipeline/harmony_demo/index/run/00_druid/00_index b/pipeline/harmony_demo/index/run/00_druid/00_index index b6db9ed4..1756dd08 100755 --- a/pipeline/harmony_demo/index/run/00_druid/00_index +++ b/pipeline/harmony_demo/index/run/00_druid/00_index @@ -5,5 +5,6 @@ set -o pipefail --data_files="${DRUID_SHARED_FOLDER:-/home/share}/data/harmony_demo/*/current/processed_rows.*" \ --task_id_file="${PIPELINE_TMP_DIR}/task_id" \ --task_hash_dir "${DRUID_SHARED_FOLDER:-/home/share}/data/logs/druid_indexing/hash" \ + --druid_server_shared_folder="${DRUID_SHARED_FOLDER:-/home/share}" \ --local_server_shared_folder "${DRUID_SHARED_FOLDER:-/home/share}" \ --min_data_date='1970-01-01' diff --git a/scripts/clean_published_pipeline.py b/scripts/clean_published_pipeline.py index f2c3ea07..a418b458 100755 --- a/scripts/clean_published_pipeline.py +++ b/scripts/clean_published_pipeline.py @@ -9,8 +9,10 @@ from util.file.directory_util import compute_dir_hash, equal_dir_content +DRUID_SHARED_FOLDER = os.environ.get('DRUID_SHARED_FOLDER', '/home/share') + # Base directory of where to begin search. -MAIN_DIR = '/home/share/data' +MAIN_DIR = f'{DRUID_SHARED_FOLDER}/data' # Directories to whitelist WHITELIST_DIRS = set(['current']) diff --git a/scripts/db/hasura/dev/start_hasura.sh b/scripts/db/hasura/dev/start_hasura.sh index a53f48fd..5340b0d4 100755 --- a/scripts/db/hasura/dev/start_hasura.sh +++ b/scripts/db/hasura/dev/start_hasura.sh @@ -32,10 +32,11 @@ if (( CONTAINER_COUNT == 0 )) ; then docker run \ -d \ -it \ + --add-host=host.docker.internal:host-gateway \ --name "${CONTAINER_NAME}" \ -v "${HASURA_METADATA_DIR}:/hasura-metadata" \ -v "${HASURA_CONTAINER_SCRIPTS_DIR}:/zenysis:ro" \ - -p '8088:8080' \ + -p '5003:8080' \ --entrypoint 'sh' \ "${IMAGE_NAME}" fi diff --git a/scripts/local_setup.sh b/scripts/local_setup.sh new file mode 100755 index 00000000..2cc5c10b --- /dev/null +++ b/scripts/local_setup.sh @@ -0,0 +1,93 @@ +#!/bin/bash +set -euo pipefail + +# This script is used to run the harmony backend locally. +# It consist of 5 arguments which are: +# 1. The environment file to be used +# 2. The email of the user to be created +# 3. The first name of the user to be created +# 4. The last name of the user to be created +# 5. The password of the user to be created + +# Function to check if all required arguments are provided +check_arguments() { + if [ $# -ne 5 ]; then + echo "Error: Missing arguments." + echo "Usage: $0 " + exit 1 + fi +} + +# Function to check if druid is running or not using curl +check_druid() { + if curl localhost:8888 >/dev/null 2>&1; then + echo "Druid running" + else + echo "Error: Druid not running" + exit 1 + fi +} + +# Function to install and configure the virtual environment +setup_virtualenv() { + if [ ! -d "venv" ]; then + echo "Virtual environment not found, creating one..." + python3.9 -m venv venv + fi + echo "Activating virtual environment..." + source ./venv/bin/activate + echo "Upgrading pip and installing dependencies..." + python -m pip install --upgrade pip + python -m pip install wheel==0.43.0 + python -m pip install --no-build-isolation -r requirements.txt -r requirements-dev.txt -r requirements-web.txt -r requirements-pipeline.txt +} + +# Function to load environment variables +load_env_vars() { + echo "Loading environment variables..." + set -o allexport + source $ENV_FILE + set +o allexport +} + +# Function to run pipeline processes +run_pipeline() { + echo "Running pipeline processes..." + ./pipeline/harmony_demo/process/process_all + ./pipeline/harmony_demo/index/index_all + ./pipeline/harmony_demo/validate/validate_all +} + +# Function to initialize the database and populate indicators +init_db() { + echo "Initializing the database..." + yarn init-db harmony_demo --populate_indicators_from_config +} + +# Function to create a user +create_user() { + echo "Creating user $EMAIL..." + ./scripts/create_user.py --username "$EMAIL" --first_name "$FIRST_NAME" --last_name "$LAST_NAME" --site_admin -o --password "$PASSWORD" +} + + +check_arguments "$@" + +ENV_FILE=$1 +EMAIL=$2 +FIRST_NAME=$3 +LAST_NAME=$4 +PASSWORD=$5 + +echo "Using environment file: $ENV_FILE" + +# Main execution starts here +check_druid +setup_virtualenv +load_env_vars +init_db +run_pipeline +create_user $EMAIL $FIRST_NAME $LAST_NAME $PASSWORD + +echo "All tasks completed successfully!" + diff --git a/web/server/routes/webpack_dev_proxy.py b/web/server/routes/webpack_dev_proxy.py index 86e033f1..60d9530f 100644 --- a/web/server/routes/webpack_dev_proxy.py +++ b/web/server/routes/webpack_dev_proxy.py @@ -22,11 +22,11 @@ def _proxy_request(url): @webpack_dev_proxy.route('/build/') def route_to_webpack_build(asset): # Built files will live in webpack-dev-server's virtual `build/` directory. - return _proxy_request(f'http://localhost:8080/build/{asset}') + return _proxy_request(f'http://localhost:5001/build/{asset}') @webpack_dev_proxy.route('/static/') def route_to_webpack_static_asset(asset): # Static assets will not be copied into webpack-dev-server's virtual # directories but will exist in the same path on the filesystem. - return _proxy_request(f'http://localhost:8080/{asset}') + return _proxy_request(f'http://localhost:5001/{asset}') diff --git a/web/webpack.config.js b/web/webpack.config.js index c9f036ed..0c45bb26 100644 --- a/web/webpack.config.js +++ b/web/webpack.config.js @@ -36,6 +36,7 @@ module.exports = { // Needed for friendly errors plugin. quiet: true, + port: 5001 }, devtool: 'source-map', entry: { @@ -184,5 +185,5 @@ module.exports = { // You can now require('file') instead of require('file.jsx'). extensions: ['.js', '.json', '.jsx'], modules: [absPath('web/client'), absPath('node_modules')], - }, + } };