Allow default override for local development - PRO-995

docker-compose.pipeline.yaml

@@ -4,7 +4,7 @@ services:
     image: ${DOCKER_NAMESPACE:-ghcr.io/zenysis}/harmony-etl-pipeline:${DOCKER_TAG:-latest}
     volumes:
       - ${DRUID_SHARED_FOLDER:-/home/share}:/home/share
-      - ${OUTPUT_PATH:-/data/output}:/zenysis/pipeline/out
+      - ${DATA_OUTPUT_FOLDER:-/data/output}:/zenysis/pipeline/out
       # Map minio config folder
       # When pipeline runs, mc should pick up $USER and point to this:
       - ${MC_CONFIG_PATH:-/home/zenysis/.mc}:/home/zenysis/.mc
@@ -25,10 +25,3 @@ services:
           memory: 50g
     user: ${PIPELINE_USER:-1000}:${PIPELINE_GROUP:-1000}
     command: [ "/bin/bash", "-c", "./docker/entrypoint_pipeline.sh" ]
-volumes:
-  mc_config:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: ${MC_CONFIG_PATH:-/home/zenysis/.mc}

pipeline/harmony_demo/index/run/00_druid/00_index

@@ -6,4 +6,5 @@ set -o pipefail
   --task_id_file="${PIPELINE_TMP_DIR}/task_id" \
   --task_hash_dir "${DRUID_SHARED_FOLDER:-/home/share}/data/logs/druid_indexing/hash" \
   --local_server_shared_folder "${DRUID_SHARED_FOLDER:-/home/share}" \
+  --druid_server_shared_folder="${DRUID_SHARED_FOLDER:-/home/share}" \
   --min_data_date='1970-01-01'

scripts/clean_published_pipeline.py

@@ -9,8 +9,10 @@
 
 from util.file.directory_util import compute_dir_hash, equal_dir_content
 
+DRUID_SHARED_FOLDER = os.environ.get('DRUID_SHARED_FOLDER', '/home/share')
+
 # Base directory of where to begin search.
-MAIN_DIR = '/home/share/data'
+MAIN_DIR = f'{DRUID_SHARED_FOLDER}/data'
 
 # Directories to whitelist
 WHITELIST_DIRS = set(['current'])