Merge pull request #35 from puckel/refacto_executor

- Modify entrypoint.sh to select the type of executor.
- Split docker-compose.yml definition for local or celery executor
- Load examples or not

Author: puckel

Dockerfile

@@ -1,4 +1,4 @@
-# VERSION 1.7.1.3
+# VERSION 1.7.1.3-2
 # AUTHOR: Matthieu "Puckel_" Roisil
 # DESCRIPTION: Basic Airflow container
 # BUILD: docker build --rm -t puckel/docker-airflow

README.md

@@ -27,9 +27,25 @@ For example, if you need to install [Extra Packages](http://pythonhosted.org/air
 
 ## Usage
 
-Start the stack (postgres, rabbitmq, airflow-webserver, airflow-scheduler airflow-flower & airflow-worker):
+By default, docker-airflow run Airflow with **SequentialExecutor** :
 
-        docker-compose up -d
+        docker run -d -p 8080:8080 puckel/docker-airflow
+
+If you want to run other executor, you've to use the docker-compose.yml files provided in this repository.
+
+For **LocalExecutor** :
+
+        docker-compose -f docker-compose-LocalExecutor.yml up -d
+
+For **CeleryExecutor** :
+
+        docker-compose -f docker-compose-CeleryExecutor.yml up -d
+
+NB : If you don't want to have DAGs example loaded (default=True), you've to set the following environment variable :
+
+`LOAD_EX=n`
+
+        docker run -d -p 8080:8080 -e LOAD_EX=n puckel/docker-airflow
 
 If you want to use Ad hoc query, make sure you've configured connections:
 Go to Admin -> Connections and Edit "mysql_default" set this values (equivalent to values in airflow.cfg/docker-compose.yml) :
@@ -48,10 +64,6 @@ Check [Airflow Documentation](http://pythonhosted.org/airflow/)
 
 When using OSX with boot2docker, use: open http://$(boot2docker ip):8080
 
-## Run the test "tutorial"
-
-        docker exec dockerairflow_webserver_1 airflow backfill tutorial -s 2015-05-01 -e 2015-06-01
-
 ## Scale the number of workers
 
 Easy scaling using docker-compose:

docker-compose.yml docker-compose-CeleryExecutor.ymldocker-compose.yml renamed to docker-compose-CeleryExecutor.yml

@@ -1,12 +1,5 @@
 version: '2'
 services:
-    postgres:
-        image: postgres
-        environment:
-            - POSTGRES_USER=airflow
-            - POSTGRES_PASSWORD=airflow
-            - POSTGRES_DB=airflow
-
     rabbitmq:
         image: rabbitmq:3-management
         restart: always
@@ -18,40 +11,58 @@ services:
             - RABBITMQ_DEFAULT_PASS=airflow
             - RABBITMQ_DEFAULT_VHOST=airflow
 
+    postgres:
+        image: postgres
+        environment:
+            - POSTGRES_USER=airflow
+            - POSTGRES_PASSWORD=airflow
+            - POSTGRES_DB=airflow
+
     webserver:
-        image: puckel/docker-airflow:1.7.1.3
+        image: puckel/docker-airflow:1.7.1.3-2
         restart: always
+        depends_on:
+            - postgres
+            - rabbitmq
         environment:
-            - AIRFLOW_HOME=/usr/local/airflow
+            # - LOAD_EX=n
+            - EXECUTOR=Celery
         # volumes:
         #     - /localpath/to/dags:/usr/local/airflow/dags
         ports:
             - "8080:8080"
         command: webserver
 
     flower:
-        image: puckel/docker-airflow:1.7.1.3
+        image: puckel/docker-airflow:1.7.1.3-2
         restart: always
+        depends_on:
+            - rabbitmq
         environment:
-            - AIRFLOW_HOME=/usr/local/airflow
+            - EXECUTOR=Celery
         ports:
             - "5555:5555"
         command: flower
 
     scheduler:
-        image: puckel/docker-airflow:1.7.1.3
+        image: puckel/docker-airflow:1.7.1.3-2
         restart: always
+        depends_on:
+            - webserver
         # volumes:
         #     - /localpath/to/dags:/usr/local/airflow/dags
         environment:
-            - AIRFLOW_HOME=/usr/local/airflow
+            # - LOAD_EX=n
+            - EXECUTOR=Celery
         command: scheduler -n 5
 
     worker:
-        image: puckel/docker-airflow:1.7.1.3
+        image: puckel/docker-airflow:1.7.1.3-2
         restart: always
+        depends_on:
+            - scheduler
         # volumes:
         #     - /localpath/to/dags:/usr/local/airflow/dags
         environment:
-            - AIRFLOW_HOME=/usr/local/airflow
+            - EXECUTOR=Celery
         command: worker

docker-compose-LocalExecutor.yml

@@ -0,0 +1,34 @@
+version: '2'
+services:
+    postgres:
+        image: postgres
+        environment:
+            - POSTGRES_USER=airflow
+            - POSTGRES_PASSWORD=airflow
+            - POSTGRES_DB=airflow
+
+    webserver:
+        image: puckel/docker-airflow:1.7.1.3-2
+        restart: always
+        depends_on:
+            - postgres
+        environment:
+            # - LOAD_EX=n
+            - EXECUTOR=Local
+        # volumes:
+        #     - /localpath/to/dags:/usr/local/airflow/dags
+        ports:
+            - "8080:8080"
+        command: webserver
+
+    scheduler:
+        image: puckel/docker-airflow:1.7.1.3-2
+        restart: always
+        depends_on:
+            - webserver
+        # volumes:
+        #     - /localpath/to/dags:/usr/local/airflow/dags
+        environment:
+            # - LOAD_EX=n
+            - EXECUTOR=Local
+        command: scheduler -n 5

script/entrypoint.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+AIRFLOW_HOME="/usr/local/airflow"
 CMD="airflow"
 TRY_LOOP="10"
 POSTGRES_HOST="postgres"
@@ -8,28 +9,19 @@ RABBITMQ_HOST="rabbitmq"
 RABBITMQ_CREDS="airflow:airflow"
 FERNET_KEY=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print FERNET_KEY")
 
-# Generate Fernet key
-sed -i "s/{FERNET_KEY}/${FERNET_KEY}/" $AIRFLOW_HOME/airflow.cfg
-
-# wait for rabbitmq
-if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] || [ "$1" = "flower" ] ; then
-  j=0
-  while ! curl -sI -u $RABBITMQ_CREDS http://$RABBITMQ_HOST:15672/api/whoami |grep '200 OK'; do
-    j=`expr $j + 1`
-    if [ $j -ge $TRY_LOOP ]; then
-      echo "$(date) - $RABBITMQ_HOST still not reachable, giving up"
-      exit 1
-    fi
-    echo "$(date) - waiting for RabbitMQ... $j/$TRY_LOOP"
-    sleep 5
-  done
+# Load DAGs exemples (default: Yes)
+if [ "x$LOAD_EX" = "xn" ]; then
+    sed -i "s/load_examples = True/load_examples = False/" "$AIRFLOW_HOME"/airflow.cfg
 fi
 
+# Generate Fernet key
+sed -i "s|\$FERNET_KEY|$FERNET_KEY|" "$AIRFLOW_HOME"/airflow.cfg
+
 # wait for DB
 if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] ; then
   i=0
   while ! nc $POSTGRES_HOST $POSTGRES_PORT >/dev/null 2>&1 < /dev/null; do
-    i=`expr $i + 1`
+    i=$((i+1))
     if [ $i -ge $TRY_LOOP ]; then
       echo "$(date) - ${POSTGRES_HOST}:${POSTGRES_PORT} still not reachable, giving up"
       exit 1
@@ -44,4 +36,34 @@ if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] ; the
   sleep 5
 fi
 
-exec $CMD "$@"
+# If we use docker-compose, we use Celery (rabbitmq container).
+if [ "x$EXECUTOR" = "xCelery" ]
+then
+# wait for rabbitmq
+  if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] || [ "$1" = "flower" ] ; then
+    j=0
+    while ! curl -sI -u $RABBITMQ_CREDS http://$RABBITMQ_HOST:15672/api/whoami |grep '200 OK'; do
+      j=$((j+1))
+      if [ $j -ge $TRY_LOOP ]; then
+        echo "$(date) - $RABBITMQ_HOST still not reachable, giving up"
+        exit 1
+      fi
+      echo "$(date) - waiting for RabbitMQ... $j/$TRY_LOOP"
+      sleep 5
+    done
+  fi
+  exec $CMD "$@"
+elif [ "x$EXECUTOR" = "xLocal" ]
+then
+  sed -i "s/executor = CeleryExecutor/executor = LocalExecutor/" "$AIRFLOW_HOME"/airflow.cfg
+  exec $CMD "$@"
+else
+  if [ "$1" = "version" ]; then
+    exec $CMD version
+  fi
+  sed -i "s/executor = CeleryExecutor/executor = SequentialExecutor/" "$AIRFLOW_HOME"/airflow.cfg
+  sed -i "s#sql_alchemy_conn = postgresql+psycopg2://airflow:airflow@postgres/airflow#sql_alchemy_conn = sqlite:////usr/local/airflow/airflow.db#" "$AIRFLOW_HOME"/airflow.cfg
+  echo "Initialize database..."
+  $CMD initdb
+  exec $CMD webserver
+fi