Replace DataPusher with XLoader (Compose and CKAN Dockerfile changes)

README.md

@@ -369,7 +369,28 @@ The base image used in the CKAN Dockerfile and Dockerfile.dev can be changed so
 
 ## 13. Replacing DataPusher with XLoader
 
-Check out the wiki page for this: https://github.com/ckan/ckan-docker/wiki/Replacing-DataPusher-with-XLoader
+Basically it's just replacing files copied from the xloader directory. Depending on if you are running in Production mode or Development mode, here are the instructions you should use:
+
+##### For Production (base)
+
+From the top (root) directory of your docker installation:
+
+ 1. `cd xloader/`
+ 2. `cp .env ..`
+ 3. `cp docker-compose.yml ..`
+ 4. `cp Dockerfile ../ckan`
+ 5. `cp -pr docker-entrypoint.d/ ../ckan`
+
+##### For Development
+
+From the top (root) directory of your docker installation:
+
+ 1. `cd xloader/`
+ 2. `cp .env ..`
+ 3. `cp docker-compose-dev.yml ..`
+ 4. `cp Dockerfile.dev ../ckan`
+ 5. `cp -pr docker-entrypoint.d/ ../ckan`
+
 
 Copying and License
 -------------------

xloader/Dockerfile

@@ -0,0 +1,32 @@
+FROM ckan/ckan-base:2.11
+
+ARG XLOADER_VERSION=${XLOADER_VERSION}
+
+# Install any extensions needed by your CKAN instance
+# See Dockerfile.dev for more details and examples
+
+USER root
+
+# Install the XLoader extension
+RUN pip3 install -U pip && \ 
+    cd ${SRC_DIR} && \
+    pip3 install -e git+https://github.com/ckan/ckanext-xloader.git@${XLOADER_VERSION}#egg=ckanext-xloader && \
+    pip3 install -r ckanext-xloader/requirements.txt && \
+    pip3 install -U requests[security]
+
+# Copy custom initialization scripts
+COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/
+
+# Apply any patches needed to CKAN core or any of the built extensions (not the
+# runtime mounted ones)
+COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches
+
+USER ckan
+
+RUN for d in $APP_DIR/patches/*; do \
+        if [ -d $d ]; then \
+            for f in `ls $d/*.patch | sort -g`; do \
+                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+            done ; \
+        fi ; \
+    done

xloader/Dockerfile.dev

@@ -0,0 +1,66 @@
+FROM ckan/ckan-dev:2.11
+
+ARG XLOADER_VERSION=${XLOADER_VERSION}
+
+# Install any extensions needed by your CKAN instance
+# - Make sure to add the plugins to CKAN__PLUGINS in the .env file
+# - Also make sure all provide all extra configuration options, either by:
+#   * Adding them to the .env file (check the ckanext-envvars syntax for env vars), or
+#   * Adding extra configuration scripts to /docker-entrypoint.d folder) to update
+#      the CKAN config file (ckan.ini) with the `ckan config-tool` command
+#
+# See README > Extending the base images for more details
+#
+# For instance:
+#
+### XLoader ###
+#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ 
+#    pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \
+#    pip3 install -U requests[security]
+
+### Harvester ###
+#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest' && \
+#    pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt
+# will also require gather_consumer and fetch_consumer processes running (please see https://github.com/ckan/ckanext-harvest)
+
+### Scheming ###
+#RUN  pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming'
+
+### Pages ###
+#RUN  pip3 install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages
+
+### DCAT ###
+#RUN  pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
+#     pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
+
+# Clone the extension(s) your are writing for your own project in the `src` folder
+# to get them mounted in this image at runtime
+
+# Install any extensions needed by your CKAN instance
+# See Dockerfile.dev for more details and examples
+
+USER root
+
+# Install the XLoader extension
+RUN pip3 install -U pip && \ 
+    cd ${SRC_DIR} && \
+    pip3 install -e git+https://github.com/ckan/ckanext-xloader.git@${XLOADER_VERSION}#egg=ckanext-xloader && \
+    pip3 install -r ckanext-xloader/requirements.txt && \
+    pip3 install -U requests[security]
+
+# Copy custom initialization scripts
+COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/
+
+# Apply any patches needed to CKAN core or any of the built extensions (not the
+# runtime mounted ones)
+COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches
+
+USER ckan
+
+RUN for d in $APP_DIR/patches/*; do \
+        if [ -d $d ]; then \
+            for f in `ls $d/*.patch | sort -g`; do \
+                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+            done ; \
+        fi ; \
+    done

xloader/docker-compose.dev.yml

@@ -0,0 +1,81 @@
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data:
+  pip_cache:
+  site_packages:
+  local_bin:
+  home_dir:
+
+services:
+
+  ckan-dev:
+    build:
+      context: ckan/
+      dockerfile: Dockerfile.dev
+      args:
+        - TZ=${TZ}
+        - XLOADER_VERSION=${XLOADER_VERSION}       
+    env_file:
+      - .env
+    links:
+      - db
+      - solr
+      - redis
+    ports:
+      - "0.0.0.0:${CKAN_PORT_HOST}:5000"
+    volumes:
+      - ckan_storage:/var/lib/ckan
+      - ./src:/srv/app/src_extensions
+      - pip_cache:/root/.cache/pip
+      - site_packages:/usr/local/lib/python3.10/site-packages
+      - local_bin:/usr/local/bin
+      - home_dir:/srv/app/
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+
+  xloader:
+    image: ckan/ckan-base-xloader:${XLOADER_VERSION}
+    env_file:
+      - .env
+    restart: unless-stopped
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
+
+  db:
+    build:
+      context: postgresql/
+    environment:
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - CKAN_DB_USER
+      - CKAN_DB_PASSWORD
+      - CKAN_DB
+      - DATASTORE_READONLY_USER
+      - DATASTORE_READONLY_PASSWORD
+      - DATASTORE_DB
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+
+  solr:
+    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    volumes:
+      - solr_data:/var/solr
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
+
+  redis:
+    image: redis:${REDIS_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "-e", "QUIT"]

xloader/docker-compose.yml

@@ -0,0 +1,116 @@
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data:
+  pip_cache:
+  site_packages:
+
+services:
+
+  nginx:
+    build:
+      context: nginx/
+      dockerfile: Dockerfile
+    networks:
+      - webnet
+      - ckannet
+    depends_on:
+      ckan:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}"
+
+  ckan:
+    build:
+      context: ckan/
+      dockerfile: Dockerfile
+      args:
+        - TZ=${TZ}
+        - XLOADER_VERSION=${XLOADER_VERSION}
+    networks:
+      - ckannet
+      - dbnet
+      - solrnet
+      - redisnet
+    env_file:
+      - .env
+    depends_on:
+      db:
+        condition: service_healthy
+      solr:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    volumes:
+      - ckan_storage:/var/lib/ckan
+      - pip_cache:/root/.cache/pip
+      - site_packages:/usr/lib/python3.10/site-packages
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000/api/action/status_show"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+
+  xloader:
+    image: ckan/ckan-base-xloader:${XLOADER_VERSION}
+    env_file:
+      - .env
+    networks:
+      - ckannet
+      - dbnet
+      - solrnet
+      - redisnet
+    restart: unless-stopped
+    depends_on:
+      ckan:
+        condition: service_healthy
+
+  db:
+    build:
+      context: postgresql/
+    networks:
+      - dbnet
+    environment:
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - CKAN_DB_USER
+      - CKAN_DB_PASSWORD
+      - CKAN_DB
+      - DATASTORE_READONLY_USER
+      - DATASTORE_READONLY_PASSWORD
+      - DATASTORE_DB
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+
+  solr:
+    networks:
+      - solrnet
+    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    volumes:
+      - solr_data:/var/solr
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
+
+  redis:
+    image: redis:${REDIS_VERSION}
+    networks:
+      - redisnet
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "-e", "QUIT"]
+
+networks:
+  webnet:
+  ckannet:
+  solrnet:
+    internal: true
+  dbnet:
+    internal: true
+  redisnet:
+    internal: true

xloader/docker-entrypoint.d/02_setup_xloader.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ $CKAN__PLUGINS == *"xloader"* ]]; then
+   # Datapusher settings have been configured in the .env file
+   # Set API token if necessary
+   if [ -z "$CKAN__XLOADER__API_TOKEN" ] ; then
+      echo "Set up ckanext.xloader.api_token in the CKAN config file"
+      ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')"
+   fi
+else
+   echo "Not configuring xloader"
+fi

xloader/docker-entrypoint.d/README.md

@@ -0,0 +1,4 @@
+Use scripts in this folder to run extra initialization steps in your custom CKAN images.
+Any file with `.sh` or `.py` extension will be executed just after the main initialization
+script (`prerun.py`) is executed and just before the web server and supervisor processes are
+started.