Enforce the load order of components defined in env.local

.github/workflows/tests.yml

@@ -0,0 +1,21 @@
+name: Unit tests
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+jobs:
+  test:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up python for testing
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: 'pip'
+      - name: Install python test dependencies
+        run: |
+          pip install -r ./tests/requirements.txt
+      - name: Test with pytest
+        run: |
+          pytest ./tests/

.gitignore

@@ -23,3 +23,9 @@ ubuntu-bionic-18.04-cloudimg-console.log
 ## Python temp files
 **/__pycache__
 **/*.py[cod]
+
+## Virtual environments
+venv/
+
+## Testing
+.pytest_cache/

CHANGES.md

@@ -56,13 +56,50 @@
 
   the load order is:
 
-  - ./config/twitcher
   - ./config/magpie (loaded as a dependency of twitcher, not loaded a second time after project-api)
-  - ./config/project-api 
-  - ./optional-components/generic_bird
+  - ./config/twitcher
+  - ./config/project-api
   - ./config/wps_outputs-volume (loaded as a dependency of generic_bird)
+  - ./optional-components/generic_bird
   - ./components/cowbird
 
+  This load order also applies to the order that docker-compose-extra.yml files are specified. If a component also
+  includes an override file for another component (eg: ./config/finch/config/proxy/docker-compose-extra.yml overrides 
+  ./config/proxy/docker-compose-extra.yml), the following additional load order rules apply:
+
+  - if the component that is being overridden has already been loaded, the override file is loaded immediately
+  - otherwise, the override files will be loaded immediately after the component that is being overridden has been loaded
+
+  For example, with the following files in place:
+
+    ```shell
+  # env.local
+  DEFAULT_CONF_DIRS="
+    ./config/finch
+    ./config/proxy
+  "
+  ```
+  ```yaml
+  # config/proxy/docker-compose-extra.yml
+    ...
+  # config/finch/docker-compose-extra.yml
+    ...
+  # config/finch/config/proxy/docker-compose-extra.yml
+    ...
+  ```
+
+  the docker compose files will be loaded in the following order: 
+
+  - config/finch/docker-compose-extra.yml
+  - config/proxy/docker-compose-extra.yml
+  - config/finch/config/proxy/docker-compose-extra.yml
+
+- Add tests to ensure override capabilities are preserved which allows all default
+  behaviors of the platform can be customized.
+
+  See [birdhouse/README.rst](birdhouse/README.rst) for instruction to run the
+  tests.
+
 [1.25.1](https://github.com/bird-house/birdhouse-deploy/tree/1.25.1) (2023-04-11)
 ------------------------------------------------------------------------------------------------------------------
 

birdhouse/README.rst

@@ -202,6 +202,18 @@ Starting and managing the lifecycle of the VM:
    vagrant provision
 
 
+Framework tests
+---------------
+
+Core features of the platform has tests to prevent regressions.
+
+To run the tests:
+
+.. code-block::
+    python3 -m pip install -r tests/requirements.txt
+    pytest tests/
+
+
 Tagging policy
 --------------
 

birdhouse/config/canarie-api/default.env

@@ -5,7 +5,6 @@ export PROXY_IMAGE="pavics/canarieapi:0.3.5"
 
 # add any component that this component requires to run
 COMPONENT_DEPENDENCIES="
-  $COMPONENT_DEPENDENCIES
   ./config/proxy
 "
 

birdhouse/config/proxy/default.env

@@ -29,8 +29,3 @@ export OPTIONAL_VARS="
   \$PROXY_READ_TIMEOUT_VALUE
   \$PROXY_ROOT_LOCATION
 "
-
-# add any component that this component requires to run
-COMPONENT_DEPENDENCIES="
-  ./config/canarie-api
-"

birdhouse/default.env

@@ -19,6 +19,7 @@ export DELAYED_EVAL=''
 
 export DEFAULT_CONF_DIRS='
   ./config/proxy
+  ./config/canarie-api
   ./config/phoenix
   ./config/geoserver
   ./config/flyingpigeon
@@ -35,5 +36,4 @@ export DEFAULT_CONF_DIRS='
   ./config/catalog
   ./config/malleefowl
   ./config/solr
-  ./config/canarie-api
 '

birdhouse/optional-components/canarie-api-full-monitoring/default.env

@@ -2,5 +2,5 @@
 
 # add any component that this component requires to run
 COMPONENT_DEPENDENCIES="
-  ./config/proxy
+  ./config/canarie-api
 "

birdhouse/pavics-compose.sh

@@ -104,30 +104,7 @@ if [ x"$1" = x"up" ]; then
   done
 fi
 
-COMPOSE_CONF_LIST="-f docker-compose.yml"
-for adir in $ALL_CONF_DIRS; do
-  if [ -f "$adir/docker-compose-extra.yml" ]; then
-    COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f $adir/docker-compose-extra.yml"
-  fi
-done
-CONFIGURED_COMPONENTS=''
-for adir in $ALL_CONF_DIRS; do
-  CONFIGURED_COMPONENTS="
-    $CONFIGURED_COMPONENTS
-    $(basename $adir)"
-done
-
-for adir in $ALL_CONF_DIRS; do
-  for conf_dir in "$adir"/config/*; do
-    service_name=$(basename "$conf_dir")
-    extra_compose="$conf_dir/docker-compose-extra.yml"
-    if [ -f "$extra_compose" ]; then
-      if echo "$CONFIGURED_COMPONENTS" | grep -q "$service_name"; then
-        COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f $extra_compose"
-      fi
-    fi
-  done
-done
+create_compose_conf_list # this sets COMPOSE_CONF_LIST
 echo "COMPOSE_CONF_LIST=${COMPOSE_CONF_LIST}"
 
 # the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script

birdhouse/read-configs.include.sh

@@ -66,17 +66,19 @@ read_default_env() {
 read_env_local() {
     # we don't use usual .env filename, because docker-compose uses it
 
-    if [ -e "$COMPOSE_DIR/env.local" ]; then
+    echo "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV:="$COMPOSE_DIR/env.local"}"
+
+    if [ -e "$BIRDHOUSE_LOCAL_ENV" ]; then
         saved_shell_options="$(set +o)"
         set +xv  # hide passwd in env.local in logs
 
-        . "$COMPOSE_DIR/env.local"
+        . "$BIRDHOUSE_LOCAL_ENV"
 
         # restore saved shell options
         eval "$saved_shell_options"
 
     else
-        echo "WARNING: '$COMPOSE_DIR/env.local' not found" 1>&2
+        echo "WARNING: '$BIRDHOUSE_LOCAL_ENV' not found" 1>&2
     fi
 
 }
@@ -93,23 +95,37 @@ source_conf_files() {
           # ignore directories that are already in ALL_CONF_DIRS
           continue
       fi
-      if [ -e "$adir" ]; then
-          ALL_CONF_DIRS="$ALL_CONF_DIRS
-            $adir
-          "
-      else
+      # push current adir onto the stack (this helps keep track of current adir when recursing)
+      _adir_stack="$_adir_stack\n$adir"
+      if [ ! -e "$adir" ]; then
           # Do not exit to not break unattended autodeploy since no human around to
           # fix immediately.
           # The new adir with typo will not be active but at least all the existing
           # will still work.
           echo "WARNING: '$adir' in $conf_locations does not exist" 1>&2
       fi
-      COMPONENT_DEFAULT_ENV="$adir/default.env"
-      if [ -f "$COMPONENT_DEFAULT_ENV" ]; then
-          echo "reading '$COMPONENT_DEFAULT_ENV'"
-          . "$COMPONENT_DEFAULT_ENV"
-          source_conf_files "$COMPONENT_DEPENDENCIES" "a dependency of $adir"
+      if [ -f "$adir/default.env" ]; then
+          # Source config settings of dependencies first if they haven't been sourced previously.
+          # Note that this will also define the order that docker-compose-extra.yml files will be loaded.
+          unset COMPONENT_DEPENDENCIES
+          dependencies="$(. "$adir/default.env" && echo "$COMPONENT_DEPENDENCIES")"
+          if [ -n "$dependencies" ]; then
+            source_conf_files "$dependencies" "a dependency of $adir"
+            # reset the adir variable in case it was changed in a recursive call
+            adir="$(printf '%b' "$_adir_stack" | tail -1)"
+          fi
+          echo "reading '$adir/default.env'"
+          . "$adir/default.env"
+      fi
+      if echo "$ALL_CONF_DIRS" | grep -qE "^\s*$adir\s*$"; then
+          # check again in case a dependency has already added this to the ALL_CONF_DIRS variable
+          continue
       fi
+      ALL_CONF_DIRS="$ALL_CONF_DIRS
+        $adir
+      "
+      # pop current adir from the stack once we're done with it
+      _adir_stack="$(printf '%b' "$_adir_stack" | sed '$d')"
   done
 }
 
@@ -151,6 +167,60 @@ process_delayed_eval() {
 }
 
 
+# Sets the COMPOSE_CONF_LIST variable to a string that contains all the docker-compose*.yml files that are
+# required to pass to the docker compose command. This is determined by the contents of the ALL_CONF_DIRS
+# variable (set using the read_components_default_env function).
+# The file order is determined by the order of the config directories in ALL_CONF_DIRS.
+# If a config directory also includes an override file for another component
+# (eg: ./config/finch/config/proxy/docker-compose-extra.yml overrides ./config/proxy/docker-compose-extra.yml),
+# the following additional order rules apply:
+#
+#  - if the component that is being overridden has already been added, the override file is added immediately
+#  - otherwise, the override files will be added immediately after the component that is being overridden
+#    has been added
+create_compose_conf_list() {
+  # ALL_CONF_DIRS relative paths are relative to COMPOSE_DIR.
+  discover_compose_dir
+  if [ -d "$COMPOSE_DIR" ]; then
+      cd "$COMPOSE_DIR" || return
+  fi
+
+  COMPOSE_CONF_LIST="-f docker-compose.yml"
+  COMPONENT_OVERRIDES=''
+  LOADED_COMPONENTS=''
+  for adir in $ALL_CONF_DIRS; do
+    service_name=$(basename "$adir")
+    LOADED_COMPONENTS="${LOADED_COMPONENTS}\n${service_name}"
+
+    if [ -f "$adir/docker-compose-extra.yml" ]; then
+      COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f $adir/docker-compose-extra.yml"
+    fi
+
+    # If previously loaded components specified overrides for the component that was just loaded, load those overrides now
+    previous_overrides=$(printf '%b' "${COMPONENT_OVERRIDES}" | grep "^$service_name " | sed "s/^${service_name}//g" | tr '\n' ' ')
+    COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} ${previous_overrides}"
+    # Load overrides for other components unless the component to be overridden hasn't been loaded yet. If the component
+    # hasn't been loaded yet, store a reference to it so that it can be added in as soon as the component is loaded.
+    for conf_dir in "$adir"/config/*; do
+      override_service_name=$(basename "$conf_dir")
+      extra_compose="$conf_dir/docker-compose-extra.yml"
+      if [ -f "$extra_compose" ]; then
+        if printf '%b' "${LOADED_COMPONENTS}" | grep -q "^$override_service_name$"; then
+          COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f $extra_compose"
+        else
+          COMPONENT_OVERRIDES="${COMPONENT_OVERRIDES}\n${override_service_name} -f ${extra_compose}"
+        fi
+      fi
+    done
+  done
+
+    # Return to previous pwd.
+  if [ -d "$COMPOSE_DIR" ]; then
+      cd - || return
+  fi
+}
+
+
 # Main function to read all config files in appropriate order and call
 # process_delayed_eval() at the appropriate moment.
 read_configs() {

tests/requirements.txt

@@ -0,0 +1 @@
+pytest==7.2.2