[Test Main]: Update python-package.yml

.github/workflows/python-package.yml

@@ -1,21 +1,15 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: Python package
-
 on:
   push:
     branches: [ main ]
   pull_request:
     branches: [ main ]
 
 jobs:
-
   build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.11]
+        python-version: [3.9, 3.11, 3.12]
 
     services:
       mysqldb_retriever:
@@ -35,8 +29,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-
-      - name: Set up Postgresql with Postgis
+      
+      - name: Set up PostgreSQL with PostGIS
         uses: huaxk/postgis-action@v1
         with:
           postgresql version: '11'
@@ -49,63 +43,52 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Install system dependencies
         env:
-          LC_ALL : en_US.UTF-8
-          ENV LANG : en_US.UTF-8
-          ENV LANGUAGE : en_US.UTF-8
-          TZ: America/New_York
           DEBIAN_FRONTEND: noninteractive
         run: |
-          sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
-          sudo apt-get update && sudo apt-get install -y --no-install-recommends apt-utils
-          sudo apt-get install -y --force-yes tzdata
-          sudo apt-get install -y --force-yes build-essential wget git locales locales-all > /dev/null
-          sudo apt-get install -y --force-yes libpq-dev
-          sudo apt-get install libgdal-dev
-          sudo apt-get install gdal-bin
-          export CPLUS_INCLUDE_PATH=/usr/include/gdal
-          export C_INCLUDE_PATH=/usr/include/gdal
-          pip install GDAL
-          sudo apt-get install -y --force-yes postgis
-
+          sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y
+          sudo apt-get update && sudo apt-get install -y \
+            build-essential wget git locales locales-all \
+            libpq-dev libgdal-dev gdal-bin postgis
+          echo "export CPLUS_INCLUDE_PATH=/usr/include/gdal" >> ~/.bashrc
+          echo "export C_INCLUDE_PATH=/usr/include/gdal" >> ~/.bashrc
+          echo "export GDAL_VERSION=$(gdalinfo --version | cut -d' ' -f2)" >> ~/.bashrc
+          source ~/.bashrc
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --no-cache-dir --find-links=https://ci.gdal.org/job/gdal-wheels/lastSuccessfulBuild/artifact/gdal/dist/ GDAL
+          pip install -r requirements.txt || true
+
       - name: Setup paths and files
-        run : |
-          chmod 0755 cli_tools/entrypoint.sh
+        run: |
+          chmod +x cli_tools/entrypoint.sh
           sh cli_tools/entrypoint.sh
-          echo "export PATH="/usr/bin/python:$PATH"" >> ~/.profile
-          echo "export PYTHONPATH="/usr/bin/python:$PYTHONPATH"" >> ~/.profile
-          echo "export PGPASSFILE="~/.pgpass"" >> ~/.profile
-          chmod 0644 ~/.profile
+          echo "export PATH=/usr/bin/python:$PATH" >> ~/.profile
+          echo "export PYTHONPATH=/usr/bin/python:$PYTHONPATH" >> ~/.profile
+          echo "export PGPASSFILE=~/.pgpass" >> ~/.profile
+          chmod 600 ~/.pgpass ~/.my.cnf
 
       - name: PostGIS check version
-        run : |
+        run: |
           export PGPASSWORD='Password12!'
           psql -d testdb_retriever -U postgres -h localhost -p 5432 -c "SELECT PostGIS_version();"
-
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8 pytest yapf codecov pytest-cov pytest-xdist openpyxl -U
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
       - name: Install retriever
         run: |
           pip install -e .
-          export PGPASSFILE="~/.pgpass"
-          chmod 600 ~/.pgpass
-          chmod 600 ~/.my.cnf
-
-      - name: Run pytest
+          chmod 600 ~/.pgpass ~/.my.cnf
+
+      - name: Run tests and linting
         run: |
-          pytest -v -k "not test_geojson2csv  and not test_mysql_integration" --cov=./retriever --cov-report=xml
-          flake8 --ignore=E501,W503,E402,F401,F403,E722,F841,W504 retriever --max-line-length=90 2>&1
-          yapf -d --recursive ./retriever/ --style=.style.yapf 2>&1
-
+          pytest -v -k "not test_geojson2csv and not test_mysql_integration" --cov=./retriever --cov-report=xml
+          flake8 --ignore=E501,W503,E402,F401,F403,E722,F841,W504 retriever --max-line-length=90
+          yapf -d --recursive ./retriever/ --style=.style.yapf
+      
       - name: Upload coverage to Codecov
-        if: matrix.python-version == 3.8
+        if: matrix.python-version == 3.9
         uses: codecov/codecov-action@v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-
-

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Download base image ubuntu .04
-FROM osgeo/gdal
+FROM osgeo/gdal:ubuntu-small-3.6.3
 
 MAINTAINER Weecology "https://github.com/weecology/retriever"
 

retriever/lib/scripts.py

@@ -1,5 +1,6 @@
 import csv
-import imp
+import importlib
+import importlib.util
 import io
 import os
 import re
@@ -80,27 +81,27 @@ def reload_scripts():
             script_name = ".".join(script.split(".")[:-1])
             if script_name not in loaded_files:
                 loaded_files.append(script_name)
-                file, pathname, desc = imp.find_module(script_name, [search_path])
-                try:
-                    new_module = imp.load_module(script_name, file, pathname, desc)
-                    if not hasattr(new_module, "SCRIPT"):
-                        continue
-                    if hasattr(new_module.SCRIPT, "retriever_minimum_version"):
-                        # a script with retriever_minimum_version should be loaded
-                        # only if its compliant with the version of the retriever
-                        if not check_retriever_minimum_version(new_module.SCRIPT):
-                            continue
-                    # if the script wasn't found in an early search path
-                    # make sure it works and then add it
-                    new_module.SCRIPT.download  # pylint: disable=W0104
-                    setattr(new_module.SCRIPT, "_file", os.path.join(search_path, script))
-                    setattr(new_module.SCRIPT, "_name", script_name)
-                    modules.append(new_module.SCRIPT)
-                except Exception as e:
-                    raise (e)
-                    sys.stderr.write("Failed to load script: {} ({})\n"
-                                     "Exception: {} \n".format(script_name, search_path,
-                                                               str(e)))
+                spec = importlib.util.find_spec(script_name, [search_path])
+                if spec:
+                    try:
+                        new_module = importlib.util.module_from_spec(spec)
+                        spec.loader.exec_module(new_module)
+                        if hasattr(new_module.SCRIPT, "retriever_minimum_version"):
+                            # a script with retriever_minimum_version should be loaded
+                            # only if its compliant with the version of the retriever
+                            if not check_retriever_minimum_version(new_module.SCRIPT):
+                                continue
+                        # if the script wasn't found in an early search path
+                        # make sure it works and then add it
+                        new_module.SCRIPT.download  # pylint: disable=W0104
+                        setattr(new_module.SCRIPT, "_file", os.path.join(search_path, script))
+                        setattr(new_module.SCRIPT, "_name", script_name)
+                        modules.append(new_module.SCRIPT)
+                    except Exception as e:
+                        raise (e)
+                        sys.stderr.write("Failed to load script: {} ({})\n"
+                                         "Exception: {} \n".format(script_name, search_path,
+                                                                  str(e)))
     if global_script_list:
         global_script_list.set_scripts(modules)
     return modules
@@ -274,11 +275,14 @@ def get_script_upstream(dataset, repo=REPOSITORY):
         setattr(read_script, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name))
         setattr(read_script, "_name", script)
         return read_script
-    file, pathname, desc = imp.find_module(script, [SCRIPT_WRITE_PATH])
-    new_module = imp.load_module(script, file, pathname, desc)
-    setattr(new_module.SCRIPT, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name))
-    setattr(new_module.SCRIPT, "_name", script)
-    return new_module.SCRIPT
+    spec = importlib.util.find_spec(script, [SCRIPT_WRITE_PATH])
+    if spec:
+        new_module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(new_module)
+        setattr(new_module.SCRIPT, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name))
+        setattr(new_module.SCRIPT, "_name", script)
+        return new_module.SCRIPT
+    return None
 
 
 def get_script_version_upstream(dataset, repo=REPOSITORY):
@@ -461,11 +465,13 @@ def read_json_version(json_file):
 
 def read_py_version(script_name, search_path):
     """Read the version of a script from a python file"""
-    file, pathname, desc = imp.find_module(script_name, [search_path])
+    spec = importlib.util.find_spec(script_name, [search_path])
     try:
-        new_module = imp.load_module(script_name, file, pathname, desc)
-        if hasattr(new_module.SCRIPT, "version"):
-            return new_module.SCRIPT.version
+        if spec:
+            new_module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(new_module)
+            if hasattr(new_module.SCRIPT, "version"):
+                return new_module.SCRIPT.version
     except:
         pass
     return None