Upgrade ubuntu base image to 24.04 LTS (#13616)

mattiagiupponi · web-flow · commit 02a761994daa · 2025-10-17T14:16:46.000+02:00
* Upgrade gdal to 3.11.4

* Upgrade gdal to 3.11.4

* Add base image with gdal upgrade

* upgrade base ubuntu image and gdal upgrade

* upgrade base ubuntu image and gdal

* minor fix on gdal exec path and geoserver upgrade

* Upgrade ubuntu to 24.04, gdal upgrade and pyton3.12

* Upgrade ubuntu to 24.04, gdal upgrade and pyton3.12

* Upgrade ubuntu to 24.04, gdal upgrade and pyton3.12
diff --git a/.env_dev b/.env_dev
@@ -210,5 +210,5 @@ RESTART_POLICY_WINDOW=120s
 
 DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5
 IMPORTER_ENABLE_DYN_MODELS=True
-UPSERT_CHUNK_SIZE= 10
+UPSERT_CHUNK_SIZE= 100
 UPSERT_LIMIT_ERROR_LOG=100
diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ '3.10', '3.11']
+        python-version: [ '3.12']
 
     runs-on: ubuntu-latest
 
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM geonode/geonode-base:latest-ubuntu-22.04
+FROM geonode/geonode-base:latest-ubuntu-24.04
 LABEL GeoNode development team
 
 # copy local geonode src inside container
@@ -22,6 +22,7 @@ RUN chmod +x /usr/bin/celery-cmd
 # RUN cd /usr/src/geonode-contribs/geonode-logstash; pip install --upgrade  -e . \
 #     cd /usr/src/geonode-contribs/ldap; pip install --upgrade  -e .
 
+RUN . /usr/src/venv/bin/activate
 RUN yes w | pip install --src /usr/src -r requirements.txt &&\
     yes w | pip install -e .
 
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -11,6 +11,7 @@ x-common-django:
   env_file:
     - .env
   volumes:
+    - .:/usr/src/geonode/
     - statics:/mnt/volumes/statics
     - geoserver-data-dir:/geoserver_data/data
     - backup-restore:/backup_restore
@@ -91,7 +92,7 @@ services:
 
   # Geoserver backend
   geoserver:
-    image: geonode/geoserver:2.27.2-latest
+    image: geonode/geoserver:2.27.x-latest
     container_name: geoserver4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows"
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -2,14 +2,15 @@
 
 # Exit script in case of error
 set -e
+source /usr/src/venv/bin/activate
 
 INVOKE_LOG_STDOUT=${INVOKE_LOG_STDOUT:-FALSE}
 invoke () {
     if [ $INVOKE_LOG_STDOUT = 'true' ] || [ $INVOKE_LOG_STDOUT = 'True' ]
     then
-        /usr/local/bin/invoke $@
+        $(which invoke) $@
     else
-        /usr/local/bin/invoke $@ > /usr/src/geonode/invoke.log 2>&1
+        $(which invoke) $@ > /usr/src/geonode/invoke.log 2>&1
     fi
     echo "$@ tasks done"
 }
diff --git a/geonode/base/bbox_utils.py b/geonode/base/bbox_utils.py
@@ -119,7 +119,7 @@ def translate_polygons(
     """
     ref.: https://towardsdatascience.com/around-the-world-in-80-lines-crossing-the-antimeridian-with-python-and-shapely-c87c9b6e1513
     """
-    for polygon in geometry_collection:
+    for polygon in [geometry_collection]:
         (minx, _, maxx, _) = polygon.bounds
         if minx < -180:
             geo_polygon = affinity.translate(polygon, xoff=360)
diff --git a/geonode/base/management/commands/thesaurus_subcommands/list.py b/geonode/base/management/commands/thesaurus_subcommands/list.py
@@ -19,7 +19,7 @@ def list_thesauri():
         else:
             # DISABLED
             # [0..n]
-            card = f'[{t.card_min}..{t.card_max if t.card_max!=-1 else "N"}]  '
+            card = f'[{t.card_min}..{t.card_max if t.card_max != -1 else "N"}]  '
         logger.info(
             f'id:{t.id:2} sort:{t.order:3} {card} name={t.identifier.ljust(max_id_len)} title="{t.title}" URI:{t.about}\n'
         )
diff --git a/geonode/br/management/commands/utils/utils.py b/geonode/br/management/commands/utils/utils.py
@@ -287,7 +287,7 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol
                 f" -h {db_host} -p {str(db_port)} -d {db_name}"
                 f" -U {db_user} --role={db_user} "
                 f' -t "{table_name}" '
-                f' {"-c" if not preserve_tables else "" } '
+                f' {"-c" if not preserve_tables else ""} '
                 f" {os.path.join(source_folder, filename)} "
             )
             ret = subprocess.call(command, env={"PGPASSWORD": db_passwd}, shell=True)
diff --git a/geonode/facets/tests.py b/geonode/facets/tests.py
@@ -649,8 +649,8 @@ def test_group_facet_api_call(self):
             },
         }
 
-        url_filtered = f"{reverse('get_facet',args=['group'])}?filter{{group.in}}={self.group_admin.id}&include_topics=true&key={self.group_admin.id}"
-        url_base = f"{reverse('get_facet',args=['group'])}"
+        url_filtered = f"{reverse('get_facet', args=['group'])}?filter{{group.in}}={self.group_admin.id}&include_topics=true&key={self.group_admin.id}"
+        url_base = f"{reverse('get_facet', args=['group'])}"
 
         response_filtered = self.client.get(url_filtered)
         response_dict_filtered = response_filtered.json()
@@ -672,23 +672,23 @@ def test_group_facet_api_call(self):
 
     def test_group_facets_are_filtered_by_words(self):
         # there are some groups and the facets return them
-        url = f"{reverse('get_facet',args=['group'])}"
+        url = f"{reverse('get_facet', args=['group'])}"
 
         response = self.client.get(url)
         self.assertEqual(200, response.status_code, response.json())
 
         self.assertTrue(response.json().get("topics", {}).get("total", 0) > 0)
 
         # topic_contains with real name should return 1
-        url = f"{reverse('get_facet',args=['group'])}?topic_contains=UserAdmin"
+        url = f"{reverse('get_facet', args=['group'])}?topic_contains=UserAdmin"
         response = self.client.get(url)
 
         self.assertEqual(200, response.status_code, response.json())
 
         self.assertEqual(1, response.json().get("topics", {}).get("total", 0))
 
         # topic_contains with a random string to be searched for should be 0
-        url = f"{reverse('get_facet',args=['group'])}?topic_contains=abc123scfuqbrwefbasascgiu"
+        url = f"{reverse('get_facet', args=['group'])}?topic_contains=abc123scfuqbrwefbasascgiu"
         response = self.client.get(url)
 
         self.assertEqual(200, response.status_code, response.json())
diff --git a/geonode/harvesting/tests/test_tasks.py b/geonode/harvesting/tests/test_tasks.py
@@ -190,7 +190,8 @@ def test_handle_harvesting_error_cleans_up_harvest_execution(self):
 
     @mock.patch("geonode.harvesting.tasks.models.Harvester")
     def test_check_harvester_available(self, mock_harvester_model):
-        mock_harvester = mock.MagicMock(spec=models.Harvester).return_value
+
+        mock_harvester = mock_harvester_model
         mock_harvester_model.objects.get.return_value = mock_harvester
         tasks.check_harvester_available(1000)
         mock_harvester.update_availability.assert_called()
diff --git a/geonode/proxy/tests.py b/geonode/proxy/tests.py
@@ -182,21 +182,19 @@ class Response:
         url = "http://example.org/test/test/../../image.tiff"
 
         response = self.client.get(f"{self.proxy_url}?url={url}")
-        self.assertDictContainsSubset(
-            {
-                "Content-Type": "image/tiff",
-                "Vary": "Authorization, Accept-Language, Cookie, origin",
-                "X-Content-Type-Options": "nosniff",
-                "X-XSS-Protection": "1; mode=block",
-                "Referrer-Policy": "strict-origin-when-cross-origin",
-                "Cross-Origin-Opener-Policy": "same-origin",
-                "X-Frame-Options": "SAMEORIGIN",
-                "Content-Language": "en",
-                "Content-Length": "11",
-                "Content-Disposition": 'attachment; filename="filename.tif"',
-            },
-            dict(response.headers.copy()),
-        )
+        expected_subset = {
+            "Content-Type": "image/tiff",
+            "Vary": "Authorization, Accept-Language, Cookie, origin",
+            "X-Content-Type-Options": "nosniff",
+            "X-XSS-Protection": "1; mode=block",
+            "Referrer-Policy": "strict-origin-when-cross-origin",
+            "Cross-Origin-Opener-Policy": "same-origin",
+            "X-Frame-Options": "SAMEORIGIN",
+            "Content-Language": "en",
+            "Content-Length": "11",
+            "Content-Disposition": 'attachment; filename="filename.tif"',
+        }
+        self.assertTrue(expected_subset.items() <= dict(response.headers.copy()).items())
 
     def test_proxy_url_forgery(self):
         import geonode.proxy.views
diff --git a/geonode/security/tests.py b/geonode/security/tests.py
@@ -272,29 +272,29 @@ def test_authenticated_api_request_with_login_middleware(self):
             MIDDLEWARE=settings.MIDDLEWARE + ("geonode.security.middleware.LoginRequiredMiddleware",)
         ):
             response = self.client.get(reverse("users-list"))
-            self.assertEquals(response.status_code, 302, "LoginRequiredMiddleware redirects the anonymouse user")
+            self.assertEqual(response.status_code, 302, "LoginRequiredMiddleware redirects the anonymouse user")
             self.assertTrue(reverse("account_login") in response.url, "LoginRequiredMiddleware login redirection")
 
             admin = get_user_model().objects.filter(is_superuser=True).first()
 
             response = self.client.get(
                 reverse("users-list"), HTTP_AUTHORIZATION=f"Basic {base64.b64encode(b'admin:admin').decode()}"
             )
-            self.assertEquals(
+            self.assertEqual(
                 response.status_code, 200, "LoginRequiredMiddleware passed the Basic Auth without any issues"
             )
 
             access_token = get_or_create_token(admin)
             response = self.client.get(reverse("users-list"), HTTP_AUTHORIZATION=f"Bearer {access_token.token}")
-            self.assertEquals(
+            self.assertEqual(
                 response.status_code, 200, "LoginRequiredMiddleware passed the Bearer token without any issues"
             )
 
             with override_settings(
                 ENABLE_APIKEY_LOGIN=True,
             ):
                 response = self.client.get(f"{reverse('users-list')}?apikey={access_token.token}")
-                self.assertEquals(
+                self.assertEqual(
                     response.status_code, 200, "LoginRequiredMiddleware passed the API Ley param without any issues"
                 )
 
@@ -307,7 +307,7 @@ def test_authenticated_api_request_with_login_middleware(self):
                         "email": "user1withapyley@email.com",
                     },
                 )
-                self.assertEquals(
+                self.assertEqual(
                     response.status_code, 201, "LoginRequiredMiddleware allowed the API Key to create a new user"
                 )
                 Profile.objects.get(username="user1withapyley").delete()  # cleanup
diff --git a/geonode/upload/handlers/common/vector.py b/geonode/upload/handlers/common/vector.py
@@ -22,6 +22,7 @@
 from datetime import datetime
 from itertools import islice
 from pathlib import Path
+import shutil
 import tempfile
 from django.db import connections
 from geonode.security.permissions import _to_compact_perms
@@ -1252,14 +1253,14 @@ def _save_feature(self, data_chunk, model_obj, model_instance, upsert_key, valid
                 # if the key is not present, we can create a new instance
                 feature_to_save.append(model_instance(**feature_as_dict))
                 valid_create += 1
-            try:
-                schema_fields = [f.name for f in model_obj.fields.filter(kwargs__primary_key__isnull=True)]
-                model_instance.objects.bulk_create(
-                    feature_to_save, update_conflicts=True, update_fields=schema_fields, unique_fields=[upsert_key]
-                )
-            except Exception:
-                logger.exception("Error occurred during bulk upsert in upsert_data.")
-                raise UpsertException("An internal error occurred during upsert operation.")
+        try:
+            schema_fields = [f.name for f in model_obj.fields.filter(kwargs__primary_key__isnull=True)]
+            model_instance.objects.bulk_create(
+                feature_to_save, update_conflicts=True, update_fields=schema_fields, unique_fields=[upsert_key]
+            )
+        except Exception:
+            logger.exception("Error occurred during bulk upsert in upsert_data.")
+            raise UpsertException("An internal error occurred during upsert operation.")
 
         return valid_update, valid_create
 
@@ -1400,7 +1401,7 @@ def import_with_ogr2ogr(
     If the layer should be overwritten, the option is appended dynamically
     """
     try:
-        ogr_exe = "/usr/bin/ogr2ogr"
+        ogr_exe = shutil.which("ogr2ogr")
 
         options = orchestrator.load_handler(handler_module_path).create_ogr2ogr_command(
             files, original_name, ovverwrite_layer, alternate
@@ -1413,7 +1414,6 @@ def import_with_ogr2ogr(
             options += f" | PGPASSWORD={_datastore['PASSWORD']} psql -d {_datastore['NAME']} -h {_datastore['HOST']} -p {_datastore.get('PORT', 5432)} -U {_datastore['USER']} -f -"
 
         commands = [ogr_exe] + options.split(" ")
-
         process = Popen(" ".join(commands), stdout=PIPE, stderr=PIPE, shell=True)
         stdout, stderr = process.communicate()
 
diff --git a/geonode/upload/handlers/csv/handler.py b/geonode/upload/handlers/csv/handler.py
@@ -132,7 +132,7 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate):
         base_command = BaseVectorFileHandler.create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate)
         additional_option = ' -oo "GEOM_POSSIBLE_NAMES=geom*,the_geom*,wkt_geom" -oo "X_POSSIBLE_NAMES=x,long*" -oo "Y_POSSIBLE_NAMES=y,lat*"'
         return (
-            f"{base_command } -oo KEEP_GEOM_COLUMNS=NO -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} "
+            f"{base_command} -oo KEEP_GEOM_COLUMNS=NO -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} "
             + additional_option
         )
 
diff --git a/geonode/upload/handlers/geojson/handler.py b/geonode/upload/handlers/geojson/handler.py
@@ -130,4 +130,4 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate):
         """
 
         base_command = BaseVectorFileHandler.create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate)
-        return f"{base_command } -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}"
+        return f"{base_command} -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}"
diff --git a/geonode/upload/handlers/kml/handler.py b/geonode/upload/handlers/kml/handler.py
@@ -156,4 +156,4 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate):
         """
 
         base_command = BaseVectorFileHandler.create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate)
-        return f"{base_command } -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} --config OGR_SKIP LibKML"
+        return f"{base_command} -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} --config OGR_SKIP LibKML"
diff --git a/geonode/upload/handlers/shapefile/handler.py b/geonode/upload/handlers/shapefile/handler.py
@@ -182,7 +182,7 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate):
             additional_options.append(f"--config SHAPE_ENCODING {encoding}")
 
         return (
-            f"{base_command } -lco precision=no -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} "
+            f"{base_command} -lco precision=no -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} "
             + " ".join(additional_options)
         )
 
diff --git a/requirements.txt b/requirements.txt
@@ -49,7 +49,7 @@ git+https://github.com/geopython/pycsw.git@3.0.0-beta1#egg=pycsw
 pyproj<3.7.0
 OWSLib==0.31.0
 SQLAlchemy==2.0.43 # required by PyCSW
-Shapely==1.8.5.post1
+Shapely==2.1.2
 mercantile==1.2.1
 numpy==1.26.*
 Jinja2==3.1.4
@@ -93,8 +93,7 @@ gn-gsimporter==2.0.4
 gisdata==0.5.4
 
 # importer dependencies
-setuptools>=59,<81
-gdal<=3.6.4
+setuptools>=59
 pdok-geopackage-validator==0.8.5
 geonode-django-dynamic-model==0.4.0
 
diff --git a/scripts/docker/base/ubuntu/Dockerfile b/scripts/docker/base/ubuntu/Dockerfile
@@ -1,12 +1,12 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 RUN mkdir -p /usr/src/geonode
 
 ## Enable postgresql-client-15
 RUN apt-get update -y && apt-get install curl wget unzip gnupg2 -y
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 # will install python3.10
-RUN apt-get install lsb-core -y
+RUN apt-get install lsb-release -y
 RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee  /etc/apt/sources.list.d/pgdg.list
 
 # Prepraing dependencies
@@ -21,17 +21,21 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
     python3-all-dev python3-dev \
     python3-gdal python3-psycopg2 python3-ldap \
     python3-pip python3-lxml \
-    uwsgi uwsgi-plugin-python3 python3-gdbm python-is-python3 gdal-bin
+    uwsgi uwsgi-plugin-python3 python3-gdbm python3-venv python-is-python3 gdal-bin
 
 RUN apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils
 # RUN git clone https://salsa.debian.org/debian-gis-team/proj.git /tmp/proj
 # RUN cd /tmp/proj && debuild -i -us -uc -b && dpkg -i ../*.deb
 
+RUN python -m venv /usr/src/venv
+ENV PATH="/usr/src/venv/bin:$PATH"
+RUN . /usr/src/venv/bin/activate
+
 # Install pip packages
 RUN pip3 install uwsgi \
     && pip install pip --upgrade \
     && pip install wheel==0.38.1 \
-    && pip install pygdal==$(gdal-config --version).*
+    && pip install GDAL==3.8.4
 
 # Install "sherlock" to be used with "memcached"
 RUN pip install sherlock
diff --git a/setup.cfg b/setup.cfg
@@ -75,7 +75,7 @@ install_requires =
     OWSLib==0.31.0
     pycsw @ git+https://github.com/geopython/pycsw.git@3.0.0-beta1#egg=pycsw
     SQLAlchemy==2.0.30 # required by PyCSW
-    Shapely==1.8.5.post1
+    Shapely==2.1.2
     mercantile==1.2.1
     numpy==1.26.*
     Jinja2==3.1.4
@@ -118,8 +118,7 @@ install_requires =
     gisdata==0.5.4
 
     # importer dependencies
-    setuptools>=59,<81
-    gdal<=3.6.4
+    setuptools>=59
     pdok-geopackage-validator==0.8.5
     geonode-django-dynamic-model==0.4.0
 

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ def list_thesauri():`
`19`	`19`	`else:`
`20`	`20`	`# DISABLED`
`21`	`21`	`# [0..n]`
`22`		`- card = f'[{t.card_min}..{t.card_max if t.card_max!=-1 else "N"}] '`
	`22`	`+ card = f'[{t.card_min}..{t.card_max if t.card_max != -1 else "N"}] '`
`23`	`23`	`logger.info(`
`24`	`24`	`f'id:{t.id:2} sort:{t.order:3} {card} name={t.identifier.ljust(max_id_len)} title="{t.title}" URI:{t.about}\n'`
`25`	`25`	`)`
Original file line number	Diff line number	Diff line change
`@@ -287,7 +287,7 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol`
`287`	`287`	`f" -h {db_host} -p {str(db_port)} -d {db_name}"`
`288`	`288`	`f" -U {db_user} --role={db_user} "`
`289`	`289`	`f' -t "{table_name}" '`
`290`		`- f' {"-c" if not preserve_tables else "" } '`
	`290`	`+ f' {"-c" if not preserve_tables else ""} '`
`291`	`291`	`f" {os.path.join(source_folder, filename)} "`
`292`	`292`	`)`
`293`	`293`	`ret = subprocess.call(command, env={"PGPASSWORD": db_passwd}, shell=True)`