Merge pull request #2838 from bsipocz/CI_add_py312

MAINT: Fixing python 3.12 compatibility

Author: bsipocz

.github/workflows/ci_crontests.yml

@@ -50,7 +50,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python

.github/workflows/ci_devtests.yml

@@ -32,13 +32,13 @@ jobs:
         include:
           - name: dev dependencies with all dependencies with coverage
             os: ubuntu-latest
-            python: '3.11'
-            toxenv: py311-test-alldeps-devdeps-cov
+            python: '3.12-dev'
+            toxenv: py312-test-alldeps-devdeps-cov
             toxargs: -v
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python

.github/workflows/ci_tests.yml

@@ -27,7 +27,7 @@ jobs:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
 
@@ -56,7 +56,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
@@ -78,7 +78,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "astropy_helpers"]
 	path = astropy_helpers
-	url = https://github.com/astropy/astropy-helpers.git
+	url = https://github.com/bsipocz/astropy-helpers.git

astropy_helpers

@@ -1,5 +1,5 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
-from datetime import datetime
+from datetime import datetime, timezone
 import os
 from pathlib import Path
 from urllib.parse import urlparse
@@ -139,7 +139,7 @@ def test_data_proprietary(self, alma):
         # public
         assert not alma.is_proprietary('uid://A001/X12a3/Xe9')
         IVOA_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
-        now = datetime.utcnow().strftime(IVOA_DATE_FORMAT)[:-3]
+        now = datetime.now(timezone.utc).strftime(IVOA_DATE_FORMAT)[:-3]
         query = "select top 1 member_ous_uid from ivoa.obscore where " \
                 "obs_release_date > '{}'".format(now)
         result = alma.query_tap(query)

astroquery/alma/tests/test_alma_remote.py

@@ -9,7 +9,7 @@
 import os
 import requests
 from pathlib import Path
-from datetime import datetime
+from datetime import datetime, timezone
 from astropy.coordinates import SkyCoord
 from astropy.io import fits
 from astropy import units as u
@@ -99,7 +99,7 @@ def test_query(self):
         assert 1000 < result[0][0]
 
         # test that no proprietary results are returned when not logged in
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         query = "select top 1 * from caom2.Plane where " \
                 "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
         result = cadc.exec_sync(query)
@@ -121,7 +121,7 @@ def test_login_with_user_password(self):
         for auth_session in [None, authsession.AuthSession(),
                              requests.Session()]:
             cadc = Cadc(auth_session=auth_session)
-            now = datetime.utcnow()
+            now = datetime.now(timezone.utc)
             query = \
                 "select top 1 * from caom2.Plane where metaRelease>'{}'".\
                 format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
@@ -152,7 +152,7 @@ def test_login_with_user_password(self):
     def test_login_with_cert(self):
         for auth_session in [requests.Session()]:
             cadc = Cadc(auth_session=auth_session)
-            now = datetime.utcnow()
+            now = datetime.now(timezone.utc)
             query = \
                 "select top 1 * from caom2.Plane where metaRelease>'{}'".\
                 format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
@@ -181,14 +181,14 @@ def test_authsession(self):
         auth_session = requests.Session()
         auth_session.cert = os.environ['CADC_CERT']
         cadc = Cadc(auth_session=auth_session)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         query = "select top 1 * from caom2.Plane where " \
                 "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
         result = cadc.exec_sync(query)
         assert len(result) == 1
         annon_session = requests.Session()
         cadc = Cadc(auth_session=annon_session)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         query = "select top 1 * from caom2.Plane where " \
                 "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
         result = cadc.exec_sync(query)

astroquery/cadc/tests/test_cadctap_remote.py

@@ -13,9 +13,9 @@
 import gzip
 import os
 import shutil
-import tarfile
+import tarfile as esatar
 import zipfile
-from datetime import datetime
+from datetime import datetime, timezone
 from urllib.parse import urlencode
 
 from astropy import log
@@ -39,6 +39,12 @@
 __all__ = ['Jwst', 'JwstClass']
 
 
+# We do trust the ESA tar files, this is to avoid the new to Python 3.12 deprecation warning
+# https://docs.python.org/3.12/library/tarfile.html#tarfile-extraction-filter
+if hasattr(esatar, "fully_trusted_filter"):
+    esatar.TarFile.extraction_filter = staticmethod(esatar.fully_trusted_filter)
+
+
 class JwstClass(BaseQuery):
 
     """
@@ -1037,8 +1043,8 @@ def __check_file_number(self, output_dir, output_file_name,
                         files.append(os.path.join(r, file))
 
     def __extract_file(self, output_file_full_path, output_dir, files):
-        if tarfile.is_tarfile(output_file_full_path):
-            with tarfile.open(output_file_full_path) as tar_ref:
+        if esatar.is_tarfile(output_file_full_path):
+            with esatar.open(output_file_full_path) as tar_ref:
                 tar_ref.extractall(path=output_dir)
         elif zipfile.is_zipfile(output_file_full_path):
             with zipfile.ZipFile(output_file_full_path, 'r') as zip_ref:
@@ -1050,7 +1056,7 @@ def __extract_file(self, output_file_full_path, output_dir, files):
 
     def __set_dirs(self, output_file, observation_id):
         if output_file is None:
-            now = datetime.now()
+            now = datetime.now(timezone.utc)
             formatted_now = now.strftime("%Y%m%d_%H%M%S")
             output_dir = os.getcwd() + os.sep + "temp_" + \
                 formatted_now

astroquery/esa/jwst/core.py

@@ -11,7 +11,7 @@
 import re
 import shutil
 from pathlib import Path
-import tarfile
+import tarfile as esatar
 import os
 import configparser
 from email.message import Message
@@ -29,6 +29,12 @@
 __all__ = ['XMMNewton', 'XMMNewtonClass']
 
 
+# We do trust the ESA tar files, this is to avoid the new to Python 3.12 deprecation warning
+# https://docs.python.org/3.12/library/tarfile.html#tarfile-extraction-filter
+if hasattr(esatar, "fully_trusted_filter"):
+    esatar.TarFile.extraction_filter = staticmethod(esatar.fully_trusted_filter)
+
+
 class XMMNewtonClass(BaseQuery):
     data_url = conf.DATA_ACTION
     data_aio_url = conf.DATA_ACTION_AIO
@@ -406,7 +412,7 @@ def get_epic_spectra(self, filename, source_number, *,
         if path != "" and os.path.exists(path):
             _path = path
         try:
-            with tarfile.open(filename, "r") as tar:
+            with esatar.open(filename, "r") as tar:
                 ret = {}
                 for member in tar.getmembers():
                     paths = os.path.split(member.name)
@@ -552,7 +558,7 @@ def get_epic_images(self, filename, band=[], instrument=[],
                     log.warning("Invalid instrument %s" % inst)
                     instrument.remove(inst)
         try:
-            with tarfile.open(filename, "r") as tar:
+            with esatar.open(filename, "r") as tar:
                 ret = {}
                 for member in tar.getmembers():
                     paths = os.path.split(member.name)
@@ -728,7 +734,7 @@ def get_epic_lightcurve(self, filename, source_number, *,
             _path = path
 
         try:
-            with tarfile.open(filename, "r") as tar:
+            with esatar.open(filename, "r") as tar:
                 ret = {}
                 for member in tar.getmembers():
                     paths = os.path.split(member.name)

astroquery/esa/xmm_newton/core.py

@@ -15,7 +15,7 @@
 """
 import zipfile
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 import shutil
 from collections.abc import Iterable
 
@@ -221,7 +221,7 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL',
         -------
         A table object
         """
-        now = datetime.now()
+        now = datetime.now(timezone.utc)
         now_formatted = now.strftime("%Y%m%d_%H%M%S")
         temp_dirname = "temp_" + now_formatted
         downloadname_formated = "download_" + now_formatted