Merge pull request #2838 from bsipocz/CI_add_py312

bsipocz · web-flow · commit 6eefe6dd1c6f · 2023-09-25T15:58:55.000-07:00
MAINT: Fixing python 3.12 compatibility
diff --git a/.github/workflows/ci_crontests.yml b/.github/workflows/ci_crontests.yml
@@ -50,7 +50,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
diff --git a/.github/workflows/ci_devtests.yml b/.github/workflows/ci_devtests.yml
@@ -32,13 +32,13 @@ jobs:
         include:
           - name: dev dependencies with all dependencies with coverage
             os: ubuntu-latest
-            python: '3.11'
-            toxenv: py311-test-alldeps-devdeps-cov
+            python: '3.12-dev'
+            toxenv: py312-test-alldeps-devdeps-cov
             toxargs: -v
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml
@@ -27,7 +27,7 @@ jobs:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
 
@@ -56,7 +56,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
@@ -78,7 +78,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "astropy_helpers"]
 	path = astropy_helpers
-	url = https://github.com/astropy/astropy-helpers.git
+	url = https://github.com/bsipocz/astropy-helpers.git
diff --git a/astropy_helpers b/astropy_helpers
@@ -1 +1 @@
-Subproject commit d2a6304a3e801bc2cb053ccf1cf09a9f1e62036c
+Subproject commit 9431edf2a6e4692bfbd9e6326f01288d61d414e9
diff --git a/astroquery/alma/tests/test_alma_remote.py b/astroquery/alma/tests/test_alma_remote.py
@@ -1,5 +1,5 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
-from datetime import datetime
+from datetime import datetime, timezone
 import os
 from pathlib import Path
 from urllib.parse import urlparse
@@ -139,7 +139,7 @@ def test_data_proprietary(self, alma):
         # public
         assert not alma.is_proprietary('uid://A001/X12a3/Xe9')
         IVOA_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
-        now = datetime.utcnow().strftime(IVOA_DATE_FORMAT)[:-3]
+        now = datetime.now(timezone.utc).strftime(IVOA_DATE_FORMAT)[:-3]
         query = "select top 1 member_ous_uid from ivoa.obscore where " \
                 "obs_release_date > '{}'".format(now)
         result = alma.query_tap(query)
diff --git a/astroquery/cadc/tests/test_cadctap_remote.py b/astroquery/cadc/tests/test_cadctap_remote.py
@@ -9,7 +9,7 @@
 import os
 import requests
 from pathlib import Path
-from datetime import datetime
+from datetime import datetime, timezone
 from astropy.coordinates import SkyCoord
 from astropy.io import fits
 from astropy import units as u
@@ -99,7 +99,7 @@ def test_query(self):
         assert 1000 < result[0][0]
 
         # test that no proprietary results are returned when not logged in
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         query = "select top 1 * from caom2.Plane where " \
                 "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
         result = cadc.exec_sync(query)
@@ -121,7 +121,7 @@ def test_login_with_user_password(self):
         for auth_session in [None, authsession.AuthSession(),
                              requests.Session()]:
             cadc = Cadc(auth_session=auth_session)
-            now = datetime.utcnow()
+            now = datetime.now(timezone.utc)
             query = \
                 "select top 1 * from caom2.Plane where metaRelease>'{}'".\
                 format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
@@ -152,7 +152,7 @@ def test_login_with_user_password(self):
     def test_login_with_cert(self):
         for auth_session in [requests.Session()]:
             cadc = Cadc(auth_session=auth_session)
-            now = datetime.utcnow()
+            now = datetime.now(timezone.utc)
             query = \
                 "select top 1 * from caom2.Plane where metaRelease>'{}'".\
                 format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
@@ -181,14 +181,14 @@ def test_authsession(self):
         auth_session = requests.Session()
         auth_session.cert = os.environ['CADC_CERT']
         cadc = Cadc(auth_session=auth_session)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         query = "select top 1 * from caom2.Plane where " \
                 "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
         result = cadc.exec_sync(query)
         assert len(result) == 1
         annon_session = requests.Session()
         cadc = Cadc(auth_session=annon_session)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         query = "select top 1 * from caom2.Plane where " \
                 "metaRelease>'{}'".format(now.strftime('%Y-%m-%dT%H:%M:%S.%f'))
         result = cadc.exec_sync(query)
diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py
@@ -13,9 +13,9 @@
 import gzip
 import os
 import shutil
-import tarfile
+import tarfile as esatar
 import zipfile
-from datetime import datetime
+from datetime import datetime, timezone
 from urllib.parse import urlencode
 
 from astropy import log
@@ -39,6 +39,12 @@
 __all__ = ['Jwst', 'JwstClass']
 
 
+# We do trust the ESA tar files, this is to avoid the new to Python 3.12 deprecation warning
+# https://docs.python.org/3.12/library/tarfile.html#tarfile-extraction-filter
+if hasattr(esatar, "fully_trusted_filter"):
+    esatar.TarFile.extraction_filter = staticmethod(esatar.fully_trusted_filter)
+
+
 class JwstClass(BaseQuery):
 
     """
@@ -1037,8 +1043,8 @@ def __check_file_number(self, output_dir, output_file_name,
                         files.append(os.path.join(r, file))
 
     def __extract_file(self, output_file_full_path, output_dir, files):
-        if tarfile.is_tarfile(output_file_full_path):
-            with tarfile.open(output_file_full_path) as tar_ref:
+        if esatar.is_tarfile(output_file_full_path):
+            with esatar.open(output_file_full_path) as tar_ref:
                 tar_ref.extractall(path=output_dir)
         elif zipfile.is_zipfile(output_file_full_path):
             with zipfile.ZipFile(output_file_full_path, 'r') as zip_ref:
@@ -1050,7 +1056,7 @@ def __extract_file(self, output_file_full_path, output_dir, files):
 
     def __set_dirs(self, output_file, observation_id):
         if output_file is None:
-            now = datetime.now()
+            now = datetime.now(timezone.utc)
             formatted_now = now.strftime("%Y%m%d_%H%M%S")
             output_dir = os.getcwd() + os.sep + "temp_" + \
                 formatted_now
diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py
@@ -11,7 +11,7 @@
 import re
 import shutil
 from pathlib import Path
-import tarfile
+import tarfile as esatar
 import os
 import configparser
 from email.message import Message
@@ -29,6 +29,12 @@
 __all__ = ['XMMNewton', 'XMMNewtonClass']
 
 
+# We do trust the ESA tar files, this is to avoid the new to Python 3.12 deprecation warning
+# https://docs.python.org/3.12/library/tarfile.html#tarfile-extraction-filter
+if hasattr(esatar, "fully_trusted_filter"):
+    esatar.TarFile.extraction_filter = staticmethod(esatar.fully_trusted_filter)
+
+
 class XMMNewtonClass(BaseQuery):
     data_url = conf.DATA_ACTION
     data_aio_url = conf.DATA_ACTION_AIO
@@ -406,7 +412,7 @@ def get_epic_spectra(self, filename, source_number, *,
         if path != "" and os.path.exists(path):
             _path = path
         try:
-            with tarfile.open(filename, "r") as tar:
+            with esatar.open(filename, "r") as tar:
                 ret = {}
                 for member in tar.getmembers():
                     paths = os.path.split(member.name)
@@ -552,7 +558,7 @@ def get_epic_images(self, filename, band=[], instrument=[],
                     log.warning("Invalid instrument %s" % inst)
                     instrument.remove(inst)
         try:
-            with tarfile.open(filename, "r") as tar:
+            with esatar.open(filename, "r") as tar:
                 ret = {}
                 for member in tar.getmembers():
                     paths = os.path.split(member.name)
@@ -728,7 +734,7 @@ def get_epic_lightcurve(self, filename, source_number, *,
             _path = path
 
         try:
-            with tarfile.open(filename, "r") as tar:
+            with esatar.open(filename, "r") as tar:
                 ret = {}
                 for member in tar.getmembers():
                     paths = os.path.split(member.name)
diff --git a/astroquery/gaia/core.py b/astroquery/gaia/core.py
@@ -15,7 +15,7 @@
 """
 import zipfile
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 import shutil
 from collections.abc import Iterable
 
@@ -221,7 +221,7 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL',
         -------
         A table object
         """
-        now = datetime.now()
+        now = datetime.now(timezone.utc)
         now_formatted = now.strftime("%Y%m%d_%H%M%S")
         temp_dirname = "temp_" + now_formatted
         downloadname_formated = "download_" + now_formatted
diff --git a/astroquery/ipac/ned/core.py b/astroquery/ipac/ned/core.py
@@ -5,7 +5,7 @@
 from collections import namedtuple
 from xml.dom.minidom import parseString
 
-from datetime import datetime
+from datetime import datetime, timezone
 
 import astropy.units as u
 import astropy.coordinates as coord
@@ -625,7 +625,7 @@ def get_table_async(self, object_name, *, table='photometry',
                 'yes' if kwargs.get('extended_search') else 'no')
             request_payload['begin_year'] = kwargs.get('from_year', 1800)
             request_payload['end_year'] = kwargs.get('to_year',
-                                                     datetime.now().year)
+                                                     datetime.now(timezone.utc).year)
         if get_query_payload:
             return request_payload
         response = self._request("GET", url=Ned.DATA_SEARCH_URL,
diff --git a/astroquery/query.py b/astroquery/query.py
@@ -12,7 +12,7 @@
 import requests
 import textwrap
 
-from datetime import datetime, timedelta
+from datetime import datetime, timezone, timedelta
 from pathlib import Path
 
 from astropy.config import paths
@@ -117,8 +117,8 @@ def from_cache(self, cache_location, cache_timeout):
             if cache_timeout is None:
                 expired = False
             else:
-                current_time = datetime.utcnow()
-                cache_time = datetime.utcfromtimestamp(request_file.stat().st_mtime)
+                current_time = datetime.now(timezone.utc)
+                cache_time = datetime.fromtimestamp(request_file.stat().st_mtime, timezone.utc)
                 expired = current_time-cache_time > timedelta(seconds=cache_timeout)
             if not expired:
                 with open(request_file, "rb") as f:
diff --git a/astroquery/utils/tap/taputils.py b/astroquery/utils/tap/taputils.py
@@ -17,7 +17,7 @@
 
 import re
 import warnings
-from datetime import datetime
+from datetime import datetime, timezone
 
 TAP_UTILS_QUERY_TOP_PATTERN = re.compile(
     r"\s*SELECT\s+(ALL\s+|DISTINCT\s+)?TOP\s+\d+\s+", re.IGNORECASE)
@@ -224,7 +224,7 @@ def get_table_name(full_qualified_table_name):
 
 def get_suitable_output_file(conn_handler, async_job, output_file, headers,
                              is_error, output_format):
-    date_time = datetime.now().strftime("%Y%m%d%H%M%S")
+    date_time = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
     if output_file is None:
         file_name = conn_handler.get_file_from_header(headers)
         if file_name is None:
diff --git a/docs/conf.py b/docs/conf.py
@@ -78,7 +78,7 @@
 project = setup_cfg['name']
 author = setup_cfg['author']
 copyright = '{0}, {1}'.format(
-    datetime.datetime.now().year, setup_cfg['author'])
+    datetime.datetime.now(datetime.timezone.utc).year, setup_cfg['author'])
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
diff --git a/tox.ini b/tox.ini
@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py{37,38,39,310,311}-test{,-alldeps,-oldestdeps,-devdeps,-predeps}{,-online}{,-cov}
+    py{37,38,39,310,311,312}-test{,-alldeps,-oldestdeps,-devdeps,-predeps}{,-online}{,-cov}
     codestyle
     linkcheck
     build_docs
