Skip to content

Allow psycopg2 2.9.10 #34893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jrmccluskey merged 5 commits into from
May 14, 2025
Merged

Allow psycopg2 2.9.10 #34893

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0ecdc3c
Allow psycopg2 2.9.10
jrmccluskey May 8, 2025
a1efa8e
try dependency specifieres
jrmccluskey May 8, 2025
8c0da77
split out logic to own function + formatting
jrmccluskey May 8, 2025
d7b1bd2
swap to more general psycopg2 bounds
jrmccluskey May 13, 2025
144995c
Merge branch 'master' into psycopg2upgrade
jrmccluskey May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 18 additions & 25 deletions sdks/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,18 +222,18 @@ def generate_external_transform_wrappers():
script_exists = os.path.exists(
os.path.join(sdk_dir, 'gen_xlang_wrappers.py'))
config_exists = os.path.exists(
os.path.join(os.path.dirname(sdk_dir),
'standard_external_transforms.yaml'))
os.path.join(
os.path.dirname(sdk_dir), 'standard_external_transforms.yaml'))
# we need both the script and the standard transforms config file.
# at build time, we don't have access to apache_beam to discover and
# retrieve external transforms, so the config file has to already exist
if not script_exists or not config_exists:
generated_transforms_dir = os.path.join(
sdk_dir, 'apache_beam', 'transforms', 'xlang')
sdk_dir, 'apache_beam', 'transforms', 'xlang')

# if exists, this directory will have at least its __init__.py file
if (not os.path.exists(generated_transforms_dir) or
len(os.listdir(generated_transforms_dir)) <= 1):
len(os.listdir(generated_transforms_dir)) <= 1):
message = 'External transform wrappers have not been generated '
if not script_exists:
message += 'and the generation script `gen_xlang_wrappers.py`'
Expand All @@ -251,13 +251,16 @@ def generate_external_transform_wrappers():
os.path.join(sdk_dir, 'gen_xlang_wrappers.py'),
'--cleanup',
'--transforms-config-source',
os.path.join(os.path.dirname(sdk_dir),
'standard_external_transforms.yaml')
], capture_output=True, check=True)
os.path.join(
os.path.dirname(sdk_dir), 'standard_external_transforms.yaml')
],
capture_output=True,
check=True)
except subprocess.CalledProcessError as err:
raise RuntimeError(
'Could not generate external transform wrappers due to '
'error: %s', err.stderr)
'error: %s',
err.stderr)


def get_portability_package_data():
Expand Down Expand Up @@ -417,7 +420,8 @@ def get_portability_package_data():
'scikit-learn>=0.20.0',
'setuptools',
'sqlalchemy>=1.3,<3.0',
'psycopg2-binary>=2.8.5,<3.0.0,!=2.9.10',
Copy link
Contributor

@tvalentyn tvalentyn May 12, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about we instead have two constraints here:

              'psycopg2-binary>=2.8.5,<2.9.10; python_version <= "3.9"',
              'psycopg2-binary>=2.8.5,<3.0; python_version >= "3.10"'

This way, it will be easier to maintain by dependabot and eventually easier to spot that this logic can be cleaned up when Python 3.9 is deprecated.

Adding an OS constraint if desired should be possible too via:

sys_platform != "darwin" 
sys_platform == "darwin"

Copy link
Contributor Author

@jrmccluskey jrmccluskey May 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to do the targeted constraint in-line like that (3.9 + darwin excluding 2.9.10 specifically) and it was quite messy. I'm okay to just split 3.9/3.10 if you'd prefer to avoid the more programatic setup

'psycopg2-binary>=2.8.5,<2.9.10; python_version <= "3.9"',
'psycopg2-binary>=2.8.5,<3.0; python_version >= "3.10"',
'testcontainers[mysql]>=3.0.3,<4.0.0',
'cryptography>=41.0.2',
'hypothesis>5.0.0,<7.0.0',
Expand Down Expand Up @@ -546,20 +550,14 @@ def get_portability_package_data():
# in https://github.com/apache/beam/blob/master/sdks/python/tox.ini
# For more info, see
# https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing
'torch': [
'torch<=1.13.0,<2.1.0'
],
'tensorflow': [
'tensorflow>=2.12rc1,<2.13'
],
'torch': ['torch<=1.13.0,<2.1.0'],
'tensorflow': ['tensorflow>=2.12rc1,<2.13'],
'transformers': [
'transformers>=4.28.0,<4.49.0',
'tensorflow==2.12.0',
'torch>=1.9.0,<2.1.0'
],
'tft': [
'tensorflow_transform>=1.14.0,<1.15.0'
],
'tft': ['tensorflow_transform>=1.14.0,<1.15.0'],
'onnx': [
'onnxruntime==1.13.1',
'torch==1.13.1',
Expand All @@ -568,13 +566,8 @@ def get_portability_package_data():
'skl2onnx==1.13',
'transformers==4.25.1'
],
'xgboost': [
'xgboost>=1.6.0,<2.1.3',
'datatable==1.0.0'
],
'tensorflow-hub': [
'tensorflow-hub>=0.14.0,<0.16.0'
]
'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'],
'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0']
},
zip_safe=False,
# PyPI package information.
Expand Down
Loading