Skip to content

Replace pip3 download with curl for Python deps #1283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tuhaihe merged 1 commit into from
Aug 6, 2025
Merged

Replace pip3 download with curl for Python deps #1283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 144 additions & 5 deletions configure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,9 @@ with_krb_srvnam
krb_srvtab
with_gssapi
with_pythonsrc_ext
PIP3
CURL
PYTHON3
with_python
with_perl
with_tcl
Expand Down Expand Up @@ -9892,6 +9895,142 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
$as_echo "$with_pythonsrc_ext" >&6; }

# Check for required dependencies when pythonsrc-ext is enabled
if test "$with_pythonsrc_ext" = yes; then
# Extract the first word of "python3", so it can be a program name with args.
set dummy python3; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_path_PYTHON3+:} false; then :
$as_echo_n "(cached) " >&6
else
case $PYTHON3 in
[\\/]* | ?:[\\/]*)
ac_cv_path_PYTHON3="$PYTHON3" # Let the user override the test with a path.
;;
*)
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_path_PYTHON3="$as_dir/$ac_word$ac_exec_ext"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS

;;
esac
fi
PYTHON3=$ac_cv_path_PYTHON3
if test -n "$PYTHON3"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON3" >&5
$as_echo "$PYTHON3" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


if test -z "$PYTHON3"; then
as_fn_error $? "python3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
fi

# Extract the first word of "curl", so it can be a program name with args.
set dummy curl; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_path_CURL+:} false; then :
$as_echo_n "(cached) " >&6
else
case $CURL in
[\\/]* | ?:[\\/]*)
ac_cv_path_CURL="$CURL" # Let the user override the test with a path.
;;
*)
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_path_CURL="$as_dir/$ac_word$ac_exec_ext"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS

;;
esac
fi
CURL=$ac_cv_path_CURL
if test -n "$CURL"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CURL" >&5
$as_echo "$CURL" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


if test -z "$CURL"; then
as_fn_error $? "curl is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
fi

# Extract the first word of "pip3", so it can be a program name with args.
set dummy pip3; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_path_PIP3+:} false; then :
$as_echo_n "(cached) " >&6
else
case $PIP3 in
[\\/]* | ?:[\\/]*)
ac_cv_path_PIP3="$PIP3" # Let the user override the test with a path.
;;
*)
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_path_PIP3="$as_dir/$ac_word$ac_exec_ext"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS

;;
esac
fi
PIP3=$ac_cv_path_PIP3
if test -n "$PIP3"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PIP3" >&5
$as_echo "$PIP3" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


if test -z "$PIP3"; then
as_fn_error $? "pip3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
fi
fi




#
Expand Down Expand Up @@ -18417,7 +18556,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
Expand Down Expand Up @@ -18463,7 +18602,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
Expand All @@ -18487,7 +18626,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
Expand Down Expand Up @@ -18532,7 +18671,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
Expand All @@ -18556,7 +18695,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
Expand Down
19 changes: 19 additions & 0 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1077,6 +1077,25 @@ AC_SUBST(with_python)
AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
AC_MSG_RESULT([$with_pythonsrc_ext])

# Check for required dependencies when pythonsrc-ext is enabled
if test "$with_pythonsrc_ext" = yes; then
AC_PATH_PROG([PYTHON3], [python3])
if test -z "$PYTHON3"; then
AC_MSG_ERROR([python3 is required for --with-pythonsrc-ext but was not found])
fi

AC_PATH_PROG([CURL], [curl])
if test -z "$CURL"; then
AC_MSG_ERROR([curl is required for --with-pythonsrc-ext but was not found])
fi

AC_PATH_PROG([PIP3], [pip3])
if test -z "$PIP3"; then
AC_MSG_ERROR([pip3 is required for --with-pythonsrc-ext but was not found])
fi
fi

AC_SUBST(with_pythonsrc_ext)


Expand Down
32 changes: 16 additions & 16 deletions gpMgmt/bin/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,14 @@ PYYAML_VERSION=5.4.1
download-python-deps:
@echo "--- Downloading Python dependencies for gpMgmt modules"
@mkdir -p $(PYLIB_SRC_EXT)
# Download psutil
pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
# Download PyYAML, wheel is for metadata vefiry;
# cython is used for building, see https://github.com/apache/cloudberry/issues/1201
pip3 install wheel "cython<3.0.0"
pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
# Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
# Download psutil using curl
curl -sSL https://files.pythonhosted.org/packages/source/p/psutil/psutil-$(PSUTIL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/psutil-$(PSUTIL_VERSION).tar.gz
# Download PyYAML using curl
curl -sSL https://files.pythonhosted.org/packages/source/P/PyYAML/PyYAML-$(PYYAML_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyYAML-$(PYYAML_VERSION).tar.gz
# Download PyGreSQL using curl
curl -sSL https://files.pythonhosted.org/packages/source/P/PyGreSQL/PyGreSQL-$(PYGRESQL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyGreSQL-$(PYGRESQL_VERSION).tar.gz
# Install wheel and cython for PyYAML building
pip3 install --user wheel "cython<3.0.0"

#
# PyGreSQL
Expand Down Expand Up @@ -148,14 +148,14 @@ $(MOCK_BIN):
@echo "--- mock for platform $(UBUNTU_PLATFORM)"
@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
pip3 install mock;\
else\
mkdir -p $(PYLIB_SRC_EXT) && \
pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
fi;
else\
mkdir -p $(PYLIB_SRC_EXT) && \
curl -sSL https://files.pythonhosted.org/packages/source/m/mock/mock-$(MOCK_VERSION).zip -o $(PYLIB_SRC_EXT)/mock-$(MOCK_VERSION).zip && \
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
fi;

# -----------------------------------------------------------------------------
#
Expand Down
Loading