Skip to content

Commit d5967fd

Browse files
tuhaihetuhaihe
committed
Replace pip3 download with curl for Python deps
Replace pip3 download commands with curl for downloading psutil, PyYAML, PyGreSQL and mock packages in gpMgmt/bin/Makefile. This change unifies the download mechanism to use curl consistently for all Python dependencies, avoiding potential pip3 download compatibility issues across different environments. Changes: - Remove pip3 download commands for psutil, PyYAML, PyGreSQL & mock - Use curl to download packages directly from PyPI - Maintain existing wheel and cython installation for PyYAML build - Keep the same package versions and download locations Also, add curl and pip3 dependency detection in configure.ac file for better user experience. When `--with-pythonsrc-ext` option is enabled, configure can check the required dependencies early.
1 parent ccd790e commit d5967fd

File tree

3 files changed

+179
-21
lines changed

3 files changed

+179
-21
lines changed

‎configure‎

Lines changed: 144 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,9 @@ with_krb_srvnam
742742
krb_srvtab
743743
with_gssapi
744744
with_pythonsrc_ext
745+
PIP3
746+
CURL
747+
PYTHON3
745748
with_python
746749
with_perl
747750
with_tcl
@@ -9892,6 +9895,142 @@ fi
98929895
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
98939896
$as_echo "$with_pythonsrc_ext" >&6; }
98949897

9898+
# Check for required dependencies when pythonsrc-ext is enabled
9899+
if test "$with_pythonsrc_ext" = yes; then
9900+
# Extract the first word of "python3", so it can be a program name with args.
9901+
set dummy python3; ac_word=$2
9902+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9903+
$as_echo_n "checking for $ac_word... " >&6; }
9904+
if ${ac_cv_path_PYTHON3+:} false; then :
9905+
$as_echo_n "(cached) " >&6
9906+
else
9907+
case $PYTHON3 in
9908+
[\\/]* | ?:[\\/]*)
9909+
ac_cv_path_PYTHON3="$PYTHON3" # Let the user override the test with a path.
9910+
;;
9911+
*)
9912+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9913+
for as_dir in $PATH
9914+
do
9915+
IFS=$as_save_IFS
9916+
test -z "$as_dir" && as_dir=.
9917+
for ac_exec_ext in '' $ac_executable_extensions; do
9918+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9919+
ac_cv_path_PYTHON3="$as_dir/$ac_word$ac_exec_ext"
9920+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9921+
break 2
9922+
fi
9923+
done
9924+
done
9925+
IFS=$as_save_IFS
9926+
9927+
;;
9928+
esac
9929+
fi
9930+
PYTHON3=$ac_cv_path_PYTHON3
9931+
if test -n "$PYTHON3"; then
9932+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON3" >&5
9933+
$as_echo "$PYTHON3" >&6; }
9934+
else
9935+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9936+
$as_echo "no" >&6; }
9937+
fi
9938+
9939+
9940+
if test -z "$PYTHON3"; then
9941+
as_fn_error $? "python3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9942+
fi
9943+
9944+
# Extract the first word of "curl", so it can be a program name with args.
9945+
set dummy curl; ac_word=$2
9946+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9947+
$as_echo_n "checking for $ac_word... " >&6; }
9948+
if ${ac_cv_path_CURL+:} false; then :
9949+
$as_echo_n "(cached) " >&6
9950+
else
9951+
case $CURL in
9952+
[\\/]* | ?:[\\/]*)
9953+
ac_cv_path_CURL="$CURL" # Let the user override the test with a path.
9954+
;;
9955+
*)
9956+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9957+
for as_dir in $PATH
9958+
do
9959+
IFS=$as_save_IFS
9960+
test -z "$as_dir" && as_dir=.
9961+
for ac_exec_ext in '' $ac_executable_extensions; do
9962+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9963+
ac_cv_path_CURL="$as_dir/$ac_word$ac_exec_ext"
9964+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9965+
break 2
9966+
fi
9967+
done
9968+
done
9969+
IFS=$as_save_IFS
9970+
9971+
;;
9972+
esac
9973+
fi
9974+
CURL=$ac_cv_path_CURL
9975+
if test -n "$CURL"; then
9976+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CURL" >&5
9977+
$as_echo "$CURL" >&6; }
9978+
else
9979+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9980+
$as_echo "no" >&6; }
9981+
fi
9982+
9983+
9984+
if test -z "$CURL"; then
9985+
as_fn_error $? "curl is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9986+
fi
9987+
9988+
# Extract the first word of "pip3", so it can be a program name with args.
9989+
set dummy pip3; ac_word=$2
9990+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9991+
$as_echo_n "checking for $ac_word... " >&6; }
9992+
if ${ac_cv_path_PIP3+:} false; then :
9993+
$as_echo_n "(cached) " >&6
9994+
else
9995+
case $PIP3 in
9996+
[\\/]* | ?:[\\/]*)
9997+
ac_cv_path_PIP3="$PIP3" # Let the user override the test with a path.
9998+
;;
9999+
*)
10000+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
10001+
for as_dir in $PATH
10002+
do
10003+
IFS=$as_save_IFS
10004+
test -z "$as_dir" && as_dir=.
10005+
for ac_exec_ext in '' $ac_executable_extensions; do
10006+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
10007+
ac_cv_path_PIP3="$as_dir/$ac_word$ac_exec_ext"
10008+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
10009+
break 2
10010+
fi
10011+
done
10012+
done
10013+
IFS=$as_save_IFS
10014+
10015+
;;
10016+
esac
10017+
fi
10018+
PIP3=$ac_cv_path_PIP3
10019+
if test -n "$PIP3"; then
10020+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PIP3" >&5
10021+
$as_echo "$PIP3" >&6; }
10022+
else
10023+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
10024+
$as_echo "no" >&6; }
10025+
fi
10026+
10027+
10028+
if test -z "$PIP3"; then
10029+
as_fn_error $? "pip3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
10030+
fi
10031+
fi
10032+
10033+
989510034

989610035

989710036
#
@@ -18417,7 +18556,7 @@ else
1841718556
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1841818557
since some C++ compilers masquerading as C compilers
1841918558
incorrectly reject 9223372036854775807. */
18420-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18559+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1842118560
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1842218561
&& LARGE_OFF_T % 2147483647 == 1)
1842318562
? 1 : -1];
@@ -18463,7 +18602,7 @@ else
1846318602
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1846418603
since some C++ compilers masquerading as C compilers
1846518604
incorrectly reject 9223372036854775807. */
18466-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18605+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1846718606
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1846818607
&& LARGE_OFF_T % 2147483647 == 1)
1846918608
? 1 : -1];
@@ -18487,7 +18626,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
1848718626
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1848818627
since some C++ compilers masquerading as C compilers
1848918628
incorrectly reject 9223372036854775807. */
18490-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18629+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1849118630
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1849218631
&& LARGE_OFF_T % 2147483647 == 1)
1849318632
? 1 : -1];
@@ -18532,7 +18671,7 @@ else
1853218671
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1853318672
since some C++ compilers masquerading as C compilers
1853418673
incorrectly reject 9223372036854775807. */
18535-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18674+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1853618675
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1853718676
&& LARGE_OFF_T % 2147483647 == 1)
1853818677
? 1 : -1];
@@ -18556,7 +18695,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
1855618695
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1855718696
since some C++ compilers masquerading as C compilers
1855818697
incorrectly reject 9223372036854775807. */
18559-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18698+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1856018699
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1856118700
&& LARGE_OFF_T % 2147483647 == 1)
1856218701
? 1 : -1];

‎configure.ac‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,6 +1077,25 @@ AC_SUBST(with_python)
10771077
AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
10781078
PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
10791079
AC_MSG_RESULT([$with_pythonsrc_ext])
1080+
1081+
# Check for required dependencies when pythonsrc-ext is enabled
1082+
if test "$with_pythonsrc_ext" = yes; then
1083+
AC_PATH_PROG([PYTHON3], [python3])
1084+
if test -z "$PYTHON3"; then
1085+
AC_MSG_ERROR([python3 is required for --with-pythonsrc-ext but was not found])
1086+
fi
1087+
1088+
AC_PATH_PROG([CURL], [curl])
1089+
if test -z "$CURL"; then
1090+
AC_MSG_ERROR([curl is required for --with-pythonsrc-ext but was not found])
1091+
fi
1092+
1093+
AC_PATH_PROG([PIP3], [pip3])
1094+
if test -z "$PIP3"; then
1095+
AC_MSG_ERROR([pip3 is required for --with-pythonsrc-ext but was not found])
1096+
fi
1097+
fi
1098+
10801099
AC_SUBST(with_pythonsrc_ext)
10811100

10821101

‎gpMgmt/bin/Makefile‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,14 @@ PYYAML_VERSION=5.4.1
9090
download-python-deps:
9191
@echo "--- Downloading Python dependencies for gpMgmt modules"
9292
@mkdir -p $(PYLIB_SRC_EXT)
93-
# Download psutil
94-
pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
95-
# Download PyYAML, wheel is for metadata vefiry;
96-
# cython is used for building, see https://github.com/apache/cloudberry/issues/1201
97-
pip3 install wheel "cython<3.0.0"
98-
pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
99-
# Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
100-
PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
93+
# Download psutil using curl
94+
curl -sSL https://files.pythonhosted.org/packages/source/p/psutil/psutil-$(PSUTIL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/psutil-$(PSUTIL_VERSION).tar.gz
95+
# Download PyYAML using curl
96+
curl -sSL https://files.pythonhosted.org/packages/source/P/PyYAML/PyYAML-$(PYYAML_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyYAML-$(PYYAML_VERSION).tar.gz
97+
# Download PyGreSQL using curl
98+
curl -sSL https://files.pythonhosted.org/packages/source/P/PyGreSQL/PyGreSQL-$(PYGRESQL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyGreSQL-$(PYGRESQL_VERSION).tar.gz
99+
# Install wheel and cython for PyYAML building
100+
pip3 install --user wheel "cython<3.0.0"
101101

102102
#
103103
# PyGreSQL
@@ -148,14 +148,14 @@ $(MOCK_BIN):
148148
@echo "--- mock for platform $(UBUNTU_PLATFORM)"
149149
@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
150150
pip3 install mock;\
151-
else\
152-
mkdir -p $(PYLIB_SRC_EXT) && \
153-
pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
154-
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155-
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
156-
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
157-
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
158-
fi;
151+
else\
152+
mkdir -p $(PYLIB_SRC_EXT) && \
153+
curl -sSL https://files.pythonhosted.org/packages/source/m/mock/mock-$(MOCK_VERSION).zip -o $(PYLIB_SRC_EXT)/mock-$(MOCK_VERSION).zip && \
154+
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155+
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
156+
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
157+
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
158+
fi;
159159

160160
# -----------------------------------------------------------------------------
161161
#

0 commit comments

Comments
 (0)