Skip to content

Commit a5b64d0

Browse files
tuhaihetuhaihe
committed
Replace pip3 download with curl for Python deps
Replace pip3 download commands with curl for downloading psutil, PyYAML, PyGreSQL and mock packages in gpMgmt/bin/Makefile. This change unifies the download mechanism to use curl consistently for all Python dependencies, avoiding potential pip3 download compatibility issues across different environments. Changes: - Remove pip3 download commands for psutil, PyYAML, PyGreSQL & mock - Use curl to download packages directly from PyPI - Maintain existing wheel and cython installation for PyYAML build - Keep the same package versions and download locations Also, add curl and pip3 dependency detection in configure.ac file for better user experience. When `--with-pythonsrc-ext` option is enabled, configure can check the required dependencies early.
1 parent 793971e commit a5b64d0

File tree

3 files changed

+128
-21
lines changed

3 files changed

+128
-21
lines changed

‎configure‎

Lines changed: 98 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,8 @@ with_krb_srvnam
742742
krb_srvtab
743743
with_gssapi
744744
with_pythonsrc_ext
745+
PIP3
746+
CURL
745747
with_python
746748
with_perl
747749
with_tcl
@@ -9892,6 +9894,97 @@ fi
98929894
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
98939895
$as_echo "$with_pythonsrc_ext" >&6; }
98949896

9897+
# Check for required dependencies when pythonsrc-ext is enabled
9898+
if test "$with_pythonsrc_ext" = yes; then
9899+
# Extract the first word of "curl", so it can be a program name with args.
9900+
set dummy curl; ac_word=$2
9901+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9902+
$as_echo_n "checking for $ac_word... " >&6; }
9903+
if ${ac_cv_path_CURL+:} false; then :
9904+
$as_echo_n "(cached) " >&6
9905+
else
9906+
case $CURL in
9907+
[\\/]* | ?:[\\/]*)
9908+
ac_cv_path_CURL="$CURL" # Let the user override the test with a path.
9909+
;;
9910+
*)
9911+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9912+
for as_dir in $PATH
9913+
do
9914+
IFS=$as_save_IFS
9915+
test -z "$as_dir" && as_dir=.
9916+
for ac_exec_ext in '' $ac_executable_extensions; do
9917+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9918+
ac_cv_path_CURL="$as_dir/$ac_word$ac_exec_ext"
9919+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9920+
break 2
9921+
fi
9922+
done
9923+
done
9924+
IFS=$as_save_IFS
9925+
9926+
;;
9927+
esac
9928+
fi
9929+
CURL=$ac_cv_path_CURL
9930+
if test -n "$CURL"; then
9931+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CURL" >&5
9932+
$as_echo "$CURL" >&6; }
9933+
else
9934+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9935+
$as_echo "no" >&6; }
9936+
fi
9937+
9938+
if test -z "$CURL"; then
9939+
as_fn_error $? "curl is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9940+
fi
9941+
9942+
# Extract the first word of "pip3", so it can be a program name with args.
9943+
set dummy pip3; ac_word=$2
9944+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
9945+
$as_echo_n "checking for $ac_word... " >&6; }
9946+
if ${ac_cv_path_PIP3+:} false; then :
9947+
$as_echo_n "(cached) " >&6
9948+
else
9949+
case $PIP3 in
9950+
[\\/]* | ?:[\\/]*)
9951+
ac_cv_path_PIP3="$PIP3" # Let the user override the test with a path.
9952+
;;
9953+
*)
9954+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
9955+
for as_dir in $PATH
9956+
do
9957+
IFS=$as_save_IFS
9958+
test -z "$as_dir" && as_dir=.
9959+
for ac_exec_ext in '' $ac_executable_extensions; do
9960+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
9961+
ac_cv_path_PIP3="$as_dir/$ac_word$ac_exec_ext"
9962+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
9963+
break 2
9964+
fi
9965+
done
9966+
done
9967+
IFS=$as_save_IFS
9968+
9969+
;;
9970+
esac
9971+
fi
9972+
PIP3=$ac_cv_path_PIP3
9973+
if test -n "$PIP3"; then
9974+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PIP3" >&5
9975+
$as_echo "$PIP3" >&6; }
9976+
else
9977+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
9978+
$as_echo "no" >&6; }
9979+
fi
9980+
9981+
9982+
if test -z "$PIP3"; then
9983+
as_fn_error $? "pip3 is required for --with-pythonsrc-ext but was not found" "$LINENO" 5
9984+
fi
9985+
fi
9986+
9987+
98959988

98969989

98979990
#
@@ -18417,7 +18510,7 @@ else
1841718510
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1841818511
since some C++ compilers masquerading as C compilers
1841918512
incorrectly reject 9223372036854775807. */
18420-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18513+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1842118514
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1842218515
&& LARGE_OFF_T % 2147483647 == 1)
1842318516
? 1 : -1];
@@ -18463,7 +18556,7 @@ else
1846318556
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1846418557
since some C++ compilers masquerading as C compilers
1846518558
incorrectly reject 9223372036854775807. */
18466-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18559+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1846718560
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1846818561
&& LARGE_OFF_T % 2147483647 == 1)
1846918562
? 1 : -1];
@@ -18487,7 +18580,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
1848718580
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1848818581
since some C++ compilers masquerading as C compilers
1848918582
incorrectly reject 9223372036854775807. */
18490-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18583+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1849118584
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1849218585
&& LARGE_OFF_T % 2147483647 == 1)
1849318586
? 1 : -1];
@@ -18532,7 +18625,7 @@ else
1853218625
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1853318626
since some C++ compilers masquerading as C compilers
1853418627
incorrectly reject 9223372036854775807. */
18535-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18628+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1853618629
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1853718630
&& LARGE_OFF_T % 2147483647 == 1)
1853818631
? 1 : -1];
@@ -18556,7 +18649,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
1855618649
We can't simply define LARGE_OFF_T to be 9223372036854775807,
1855718650
since some C++ compilers masquerading as C compilers
1855818651
incorrectly reject 9223372036854775807. */
18559-
#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
18652+
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
1856018653
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
1856118654
&& LARGE_OFF_T % 2147483647 == 1)
1856218655
? 1 : -1];

‎configure.ac‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,6 +1077,20 @@ AC_SUBST(with_python)
10771077
AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
10781078
PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
10791079
AC_MSG_RESULT([$with_pythonsrc_ext])
1080+
1081+
# Check for required dependencies when pythonsrc-ext is enabled
1082+
if test "$with_pythonsrc_ext" = yes; then
1083+
AC_PATH_PROG([CURL], [curl])
1084+
if test -z "$CURL"; then
1085+
AC_MSG_ERROR([curl is required for --with-pythonsrc-ext but was not found])
1086+
fi
1087+
1088+
AC_PATH_PROG([PIP3], [pip3])
1089+
if test -z "$PIP3"; then
1090+
AC_MSG_ERROR([pip3 is required for --with-pythonsrc-ext but was not found])
1091+
fi
1092+
fi
1093+
10801094
AC_SUBST(with_pythonsrc_ext)
10811095

10821096

‎gpMgmt/bin/Makefile‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,14 @@ PYYAML_VERSION=5.4.1
9090
download-python-deps:
9191
@echo "--- Downloading Python dependencies for gpMgmt modules"
9292
@mkdir -p $(PYLIB_SRC_EXT)
93-
# Download psutil
94-
pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
95-
# Download PyYAML, wheel is for metadata vefiry;
96-
# cython is used for building, see https://github.com/apache/cloudberry/issues/1201
97-
pip3 install wheel "cython<3.0.0"
98-
pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
99-
# Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
100-
PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
93+
# Download psutil using curl
94+
curl -sSL https://files.pythonhosted.org/packages/source/p/psutil/psutil-$(PSUTIL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/psutil-$(PSUTIL_VERSION).tar.gz
95+
# Download PyYAML using curl
96+
curl -sSL https://files.pythonhosted.org/packages/source/P/PyYAML/PyYAML-$(PYYAML_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyYAML-$(PYYAML_VERSION).tar.gz
97+
# Download PyGreSQL using curl
98+
curl -sSL https://files.pythonhosted.org/packages/source/P/PyGreSQL/PyGreSQL-$(PYGRESQL_VERSION).tar.gz -o $(PYLIB_SRC_EXT)/PyGreSQL-$(PYGRESQL_VERSION).tar.gz
99+
# Install wheel and cython for PyYAML building
100+
pip3 install --user wheel "cython<3.0.0"
101101

102102
#
103103
# PyGreSQL
@@ -148,14 +148,14 @@ $(MOCK_BIN):
148148
@echo "--- mock for platform $(UBUNTU_PLATFORM)"
149149
@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
150150
pip3 install mock;\
151-
else\
152-
mkdir -p $(PYLIB_SRC_EXT) && \
153-
pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
154-
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155-
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
156-
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
157-
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
158-
fi;
151+
else\
152+
mkdir -p $(PYLIB_SRC_EXT) && \
153+
curl -sSL https://files.pythonhosted.org/packages/source/m/mock/mock-$(MOCK_VERSION).zip -o $(PYLIB_SRC_EXT)/mock-$(MOCK_VERSION).zip && \
154+
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155+
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
156+
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
157+
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
158+
fi;
159159

160160
# -----------------------------------------------------------------------------
161161
#

0 commit comments

Comments
 (0)