Skip to content

Commit 8aa5b4a

Browse files
seanshparkseanshpark
committed
DRAFT upgrade to python 3.10
on-going draft to use upgrade to python 3.10. Signed-off-by: SaeHie Park <saehie.park@gmail.com>
1 parent 858d967 commit 8aa5b4a

File tree

13 files changed

+166
-56
lines changed

13 files changed

+166
-56
lines changed

‎.github/workflows/run-onecc-build.yml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@ jobs:
4949
include:
5050
- ubuntu_code: focal
5151
ubuntu_ver: 20.04
52+
python_exec: python3.10
5253
- ubuntu_code: jammy
5354
ubuntu_ver: 22.04
55+
python_exec: python3
5456
runs-on: one-x64-linux
5557
container:
5658
image: samsungonedev.azurecr.io/nnfw/one-devtools:${{ matrix.ubuntu_code }}
@@ -78,12 +80,6 @@ jobs:
7880
apt-get -qqy install python3.10 python3.10-dev python3.10-venv
7981
python3.10 -m ensurepip --upgrade
8082
81-
# dalgona uses pybind11, but pybind11 cannot bind packages in virtualenv.
82-
# So we need to install packages for dalgona-test globally.
83-
- name: Install required packages
84-
run: |
85-
python3 -m pip install numpy h5py==3.11.0 flatbuffers==23.5.26
86-
8783
- name: Caching externals
8884
uses: actions/cache@v4
8985
with:

‎compiler/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,11 @@ function(add_compiler_directory DIR)
9292
message(STATUS "Configure ${PREFIX}")
9393
add_subdirectory(${DIR})
9494
message(STATUS "Configure ${PREFIX} - Done")
95+
# NOTE ONE_PYTHON_VERSION_MINOR is to provide python3.x minor version number
96+
# to modules that use common artifacts.
97+
if(DEFINED ONE_PYTHON_VERSION_MINOR)
98+
set(ONE_PYTHON_VERSION_MINOR ${ONE_PYTHON_VERSION_MINOR} PARENT_SCOPE)
99+
endif()
95100
endif(ENABLE)
96101
endfunction(add_compiler_directory)
97102

‎compiler/common-artifacts/CMakeLists.txt‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,23 @@
11
#[[ Generate common python virtual enviornment ]]
2-
# NOTE find_package try to use at least python3.8 as follows depending on platform version
3-
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
4-
# Ubuntu20.04; default python3.8
2+
# NOTE find_package try to use at least python3.10 as follows depending on platform version
3+
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
54
# Ubuntu22.04; default python3.10
65
# Ubuntu24.04; default python3.12
76
# refer https://github.com/Samsung/ONE/issues/9962
8-
# find python 3.8 or above
9-
find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET)
7+
# refer https://github.com/Samsung/ONE/issues/15226
8+
# find python 3.10 or above
9+
find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET)
1010
if(NOT Python_FOUND)
11-
find_package(Python 3.8 COMPONENTS Interpreter QUIET)
11+
find_package(Python 3.10 COMPONENTS Interpreter QUIET)
1212
endif()
1313

1414
if(NOT Python_Interpreter_FOUND)
1515
message(STATUS "Build common-artifacts: FAILED (Python3 is missing)")
1616
return()
1717
endif()
1818

19-
# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
20-
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
21-
set(PYTHON_VERSION_MINOR 8)
22-
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
19+
# NOTE assume only use 3.10.x or 3.12.x
20+
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
2321
set(PYTHON_VERSION_MINOR 10)
2422
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
2523
set(PYTHON_VERSION_MINOR 12)
@@ -29,6 +27,10 @@ else()
2927
return()
3028
endif()
3129

30+
# NOTE ONE_PYTHON_VERSION_MINOR is to provide current python3.x minor version
31+
# number to modules that use common artifacts.
32+
set(ONE_PYTHON_VERSION_MINOR ${PYTHON_VERSION_MINOR} PARENT_SCOPE)
33+
3234
set(PYTHON_EXECUTABLE ${Python_EXECUTABLE})
3335

3436
# Create python virtual environment
@@ -44,8 +46,8 @@ set(REQUIREMENTS_FILE "requirements.txt")
4446
set(REQUIREMENTS_OVERLAY_PATH "${VIRTUALENV_OVERLAY}/${REQUIREMENTS_FILE}")
4547

4648
set(PYTHON_OVERLAY python3)
47-
if(PYTHON_EXECUTABLE MATCHES python3.8)
48-
set(PYTHON_OVERLAY python3.8)
49+
if(PYTHON_EXECUTABLE MATCHES python3.10)
50+
set(PYTHON_OVERLAY python3.10)
4951
endif()
5052

5153
# NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options

‎compiler/dalgona/CMakeLists.txt‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,23 @@
1-
# NOTE find_package will try to use at least python3.8 as follows depending on platform version
2-
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
3-
# Ubuntu20.04; default python3.8
1+
# NOTE find_package will try to use at least python3.10 as follows depending on platform version
2+
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
43
# Ubuntu22.04; default python3.10
54
# Ubuntu24.04; default python3.12
65
# refer https://github.com/Samsung/ONE/issues/9962
6+
# refer https://github.com/Samsung/ONE/issues/15226
77
# NOTE Require same python version of common-artifacts
8-
# find python 3.8 or above
9-
find_package(Python 3.8 EXACT COMPONENTS Interpreter Development QUIET)
8+
# find python 3.10 or above
9+
find_package(Python 3.10 EXACT COMPONENTS Interpreter Development QUIET)
1010
if(NOT Python_FOUND)
11-
find_package(Python 3.8 COMPONENTS Interpreter Development QUIET)
11+
find_package(Python 3.10 COMPONENTS Interpreter Development QUIET)
1212
endif()
1313

1414
if(NOT Python_Development_FOUND)
1515
message(STATUS "Build dalgona: FAILED (Python3 development package is missing)")
1616
return()
1717
endif()
1818

19-
# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
20-
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
21-
set(PYTHON_VERSION_MINOR 8)
22-
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
19+
# NOTE assume only use 3.10.x or 3.12.x
20+
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
2321
set(PYTHON_VERSION_MINOR 10)
2422
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
2523
set(PYTHON_VERSION_MINOR 12)

‎compiler/fm-equalize-value-py-test/CMakeLists.txt‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@ if(NOT ENABLE_TEST)
22
return()
33
endif(NOT ENABLE_TEST)
44

5+
if (NOT DEFINED ONE_PYTHON_VERSION_MINOR)
6+
message(STATUS "Warning: ONE_PYTHON_VERSION_MINOR not set")
7+
return()
8+
endif()
9+
set(PYTHON_SITE_PACKAGES "${VIRTUALENV}/lib/python3.${ONE_PYTHON_VERSION_MINOR}/site-packages")
10+
511
set(VIRTUALENV "${NNCC_OVERLAY_DIR}/venv")
612
set(TEST_LIST_FILE "test.lst")
713

@@ -33,14 +39,6 @@ macro(eval RECIPE)
3339
set(AFTER_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${AFTER_CIRCLE_FILE}")
3440
set(AFTER_CIRCLE_PATTERN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${AFETR_PATTERN_FILE}")
3541

36-
execute_process(
37-
COMMAND ${VIRTUALENV}/bin/python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
38-
OUTPUT_VARIABLE PYTHON_VERSION
39-
OUTPUT_STRIP_TRAILING_WHITESPACE
40-
)
41-
42-
set(PYTHON_SITE_PACKAGES "${NNCC_OVERLAY_DIR}/venv/lib/python${PYTHON_VERSION}/site-packages")
43-
4442
# Apply fm-equalize
4543
add_custom_command(OUTPUT ${AFTER_CIRCLE_OUTPUT_PATH} ${AFTER_CIRCLE_PATTERN_PATH}
4644
COMMAND ${CMAKE_COMMAND} -E env

‎compiler/one-cmds/CMakeLists.txt‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,22 @@
1-
# NOTE find_package try to use at least python3.8 as follows depending on platform version
2-
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
3-
# Ubuntu20.04; default python3.8
1+
# NOTE find_package will try to use at least python3.10 as follows depending on platform version
2+
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
43
# Ubuntu22.04; default python3.10
54
# Ubuntu24.04; explicitly installed python3.8 (default is python3.12)
65
# refer https://github.com/Samsung/ONE/issues/9962
7-
find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET)
6+
# refer https://github.com/Samsung/ONE/issues/15226
7+
find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET)
88
if(NOT Python_FOUND)
9-
find_package(Python 3.8 COMPONENTS Interpreter QUIET)
9+
find_package(Python 3.10 COMPONENTS Interpreter QUIET)
1010
endif()
1111

1212
if(NOT Python_Interpreter_FOUND)
1313
message(STATUS "Build one-cmds: FAILED (Python3 is missing)")
1414
return()
1515
endif()
1616

17-
# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
17+
# NOTE assume only use 3.10.x or 3.12.x
1818
# NOTE PYTHON_VERSION_MINOR is not used but added for consistancy with common-artifacts and dalgona
19-
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
20-
set(PYTHON_VERSION_MINOR 8)
21-
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
19+
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
2220
set(PYTHON_VERSION_MINOR 10)
2321
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
2422
set(PYTHON_VERSION_MINOR 12)

‎compiler/one-cmds/one-prepare-venv‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,7 @@ VER_PROTOBUF=4.23.3
6969
VER_NUMPY=1.24.3
7070

7171
echo "Setting version for '$DISTRIB_CODENAME'"
72-
if [[ "$DISTRIB_CODENAME" == "bionic" ]]; then
73-
: # use as is
74-
elif [[ "$DISTRIB_CODENAME" == "focal" ]]; then
72+
if [[ "$DISTRIB_CODENAME" == "focal" ]]; then
7573
: # use as is
7674
elif [[ "$DISTRIB_CODENAME" == "jammy" ]]; then
7775
: # TODO change vesions
@@ -82,6 +80,32 @@ else
8280
exit 1
8381
fi
8482

83+
PYTHON_VER=$(${PYTHON3_EXEC} -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" )
84+
if [[ "$PYTHON_VER" == "3.10" ]]; then
85+
VER_TENSORFLOW=2.19.0
86+
VER_ONNX=1.18.0
87+
VER_ONNXRUNTIME=1.21.1
88+
VER_TORCH="2.7.0+cpu"
89+
VER_NUMPY="1.26.4"
90+
VER_PROTOBUF="5.29.4"
91+
VER_TF_PROB=0
92+
VER_TF_ADON=0
93+
VER_ONNX_TF=0
94+
elif [[ "$PYTHON_VER" == "3.12" ]]; then
95+
VER_TENSORFLOW=2.19.0
96+
VER_ONNX=1.18.0
97+
VER_ONNXRUNTIME=1.21.1
98+
VER_TORCH="2.7.0+cpu"
99+
VER_NUMPY="1.26.4"
100+
VER_PROTOBUF="5.29.4"
101+
VER_TF_PROB=0
102+
VER_TF_ADON=0
103+
VER_ONNX_TF=0
104+
else
105+
echo "Error one-prepare-venv: Unsupported python $PYTHON_VER"
106+
exit 1
107+
fi
108+
85109
PIP_TRUSTED_HOST="--trusted-host pypi.org "
86110
PIP_TRUSTED_HOST+="--trusted-host pypi.python.org "
87111
PIP_TRUSTED_HOST+="--trusted-host files.pythonhosted.org "
@@ -111,15 +135,22 @@ ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade pip setuptools
111135

112136
PYTHON_PACKAGES="tensorflow-cpu==${VER_TENSORFLOW} "
113137
PYTHON_PACKAGES+="Pillow "
114-
PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} "
115-
PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} "
116138
PYTHON_PACKAGES+="torch==${VER_TORCH} "
117139
PYTHON_PACKAGES+="onnx==${VER_ONNX} "
118140
PYTHON_PACKAGES+="onnxruntime==${VER_ONNXRUNTIME} "
119-
PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} "
120141
PYTHON_PACKAGES+="protobuf==${VER_PROTOBUF} "
121142
PYTHON_PACKAGES+="fsspec==2024.6.1 "
122143
PYTHON_PACKAGES+="pydot==${VER_PYDOT} "
123144
PYTHON_PACKAGES+="numpy==${VER_NUMPY} "
124145

146+
if [[ "${VER_TF_PROB}" != "0" ]]; then
147+
PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} "
148+
fi
149+
if [[ "${VER_TF_ADON}" != "0" ]]; then
150+
PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} "
151+
fi
152+
if [[ "${VER_ONNX_TF}" != "0" ]]; then
153+
PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} "
154+
fi
155+
125156
${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade ${PYTHON_PACKAGES} ${TORCH_SOURCE_OPTION}

‎compiler/one-cmds/tests/one-optimize_002.test‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,22 @@ rm -f ${filename}.log
3232
rm -rf ${outputfile}
3333

3434
# run test
35+
# NOTE TF2.19.0 generates with "InceptionV3/Logits/SpatialSqueeze"
3536
one-optimize --resolve_customop_add \
36-
--change_outputs InceptionV3/Logits/SpatialSqueeze1 \
37+
--change_outputs InceptionV3/Logits/SpatialSqueeze \
3738
--input_path ${inputfile} \
3839
--output_path ${outputfile} > ${filename}.log 2>&1
3940

4041
if [[ ! -s "${outputfile}" ]]; then
41-
trap_err_onexit
42+
# TODO remove change_outputs with old TF2.12.1
43+
one-optimize --resolve_customop_add \
44+
--change_outputs InceptionV3/Logits/SpatialSqueeze1 \
45+
--input_path ${inputfile} \
46+
--output_path ${outputfile} > ${filename}.log 2>&1
47+
48+
if [[ ! -s "${outputfile}" ]]; then
49+
trap_err_onexit
50+
fi
4251
fi
4352

4453
echo "${filename_ext} SUCCESS"

‎compiler/one-cmds/tests/one-quantize_009.qconf.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
"layers" : [
55
{
66
"name" : "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Mixed_6a/Branch_1/Conv2d_0a_1x1/Conv2D;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D",
7+
"alternate": {
8+
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
9+
},
710
"dtype" : "int16",
811
"granularity" : "channel"
912
},

‎compiler/one-cmds/tests/one-quantize_012.qconf.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
"InceptionV3/InceptionV3/Mixed_5b/Branch_3/AvgPool_0a_3x3/AvgPool",
1010
"InceptionV3/InceptionV3/Mixed_7c/concat",
1111
"InceptionV3/Predictions/Reshape_1"],
12+
"alternate": {
13+
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
14+
},
1215
"dtype" : "int16",
1316
"granularity" : "channel"
1417
}

0 commit comments

Comments
 (0)