Skip to content

Commit 4fbfba9

Browse files
seanshparkseanshpark
committed
DRAFT upgrade to python 3.10
on-going draft to use upgrade to python 3.10. Signed-off-by: SaeHie Park <saehie.park@gmail.com>
1 parent f88ac50 commit 4fbfba9

File tree

11 files changed

+187
-66
lines changed

11 files changed

+187
-66
lines changed

‎.github/workflows/run-onecc-build.yml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@ jobs:
4949
include:
5050
- ubuntu_code: focal
5151
ubuntu_ver: 20.04
52+
python_exec: python3.10
5253
- ubuntu_code: jammy
5354
ubuntu_ver: 22.04
55+
python_exec: python3
5456
runs-on: one-x64-linux
5557
container:
5658
image: samsungonedev.azurecr.io/nnfw/one-devtools:${{ matrix.ubuntu_code }}
@@ -78,12 +80,6 @@ jobs:
7880
apt-get -qqy install python3.10 python3.10-dev python3.10-venv
7981
python3.10 -m ensurepip --upgrade
8082
81-
# dalgona uses pybind11, but pybind11 cannot bind packages in virtualenv.
82-
# So we need to install packages for dalgona-test globally.
83-
- name: Install required packages
84-
run: |
85-
python3 -m pip install numpy h5py==3.11.0 flatbuffers==23.5.26
86-
8783
- name: Caching externals
8884
uses: actions/cache@v4
8985
with:

‎compiler/common-artifacts/CMakeLists.txt‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,23 @@
11
#[[ Generate common python virtual enviornment ]]
2-
# NOTE find_package try to use at least python3.8 as follows depending on platform version
3-
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
4-
# Ubuntu20.04; default python3.8
2+
# NOTE find_package try to use at least python3.10 as follows depending on platform version
3+
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
54
# Ubuntu22.04; default python3.10
65
# Ubuntu24.04; default python3.12
76
# refer https://github.com/Samsung/ONE/issues/9962
8-
# find python 3.8 or above
9-
find_package(Python 3.8 COMPONENTS Interpreter QUIET)
7+
# refer https://github.com/Samsung/ONE/issues/15226
8+
# find python 3.10 or above
9+
find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET)
10+
if(NOT Python_FOUND)
11+
find_package(Python 3 COMPONENTS Interpreter QUIET)
12+
endif()
1013

1114
if(NOT Python_Interpreter_FOUND)
1215
message(STATUS "Build common-artifacts: FAILED (Python3 is missing)")
1316
return()
1417
endif()
1518

16-
if(Python_VERSION VERSION_LESS 3.8)
17-
message(STATUS "Build common-artifacts: FAILED (Need Python version 3.8 or higher)")
18-
return()
19-
endif()
20-
21-
# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
22-
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
23-
set(PYTHON_VERSION_MINOR 8)
24-
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
19+
# NOTE assume only use 3.10.x or 3.12.x
20+
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
2521
set(PYTHON_VERSION_MINOR 10)
2622
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
2723
set(PYTHON_VERSION_MINOR 12)
@@ -46,8 +42,8 @@ set(REQUIREMENTS_FILE "requirements.txt")
4642
set(REQUIREMENTS_OVERLAY_PATH "${VIRTUALENV_OVERLAY}/${REQUIREMENTS_FILE}")
4743

4844
set(PYTHON_OVERLAY python3)
49-
if(PYTHON_EXECUTABLE MATCHES python3.8)
50-
set(PYTHON_OVERLAY python3.8)
45+
if(PYTHON_EXECUTABLE MATCHES python3.10)
46+
set(PYTHON_OVERLAY python3.10)
5147
endif()
5248

5349
# NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options

‎compiler/dalgona/CMakeLists.txt‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,29 @@
1-
# NOTE find_package will try to use at least python3.8 as follows depending on platform version
2-
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
3-
# Ubuntu20.04; default python3.8
1+
# NOTE find_package will try to use at least python3.10 as follows depending on platform version
2+
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
43
# Ubuntu22.04; default python3.10
54
# Ubuntu24.04; default python3.12
65
# refer https://github.com/Samsung/ONE/issues/9962
6+
# refer https://github.com/Samsung/ONE/issues/15226
77
# NOTE Require same python version of common-artifacts
8-
# find python 3.8 or above
9-
find_package(Python 3.8 COMPONENTS Development QUIET)
8+
# find python 3.10 or above
9+
find_package(Python 3.10 EXACT COMPONENTS Interpreter Development QUIET)
10+
if(NOT Python_FOUND)
11+
find_package(Python 3 COMPONENTS Interpreter Development QUIET)
12+
endif()
1013

1114
if(NOT Python_Development_FOUND)
1215
message(STATUS "Build dalgona: FAILED (Python3 development package is missing)")
1316
return()
1417
endif()
1518

16-
if(Python_VERSION VERSION_LESS 3.8)
17-
message(STATUS "Build dalgona: FAILED (Need Python version 3.8 or higher)")
19+
# NOTE assume only use 3.10.x or 3.12.x
20+
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
21+
set(PYTHON_VERSION_MINOR 10)
22+
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
23+
set(PYTHON_VERSION_MINOR 12)
24+
else()
25+
# TODO support more
26+
message(STATUS "Build dalgona: FAILED (Unsupported python: ${Python_VERSION})")
1827
return()
1928
endif()
2029

‎compiler/one-cmds/CMakeLists.txt‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,27 @@
1-
# NOTE find_package try to use at least python3.8 as follows depending on platform version
2-
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
3-
# Ubuntu20.04; default python3.8
1+
# NOTE find_package will try to use at least python3.10 as follows depending on platform version
2+
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
43
# Ubuntu22.04; default python3.10
54
# Ubuntu24.04; explicitly installed python3.8 (default is python3.12)
65
# refer https://github.com/Samsung/ONE/issues/9962
7-
find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET)
6+
# refer https://github.com/Samsung/ONE/issues/15226
7+
find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET)
88
if(NOT Python_FOUND)
9-
find_package(Python 3.8 COMPONENTS Interpreter QUIET)
9+
find_package(Python 3 COMPONENTS Interpreter QUIET)
1010
endif()
1111

12-
# tensorflow 2.12.1 supports Python 3.8 ~ 3.11
13-
if(Python_VERSION VERSION_GREATER_EQUAL 3.12)
14-
message(STATUS "Build one-cmds: FALSE (Python version 3.12 or higher is not supported yet)")
15-
return()
16-
endif()
17-
if(Python_VERSION VERSION_LESS 3.8)
18-
message(STATUS "Build one-cmds: FAILED (Install Python version 3.8 or 3.10)")
12+
if(NOT Python_Interpreter_FOUND)
13+
message(STATUS "Build one-cmds: FAILED (Python3 is missing)")
1914
return()
2015
endif()
2116

22-
if(NOT Python_Interpreter_FOUND)
23-
message(STATUS "Build one-cmds: FAILED (Python3 is missing)")
17+
# NOTE assume only use 3.10.x or 3.12.x
18+
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
19+
set(PYTHON_VERSION_MINOR 10)
20+
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
21+
set(PYTHON_VERSION_MINOR 12)
22+
else()
23+
# TODO support more
24+
message(STATUS "Build one-cmds: FAILED (Unsupported python: ${Python_VERSION})")
2425
return()
2526
endif()
2627

@@ -91,25 +92,31 @@ foreach(ONE_COMMAND IN ITEMS ${ONE_COMMAND_FILES})
9192

9293
endforeach(ONE_COMMAND)
9394

95+
unset(ONNX2CIRCLE_SOURCE_PATH)
9496
# if ONE_ONNX2CIRCLE_PATH is set, install that file to bin folder
9597
if(DEFINED ENV{ONE_ONNX2CIRCLE_PATH})
96-
message(STATUS "ONNX2CIRCLE from env-var $ENV{ONE_ONNX2CIRCLE_PATH}")
97-
install(FILES $ENV{ONE_ONNX2CIRCLE_PATH}
98-
PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE
99-
GROUP_READ GROUP_EXECUTE
100-
WORLD_READ WORLD_EXECUTE
101-
DESTINATION bin)
98+
set(ONNX2CIRCLE_SOURCE_PATH $ENV{ONE_ONNX2CIRCLE_PATH})
10299
else()
100+
# if local build/install of circle-mlir exist, use it
103101
set(ONNX2CIRCLE_PATH "${NNAS_PROJECT_SOURCE_DIR}/circle-mlir/build/install/bin/onnx2circle")
104102
if (EXISTS ${ONNX2CIRCLE_PATH})
105-
message(STATUS "ONNX2CIRCLE from ${ONNX2CIRCLE_PATH}")
106-
install(FILES ${ONNX2CIRCLE_PATH}
107-
PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE
108-
GROUP_READ GROUP_EXECUTE
109-
WORLD_READ WORLD_EXECUTE
110-
DESTINATION bin)
103+
set(ONNX2CIRCLE_SOURCE_PATH ${ONNX2CIRCLE_PATH})
104+
else()
105+
# if circletools/onnx2circle package is installed, use it
106+
set(ONNX2CIRCLE_PATH "/usr/share/circletools/bin/onnx2circle")
107+
if (EXISTS ${ONNX2CIRCLE_PATH})
108+
set(ONNX2CIRCLE_SOURCE_PATH ${ONNX2CIRCLE_PATH})
109+
endif()
111110
endif()
112111
endif()
112+
if(DEFINED ONNX2CIRCLE_SOURCE_PATH AND EXISTS ${ONNX2CIRCLE_SOURCE_PATH})
113+
message(STATUS "ONNX2CIRCLE from ${ONNX2CIRCLE_SOURCE_PATH}")
114+
install(FILES ${ONNX2CIRCLE_SOURCE_PATH}
115+
PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE
116+
GROUP_READ GROUP_EXECUTE
117+
WORLD_READ WORLD_EXECUTE
118+
DESTINATION bin)
119+
endif()
113120

114121
set(ONE_UTILITY_FILES
115122
one-build.template.cfg

‎compiler/one-cmds/one-prepare-venv‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,7 @@ VER_PROTOBUF=4.23.3
6969
VER_NUMPY=1.24.3
7070

7171
echo "Setting version for '$DISTRIB_CODENAME'"
72-
if [[ "$DISTRIB_CODENAME" == "bionic" ]]; then
73-
: # use as is
74-
elif [[ "$DISTRIB_CODENAME" == "focal" ]]; then
72+
if [[ "$DISTRIB_CODENAME" == "focal" ]]; then
7573
: # use as is
7674
elif [[ "$DISTRIB_CODENAME" == "jammy" ]]; then
7775
: # TODO change vesions
@@ -82,6 +80,32 @@ else
8280
exit 1
8381
fi
8482

83+
PYTHON_VER=$(${PYTHON3_EXEC} -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" )
84+
if [[ "$PYTHON_VER" == "3.10" ]]; then
85+
VER_TENSORFLOW=2.19.0
86+
VER_ONNX=1.18.0
87+
VER_ONNXRUNTIME=1.21.1
88+
VER_TORCH="2.7.0+cpu"
89+
VER_NUMPY="1.26.4"
90+
VER_PROTOBUF="5.29.4"
91+
VER_TF_PROB=0
92+
VER_TF_ADON=0
93+
VER_ONNX_TF=0
94+
elif [[ "$PYTHON_VER" == "3.12" ]]; then
95+
VER_TENSORFLOW=2.19.0
96+
VER_ONNX=1.18.0
97+
VER_ONNXRUNTIME=1.21.1
98+
VER_TORCH="2.7.0+cpu"
99+
VER_NUMPY="1.26.4"
100+
VER_PROTOBUF="5.29.4"
101+
VER_TF_PROB=0
102+
VER_TF_ADON=0
103+
VER_ONNX_TF=0
104+
else
105+
echo "Error one-prepare-venv: Unsupported python $PYTHON_VER"
106+
exit 1
107+
fi
108+
85109
PIP_TRUSTED_HOST="--trusted-host pypi.org "
86110
PIP_TRUSTED_HOST+="--trusted-host pypi.python.org "
87111
PIP_TRUSTED_HOST+="--trusted-host files.pythonhosted.org "
@@ -111,15 +135,22 @@ ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade pip setuptools
111135

112136
PYTHON_PACKAGES="tensorflow-cpu==${VER_TENSORFLOW} "
113137
PYTHON_PACKAGES+="Pillow "
114-
PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} "
115-
PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} "
116138
PYTHON_PACKAGES+="torch==${VER_TORCH} "
117139
PYTHON_PACKAGES+="onnx==${VER_ONNX} "
118140
PYTHON_PACKAGES+="onnxruntime==${VER_ONNXRUNTIME} "
119-
PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} "
120141
PYTHON_PACKAGES+="protobuf==${VER_PROTOBUF} "
121142
PYTHON_PACKAGES+="fsspec==2024.6.1 "
122143
PYTHON_PACKAGES+="pydot==${VER_PYDOT} "
123144
PYTHON_PACKAGES+="numpy==${VER_NUMPY} "
124145

146+
if [[ "${VER_TF_PROB}" != "0" ]]; then
147+
PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} "
148+
fi
149+
if [[ "${VER_TF_ADON}" != "0" ]]; then
150+
PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} "
151+
fi
152+
if [[ "${VER_ONNX_TF}" != "0" ]]; then
153+
PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} "
154+
fi
155+
125156
${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade ${PYTHON_PACKAGES} ${TORCH_SOURCE_OPTION}

‎compiler/one-cmds/tests/one-optimize_002.test‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,22 @@ rm -f ${filename}.log
3232
rm -rf ${outputfile}
3333

3434
# run test
35+
# NOTE TF2.19.0 generates with "InceptionV3/Logits/SpatialSqueeze"
3536
one-optimize --resolve_customop_add \
36-
--change_outputs InceptionV3/Logits/SpatialSqueeze1 \
37+
--change_outputs InceptionV3/Logits/SpatialSqueeze \
3738
--input_path ${inputfile} \
3839
--output_path ${outputfile} > ${filename}.log 2>&1
3940

4041
if [[ ! -s "${outputfile}" ]]; then
41-
trap_err_onexit
42+
# TODO remove change_outputs with old TF2.12.1
43+
one-optimize --resolve_customop_add \
44+
--change_outputs InceptionV3/Logits/SpatialSqueeze1 \
45+
--input_path ${inputfile} \
46+
--output_path ${outputfile} > ${filename}.log 2>&1
47+
48+
if [[ ! -s "${outputfile}" ]]; then
49+
trap_err_onexit
50+
fi
4251
fi
4352

4453
echo "${filename_ext} SUCCESS"

‎compiler/one-cmds/tests/one-quantize_009.qconf.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
"layers" : [
55
{
66
"name" : "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Mixed_6a/Branch_1/Conv2d_0a_1x1/Conv2D;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D",
7+
"alternate": {
8+
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
9+
},
710
"dtype" : "int16",
811
"granularity" : "channel"
912
},

‎compiler/one-cmds/tests/one-quantize_012.qconf.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
"InceptionV3/InceptionV3/Mixed_5b/Branch_3/AvgPool_0a_3x3/AvgPool",
1010
"InceptionV3/InceptionV3/Mixed_7c/concat",
1111
"InceptionV3/Predictions/Reshape_1"],
12+
"alternate": {
13+
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
14+
},
1215
"dtype" : "int16",
1316
"granularity" : "channel"
1417
}

‎compiler/one-cmds/tests/one-quantize_013.qconf.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
"InceptionV3/InceptionV3/Mixed_5b/Branch_3/AvgPool_0a_3x3/AvgPool",
1010
"InceptionV3/InceptionV3/Mixed_7c/concat",
1111
"InceptionV3/Predictions/Reshape_1"],
12+
"alternate": {
13+
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
14+
},
1215
"dtype" : "int16",
1316
"granularity" : "channel"
1417
}

‎compiler/tf2tfliteV2/tf2tfliteV2.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -263,9 +263,8 @@ def _convert(flags):
263263
_apply_verbosity(flags.verbose)
264264

265265
if (flags.v1):
266-
_v1_convert(flags)
267-
else:
268-
_v2_convert(flags)
266+
print("tf2tfliteV2: v1 is not supported anymore. force to use v2.")
267+
_v2_convert(flags)
269268

270269

271270
"""

0 commit comments

Comments
 (0)