Skip to content

DRAFT upgrade to python 3.10 #15503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
seanshpark wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

DRAFT upgrade to python 3.10 #15503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 11 additions & 27 deletions compiler/common-artifacts/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,23 @@
#[[ Generate common python virtual enviornment ]]
# NOTE find_package try to use at least python3.8 as follows depending on platform version
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
# Ubuntu20.04; default python3.8
# NOTE find_package try to use at least python3.10 as follows depending on platform version
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
# Ubuntu22.04; default python3.10
# Ubuntu24.04; default python3.12
# refer https://github.com/Samsung/ONE/issues/9962
# find python 3.8 or above
find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET)
# refer https://github.com/Samsung/ONE/issues/15226
# find python 3.10 or above
find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET)
if(NOT Python_FOUND)
find_package(Python 3.8 COMPONENTS Interpreter QUIET)
find_package(Python 3.10 COMPONENTS Interpreter QUIET)
endif()

if(NOT Python_Interpreter_FOUND)
message(STATUS "Build common-artifacts: FAILED (Python3 is missing)")
return()
endif()

# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
set(PYTHON_VERSION_MINOR 8)
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
# NOTE assume only use 3.10.x or 3.12.x
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
set(PYTHON_VERSION_MINOR 10)
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
set(PYTHON_VERSION_MINOR 12)
Expand Down Expand Up @@ -48,8 +46,8 @@ set(REQUIREMENTS_FILE "requirements.txt")
set(REQUIREMENTS_OVERLAY_PATH "${VIRTUALENV_OVERLAY}/${REQUIREMENTS_FILE}")

set(PYTHON_OVERLAY python3)
if(PYTHON_EXECUTABLE MATCHES python3.8)
set(PYTHON_OVERLAY python3.8)
if(PYTHON_EXECUTABLE MATCHES python3.10)
set(PYTHON_OVERLAY python3.10)
endif()

# NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options
Expand All @@ -61,21 +59,7 @@ endif()
# NOTE `tensorflow-cpu` package is not available for aarch64, so we use `tensorflow` package.
# NOTE refer https://github.com/Samsung/ONE/issues/15226 for versions
set(PY_PKG_LIST )
if (PYTHON_VERSION_MINOR EQUAL 8)
# python3.8; Ubuntu 18.04, 20.04
if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
list(APPEND PY_PKG_LIST "tensorflow==2.12.1")
else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
list(APPEND PY_PKG_LIST "tensorflow-cpu==2.12.1")
endif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
list(APPEND PY_PKG_LIST "flatbuffers==23.5.26")
list(APPEND PY_PKG_LIST "protobuf==4.23.3")
list(APPEND PY_PKG_LIST "pydot==1.4.2")
list(APPEND PY_PKG_LIST "pytest==7.4.3")
list(APPEND PY_PKG_LIST "h5py==3.11.0")
list(APPEND PY_PKG_LIST "cffi==1.16.0")
list(APPEND PY_PKG_LIST "numpy==1.24.3")
elseif (PYTHON_VERSION_MINOR EQUAL 10)
if (PYTHON_VERSION_MINOR EQUAL 10)
# python 3.10; Ubuntu 22.04
# TF2.19.0 release at 2025-03-12
if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
Expand Down
18 changes: 8 additions & 10 deletions compiler/dalgona/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,23 @@
# NOTE find_package will try to use at least python3.8 as follows depending on platform version
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
# Ubuntu20.04; default python3.8
# NOTE find_package will try to use at least python3.10 as follows depending on platform version
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
# Ubuntu22.04; default python3.10
# Ubuntu24.04; default python3.12
# refer https://github.com/Samsung/ONE/issues/9962
# refer https://github.com/Samsung/ONE/issues/15226
# NOTE Require same python version of common-artifacts
# find python 3.8 or above
find_package(Python 3.8 EXACT COMPONENTS Interpreter Development QUIET)
# find python 3.10 or above
find_package(Python 3.10 EXACT COMPONENTS Interpreter Development QUIET)
if(NOT Python_FOUND)
find_package(Python 3.8 COMPONENTS Interpreter Development QUIET)
find_package(Python 3.10 COMPONENTS Interpreter Development QUIET)
endif()

if(NOT Python_Development_FOUND)
message(STATUS "Build dalgona: FAILED (Python3 development package is missing)")
return()
endif()

# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
set(PYTHON_VERSION_MINOR 8)
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
# NOTE assume only use 3.10.x or 3.12.x
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
set(PYTHON_VERSION_MINOR 10)
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
set(PYTHON_VERSION_MINOR 12)
Expand Down
16 changes: 7 additions & 9 deletions compiler/one-cmds/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
# NOTE find_package try to use at least python3.8 as follows depending on platform version
# Ubuntu18.04; explictly installed python3.8 (default is python3.6)
# Ubuntu20.04; default python3.8
# NOTE find_package will try to use at least python3.10 as follows depending on platform version
# Ubuntu20.04; explictly installed python3.10 (default is python3.8)
# Ubuntu22.04; default python3.10
# Ubuntu24.04; explicitly installed python3.8 (default is python3.12)
# refer https://github.com/Samsung/ONE/issues/9962
find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET)
# refer https://github.com/Samsung/ONE/issues/15226
find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET)
if(NOT Python_FOUND)
find_package(Python 3.8 COMPONENTS Interpreter QUIET)
find_package(Python 3.10 COMPONENTS Interpreter QUIET)
endif()

if(NOT Python_Interpreter_FOUND)
message(STATUS "Build one-cmds: FAILED (Python3 is missing)")
return()
endif()

# NOTE assume only use 3.8.x or 3.10.x or 3.12.x
# NOTE assume only use 3.10.x or 3.12.x
# NOTE PYTHON_VERSION_MINOR is not used but added for consistancy with common-artifacts and dalgona
if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9))
set(PYTHON_VERSION_MINOR 8)
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11))
set(PYTHON_VERSION_MINOR 10)
elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13))
set(PYTHON_VERSION_MINOR 12)
Expand Down
23 changes: 7 additions & 16 deletions compiler/one-cmds/one-prepare-venv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,22 +57,16 @@ source /etc/lsb-release
# - https://github.com/onnx/onnx/blob/master/docs/Versioning.md
# - https://github.com/onnx/onnx-tensorflow/blob/master/Versioning.md

VER_TENSORFLOW=2.12.1
VER_TF_PROB=0.20.1
VER_TF_ADON=0.20.0
VER_ONNX=1.16.0
VER_ONNXRUNTIME=1.18.0
VER_ONNX_TF=1.10.0
VER_TENSORFLOW=2.19.0
VER_ONNX=1.18.0
VER_ONNXRUNTIME=1.21.1
VER_PYDOT=1.4.2
VER_TORCH="2.1.2+cpu"
VER_PROTOBUF=4.23.3
VER_NUMPY=1.24.3
VER_TORCH="2.7.0+cpu"
VER_PROTOBUF="5.29.4"
VER_NUMPY="1.26.4"

PYTHON_VER=$(${PYTHON3_EXEC} -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" )
echo "Setting package version for python $PYTHON_VER"
if [[ "$PYTHON_VER" == "3.8" ]]; then
: # use as is
elif [[ "$PYTHON_VER" == "3.10" ]]; then
if [[ "$PYTHON_VER" == "3.10" ]]; then
: # TODO change vesions
elif [[ "$PYTHON_VER" == "3.12" ]]; then
: # TODO change vesions
Expand Down Expand Up @@ -110,12 +104,9 @@ ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade pip setuptools

PYTHON_PACKAGES="tensorflow-cpu==${VER_TENSORFLOW} "
PYTHON_PACKAGES+="Pillow "
PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} "
PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} "
PYTHON_PACKAGES+="torch==${VER_TORCH} "
PYTHON_PACKAGES+="onnx==${VER_ONNX} "
PYTHON_PACKAGES+="onnxruntime==${VER_ONNXRUNTIME} "
PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} "
PYTHON_PACKAGES+="protobuf==${VER_PROTOBUF} "
PYTHON_PACKAGES+="fsspec==2024.6.1 "
PYTHON_PACKAGES+="pydot==${VER_PYDOT} "
Expand Down
4 changes: 3 additions & 1 deletion compiler/one-cmds/tests/one-quantize_009.qconf.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
{
"name" : "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Mixed_6a/Branch_1/Conv2d_0a_1x1/Conv2D;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D",
"alternate": {
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
"tf2190": [
"InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
]
},
"dtype" : "int16",
"granularity" : "channel"
Expand Down
4 changes: 3 additions & 1 deletion compiler/one-cmds/tests/one-quantize_012.qconf.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
"InceptionV3/InceptionV3/Mixed_7c/concat",
"InceptionV3/Predictions/Reshape_1"],
"alternate": {
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
"tf2190": [
"InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
]
},
"dtype" : "int16",
"granularity" : "channel"
Expand Down
4 changes: 3 additions & 1 deletion compiler/one-cmds/tests/one-quantize_013.qconf.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
"InceptionV3/InceptionV3/Mixed_7c/concat",
"InceptionV3/Predictions/Reshape_1"],
"alternate": {
"tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
"tf2190": [
"InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D"
]
},
"dtype" : "int16",
"granularity" : "channel"
Expand Down
5 changes: 2 additions & 3 deletions compiler/tf2tfliteV2/tf2tfliteV2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,9 +263,8 @@ def _convert(flags):
_apply_verbosity(flags.verbose)

if (flags.v1):
_v1_convert(flags)
else:
_v2_convert(flags)
print("tf2tfliteV2: v1 not supported anymore. convert with v2.")
_v2_convert(flags)


"""
Expand Down
65 changes: 65 additions & 0 deletions docs/howto/how-to-build-compiler.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,71 @@ $ NNCC_WORKSPACE=build/release ./nncc build
```
will build release version in `build/release` folder.

## Local install and command line tests

Local install test is to make an local installation in `build/install` path,
check the build artifacts and run compiler command line tests of `onecc`
and others.

### Prepare onnx2circle

`one-import-onnx` tool now uses `onnx2circle` tool from `circle-mlir`.

`onnx2cicle` tool can be prepared with (1) build from source or (2) install
from [launchpad.net](https://launchpad.net/~circletools).

Please read [README.md](../circle-mlir/README.md) for details how to build.

As `circle-mlir/externals` uses lots of storage(about 150~200GB) and time,
we recommand to use build with Docker image as described in above README file.

To use Debian package from [launchpad.net](https://launchpad.net/~circletools),
```
sudo apt-get update
sudo apt-get install software-properties-common
sudo -E add-apt-repository ppa:circletools/nightly
sudo apt-get install onnx2circle
```
- this will install `onnx2circle` tool to `/usr/share/circletools` folder

`one-cmds/CMakeLists.txt` will install `onnx2circle` from (1) or (2) to
`buid/install` as conversion tool of `one-import-onnx` command.

### How to make local installation
```
NNAS_BUILD_PREFIX=build/setup \
BUILD_TYPE=Release \
./nnas create-package --preset 20230907 --prefix build/install
```
This will configure and build necessary modules and install files in
`build/install` path.

### Local command line tests

To run local command line tests, python virtual envirnment is necessary with
test models.

Prepare python virtual environment:
```
pushd build/install/bin
bash ./one-prepare-venv
popd
```

Download test models:
```
pushd build/install/test
bash ./prepare_test_materials.sh
popd
```

Run command line tests:
```
pushd build/install/test
bash ./runtestall.sh
popd
```

## Build for Windows

To build for Windows, we use MinGW(Minimalist GNU for Windows). [Here](https://github.com/git-for-windows/build-extra/releases) you can download a tool that includes it.
Expand Down