Start on improving Python project structure

Author: multiplemonomials

tools/cmake/mbed_python_interpreter.cmake

@@ -6,30 +6,54 @@
 
 option(MBED_CREATE_PYTHON_VENV "If true, Mbed OS will create its own virtual environment (venv) and install its Python packages there.  This removes the need to manually install Python packages." TRUE)
 
+get_filename_component(MBED_CE_TOOLS_BASE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)
+
 if(MBED_CREATE_PYTHON_VENV)
     # Use the venv.
 
     # Note: venv is stored in the source directory as it can be shared between all the build directories
     # (not target specific)
     set(MBED_VENV_LOCATION ${MBED_SOURCE_DIR}/venv)
     set(VENV_STAMP_FILE ${MBED_VENV_LOCATION}/mbed-venv.stamp)
-    set(MBED_REQUIREMENTS_TXT_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../requirements.txt")
+    set(MBED_PYPROJECT_TOML_LOCATION "${MBED_CE_TOOLS_BASE_DIR}/pyproject.toml")
 
-    # Make it so modifying requirements.txt will trigger a reconfigure
-    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_REQUIREMENTS_TXT_LOCATION})
+    # Make it so modifying pyproject.toml will trigger a reconfigure
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_PYPROJECT_TOML_LOCATION})
 
-    # Find Python3, using the venv if it already exists
-    set (ENV{VIRTUAL_ENV} ${MBED_VENV_LOCATION})
-    set (Python3_FIND_VIRTUALENV FIRST)
-    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    # Find Python3 (this will get the one in the venv if we already found it)
+    set(ENV{VIRTUAL_ENV} ${MBED_VENV_LOCATION})
+    set(Python3_FIND_VIRTUALENV FIRST)
+    find_package(Python3 COMPONENTS Interpreter)
     include(CheckPythonPackage)
 
     set(NEED_TO_CREATE_VENV FALSE)
     set(NEED_TO_INSTALL_PACKAGES FALSE)
-    if(NOT EXISTS "${VENV_STAMP_FILE}")
+
+    # Special situation: if we have a cached interpreter location in the venv dir, but Python could be found,
+    # this means that the venv was deleted or symlinks to a missing python install location.
+    # So, use the system python and recreate it.
+    if("${Python3_EXECUTABLE}" MATCHES "${MBED_VENV_LOCATION}" AND NOT Python3_FOUND)
+        message(STATUS "Python venv deleted or unusable. Recreating using system Python...")
+
+        # Launch a new search for Python3
+        unset(Python3_EXECUTABLE)
+        unset(_Python3_EXECUTABLE CACHE)
+        unset(_Python3_INTERPRETER_PROPERTIES CACHE)
+        unset(_Python3_INTERPRETER_SIGNATURE CACHE)
+        set (Python3_FIND_VIRTUALENV STANDARD)
+        find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+        set(NEED_TO_CREATE_VENV TRUE)
+        set(NEED_TO_INSTALL_PACKAGES TRUE)
+    elseif(NOT EXISTS "${VENV_STAMP_FILE}")
         set(NEED_TO_CREATE_VENV TRUE)
         set(NEED_TO_INSTALL_PACKAGES TRUE)
-    elseif("${MBED_REQUIREMENTS_TXT_LOCATION}" IS_NEWER_THAN "${VENV_STAMP_FILE}")
+    elseif(NOT ("${Python3_EXECUTABLE}" MATCHES "${MBED_VENV_LOCATION}"))
+        # Alternately if we think we have the venv but FindPython didn't use it, that likely means it's
+        # missing or corrupted and we need to recreate it
+        set(NEED_TO_CREATE_VENV TRUE)
+        set(NEED_TO_INSTALL_PACKAGES TRUE)
+    elseif("${MBED_PYPROJECT_TOML_LOCATION}" IS_NEWER_THAN "${VENV_STAMP_FILE}")
         set(NEED_TO_INSTALL_PACKAGES TRUE)
     endif()
 
@@ -47,7 +71,8 @@ if(MBED_CREATE_PYTHON_VENV)
         unset(_Python3_EXECUTABLE CACHE)
         unset(_Python3_INTERPRETER_PROPERTIES CACHE)
         unset(_Python3_INTERPRETER_SIGNATURE CACHE)
-        ## Launch a new search for Python3
+
+        ## Launch a new search for Python3 in the venv
         find_package (Python3 REQUIRED COMPONENTS Interpreter)
     endif()
 
@@ -59,7 +84,7 @@ if(MBED_CREATE_PYTHON_VENV)
             COMMAND_ERROR_IS_FATAL ANY
         )
         execute_process(
-            COMMAND ${Python3_EXECUTABLE} -m pip install -r ${MBED_REQUIREMENTS_TXT_LOCATION}
+            COMMAND ${Python3_EXECUTABLE} -m pip install ${MBED_CE_TOOLS_BASE_DIR}
             COMMAND_ERROR_IS_FATAL ANY
         )
 
@@ -75,25 +100,12 @@ else()
     find_package(Python3 REQUIRED COMPONENTS Interpreter)
     include(CheckPythonPackage)
 
+    # The cmsis_mcu_descr module was written from scratch by Mbed CE.
+    # So, this check will ensure that the user has installed the Mbed CE version of mbed_tools
+    # and not the PyPI version (which we cannot do anything with because it's owned by ARM)
+    check_python_package(mbed_tools.cli.cmsis_mcu_descr HAVE_MBED_CE_TOOLS)
 
-    # Check python packages
-    set(PYTHON_PACKAGES_TO_CHECK intelhex prettytable future jinja2)
-    foreach(PACKAGE_NAME ${PYTHON_PACKAGES_TO_CHECK})
-        string(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UCASE) # Ucase name needed for CMake variable
-        string(TOLOWER ${PACKAGE_NAME} PACKAGE_NAME_LCASE) # Lcase name needed for import statement
-
-        check_python_package(${PACKAGE_NAME_LCASE} HAVE_PYTHON_${PACKAGE_NAME_UCASE})
-        if(NOT HAVE_PYTHON_${PACKAGE_NAME_UCASE})
-            message(WARNING "Missing Python dependency ${PACKAGE_NAME}")
-        endif()
-    endforeach()
-
-    # Check deps for memap
-    if(Python3_FOUND AND HAVE_PYTHON_INTELHEX AND HAVE_PYTHON_PRETTYTABLE)
-        set(HAVE_MEMAP_DEPS TRUE)
-    else()
-        set(HAVE_MEMAP_DEPS FALSE)
-        message(STATUS "Missing Python dependencies (at least one of: python3, intelhex, prettytable) so the memory map cannot be printed")
+    if(NOT HAVE_MBED_CE_TOOLS)
+        message(FATAL_ERROR "Did not detect the Mbed CE Python tools installed into the python interpreter ${Python3_EXECUTABLE}. Install them with a command like: ${Python3_EXECUTABLE} -m pip install ${MBED_CE_TOOLS_BASE_DIR}")
     endif()
-
 endif()

tools/pyproject.toml

@@ -0,0 +1,62 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "mbed-ce-tools"
+version = "0.0.1"
+description = 'Tools for building and testing Mbed OS Community Edition'
+requires-python = ">=3.8"
+license = "Apache-2.0"
+dependencies = [
+    # memap dependencies ---------------------------------------------------------------------
+    "intelhex>=2.3.0,<3.0.0",
+
+    # 3.12.0 both added HRuleStyle and deprecated the old constants in one single release :/
+    "prettytable>=2.0,<3.12.0; python_version < '3.9'",
+    "prettytable>=3.12.0; python_version >= '3.9'",
+
+    # mbed_tools dependencies ----------------------------------------------------------------
+    "jinja2>=2.11.3",
+    "python-dotenv",
+    "Click>=7.1",
+    "GitPython",
+    "tqdm",
+    "tabulate",
+    "requests>=2.20",
+    "typing-extensions",
+    "pyserial",
+    "appdirs",
+    "pyjson5>=1.6",
+    "humanize>=4.9.0",
+
+    # Needed for downloading CMSIS MCU descriptions
+    "cmsis-pack-manager>=0.5.0",
+
+    # USB device detection on Mac
+    "pywin32; platform_system=='Windows'",
+
+    # USB device detection on Linux
+    "psutil; platform_system=='Linux'",
+    "pyudev; platform_system=='Linux'",
+
+    # USB device detection on Mac
+    "beautifulsoup4; sys_platform == 'darwin'",
+    "lxml; sys_platform == 'darwin'",
+
+    # mbed_os_tools.test dependencies ---------------------------------------------------------
+    "junit-xml>=1.0",
+    "lockfile",
+    "colorama",
+
+    # needed for signing secure images --------------------------------------------------------
+    "cryptography",
+    "cbor",
+    "cysecuretools~=6.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["python"]
+
+[project.scripts]
+memap = "memap.memap:main"

tools/python/mbed_os_tools/test/host_tests/base_host_test.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import inspect
-import six
 from time import time
 from inspect import isfunction, ismethod
 
@@ -227,15 +226,15 @@ def register_callback(self, key, callback, force=False):
         # Check if callback has all three required parameters (key, value, timestamp)
         # When callback is class method should have 4 arguments (self, key, value, timestamp)
         if ismethod(callback):
-            arg_count = six.get_function_code(callback).co_argcount
+            arg_count = callback.__code__.co_argcount
             if arg_count != 4:
                 err_msg = "callback 'self.%s('%s', ...)' defined with %d arguments"% (callback.__name__, key, arg_count)
                 err_msg += ", should have 4 arguments: self.%s(self, key, value, timestamp)"% callback.__name__
                 raise TypeError(err_msg)
 
         # When callback is just a function should have 3 arguments func(key, value, timestamp)
         if isfunction(callback):
-            arg_count = six.get_function_code(callback).co_argcount
+            arg_count = callback.__code__.co_argcount
             if arg_count != 3:
                 err_msg = "callback '%s('%s', ...)' defined with %d arguments"% (callback.__name__, key, arg_count)
                 err_msg += ", should have 3 arguments: %s(key, value, timestamp)"% callback.__name__