bricks/ev3dev: merge changes from upstream unix port

This merges some changes to makefiles and config files from the upstream
unix port.

Author: dlech

bricks/ev3dev/Makefile

@@ -39,15 +39,17 @@ USER_C_MODULES = ../..
 -include mpconfigport.mk
 include ../../micropython/py/mkenv.mk
 
-FROZEN_DIR = scripts
-FROZEN_MPY_DIR = modules
-QSTR_GLOBAL_DEPENDENCIES = brickconfig.h
+# use FROZEN_MANIFEST for new projects, others are legacy
+FROZEN_MANIFEST ?= manifest.py
+FROZEN_DIR =
+FROZEN_MPY_DIR =
 
 # define main target
 PROG = pybricks-micropython
 
 # qstr definitions (must come before including py.mk)
 #QSTR_DEFS = qstrdefsport.h
+QSTR_GLOBAL_DEPENDENCIES =
 
 # OS name, for simple autoconfig
 UNAME_S := $(shell uname -s)
@@ -72,15 +74,23 @@ INC += -I$(TOP)/ports/unix
 
 # compiler settings
 CWARN = -Wall -Werror
-CWARN += -Wpointer-arith -Wuninitialized
-CFLAGS = $(INC) $(CWARN) -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA)
+CWARN += -Wpointer-arith -Wuninitialized -Wdouble-promotion -Wsign-compare -Wfloat-conversion
+CFLAGS += $(INC) $(CWARN) -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA)
 
 # Debugging/Optimization
 ifdef DEBUG
-CFLAGS += -g
-COPT = -O0
+COPT ?= -O0
 else
-COPT = -Os -fdata-sections -ffunction-sections -DNDEBUG
+COPT ?= -Os
+COPT += -fdata-sections -ffunction-sections
+COPT += -DNDEBUG
+endif
+
+# Always enable symbols -- They're occasionally useful, and don't make it into the
+# final .bin/.hex/.dfu so the extra size doesn't matter.
+CFLAGS += -g
+
+ifndef DEBUG
 # _FORTIFY_SOURCE is a feature in gcc/glibc which is intended to provide extra
 # security for detecting buffer overflows. Some distros (Ubuntu at the very least)
 # have it enabled by default.
@@ -124,7 +134,7 @@ else
 # Use gcc syntax for map file
 LDFLAGS_ARCH = -Wl,-Map=$@.map,--cref -Wl,--gc-sections
 endif
-LDFLAGS = $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
+LDFLAGS += $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
 
 # Flags to link with pthread library
 LDFLAGS += -lpthread
@@ -183,20 +193,19 @@ endif
 
 # source files
 SRC_C = $(addprefix micropython/ports/unix/,\
-	main.c \
+	alloc.c \
+	coverage.c \
 	gccollect.c \
-	mpthreadport.c \
 	input.c \
-	file.c \
+	main.c \
 	modmachine.c \
 	modos.c \
-	modmmap.c \
-	moduos_vfs.c \
 	modtime.c \
 	modufcntl.c \
+	modummap.c \
+	moduos_vfs.c \
 	moduselect.c \
-	alloc.c \
-	coverage.c \
+	mpthreadport.c \
 	)
 
 SRC_C += $(SRC_MOD)
@@ -216,6 +225,7 @@ PYBRICKS_SRC_C += \
 LIB_SRC_C = $(addprefix micropython/lib/,\
 	$(LIB_SRC_C_EXTRA) \
 	timeutils/timeutils.c \
+	utils/gchelper_generic.c \
 	utils/pyexec.c \
 	)
 
@@ -288,26 +298,32 @@ PYBRICKS_LIB_SRC_C = $(addprefix lib/,\
 OBJ = $(PY_O)
 OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(LIB_SRC_C:.c=.o))
+OBJ += $(addprefix $(BUILD)/, $(EXTMOD_SRC_C:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(PYBRICKS_SRC_C:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(PYBRICKS_PY_SRC_C:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(PYBRICKS_EXTMOD_SRC_C:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(PYBRICKS_LIB_SRC_C:.c=.o))
 
 # List of sources for qstr extraction
-SRC_QSTR += $(SRC_C) $(LIB_SRC_C) $(PYBRICKS_SRC_C) $(PYBRICKS_PY_SRC_C) $(PYBRICKS_EXTMOD_SRC_C)
+SRC_QSTR += $(SRC_C) $(LIB_SRC_C) $(EXTMOD_SRC_C) $(PYBRICKS_SRC_C) $(PYBRICKS_PY_SRC_C) $(PYBRICKS_EXTMOD_SRC_C)
 # Append any auto-generated sources that are needed by sources listed in
 # SRC_QSTR
 SRC_QSTR_AUTO_DEPS +=
 
-ifneq ($(FROZEN_MPY_DIR),)
-# To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and
-# then invoke make with FROZEN_MPY_DIR=frozen (be sure to build from scratch).
+ifneq ($(FROZEN_MANIFEST)$(FROZEN_MPY_DIR),)
+# To use frozen code create a manifest.py file with a description of files to
+# freeze, then invoke make with FROZEN_MANIFEST=manifest.py (be sure to build from scratch).
 CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool
 CFLAGS += -DMICROPY_MODULE_FROZEN_MPY
 CFLAGS += -DMPZ_DIG_SIZE=16 # force 16 bits to work on both 32 and 64 bit archs
 MPY_CROSS_FLAGS += -mcache-lookup-bc
 endif
 
+ifneq ($(FROZEN_MANIFEST)$(FROZEN_DIR),)
+CFLAGS += -DMICROPY_MODULE_FROZEN_STR
+endif
+
+RUN_TESTS_MPY_CROSS_FLAGS = --mpy-cross-flags='-mcache-lookup-bc'
 
 include $(TOP)/py/mkrules.mk
 
@@ -342,15 +358,13 @@ test: $(PROG) $(TOP)/tests/run-tests $(GRX_TEST_PLUGIN_LIB)
 		GRX_PLUGIN_PATH=$(realpath $(BUILD)) GRX_DRIVER=test \
 		./run-tests
 
-# install micropython in /usr/local/bin
 PREFIX = /usr/local
 BINDIR = $(DESTDIR)$(PREFIX)/bin
 
 install: $(PROG)
 	install -d $(BINDIR)
 	install $(PROG) $(BINDIR)/$(PROG)
 
-# uninstall micropython
 uninstall:
 	-rm $(BINDIR)/$(PROG)
 

bricks/ev3dev/brickconfig.h

@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2020 The Pybricks Authors
 
-#include <glib.h>
-
 #include <pbdrv/config.h>
 #include "pbinit.h"
 
@@ -25,20 +23,6 @@
 #define MICROPY_PORT_INIT_FUNC pybricks_init()
 #define MICROPY_PORT_DEINIT_FUNC pybricks_deinit()
 #define MICROPY_MPHALPORT_H "ev3dev_mphal.h"
-#define MICROPY_VM_HOOK_LOOP do { \
-        extern int pbio_do_one_event(void); \
-        pbio_do_one_event(); \
-} while (0);
-#define MICROPY_EVENT_POLL_HOOK do { \
-        extern void mp_handle_pending(bool); \
-        mp_handle_pending(true); \
-        extern int pbio_do_one_event(void); \
-        while (pbio_do_one_event()) { } \
-        MP_THREAD_GIL_EXIT(); \
-        g_main_context_iteration(g_main_context_get_thread_default(), TRUE); \
-        MP_THREAD_GIL_ENTER(); \
-} while (0);
-
 #define MICROPY_PY_SYS_PATH_DEFAULT (":~/.pybricks-micropython/lib:/usr/lib/pybricks-micropython")
 
 extern const struct _mp_obj_module_t pb_module_bluetooth;

bricks/ev3dev/ev3dev_mphal.h

@@ -57,3 +57,21 @@ static inline void mp_hal_delay_us(mp_uint_t us) {
 #define RAISE_ERRNO(err_flag, error_val) \
     { if (err_flag == -1) \
       { mp_raise_OSError(error_val); } }
+
+// This macro is used to implement PEP 475 to retry specified syscalls on EINTR
+#define MP_HAL_RETRY_SYSCALL(ret, syscall, raise) { \
+        for (;;) { \
+            MP_THREAD_GIL_EXIT(); \
+            ret = syscall; \
+            MP_THREAD_GIL_ENTER(); \
+            if (ret == -1) { \
+                int err = errno; \
+                if (err == EINTR) { \
+                    mp_handle_pending(true); \
+                    continue; \
+                } \
+                raise; \
+            } \
+            break; \
+        } \
+}

bricks/ev3dev/manifest.py

@@ -0,0 +1,3 @@
+freeze_as_mpy("./modules")
+freeze_as_mpy("$(MPY_DIR)/tools", "upip.py")
+freeze_as_mpy("$(MPY_DIR)/tools", "upip_utarfile.py", opt=3)

bricks/ev3dev/mpconfigport.h

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) 2013, 2014 Damien P. George
+// Copyright (c) 2018-2020 The Pybricks Authors
 
 // Pybricks brick specific definitions
 #include "brickconfig.h"
@@ -64,7 +65,9 @@ static const char pybricks_ev3dev_help_text[] =
 #ifndef MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE
 #define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1)
 #endif
+#define MICROPY_MODULE_WEAK_LINKS   (1)
 #define MICROPY_CAN_OVERRIDE_BUILTINS (1)
+#define MICROPY_VFS_POSIX_FILE      (1)
 #define MICROPY_PY_FUNCTION_ATTRS   (1)
 #define MICROPY_PY_DESCRIPTORS      (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE (1)
@@ -86,13 +89,21 @@ static const char pybricks_ev3dev_help_text[] =
 #define MICROPY_PY_REVERSE_SPECIAL_METHODS (1)
 #define MICROPY_PY_ARRAY_SLICE_ASSIGN (1)
 #define MICROPY_PY_BUILTINS_SLICE_ATTRS (1)
+#define MICROPY_PY_BUILTINS_SLICE_INDICES (1)
 #define MICROPY_PY_INSTANCE_ATTRS   (1)
 #define MICROPY_PY_SYS_EXIT         (1)
+#define MICROPY_PY_SYS_ATEXIT       (1)
+#if MICROPY_PY_SYS_SETTRACE
+#define MICROPY_PERSISTENT_CODE_SAVE (1)
+#define MICROPY_COMP_CONST (0)
+#endif
+#ifndef MICROPY_PY_SYS_PLATFORM
 #if defined(__APPLE__) && defined(__MACH__)
     #define MICROPY_PY_SYS_PLATFORM  "darwin"
 #else
     #define MICROPY_PY_SYS_PLATFORM  "linux"
 #endif
+#endif
 #define MICROPY_PY_SYS_MAXSIZE      (1)
 #define MICROPY_PY_SYS_STDFILES     (1)
 #define MICROPY_PY_SYS_EXC_INFO     (0)
@@ -105,7 +116,6 @@ static const char pybricks_ev3dev_help_text[] =
 #define MICROPY_PY_IO_IOBASE        (1)
 #define MICROPY_PY_IO_FILEIO        (1)
 #define MICROPY_PY_GC_COLLECT_RETVAL (1)
-#define MICROPY_MODULE_FROZEN_STR   (1)
 #define MICROPY_MODULE_BUILTIN_INIT (1)
 
 #define MICROPY_PY_THREAD           (1)
@@ -200,6 +210,7 @@ static const char pybricks_ev3dev_help_text[] =
 #define MICROPY_PY_UTIMEQ           (1)
 #define MICROPY_PY_UHASHLIB         (1)
 #if MICROPY_PY_USSL
+#define MICROPY_PY_UHASHLIB_MD5     (1)
 #define MICROPY_PY_UHASHLIB_SHA1    (1)
 #define MICROPY_PY_UCRYPTOLIB       (1)
 #endif
@@ -233,23 +244,20 @@ static const char pybricks_ev3dev_help_text[] =
 
 extern const struct _mp_print_t mp_stderr_print;
 
-// Define to 1 to use undertested inefficient GC helper implementation
-// (if more efficient arch-specific one is not available).
-#ifndef MICROPY_GCREGS_SETJMP
-    #ifdef __mips__
-        #define MICROPY_GCREGS_SETJMP (1)
-    #else
-        #define MICROPY_GCREGS_SETJMP (0)
-    #endif
+#if !(defined(MICROPY_GCREGS_SETJMP) || defined(__x86_64__) || defined(__i386__) || defined(__thumb2__) || defined(__thumb__) || defined(__arm__))
+// Fall back to setjmp() implementation for discovery of GC pointers in registers.
+#define MICROPY_GCREGS_SETJMP (1)
 #endif
 
 #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF   (1)
 #define MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE  (256)
 #define MICROPY_KBD_EXCEPTION       (1)
 #define MICROPY_ASYNC_KBD_INTR      (0)
 
+#define mp_type_fileio mp_type_vfs_posix_fileio
+#define mp_type_textio mp_type_vfs_posix_textio
+
 extern const struct _mp_obj_module_t mp_module_machine;
-extern const struct _mp_obj_module_t mp_module_mmap;
 extern const struct _mp_obj_module_t mp_module_os;
 extern const struct _mp_obj_module_t mp_module_uos_vfs;
 extern const struct _mp_obj_module_t mp_module_uselect;
@@ -259,6 +267,7 @@ extern const struct _mp_obj_module_t mp_module_socket;
 extern const struct _mp_obj_module_t mp_module_ffi;
 extern const struct _mp_obj_module_t mp_module_jni;
 extern const struct _mp_obj_module_t mp_module_ufcntl;
+extern const struct _mp_obj_module_t mp_module_ummap;
 
 #if MICROPY_PY_UOS_VFS
 #define MICROPY_PY_UOS_DEF { MP_ROM_QSTR(MP_QSTR_uos), MP_ROM_PTR(&mp_module_uos_vfs) },
@@ -306,8 +315,8 @@ extern const struct _mp_obj_module_t mp_module_ufcntl;
     MICROPY_PY_UOS_DEF \
     MICROPY_PY_USELECT_DEF \
     MICROPY_PY_TERMIOS_DEF \
-    { MP_ROM_QSTR(MP_QSTR_mmap), MP_ROM_PTR(&mp_module_mmap) }, \
     { MP_ROM_QSTR(MP_QSTR_ufcntl), MP_ROM_PTR(&mp_module_ufcntl) }, \
+    { MP_ROM_QSTR(MP_QSTR_ummap), MP_ROM_PTR(&mp_module_ummap) }, \
 
 // type definitions for the specific machine
 
@@ -345,17 +354,6 @@ void mp_unix_mark_exec(void);
 #define MICROPY_FORCE_PLAT_ALLOC_EXEC (1)
 #endif
 
-#if MICROPY_PY_OS_DUPTERM
-#define MP_PLAT_PRINT_STRN(str, len) mp_hal_stdout_tx_strn_cooked(str, len)
-#else
-#define MP_PLAT_PRINT_STRN(str, len) do { \
-        MP_THREAD_GIL_EXIT(); \
-        ssize_t ret = write(1, str, len); \
-        MP_THREAD_GIL_ENTER(); \
-        (void)ret; \
-} while (0)
-#endif
-
 #ifdef __linux__
 // Can access physical memory using /dev/mem
 #define MICROPY_PLAT_DEV_MEM  (1)
@@ -380,9 +378,17 @@ void mp_unix_mark_exec(void);
 
 #define MP_STATE_PORT MP_STATE_VM
 
+#if MICROPY_PY_BLUETOOTH && MICROPY_BLUETOOTH_BTSTACK
+struct _mp_bluetooth_btstack_root_pointers_t;
+#define MICROPY_BLUETOOTH_ROOT_POINTERS struct _mp_bluetooth_btstack_root_pointers_t *bluetooth_btstack_root_pointers;
+#else
+#define MICROPY_BLUETOOTH_ROOT_POINTERS
+#endif
+
 #define MICROPY_PORT_ROOT_POINTERS \
     const char *readline_hist[50]; \
     void *mmap_region_head; \
+    MICROPY_BLUETOOTH_ROOT_POINTERS \
 
 // We need to provide a declaration/definition of alloca()
 // unless support for it is disabled.
@@ -413,4 +419,30 @@ void mp_unix_mark_exec(void);
 #include <stdio.h>
 #endif
 
+#if MICROPY_PY_THREAD
+#define MICROPY_BEGIN_ATOMIC_SECTION() (mp_thread_unix_begin_atomic_section(), 0)
+#define MICROPY_END_ATOMIC_SECTION(x) (void)x; mp_thread_unix_end_atomic_section()
+#endif
+
+#define MICROPY_VM_HOOK_LOOP do { \
+        extern int pbio_do_one_event(void); \
+        pbio_do_one_event(); \
+} while (0);
+
+#include <glib.h>
+
+#define MICROPY_EVENT_POLL_HOOK do { \
+        extern void mp_handle_pending(bool); \
+        mp_handle_pending(true); \
+        extern int pbio_do_one_event(void); \
+        while (pbio_do_one_event()) { } \
+        MP_THREAD_GIL_EXIT(); \
+        g_main_context_iteration(g_main_context_get_thread_default(), TRUE); \
+        MP_THREAD_GIL_ENTER(); \
+} while (0);
+
+
+#include <sched.h>
+#define MICROPY_UNIX_MACHINE_IDLE sched_yield();
+
 #include "../pybricks_config.h"

bricks/ev3dev/mpconfigport.mk

@@ -25,13 +25,11 @@ MICROPY_PY_FFI = 1
 
 # ussl module requires one of the TLS libraries below
 MICROPY_PY_USSL = 1
-# axTLS has minimal size and fully integrated with MicroPython, but
-# implements only a subset of modern TLS functionality, so may have
-# problems with some servers.
+# axTLS has minimal size but implements only a subset of modern TLS
+# functionality, so may have problems with some servers.
 MICROPY_SSL_AXTLS = 1
 # mbedTLS is more up to date and complete implementation, but also
-# more bloated. Configuring and building of mbedTLS should be done
-# outside of MicroPython, it can just link with mbedTLS library.
+# more bloated.
 MICROPY_SSL_MBEDTLS = 0
 
 # jni module requires JVM/JNI