Merge pull request #7321 from jepler/dotenv-becomes-toml

Dotenv becomes toml

Author: dhalbert

.gitignore

@@ -9,6 +9,7 @@
 !atmel-samd/asf/**/*.a
 *.elf
 *.bin
+!*.toml.bin
 *.map
 *.hex
 *.dis

devices/ble_hci/common-hal/_bleio/Adapter.c

@@ -49,8 +49,8 @@
 #include "shared-bindings/_bleio/ScanEntry.h"
 #include "shared-bindings/time/__init__.h"
 
-#if CIRCUITPY_DOTENV
-#include "shared-module/dotenv/__init__.h"
+#if CIRCUITPY_OS_GETENV
+#include "shared-bindings/os/__init__.h"
 #endif
 
 #define MSEC_TO_UNITS(TIME, RESOLUTION) (((TIME) * 1000) / (RESOLUTION))
@@ -284,15 +284,15 @@ char default_ble_name[] = { 'C', 'I', 'R', 'C', 'U', 'I', 'T', 'P', 'Y', 0, 0, 0
 STATIC void bleio_adapter_hci_init(bleio_adapter_obj_t *self) {
     mp_int_t name_len = 0;
 
-    #if CIRCUITPY_DOTENV
-    char ble_name[32];
-    name_len = dotenv_get_key("/.env", "CIRCUITPY_BLE_NAME", ble_name, sizeof(ble_name) - 1);
-    if (name_len > 0) {
-        self->name = mp_obj_new_str(ble_name, (size_t)name_len);
+    #if CIRCUITPY_OS_GETENV
+    mp_obj_t name = common_hal_os_getenv("CIRCUITPY_BLE_NAME", mp_const_none);
+    if (name != mp_const_none) {
+        mp_arg_validate_type_string(name, MP_QSTR_CIRCUITPY_BLE_NAME);
+        self->name = name;
     }
     #endif
 
-    if (name_len <= 0) {
+    if (!self->name) {
         name_len = sizeof(default_ble_name);
         bt_addr_t addr;
         hci_check_error(hci_read_bd_addr(&addr));

docs/environment.rst

@@ -6,24 +6,49 @@ variables are commonly used to store "secrets" such as Wi-Fi passwords and API
 keys. This method *does not* make them secure. It only separates them from the
 code.
 
-CircuitPython supports these by mimicking the `dotenv <https://github.com/theskumar/python-dotenv>`_
-CPython library. Other languages such as Javascript, PHP and Ruby also have
-dotenv libraries.
+CircuitPython uses a file called ``settings.toml`` at the drive root (no
+folder) as the environment.  User code can access the values from the file
+using `os.getenv()`. It is recommended to save any values used repeatedly in a
+variable because `os.getenv()` will parse the ``settings.toml`` file contents
+on every access.
 
-These libraries store environment variables in a ``.env`` file. Here is a simple
-example:
+CircuitPython only supports a subset of the full toml specification, see below
+for more details. The subset is very "Python-like", which is a key reason we
+selected the format.
 
-.. code-block:: bash
+Due to technical limitations it probably also accepts some files that are
+not valid TOML files; bugs of this nature are subject to change (i.e., be
+fixed) without the usual deprecation period for incompatible changes.
 
-  KEY1='value1'
-  # Comment
-  KEY2='value2
-  is multiple lines'
+File format example:
 
-CircuitPython uses the ``.env`` at the drive root (no folder) as the environment.
-User code can access the values from the file using `os.getenv()`. It is
-recommended to save any values used repeatedly in a variable because `os.getenv()`
-will parse the ``/.env`` on every access.
+.. code-block::
+
+   str_key="Hello world" # with trailing comment
+   int_key = 7
+   unicode_key="œuvre"
+   unicode_key2="\\u0153uvre" # same as above
+   unicode_key3="\\U00000153uvre" # same as above
+   escape_codes="supported, including \\r\\n\\"\\\\"
+   # comment
+   [subtable]
+   subvalue="cannot retrieve this using getenv"
+
+
+Details of the toml language subset
+-----------------------------------
+
+* The content is required to be in UTF-8 encoding
+* The supported data types are string and integer
+* Only basic strings are supported, not triple-quoted strings
+* Only integers supported by strtol. (no 0o, no 0b, no underscores 1_000, 011
+  is 9, not 11)
+* Only bare keys are supported
+* Duplicate keys are not diagnosed.
+* Comments are supported
+* Only values from the "root table" can be retrieved
+* due to technical limitations, the content of multi-line
+  strings can erroneously be parsed as a value.
 
 CircuitPython behavior
 ----------------------

docs/workflows.md

@@ -46,7 +46,7 @@ connection, the central device can discover two default services. One for file t
 CircuitPython specifically that includes serial characteristics.
 
 To change the default BLE advertising name without (or before) running user code, the desired name
-can be put in the `/.env` file. The key is `CIRCUITPY_BLE_NAME`. It's limited to approximately
+can be put in the `settings.toml` file. The key is `CIRCUITPY_BLE_NAME`. It's limited to approximately
 30 characters depending on the port's settings and will be truncated if longer.
 
 ### File Transfer API
@@ -69,21 +69,21 @@ Read-only characteristic that returns the UTF-8 encoded version string.
 
 ## Web
 
-The web workflow is depends on adding Wi-Fi credentials into the `/.env` file. The keys are
+The web workflow is depends on adding Wi-Fi credentials into the `settings.toml` file. The keys are
 `CIRCUITPY_WIFI_SSID` and `CIRCUITPY_WIFI_PASSWORD`. Once these are defined, CircuitPython will
 automatically connect to the network and start the webserver used for the workflow. The webserver
 is on port 80 unless overridden by `CIRCUITPY_WEB_API_PORT`. It also enables MDNS.
 
-Here is an example `/.env`:
+Here is an example `/settings.toml`:
 
 ```bash
 # To auto-connect to Wi-Fi
-CIRCUITPY_WIFI_SSID='scottswifi'
-CIRCUITPY_WIFI_PASSWORD='secretpassword'
+CIRCUITPY_WIFI_SSID="scottswifi"
+CIRCUITPY_WIFI_PASSWORD="secretpassword"
 
 # To enable modifying files from the web. Change this too!
 # Leave the User field blank in the browser.
-CIRCUITPY_WEB_API_PASSWORD='passw0rd'
+CIRCUITPY_WEB_API_PASSWORD="passw0rd"
 
 CIRCUITPY_WEB_API_PORT=80
 ```
@@ -124,7 +124,7 @@ All file system related APIs are protected by HTTP basic authentication. It is *
 hopefully prevent some griefing in shared settings. The password is sent unencrypted so do not reuse
 a password with something important. The user field is left blank.
 
-The password is taken from `/.env` with the key `CIRCUITPY_WEB_API_PASSWORD`. If this is unset, the
+The password is taken from `settings.toml` with the key `CIRCUITPY_WEB_API_PASSWORD`. If this is unset, the
 server will respond with `403 Forbidden`. When a password is set, but not provided in a request, it
 will respond `401 Unauthorized`.
 

locale/circuitpython.pot

@@ -1002,6 +1002,10 @@ msgstr ""
 msgid "File exists"
 msgstr ""
 
+#: shared-module/os/getenv.c
+msgid "File not found"
+msgstr ""
+
 #: ports/atmel-samd/common-hal/canio/Listener.c
 #: ports/espressif/common-hal/canio/Listener.c
 #: ports/stm/common-hal/canio/Listener.c
@@ -1058,6 +1062,8 @@ msgstr ""
 #: shared-bindings/displayio/Display.c
 #: shared-bindings/displayio/EPaperDisplay.c
 #: shared-bindings/framebufferio/FramebufferDisplay.c
+#: shared-module/displayio/Display.c
+#: shared-module/framebufferio/FramebufferDisplay.c
 msgid "Group already used"
 msgstr ""
 
@@ -1184,6 +1190,7 @@ msgid "Internal define error"
 msgstr ""
 
 #: ports/espressif/common-hal/paralleldisplay/ParallelBus.c
+#: shared-module/os/getenv.c
 msgid "Internal error"
 msgstr ""
 
@@ -1230,6 +1237,11 @@ msgstr ""
 msgid "Invalid bits per value"
 msgstr ""
 
+#: shared-module/os/getenv.c
+#, c-format
+msgid "Invalid byte %.*s"
+msgstr ""
+
 #: ports/atmel-samd/common-hal/imagecapture/ParallelImageCapture.c
 #, c-format
 msgid "Invalid data_pins[%d]"
@@ -1260,10 +1272,18 @@ msgstr ""
 msgid "Invalid state"
 msgstr ""
 
+#: shared-module/os/getenv.c
+msgid "Invalid unicode escape"
+msgstr ""
+
 #: shared-bindings/aesio/aes.c
 msgid "Key must be 16, 24, or 32 bytes long"
 msgstr ""
 
+#: shared-module/os/getenv.c
+msgid "Key not found"
+msgstr ""
+
 #: shared-module/is31fl3741/FrameBuffer.c
 msgid "LED mappings must match display size"
 msgstr ""
@@ -2262,7 +2282,7 @@ msgid "Unkown error code %d"
 msgstr ""
 
 #: shared-bindings/adafruit_pixelbuf/PixelBuf.c
-#: shared-module/adafruit_pixelbuf/PixelMap.c
+#: shared-module/_pixelmap/PixelMap.c
 #, c-format
 msgid "Unmatched number of items on RHS (expected %d, got %d)."
 msgstr ""
@@ -3143,7 +3163,7 @@ msgstr ""
 msgid "index is out of bounds"
 msgstr ""
 
-#: shared-bindings/adafruit_pixelbuf/PixelMap.c
+#: shared-bindings/_pixelmap/PixelMap.c
 msgid "index must be tuple or int"
 msgstr ""
 
@@ -3522,7 +3542,7 @@ msgstr ""
 msgid "negative shift count"
 msgstr ""
 
-#: shared-bindings/adafruit_pixelbuf/PixelMap.c
+#: shared-bindings/_pixelmap/PixelMap.c
 msgid "nested index must be int"
 msgstr ""
 

ports/atmel-samd/mpconfigport.mk

@@ -24,13 +24,13 @@ ifeq ($(CHIP_FAMILY),samd21)
 CIRCUITPY_AESIO ?= 0
 CIRCUITPY_ATEXIT ?= 0
 CIRCUITPY_AUDIOMIXER ?= 0
+CIRCUITPY_AUDIOMP3 ?= 0
 CIRCUITPY_BINASCII ?= 0
 CIRCUITPY_BITBANGIO ?= 0
 CIRCUITPY_BITMAPTOOLS ?= 0
-CIRCUITPY_BUSDEVICE ?= 0
-CIRCUITPY_AUDIOMP3 ?= 0
 CIRCUITPY_BLEIO_HCI = 0
 CIRCUITPY_BUILTINS_POW3 ?= 0
+CIRCUITPY_BUSDEVICE ?= 0
 CIRCUITPY_COMPUTED_GOTO_SAVE_SPACE ?= 1
 CIRCUITPY_COUNTIO ?= 0
 # Not enough RAM for framebuffers
@@ -42,6 +42,7 @@ CIRCUITPY_I2CTARGET ?= 0
 CIRCUITPY_JSON ?= 0
 CIRCUITPY_KEYPAD ?= 0
 CIRCUITPY_MSGPACK ?= 0
+CIRCUITPY_OS_GETENV ?= 0
 CIRCUITPY_PIXELMAP ?= 0
 CIRCUITPY_RE ?= 0
 CIRCUITPY_SDCARDIO ?= 0

ports/cxd56/Makefile

@@ -134,7 +134,8 @@ SRC_COMMON_HAL_EXPANDED = $(addprefix shared-bindings/, $(SRC_COMMON_HAL)) \
                           $(addprefix common-hal/, $(SRC_COMMON_HAL))
 
 SRC_SHARED_MODULE_EXPANDED = $(addprefix shared-bindings/, $(SRC_SHARED_MODULE)) \
-                             $(addprefix shared-module/, $(SRC_SHARED_MODULE))
+                             $(addprefix shared-module/, $(SRC_SHARED_MODULE)) \
+                             $(addprefix shared-module/, $(SRC_SHARED_MODULE_INTERNAL))
 
 SRC_S = supervisor/cpu.s
 

ports/espressif/bindings/esp32_camera/Camera.c

@@ -63,7 +63,7 @@
 //|         """
 //|         Configure and initialize a camera with the given properties
 //|
-//|         This driver requires that the ``CIRCUITPY_RESERVED_PSRAM`` in ``/.env`` be large enough to hold the camera frambuffer(s). Generally, boards with built-in cameras will have a default setting that is large enough. If the constructor raises a MemoryError or an IDFError, this probably indicates the setting is too small and should be increased.
+//|         This driver requires that the ``CIRCUITPY_RESERVED_PSRAM`` in ``settings.toml`` be large enough to hold the camera frambuffer(s). Generally, boards with built-in cameras will have a default setting that is large enough. If the constructor raises a MemoryError or an IDFError, this probably indicates the setting is too small and should be increased.
 //|
 //|
 //|         .. important::

ports/espressif/bindings/espidf/__init__.c

@@ -132,7 +132,7 @@ STATIC mp_obj_t espidf_get_total_psram(void) {
 MP_DEFINE_CONST_FUN_OBJ_0(espidf_get_total_psram_obj, espidf_get_total_psram);
 
 //| def get_reserved_psram() -> int:
-//|     """Returns number of bytes of psram reserved for use by esp-idf, either a board-specific default value or the value defined in ``/.env``."""
+//|     """Returns number of bytes of psram reserved for use by esp-idf, either a board-specific default value or the value defined in ``settings.toml``."""
 //|
 STATIC mp_obj_t espidf_get_reserved_psram(void) {
     return MP_OBJ_NEW_SMALL_INT(common_hal_espidf_get_reserved_psram());

ports/espressif/common-hal/_bleio/Adapter.c

@@ -59,8 +59,8 @@
 #include "esp_bt.h"
 #include "esp_nimble_hci.h"
 
-#if CIRCUITPY_DOTENV
-#include "shared-module/dotenv/__init__.h"
+#if CIRCUITPY_OS_GETENV
+#include "shared-module/os/__init__.h"
 #endif
 
 bleio_connection_internal_t bleio_connections[BLEIO_TOTAL_CONNECTION_COUNT];
@@ -101,22 +101,16 @@ void common_hal_bleio_adapter_set_enabled(bleio_adapter_obj_t *self, bool enable
         ble_hs_cfg.sync_cb = _on_sync;
         // ble_hs_cfg.store_status_cb = ble_store_util_status_rr;
 
-        #if CIRCUITPY_DOTENV
-        mp_int_t name_len = 0;
-        char ble_name[32];
-        name_len = dotenv_get_key("/.env", "CIRCUITPY_BLE_NAME", ble_name, sizeof(ble_name) - 1);
-        if (name_len > 0) {
-            if (name_len > MYNEWT_VAL_BLE_SVC_GAP_DEVICE_NAME_MAX_LENGTH) {
-                name_len = MYNEWT_VAL_BLE_SVC_GAP_DEVICE_NAME_MAX_LENGTH;
-            }
-            ble_name[name_len] = '\0';
+        #if CIRCUITPY_OS_GETENV
+        char ble_name[1 + MYNEWT_VAL_BLE_SVC_GAP_DEVICE_NAME_MAX_LENGTH];
+        os_getenv_err_t result = common_hal_os_getenv_str("CIRCUITPY_BLE_NAME", ble_name, sizeof(ble_name));
+        if (result == GETENV_OK) {
             ble_svc_gap_device_name_set(ble_name);
-        } else {
+        } else
+        #endif
+        {
             ble_svc_gap_device_name_set("CIRCUITPY");
         }
-        #else
-        ble_svc_gap_device_name_set("CIRCUITPY");
-        #endif
 
         // Clear all of the internal connection objects.
         for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {