diff --git a/docs/esp32/quickref.rst b/docs/esp32/quickref.rst index 79e61a10b6dfd..c88eff9a4373a 100644 --- a/docs/esp32/quickref.rst +++ b/docs/esp32/quickref.rst @@ -537,3 +537,59 @@ the corresponding functions, or you can use the command-line client See the MicroPython forum for other community-supported alternatives to transfer files to an ESP32 board. + +Controlling the Python heap size +-------------------------------- + +By default MicroPython allocates the largest contiguous chunk of memory to the python heap. +On a "simple" esp32 this comes out to around 100KB and on an esp32 with external SPIRAM this +ends up being the full SPIRAM, typically 4MB. This default allocation may not be desirable +and can be reduced for two use-cases by setting one of two variables in the ``micropython`` NVS +namespace, see :ref:`esp32.NVS ` for details about accessing NVS (Non-Volatile +Storage). + +Because MicroPython allocates the heap as one of the very first actions it is not possible to run +python code to set the heap size. This is the reason that NVS variables are used and it also means +that a hard reset is necessary after setting the variables before they take effect. +A typical use is for ``main.py`` to check heap sizes and, if they're not appropriate, +to set an NVS variable and perform a hard reset. + +The first use-case for this feature is to guarantee that ESP-IDF has some minimum amount of memory +to work with. For example, by default without SPIRAM there is around 90KB left for ESP-IDF +right at boot time. This is not enough for two TLS connections and not enough for one +TLS connection and BLE either. (While 90KB might seem like a lot, it disappears quickly once +Wifi is started and sockets are connected.) + +To give esp-idf a bit more memory, use something like:: + + import machine + from esp32 import NVS, idf_heap_info, HEAP_DATA + + idf_free = sum([h[2] for h in idf_heap_info(HEAP_DATA)]) + print("IDF heap free:", idf_free) + + nvs = NVS("micropython") + nvs.set_i32("min_idf_heap", 120000) + nvs.commit() + machine.reset() + +Setting the ``min_idf_heap`` NVS variable to 120000 tells MicroPython to reduce its heap allocation +from the default such that at least 120000 bytes are left for esp-idf. + +A second use case is to reduce GC times when using SPIRAM. A GC collection has to read sequentially +though all of RAM during its sweep phase. When using a SPIRAM with the default 4MB allocation this +takes about 90ms (assuming 240Mhz cpu and 80Mhz QIO SPIRAM), which is very impactful in a not so +good way. Often applications only need a few hundred KB and this can be accomplished by setting the +``max_mp_heap`` NVS variable to the desired size in bytes. + +A similar use-case is with an SPIRAM where it is desired to leave memory to a native module, for +example to allocate a camera framebuffer. The size of the MP heap can be limited to at most 300KB +using something like:: + + import machine + from esp32 import NVS + + nvs = NVS("micropython") + nvs.set_i32("max_mp_heap", 300*1024) + nvs.commit() + machine.reset() diff --git a/docs/library/esp32.rst b/docs/library/esp32.rst index f179a31ef65e3..e58617731f5cc 100644 --- a/docs/library/esp32.rst +++ b/docs/library/esp32.rst @@ -270,6 +270,8 @@ Constants Selects the wake level for pins. +.. _esp32.NVS: + Non-Volatile Storage -------------------- diff --git a/ports/esp32/boards/sdkconfig.spiram b/ports/esp32/boards/sdkconfig.spiram index 5b4ce118b890e..9667591c68730 100644 --- a/ports/esp32/boards/sdkconfig.spiram +++ b/ports/esp32/boards/sdkconfig.spiram @@ -3,4 +3,5 @@ CONFIG_ESP32_SPIRAM_SUPPORT=y CONFIG_SPIRAM_CACHE_WORKAROUND=y CONFIG_SPIRAM_IGNORE_NOTFOUND=y -CONFIG_SPIRAM_USE_MEMMAP=y +CONFIG_SPIRAM_USE_MEMMAP=n +CONFIG_SPIRAM_USE_CAPS_ALLOC=y diff --git a/ports/esp32/esp32_nvs.c b/ports/esp32/esp32_nvs.c index d13151d3ce774..85bf31989456e 100644 --- a/ports/esp32/esp32_nvs.c +++ b/ports/esp32/esp32_nvs.c @@ -33,6 +33,7 @@ #include "nvs_flash.h" #include "nvs.h" + // This file implements the NVS (Non-Volatile Storage) class in the esp32 module. // It provides simple access to the NVS feature provided by ESP-IDF. diff --git a/ports/esp32/main.c b/ports/esp32/main.c index 7413798d0c302..71ec395d66847 100644 --- a/ports/esp32/main.c +++ b/ports/esp32/main.c @@ -75,30 +75,43 @@ void mp_task(void *pvParameter) { uart_init(); machine_init(); - // TODO: CONFIG_SPIRAM_SUPPORT is for 3.3 compatibility, remove after move to 4.0. - #if CONFIG_ESP32_SPIRAM_SUPPORT || CONFIG_SPIRAM_SUPPORT - // Try to use the entire external SPIRAM directly for the heap - size_t mp_task_heap_size; - void *mp_task_heap = (void *)0x3f800000; - switch (esp_spiram_get_chip_size()) { - case ESP_SPIRAM_SIZE_16MBITS: - mp_task_heap_size = 2 * 1024 * 1024; - break; - case ESP_SPIRAM_SIZE_32MBITS: - case ESP_SPIRAM_SIZE_64MBITS: - mp_task_heap_size = 4 * 1024 * 1024; - break; - default: - // No SPIRAM, fallback to normal allocation - mp_task_heap_size = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); - mp_task_heap = malloc(mp_task_heap_size); - break; + // Allocate MicroPython heap. By default we grab the largest contiguous chunk (GC requires + // having a contiguous heap). But this can be customized by setting two NVS variables. + // There's nothing special here about SPIRAM because the SDK config should set + // CONFIG_SPIRAM_USE_CAPS_ALLOC=y which makes any external SPIRAM automatically + // show up here under MALLOC_CAP_8BIT memory. +#define MIN_HEAP_SIZE (20 * 1024) // simple safety to avoid ending up with a non-functional heap + size_t avail_heap_size = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); // in bytes + size_t total_free = heap_caps_get_free_size(MALLOC_CAP_8BIT); + size_t mp_task_heap_size = avail_heap_size; // proposed MP heap size + nvs_handle_t mp_nvs; + if (nvs_open("micropython", NVS_READONLY, &mp_nvs) == ESP_OK) { + // implement minimum heap left for esp-idf + int32_t min_idf_heap_size = 0; // minimum we leave to esp-idf, in bytes + if (nvs_get_i32(mp_nvs, "min_idf_heap", &min_idf_heap_size) == ESP_OK) { + if (total_free - mp_task_heap_size < min_idf_heap_size) { + // we can't take the largest contig chunk, need to leave more to esp-idf + mp_task_heap_size = total_free - min_idf_heap_size; + if (mp_task_heap_size < MIN_HEAP_SIZE) { + mp_task_heap_size = MIN_HEAP_SIZE; + } + } + } + // implement maximum MP heap size + int32_t max_mp_heap_size = mp_task_heap_size; // max we alllocate to MicroPython, in bytes + if (nvs_get_i32(mp_nvs, "max_mp_heap", &max_mp_heap_size) == ESP_OK) { + if (max_mp_heap_size > MIN_HEAP_SIZE && mp_task_heap_size > max_mp_heap_size) { + // we're about to create too large a heap, be more modest + mp_task_heap_size = max_mp_heap_size; + } + } + nvs_close(mp_nvs); + } + void *mp_task_heap = heap_caps_malloc(mp_task_heap_size, MALLOC_CAP_8BIT); + if (avail_heap_size != mp_task_heap_size) { + printf("Heap limited: avail=%d, actual=%d, left to idf=%d\n", + avail_heap_size, mp_task_heap_size, total_free - mp_task_heap_size); } - #else - // Allocate the uPy heap using malloc and get the largest available region - size_t mp_task_heap_size = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); - void *mp_task_heap = malloc(mp_task_heap_size); - #endif soft_reset: // initialise the stack pointer for the main thread diff --git a/ports/esp32/modesp32.c b/ports/esp32/modesp32.c index 53ca7fdc60fcc..ea719940c40f5 100644 --- a/ports/esp32/modesp32.c +++ b/ports/esp32/modesp32.c @@ -46,8 +46,8 @@ #include "modesp32.h" // These private includes are needed for idf_heap_info. -#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(4, 3, 0) -#define MULTI_HEAP_FREERTOS +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(4, 1, 0) +#define MULTI_HEAP_FREERTOS // see esp-idf/components/heap/component.mk #include "../multi_heap_platform.h" #endif #include "../heap_private.h" @@ -190,6 +190,7 @@ STATIC const mp_rom_map_elem_t esp32_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_Partition), MP_ROM_PTR(&esp32_partition_type) }, { MP_ROM_QSTR(MP_QSTR_RMT), MP_ROM_PTR(&esp32_rmt_type) }, { MP_ROM_QSTR(MP_QSTR_ULP), MP_ROM_PTR(&esp32_ulp_type) }, + { MP_ROM_QSTR(MP_QSTR_NVS), MP_ROM_PTR(&esp32_nvs_type) }, { MP_ROM_QSTR(MP_QSTR_WAKEUP_ALL_LOW), MP_ROM_FALSE }, { MP_ROM_QSTR(MP_QSTR_WAKEUP_ANY_HIGH), MP_ROM_TRUE }, diff --git a/ports/esp32/modesp32.h b/ports/esp32/modesp32.h index 18bd62ee41640..2d7ff29b584ec 100644 --- a/ports/esp32/modesp32.h +++ b/ports/esp32/modesp32.h @@ -30,5 +30,6 @@ extern const mp_obj_type_t esp32_nvs_type; extern const mp_obj_type_t esp32_partition_type; extern const mp_obj_type_t esp32_rmt_type; extern const mp_obj_type_t esp32_ulp_type; +extern const mp_obj_type_t esp32_nvs_type; #endif // MICROPY_INCLUDED_ESP32_MODESP32_H diff --git a/tests/esp32/limit_heap.py b/tests/esp32/limit_heap.py new file mode 100644 index 0000000000000..84b6986a0e11a --- /dev/null +++ b/tests/esp32/limit_heap.py @@ -0,0 +1,44 @@ +# Test MicroPython heap limits on the esp32. +# This test requires resetting the device and thus does not simply run in run-tests. +# You can run this test manually using pyboard and hitting ctrl-c after each reset and rerunning +# the script. It will cycle through the various settings. +import gc +import machine +from esp32 import NVS, idf_heap_info, HEAP_DATA + +nvs = NVS("micropython") +try: + min_idf = nvs.get_i32("min_idf_heap") +except OSError: + min_idf = 0 +try: + max_mp = nvs.get_i32("max_mp_heap") +except OSError: + max_mp = None + +mp_total = gc.mem_alloc() + gc.mem_free() +print("MP heap:", mp_total) +idf_free = sum([h[2] for h in idf_heap_info(HEAP_DATA)]) +print("IDF heap free:", idf_free) + +if min_idf == 0: + nvs.set_i32("min_idf_heap", 100000) + nvs.commit() + print("IDF MIN heap changed to 100000") + machine.reset() +elif max_mp is None: + nvs.set_i32("max_mp_heap", 50000) + nvs.commit() + print("MAX heap changed to 50000") + machine.reset() +else: + try: + nvs.erase_key("min_idf_heap") + except OSError: + pass + try: + nvs.erase_key("max_mp_heap") + except OSError: + pass + print("Everything reset to default") + machine.reset()