diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99d6d5c9d..ede03ce8f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,12 @@ variables: IDF_PATH: "$CI_PROJECT_DIR" + # Versioned esp-idf-doc env image to use for all document building jobs + ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v8" + + GIT_SUBMODULE_STRATEGY: recursive + ESPCI_TOKEN: $GITLAB_KEY + # before each job, we need to check if this job is filtered by bot stage/job filter .apply_bot_filter: &apply_bot_filter python $APPLY_BOT_FILTER_SCRIPT || exit 0 @@ -45,7 +51,7 @@ build_ssc: - git clone $GITLAB_SSH_SERVER/yinling/SSC.git - cd SSC # try checkout same branch - - git checkout "${CI_BUILD_REF_NAME}_8266" || echo "Using default branch..." + - git checkout "release/v3.4_8266" || echo "Using default branch..." - ./gen_misc_rtos.sh push_master_to_github: @@ -79,7 +85,7 @@ push_master_to_github: build_docs: stage: build - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + image: $ESP_IDF_DOC_ENV_IMAGE tags: - build_docs artifacts: @@ -99,13 +105,14 @@ build_docs: - $BOT_LABEL_BUILD_DOCS - $BOT_LABEL_REGULAR_TEST script: + - source /opt/pyenv/activate && pyenv global 3.6.10 + - /opt/pyenv/pyenv-1.2.16/versions/3.6.10/bin/python -m pip install --user -r $CI_PROJECT_DIR/docs/requirements.txt - cd docs - cd en - make gh-linkcheck - make html - ../check_doc_warnings.sh - .build_examples_make_template: &build_examples_make_template <<: *build_template # This is a workaround for a rarely encountered issue with building examples in CI. diff --git a/.gitmodules b/.gitmodules index d36acbfe5..5051ce92e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,4 +16,4 @@ [submodule "components/coap/libcoap"] path = components/coap/libcoap - url = ../../obgm/libcoap.git + url = https://jihulab.com/esp-mirror/obgm/libcoap.git diff --git a/.readthedocs.yml b/.readthedocs.yaml similarity index 55% rename from .readthedocs.yml rename to .readthedocs.yaml index 569fae4c4..3704e9469 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yaml @@ -5,6 +5,12 @@ # Required version: 2 +# Set the version of Python and other tools you might need +build: + os: "ubuntu-20.04" + tools: + python: "3.6" + # Optionally build your docs in additional formats such as PDF and ePub formats: - htmlzip @@ -12,6 +18,10 @@ formats: # Optionally set the version of Python and requirements required to build your docs python: - version: 2.7 install: - - requirements: docs/requirements.txt \ No newline at end of file + - requirements: docs/setuptools.requirements.txt + - requirements: docs/requirements.txt + +# Specify the path for conf.py +sphinx: + configuration: docs/en/conf.py diff --git a/components/bootloader/Kconfig.projbuild b/components/bootloader/Kconfig.projbuild index 50c7633cf..2e108b512 100644 --- a/components/bootloader/Kconfig.projbuild +++ b/components/bootloader/Kconfig.projbuild @@ -174,6 +174,15 @@ config BOOTLOADER_STORE_OFFSET bootloader of the SDK's bootloader, you can set the option to store SDK's bootloader to other space in the flash instead of "0x0". +config BOOTLOADER_FLASH_XMC_SUPPORT + bool "Enable the support for flash chips of XMC (READ HELP FIRST)" + default y + help + Perform the startup flow recommended by XMC. Please consult XMC for the details of this flow. + XMC chips will be forbidden to be used, when this option is disabled. + + DON'T DISABLE THIS UNLESS YOU KNOW WHAT YOU ARE DOING. + endmenu # Bootloader diff --git a/components/bootloader/subproject/main/bootloader_start.c b/components/bootloader/subproject/main/bootloader_start.c index 3ba0addee..d79496db8 100644 --- a/components/bootloader/subproject/main/bootloader_start.c +++ b/components/bootloader/subproject/main/bootloader_start.c @@ -22,6 +22,7 @@ #include "esp_image_format.h" #include "esp_spi_flash.h" #include "esp_log.h" +#include "driver/soc.h" static const char* TAG = "boot"; @@ -30,6 +31,11 @@ static int selected_boot_partition(const bootloader_state_t *bs); void call_start_cpu(void) { + esp_irqflag_t irq; + + irq = soc_save_local_irq(); + ESP_LOGD(TAG, "CPU local irq: 0x%x", irq); + #ifdef CONFIG_BOOTLOADER_FAST_BOOT REG_SET_BIT(DPORT_CTL_REG, DPORT_CTL_DOUBLE_CLK); #endif diff --git a/components/bootloader/subproject/main/esp8266.bootloader.rom.ld b/components/bootloader/subproject/main/esp8266.bootloader.rom.ld index 3ff8f2710..27d35bbee 100644 --- a/components/bootloader/subproject/main/esp8266.bootloader.rom.ld +++ b/components/bootloader/subproject/main/esp8266.bootloader.rom.ld @@ -8,4 +8,5 @@ PROVIDE ( gpio_input_get = 0x40004cf0 ); PROVIDE ( xthal_get_ccount = 0x4000dd38 ); PROVIDE ( uart_div_modify = 0x400039d8 ); -PROVIDE ( ets_io_vprintf = 0x40001f00 ); \ No newline at end of file +PROVIDE ( ets_io_vprintf = 0x40001f00 ); +PROVIDE ( ets_rom_delay_us = 0x40002ecc ); \ No newline at end of file diff --git a/components/bootloader_support/include_priv/bootloader_flash.h b/components/bootloader_support/include_priv/bootloader_flash.h index 763136e03..101a32752 100644 --- a/components/bootloader_support/include_priv/bootloader_flash.h +++ b/components/bootloader_support/include_priv/bootloader_flash.h @@ -22,6 +22,11 @@ #define FLASH_SECTOR_SIZE 0x1000 +#define CMD_RDSFDP 0x5A /* Read the SFDP of the flash */ +#define CMD_RDJEDECID 0x9F /* Read the JEDEC ID of the flash */ + +#define XMC_VENDOR_ID 0x20 + /* Provide a Flash API for bootloader_support code, that can be used from bootloader or app code. @@ -100,4 +105,20 @@ esp_err_t bootloader_flash_write(size_t dest_addr, void *src, size_t size, bool */ esp_err_t bootloader_flash_erase_sector(size_t sector); +/** + * @brief Read the SFDP of the flash + * + * @param sfdp_addr Address of the parameter to read + * @param miso_byte_num Bytes to read + * @return The read SFDP, little endian, 4 bytes at most + */ +uint32_t bootloader_flash_read_sfdp(uint32_t sfdp_addr, unsigned int miso_byte_num); + +/** + * @brief Startup flow recommended by XMC. Call at startup before any erase/write operation. + * + * @return ESP_OK When startup successfully, otherwise ESP_FAIL (indiciating you should reboot before erase/write). + */ +esp_err_t bootloader_flash_xmc_startup(void); + #endif diff --git a/components/bootloader_support/src/bootloader_flash.c b/components/bootloader_support/src/bootloader_flash.c index f93226b3a..47edf2126 100644 --- a/components/bootloader_support/src/bootloader_flash.c +++ b/components/bootloader_support/src/bootloader_flash.c @@ -263,9 +263,13 @@ esp_err_t bootloader_flash_erase_sector(size_t sector) #include "esp_err.h" #include "esp_log.h" +#include "esp8266/rom_functions.h" #ifndef BOOTLOADER_BUILD #include "esp_spi_flash.h" +#else +#include "bootloader_flash.h" +#include "priv/esp_spi_flash_raw.h" #endif #ifdef CONFIG_SOC_FULL_ICACHE @@ -274,18 +278,11 @@ esp_err_t bootloader_flash_erase_sector(size_t sector) #define SOC_CACHE_SIZE 0 // 16KB #endif -extern void Cache_Read_Disable(); -extern void Cache_Read_Enable(uint8_t map, uint8_t p, uint8_t v); - -static const char *TAG = "bootloader_flash"; +#define XMC_SUPPORT CONFIG_BOOTLOADER_FLASH_XMC_SUPPORT -typedef enum { SPI_FLASH_RESULT_OK = 0, - SPI_FLASH_RESULT_ERR = 1, - SPI_FLASH_RESULT_TIMEOUT = 2 } SpiFlashOpResult; +#define BYTESHIFT(VAR, IDX) (((VAR) >> ((IDX) * 8)) & 0xFF) -SpiFlashOpResult SPIRead(uint32_t addr, void *dst, uint32_t size); -SpiFlashOpResult SPIWrite(uint32_t addr, const uint8_t *src, uint32_t size); -SpiFlashOpResult SPIEraseSector(uint32_t sector_num); +static const char *TAG = "bootloader_flash"; static bool mapped; @@ -406,4 +403,150 @@ esp_err_t bootloader_flash_erase_sector(size_t sector) return ESP_OK; } +#ifdef BOOTLOADER_BUILD +uint32_t bootloader_read_flash_id(void) +{ + uint32_t id = spi_flash_get_id_raw(&g_rom_flashchip); + id = ((id & 0xff) << 16) | ((id >> 16) & 0xff) | (id & 0xff00); + return id; +} + +#if XMC_SUPPORT +static bool is_xmc_chip_strict(uint32_t rdid) +{ + uint32_t vendor_id = BYTESHIFT(rdid, 2); + uint32_t mfid = BYTESHIFT(rdid, 1); + uint32_t cpid = BYTESHIFT(rdid, 0); + + if (vendor_id != XMC_VENDOR_ID) { + return false; + } + + bool matched = false; + if (mfid == 0x40) { + if (cpid >= 0x13 && cpid <= 0x20) { + matched = true; + } + } else if (mfid == 0x41) { + if (cpid >= 0x17 && cpid <= 0x20) { + matched = true; + } + } else if (mfid == 0x50) { + if (cpid >= 0x15 && cpid <= 0x16) { + matched = true; + } + } + return matched; +} + +bool bootloader_execute_flash_command(uint8_t command, uint32_t mosi_data, uint8_t mosi_len, uint8_t miso_len) +{ + bool ret; + spi_cmd_t cmd; + + cmd.cmd = command; + cmd.cmd_len = 1; + cmd.addr = NULL; + cmd.addr_len = 0; + cmd.dummy_bits = 0; + cmd.data = NULL; + cmd.data_len = 0; + + ret = spi_user_cmd_raw(&g_rom_flashchip, SPI_TX, &cmd); + if (!ret) { + ESP_LOGE(TAG, "failed to write cmd=%02x", command); + } + + return ret; +} + +uint32_t bootloader_flash_read_sfdp(uint32_t sfdp_addr, unsigned int miso_byte_num) +{ + bool ret; + spi_cmd_t cmd; + uint32_t data = 0; + uint32_t addr = sfdp_addr << 8; + + cmd.cmd = CMD_RDSFDP; + cmd.cmd_len = 1; + cmd.addr = &addr; + cmd.addr_len = 3; + cmd.dummy_bits = 8; + cmd.data = &data; + cmd.data_len = miso_byte_num; + + ret = spi_user_cmd_raw(&g_rom_flashchip, SPI_RX, &cmd); + if (!ret) { + ESP_LOGE(TAG, "failed to read sfdp"); + } + + return data; +} + +esp_err_t bootloader_flash_xmc_startup(void) +{ + extern void ets_rom_delay_us(uint16_t us); + + uint32_t id = bootloader_read_flash_id(); + + // If the RDID value is a valid XMC one, may skip the flow + const bool fast_check = true; + if (fast_check && is_xmc_chip_strict(id)) { + ESP_LOGD(TAG, "XMC chip detected by RDID (%08X), skip.", id); + return ESP_OK; + } + + // Check the Manufacturer ID in SFDP registers (JEDEC standard). If not XMC chip, no need to run the flow + const int sfdp_mfid_addr = 0x10; + uint8_t mf_id = (bootloader_flash_read_sfdp(sfdp_mfid_addr, 1) & 0xff); + if (mf_id != XMC_VENDOR_ID) { + ESP_LOGD(TAG, "non-XMC chip detected by SFDP Read (%02X), skip.", mf_id); + return ESP_OK; + } + + ESP_LOGI(TAG, "XM25QHxxC startup flow"); + // Enter DPD + bootloader_execute_flash_command(0xB9, 0, 0, 0); + // Enter UDPD + bootloader_execute_flash_command(0x79, 0, 0, 0); + // Exit UDPD + bootloader_execute_flash_command(0xFF, 0, 0, 0); + // Delay tXUDPD + ets_rom_delay_us(2000); + // Release Power-down + bootloader_execute_flash_command(0xAB, 0, 0, 0); + ets_rom_delay_us(20); + // Read flash ID and check again + id = bootloader_read_flash_id(); + if (!is_xmc_chip_strict(id)) { + ESP_LOGE(TAG, "XMC flash startup fail"); + return ESP_FAIL; + } + + return ESP_OK; +} +#else +static bool is_xmc_chip(uint32_t rdid) +{ + uint32_t vendor_id = (rdid >> 16) &0xff; + + return vendor_id == XMC_VENDOR_ID; +} + +esp_err_t bootloader_flash_xmc_startup(void) +{ + uint32_t id = bootloader_read_flash_id(); + + if (is_xmc_chip(id)) { + ESP_LOGE(TAG, "XMC chip detected(%08X) while support disable.", id); + return ESP_FAIL; + } else { + ESP_LOGI(TAG, "flash chip is %08X", id); + } + + return ESP_OK; +} +#endif +#endif + #endif diff --git a/components/bootloader_support/src/bootloader_init.c b/components/bootloader_support/src/bootloader_init.c index fda2a49f7..2ca04d11f 100644 --- a/components/bootloader_support/src/bootloader_init.c +++ b/components/bootloader_support/src/bootloader_init.c @@ -642,6 +642,8 @@ esp_err_t bootloader_init() static esp_err_t bootloader_main() { + esp_err_t ret; + #ifdef CONFIG_BOOTLOADER_DISABLE_JTAG_IO /* Set GPIO 12-15 to be normal GPIO */ PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDI_U, FUNC_GPIO12); @@ -655,6 +657,11 @@ static esp_err_t bootloader_main() uart_console_configure(); + if ((ret = bootloader_flash_xmc_startup()) != ESP_OK) { + ESP_LOGE(TAG, "failed when running XMC startup flow, reboot!"); + return ESP_FAIL; + } + esp_image_header_t fhdr; if (bootloader_flash_read(ESP_BOOTLOADER_OFFSET, &fhdr, sizeof(esp_image_header_t), true) != ESP_OK) { ESP_LOGE(TAG, "failed to load bootloader header!"); diff --git a/components/coap/libcoap b/components/coap/libcoap index cfec0d072..0dbccca06 160000 --- a/components/coap/libcoap +++ b/components/coap/libcoap @@ -1 +1 @@ -Subproject commit cfec0d072c5b99ed3e54828ca50ea2f6b91e1f50 +Subproject commit 0dbccca06c3ea75a20ef0a51633eb424a35b5e2a diff --git a/components/esp-tls/Kconfig b/components/esp-tls/Kconfig index dc9154170..e61145974 100644 --- a/components/esp-tls/Kconfig +++ b/components/esp-tls/Kconfig @@ -31,6 +31,25 @@ menu "ESP-TLS" Enable support for pre shared key ciphers, supported for both mbedTLS as well as wolfSSL TLS library. + config ESP_TLS_INSECURE + bool "Allow potentially insecure options" + help + You can enable some potentially insecure options. These options should only be used for testing pusposes. + Only enable these options if you are very sure. + + config ESP_TLS_SKIP_SERVER_CERT_VERIFY + bool "Skip server certificate verification by default (WARNING: ONLY FOR TESTING PURPOSE, READ HELP)" + depends on ESP_TLS_INSECURE + help + After enabling this option the esp-tls client will skip the server certificate verification + by default. Note that this option will only modify the default behaviour of esp-tls client + regarding server cert verification. The default behaviour should only be applicable when + no other option regarding the server cert verification is opted in the esp-tls config + (e.g. crt_bundle_attach, use_global_ca_store etc.). + WARNING : Enabling this option comes with a potential risk of establishing a TLS connection + with a server which has a fake identity, provided that the server certificate + is not provided either through API or other mechanism like ca_store etc. + config ESP_WOLFSSL_SMALL_CERT_VERIFY bool "Enable SMALL_CERT_VERIFY" depends on ESP_TLS_USING_WOLFSSL diff --git a/components/esp-tls/esp_tls.c b/components/esp-tls/esp_tls.c index da4dfe291..a5daf24f9 100644 --- a/components/esp-tls/esp_tls.c +++ b/components/esp-tls/esp_tls.c @@ -54,6 +54,7 @@ static const char *TAG = "esp-tls"; #define _esp_tls_set_global_ca_store esp_mbedtls_set_global_ca_store /*!< Callback function for setting global CA store data for TLS/SSL */ #define _esp_tls_get_global_ca_store esp_mbedtls_get_global_ca_store #define _esp_tls_free_global_ca_store esp_mbedtls_free_global_ca_store /*!< Callback function for freeing global ca store for TLS/SSL */ +#define _esp_tls_get_ciphersuites_list esp_mbedtls_get_ciphersuites_list #elif CONFIG_ESP_TLS_USING_WOLFSSL /* CONFIG_ESP_TLS_USING_MBEDTLS */ #define _esp_create_ssl_handle esp_create_wolfssl_handle #define _esp_tls_handshake esp_wolfssl_handshake @@ -437,6 +438,10 @@ mbedtls_x509_crt *esp_tls_get_global_ca_store(void) return _esp_tls_get_global_ca_store(); } +const int *esp_tls_get_ciphersuites_list(void) +{ + return _esp_tls_get_ciphersuites_list(); +} #endif /* CONFIG_ESP_TLS_USING_MBEDTLS */ #ifdef CONFIG_ESP_TLS_SERVER /** diff --git a/components/esp-tls/esp_tls.h b/components/esp-tls/esp_tls.h index a5983b2e3..a9ff17d5d 100644 --- a/components/esp-tls/esp_tls.h +++ b/components/esp-tls/esp_tls.h @@ -200,7 +200,8 @@ typedef struct esp_tls_cfg { esp_err_t (*crt_bundle_attach)(void *conf); /*!< Function pointer to esp_crt_bundle_attach. Enables the use of certification bundle for server verification, must be enabled in menuconfig */ - + const int *ciphersuites_list; /*!< Pointer to a zero-terminated array of IANA identifiers of TLS ciphersuites. + Please check the list validity by esp_tls_get_ciphersuites_list() API */ } esp_tls_cfg_t; #ifdef CONFIG_ESP_TLS_SERVER @@ -574,6 +575,15 @@ esp_err_t esp_tls_get_and_clear_last_error(esp_tls_error_handle_t h, int *esp_tl */ mbedtls_x509_crt *esp_tls_get_global_ca_store(void); +/** + * @brief Get supported TLS ciphersuites list. + * + * See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 for the list of ciphersuites + * + * @return Pointer to a zero-terminated array of IANA identifiers of TLS ciphersuites. + * + */ +const int *esp_tls_get_ciphersuites_list(void); #endif /* CONFIG_ESP_TLS_USING_MBEDTLS */ #ifdef CONFIG_ESP_TLS_SERVER /** @@ -602,7 +612,6 @@ int esp_tls_server_session_create(esp_tls_cfg_server_t *cfg, int sockfd, esp_tls */ void esp_tls_server_session_delete(esp_tls_t *tls); #endif /* ! CONFIG_ESP_TLS_SERVER */ - #ifdef __cplusplus } #endif diff --git a/components/esp-tls/esp_tls_mbedtls.c b/components/esp-tls/esp_tls_mbedtls.c index 245921fd0..3fb0678fc 100644 --- a/components/esp-tls/esp_tls_mbedtls.c +++ b/components/esp-tls/esp_tls_mbedtls.c @@ -445,7 +445,12 @@ esp_err_t set_client_config(const char *hostname, size_t hostlen, esp_tls_cfg_t return ESP_ERR_INVALID_STATE; #endif } else { +#ifdef CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY mbedtls_ssl_conf_authmode(&tls->conf, MBEDTLS_SSL_VERIFY_NONE); +#else + ESP_LOGE(TAG, "No server verification option set in esp_tls_cfg_t structure. Check esp_tls API reference"); + return ESP_ERR_MBEDTLS_SSL_SETUP_FAILED; +#endif } if (cfg->clientcert_buf != NULL && cfg->clientkey_buf != NULL) { @@ -468,6 +473,11 @@ esp_err_t set_client_config(const char *hostname, size_t hostlen, esp_tls_cfg_t ESP_LOGE(TAG, "You have to provide both clientcert_buf and clientkey_buf for mutual authentication"); return ESP_ERR_INVALID_STATE; } + + if (cfg->ciphersuites_list != NULL && cfg->ciphersuites_list[0] != 0) { + ESP_LOGD(TAG, "Set the ciphersuites list"); + mbedtls_ssl_conf_ciphersuites(&tls->conf, cfg->ciphersuites_list); + } return ESP_OK; } @@ -564,3 +574,8 @@ void esp_mbedtls_free_global_ca_store(void) global_cacert = NULL; } } + +const int *esp_mbedtls_get_ciphersuites_list(void) +{ + return mbedtls_ssl_list_ciphersuites(); +} diff --git a/components/esp-tls/esp_tls_wolfssl.c b/components/esp-tls/esp_tls_wolfssl.c index 7ec5ee383..d44198b41 100644 --- a/components/esp-tls/esp_tls_wolfssl.c +++ b/components/esp-tls/esp_tls_wolfssl.c @@ -201,7 +201,12 @@ static esp_err_t set_client_config(const char *hostname, size_t hostlen, esp_tls return ESP_ERR_INVALID_STATE; #endif } else { +#ifdef CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY wolfSSL_CTX_set_verify( (WOLFSSL_CTX *)tls->priv_ctx, WOLFSSL_VERIFY_NONE, NULL); +#else + ESP_LOGE(TAG, "No server verification option set in esp_tls_cfg_t structure. Check esp_tls API reference"); + return ESP_ERR_WOLFSSL_SSL_SETUP_FAILED; +#endif } if (cfg->clientcert_buf != NULL && cfg->clientkey_buf != NULL) { @@ -242,8 +247,8 @@ static esp_err_t set_client_config(const char *hostname, size_t hostlen, esp_tls return ESP_ERR_NO_MEM; } /* Hostname set here should match CN in server certificate */ - if ((ret = wolfSSL_set_tlsext_host_name( (WOLFSSL *)tls->priv_ssl, use_host))!= WOLFSSL_SUCCESS) { - ESP_LOGE(TAG, "wolfSSL_set_tlsext_host_name returned -0x%x", -ret); + if ((ret = wolfSSL_check_domain_name( (WOLFSSL *)tls->priv_ssl, use_host))!= WOLFSSL_SUCCESS) { + ESP_LOGE(TAG, "wolfSSL_check_domain_name returned -0x%x", -ret); ESP_INT_EVENT_TRACKER_CAPTURE(tls->error_handle, ERR_TYPE_WOLFSSL, -ret); free(use_host); return ESP_ERR_WOLFSSL_SSL_SET_HOSTNAME_FAILED; diff --git a/components/esp-tls/private_include/esp_tls_mbedtls.h b/components/esp-tls/private_include/esp_tls_mbedtls.h index 362a71fd6..4e542cbe6 100644 --- a/components/esp-tls/private_include/esp_tls_mbedtls.h +++ b/components/esp-tls/private_include/esp_tls_mbedtls.h @@ -102,3 +102,8 @@ mbedtls_x509_crt *esp_mbedtls_get_global_ca_store(void); * Callback function for freeing global ca store for TLS/SSL using mbedtls */ void esp_mbedtls_free_global_ca_store(void); + +/** + * Internal Callback for esp_tls_get_ciphersuites_list + */ +const int *esp_mbedtls_get_ciphersuites_list(void); diff --git a/components/esp8266/CMakeLists.txt b/components/esp8266/CMakeLists.txt index 26120f438..887f9b6e6 100644 --- a/components/esp8266/CMakeLists.txt +++ b/components/esp8266/CMakeLists.txt @@ -69,7 +69,12 @@ else() target_link_libraries(${COMPONENT_LIB} PUBLIC "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib" "-lstdc++") target_compile_definitions(${COMPONENT_LIB} PUBLIC -DUSING_IBUS_FASTER_GET) if(NOT CONFIG_NO_BLOBS) - set(blobs "gcc" "hal" "core" "net80211" "phy" "rtc" "clk" "pp" "smartconfig" "ssc" "espnow") + set(blobs "gcc" "hal" "phy" "rtc" "clk" "smartconfig" "ssc") + if(CONFIG_ESP8266_WIFI_DEBUG_LOG_ENABLE) + list(APPEND blobs "core_dbg" "net80211_dbg" "pp_dbg" "espnow_dbg") + else() + list(APPEND blobs "core" "net80211" "pp" "espnow") + endif() foreach(blob ${blobs}) add_library(${blob} STATIC IMPORTED) set_property(TARGET ${blob} PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/lib/lib${blob}.a) diff --git a/components/esp8266/driver/uart.c b/components/esp8266/driver/uart.c index 63d2a92bb..ae067dd07 100644 --- a/components/esp8266/driver/uart.c +++ b/components/esp8266/driver/uart.c @@ -257,7 +257,18 @@ esp_err_t uart_wait_tx_done(uart_port_t uart_num, TickType_t ticks_to_wait) uint32_t baudrate; uint32_t byte_delay_us = 0; BaseType_t res; + portTickType ticks_cur; + portTickType ticks_start = xTaskGetTickCount(); portTickType ticks_end = xTaskGetTickCount() + ticks_to_wait; + /** + * Considering the overflow of the ticks_end and the ticks_cur (xTaskGetTickCount()), + * the possible tick timestamp is as follows: + * (one start tick timestamp, two end tick timestamps, four current tick timestamps) + * + * ticks: 0 0xFFFFFFFF + * |_______._______._______._______._______._______._______._______| + * cur1 end1 cur2 start cur3 end2 cur4 + */ // Take tx_mux res = xSemaphoreTake(p_uart_obj[uart_num]->tx_mux, (portTickType)ticks_to_wait); @@ -273,10 +284,22 @@ esp_err_t uart_wait_tx_done(uart_port_t uart_num, TickType_t ticks_to_wait) uart_get_baudrate(uart_num, &baudrate); byte_delay_us = (uint32_t)(10000000 / baudrate); // (1/baudrate)*10*1000_000 us - ticks_to_wait = ticks_end - xTaskGetTickCount(); + ticks_cur = xTaskGetTickCount(); + if (ticks_start <= ticks_cur) { + ticks_to_wait = ticks_to_wait - (ticks_cur - ticks_start); + } else { + ticks_to_wait = ticks_to_wait - (portMAX_DELAY - ticks_start + ticks_cur); + } // wait for tx done sem. if (pdTRUE == xSemaphoreTake(p_uart_obj[uart_num]->tx_done_sem, ticks_to_wait)) { while (1) { + ticks_cur = xTaskGetTickCount(); + bool end1_timeout = (ticks_end < ticks_start && ticks_cur < ticks_start && ticks_cur > ticks_end); + bool end2_timeout = (ticks_start < ticks_end && (ticks_cur < ticks_start || ticks_end < ticks_cur)); + if (end1_timeout || end2_timeout) { + xSemaphoreGive(p_uart_obj[uart_num]->tx_mux); + return ESP_ERR_TIMEOUT; + } if (UART[uart_num]->status.txfifo_cnt == 0) { ets_delay_us(byte_delay_us); // Delay one byte time to guarantee transmission completion break; diff --git a/components/esp8266/include/driver/soc.h b/components/esp8266/include/driver/soc.h index ac3e330de..894f64e95 100644 --- a/components/esp8266/include/driver/soc.h +++ b/components/esp8266/include/driver/soc.h @@ -31,7 +31,11 @@ static inline esp_irqflag_t soc_save_local_irq(void) esp_irqflag_t flag; __asm__ __volatile__( +#ifdef BOOTLOADER_BUILD + "rsil %0, 3\n" +#else "rsil %0, 1\n" +#endif : "=a"(flag) : : "memory" diff --git a/components/esp8266/include/esp_attr.h b/components/esp8266/include/esp_attr.h index f21fb7a54..d5b14d38b 100644 --- a/components/esp8266/include/esp_attr.h +++ b/components/esp8266/include/esp_attr.h @@ -68,4 +68,10 @@ #define _COUNTER_STRINGIFY(COUNTER) #COUNTER +#ifdef IDF_CI_BUILD +#define IDF_DEPRECATED(REASON) __attribute__((deprecated(REASON))) +#else +#define IDF_DEPRECATED(REASON) +#endif + #endif /* __ESP_ATTR_H__ */ diff --git a/components/esp8266/lib/VERSION b/components/esp8266/lib/VERSION index 49d4d2e10..2873e0c84 100644 --- a/components/esp8266/lib/VERSION +++ b/components/esp8266/lib/VERSION @@ -1,8 +1,8 @@ gwen: - core: cebfb0e - net80211: 8114f9c - pp: 52bb147 - espnow: 7a93d71 + core: 231e0e2 + net80211: 9ad2f23 + pp: 25e5ef2 + espnow: 231e0e2 - smartconfig: 3.0.0/5f550c40 - phy: 1163.0 + smartconfig: 3.0.0/af78f443 + phy: 1166.0 diff --git a/components/esp8266/lib/libcore.a b/components/esp8266/lib/libcore.a index 75f2cd848..8beb25fae 100755 Binary files a/components/esp8266/lib/libcore.a and b/components/esp8266/lib/libcore.a differ diff --git a/components/esp8266/lib/libcore_dbg.a b/components/esp8266/lib/libcore_dbg.a index c59cd3276..45b00afe0 100755 Binary files a/components/esp8266/lib/libcore_dbg.a and b/components/esp8266/lib/libcore_dbg.a differ diff --git a/components/esp8266/lib/libespnow.a b/components/esp8266/lib/libespnow.a index 807bac791..e1af2020a 100755 Binary files a/components/esp8266/lib/libespnow.a and b/components/esp8266/lib/libespnow.a differ diff --git a/components/esp8266/lib/libespnow_dbg.a b/components/esp8266/lib/libespnow_dbg.a index e3c2f9d3c..2c7346c7b 100755 Binary files a/components/esp8266/lib/libespnow_dbg.a and b/components/esp8266/lib/libespnow_dbg.a differ diff --git a/components/esp8266/lib/libnet80211.a b/components/esp8266/lib/libnet80211.a old mode 100755 new mode 100644 index ed88b9dcb..a6da69b90 Binary files a/components/esp8266/lib/libnet80211.a and b/components/esp8266/lib/libnet80211.a differ diff --git a/components/esp8266/lib/libnet80211_dbg.a b/components/esp8266/lib/libnet80211_dbg.a old mode 100755 new mode 100644 index 7f60cdb35..d74f6f5c7 Binary files a/components/esp8266/lib/libnet80211_dbg.a and b/components/esp8266/lib/libnet80211_dbg.a differ diff --git a/components/esp8266/lib/libphy.a b/components/esp8266/lib/libphy.a old mode 100755 new mode 100644 index 3113a722c..6f3b75e93 Binary files a/components/esp8266/lib/libphy.a and b/components/esp8266/lib/libphy.a differ diff --git a/components/esp8266/lib/libpp.a b/components/esp8266/lib/libpp.a old mode 100755 new mode 100644 index 761b8be80..de9333380 Binary files a/components/esp8266/lib/libpp.a and b/components/esp8266/lib/libpp.a differ diff --git a/components/esp8266/lib/libpp_dbg.a b/components/esp8266/lib/libpp_dbg.a old mode 100755 new mode 100644 index 26775d3d4..89786fcc4 Binary files a/components/esp8266/lib/libpp_dbg.a and b/components/esp8266/lib/libpp_dbg.a differ diff --git a/components/esp8266/lib/librtc.a b/components/esp8266/lib/librtc.a old mode 100755 new mode 100644 index cfb64443c..8a809ee92 Binary files a/components/esp8266/lib/librtc.a and b/components/esp8266/lib/librtc.a differ diff --git a/components/esp8266/lib/libsmartconfig.a b/components/esp8266/lib/libsmartconfig.a index ab3ccef63..cdadb3a39 100755 Binary files a/components/esp8266/lib/libsmartconfig.a and b/components/esp8266/lib/libsmartconfig.a differ diff --git a/components/esp8266/source/phy_init.c b/components/esp8266/source/phy_init.c index 722dec55c..21563f8bb 100644 --- a/components/esp8266/source/phy_init.c +++ b/components/esp8266/source/phy_init.c @@ -204,15 +204,15 @@ static const char* PHY_NAMESPACE = "phy"; static const char* PHY_CAL_DATA_KEY = "cal_data"; static const char* PHY_RX_GAIN_DC_TABLE_KEY = "dc_table"; -static esp_err_t load_cal_data_from_nvs_handle(nvs_handle handle, +static esp_err_t load_cal_data_from_nvs_handle(nvs_handle_t handle, esp_phy_calibration_data_t* out_cal_data); -static esp_err_t store_cal_data_to_nvs_handle(nvs_handle handle, +static esp_err_t store_cal_data_to_nvs_handle(nvs_handle_t handle, const esp_phy_calibration_data_t* cal_data); esp_err_t esp_phy_load_cal_data_from_nvs(esp_phy_calibration_data_t* out_cal_data) { - nvs_handle handle; + nvs_handle_t handle; esp_err_t err = nvs_open(PHY_NAMESPACE, NVS_READONLY, &handle); if (err == ESP_ERR_NVS_NOT_INITIALIZED) { @@ -230,7 +230,7 @@ esp_err_t esp_phy_load_cal_data_from_nvs(esp_phy_calibration_data_t* out_cal_dat esp_err_t esp_phy_store_cal_data_to_nvs(const esp_phy_calibration_data_t* cal_data) { - nvs_handle handle; + nvs_handle_t handle; esp_err_t err = nvs_open(PHY_NAMESPACE, NVS_READWRITE, &handle); if (err != ESP_OK) { @@ -243,7 +243,7 @@ esp_err_t esp_phy_store_cal_data_to_nvs(const esp_phy_calibration_data_t* cal_da } } -static esp_err_t load_cal_data_from_nvs_handle(nvs_handle handle, +static esp_err_t load_cal_data_from_nvs_handle(nvs_handle_t handle, esp_phy_calibration_data_t* out_cal_data) { esp_err_t err; @@ -278,7 +278,7 @@ static esp_err_t load_cal_data_from_nvs_handle(nvs_handle handle, return ESP_OK; } -static esp_err_t store_cal_data_to_nvs_handle(nvs_handle handle, +static esp_err_t store_cal_data_to_nvs_handle(nvs_handle_t handle, const esp_phy_calibration_data_t* cal_data) { esp_err_t err; diff --git a/components/esp8266/source/smartconfig_ack.c b/components/esp8266/source/smartconfig_ack.c index 7c4347470..34bd8eb30 100644 --- a/components/esp8266/source/smartconfig_ack.c +++ b/components/esp8266/source/smartconfig_ack.c @@ -31,6 +31,7 @@ #include "esp_wifi.h" #include "esp_smartconfig.h" #include "smartconfig_ack.h" +#include "lwip/inet.h" #define SC_ACK_TASK_PRIORITY 2 /*!< Priority of sending smartconfig ACK task */ #define SC_ACK_TASK_STACK_SIZE 2048 /*!< Stack size of sending smartconfig ACK task */ @@ -40,7 +41,7 @@ #define SC_ACK_TOUCH_V2_SERVER_PORT(i) (18266+i*10000) /*!< ESP touch_v2 UDP port of server on cellphone */ #define SC_ACK_AIRKISS_SERVER_PORT 10000 /*!< Airkiss UDP port of server on cellphone */ #define SC_ACK_AIRKISS_DEVICE_PORT 10001 /*!< Airkiss UDP port of server on device */ -#define SC_ACK_AIRKISS_TIMEOUT 1500 /*!< Airkiss read data timout millisecond */ +#define SC_ACK_TIMEOUT 1500 /*!< Airkiss and ESP touch_v2 read data timout millisecond */ #define SC_ACK_TOUCH_LEN 11 /*!< Length of ESP touch ACK context */ #define SC_ACK_AIRKISS_LEN 7 /*!< Length of Airkiss ACK context */ @@ -79,7 +80,7 @@ static void sc_ack_send_task(void* pvParameters) sc_ack_t* ack = (sc_ack_t*)pvParameters; tcpip_adapter_ip_info_t local_ip; uint8_t remote_ip[4]; - memset(remote_ip, 0xFF, sizeof(remote_ip)); + memcpy(remote_ip, ack->ctx.ip, sizeof(remote_ip)); struct sockaddr_in server_addr; socklen_t sin_size = sizeof(server_addr); int send_sock = -1; @@ -99,6 +100,7 @@ static void sc_ack_send_task(void* pvParameters) port_bit = 0; } remote_port = SC_ACK_TOUCH_V2_SERVER_PORT(port_bit); + memset(remote_ip, 0xFF, sizeof(remote_ip)); } else { remote_port = SC_ACK_AIRKISS_SERVER_PORT; } @@ -132,21 +134,24 @@ static void sc_ack_send_task(void* pvParameters) setsockopt(send_sock, SOL_SOCKET, SO_BROADCAST | SO_REUSEADDR, &optval, sizeof(int)); - if (ack->type == SC_TYPE_AIRKISS) { + if (ack->type == SC_TYPE_AIRKISS || ack->type == SC_TYPE_ESPTOUCH_V2) { char data = 0; struct sockaddr_in local_addr, from; socklen_t sockadd_len = sizeof(struct sockaddr); struct timeval timeout = { - SC_ACK_AIRKISS_TIMEOUT / 1000, - SC_ACK_AIRKISS_TIMEOUT % 1000 * 1000 + SC_ACK_TIMEOUT / 1000, + SC_ACK_TIMEOUT % 1000 * 1000 }; bzero(&local_addr, sizeof(struct sockaddr_in)); bzero(&from, sizeof(struct sockaddr_in)); local_addr.sin_family = AF_INET; local_addr.sin_addr.s_addr = INADDR_ANY; - local_addr.sin_port = htons(SC_ACK_AIRKISS_DEVICE_PORT); - + if (ack->type == SC_TYPE_AIRKISS) { + local_addr.sin_port = htons(SC_ACK_AIRKISS_DEVICE_PORT); + } else { + local_addr.sin_port = htons(SC_ACK_TOUCH_DEVICE_PORT); + } bind(send_sock, (struct sockaddr*)&local_addr, sockadd_len); setsockopt(send_sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); @@ -155,31 +160,34 @@ static void sc_ack_send_task(void* pvParameters) if (from.sin_addr.s_addr != INADDR_ANY) { memcpy(remote_ip, &from.sin_addr, 4); server_addr.sin_addr.s_addr = from.sin_addr.s_addr; + ESP_LOGI(TAG, "cellphone_ip: %s", inet_ntoa(server_addr.sin_addr)); } else { - goto _end; + server_addr.sin_addr.s_addr = INADDR_BROADCAST; } } + uint32_t ip_addr = server_addr.sin_addr.s_addr; while (s_sc_ack_send) { /* Send smartconfig ACK every 100ms. */ vTaskDelay(100 / portTICK_RATE_MS); - sendlen = sendto(send_sock, &ack->ctx, ack_len, 0, (struct sockaddr*) &server_addr, sin_size); - - if (sendlen > 0) { - /* Totally send 60 smartconfig ACKs. Then smartconfig is successful. */ - if (packet_count++ >= SC_ACK_MAX_COUNT) { - esp_event_post(SC_EVENT, SC_EVENT_SEND_ACK_DONE, NULL, 0, portMAX_DELAY); - goto _end; - } + if (ip_addr != INADDR_BROADCAST) { + sendto(send_sock, &ack->ctx, ack_len, 0, (struct sockaddr*) &server_addr, sin_size); + server_addr.sin_addr.s_addr = INADDR_BROADCAST; + sendlen = sendto(send_sock, &ack->ctx, ack_len, 0, (struct sockaddr*) &server_addr, sin_size); + server_addr.sin_addr.s_addr = ip_addr; } else { - err = sc_ack_send_get_errno(send_sock); - - if (err == ENOMEM || err == EAGAIN) { - ESP_LOGD(TAG, "send failed, errno %d", err); - continue; - } + sendlen = sendto(send_sock, &ack->ctx, ack_len, 0, (struct sockaddr*) &server_addr, sin_size); + } + if (sendlen <= 0) { + err = sc_ack_send_get_errno(send_sock); ESP_LOGE(TAG, "send failed, errno %d", err); + vTaskDelay(200 / portTICK_RATE_MS); + } + + /* Send 60 smartconfig ACKs, exit regardless of failure or success. */ + if (packet_count++ >= SC_ACK_MAX_COUNT) { + esp_event_post(SC_EVENT, SC_EVENT_SEND_ACK_DONE, NULL, 0, portMAX_DELAY); goto _end; } } diff --git a/components/esp8266/source/startup.c b/components/esp8266/source/startup.c index b9e582cec..42fc18825 100644 --- a/components/esp8266/source/startup.c +++ b/components/esp8266/source/startup.c @@ -93,6 +93,10 @@ static void user_init_entry(void *param) esp_set_cpu_freq(ESP_CPU_FREQ_160M); #endif +#ifdef CONFIG_ENABLE_TH25Q16HB_PATCH_0 + assert(th25q16hb_apply_patch_0() == 0); +#endif + app_main(); vTaskDelete(NULL); diff --git a/components/esp8266/source/system_api.c b/components/esp8266/source/system_api.c index 080514e47..1a0c1aad3 100644 --- a/components/esp8266/source/system_api.c +++ b/components/esp8266/source/system_api.c @@ -136,7 +136,7 @@ static const char *BACKUP_MAC_DATA_KEY = "backup_mac_data"; static esp_err_t load_backup_mac_data(uint8_t *mac) { esp_err_t err; - nvs_handle handle; + nvs_handle_t handle; uint32_t efuse[4]; uint8_t efuse_crc = 0; uint8_t calc_crc = 0; @@ -221,7 +221,7 @@ static esp_err_t load_backup_mac_data(uint8_t *mac) static esp_err_t store_backup_mac_data() { esp_err_t err; - nvs_handle handle; + nvs_handle_t handle; uint32_t efuse[4]; efuse[0] = REG_READ(EFUSE_DATA0_REG); efuse[1] = REG_READ(EFUSE_DATA1_REG); diff --git a/components/esp_http_server/src/httpd_main.c b/components/esp_http_server/src/httpd_main.c index a67513195..5724383aa 100644 --- a/components/esp_http_server/src/httpd_main.c +++ b/components/esp_http_server/src/httpd_main.c @@ -262,6 +262,16 @@ static esp_err_t httpd_server_init(struct httpd_data *hd) .sin_port = htons(hd->config.server_port) }; #endif /* CONFIG_LWIP_IPV6 */ + + /* Enable SO_REUSEADDR to allow binding to the same + * address and port when restarting the server */ + int enable = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable)) < 0) { + /* This will fail if CONFIG_LWIP_SO_REUSE is not enabled. But + * it does not affect the normal working of the HTTP Server */ + ESP_LOGW(TAG, LOG_FMT("error in setsockopt SO_REUSEADDR (%d)"), errno); + } + int ret = bind(fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)); if (ret < 0) { ESP_LOGE(TAG, LOG_FMT("error in bind (%d)"), errno); diff --git a/components/freertos/freertos/queue.c b/components/freertos/freertos/queue.c index c37d285f3..a87d6d62e 100644 --- a/components/freertos/freertos/queue.c +++ b/components/freertos/freertos/queue.c @@ -360,6 +360,7 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue; Queue_t *pxNewQueue; size_t xQueueSizeInBytes; uint8_t *pucQueueStorage; + BaseType_t overflow; configASSERT( uxQueueLength > ( UBaseType_t ) 0 ); @@ -375,7 +376,29 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue; xQueueSizeInBytes = ( size_t ) ( uxQueueLength * uxItemSize ); /*lint !e961 MISRA exception as the casts are only redundant for some ports. */ } - pxNewQueue = ( Queue_t * ) pvPortMalloc( sizeof( Queue_t ) + xQueueSizeInBytes ); + /* Check for multiplication overflow. */ + overflow = ( uxItemSize != 0 ) && ( uxQueueLength != ( xQueueSizeInBytes / uxItemSize ) ); + + /* Check for addition overflow. */ + overflow = overflow || ( ( sizeof( Queue_t ) + xQueueSizeInBytes ) < xQueueSizeInBytes ); + + if ( overflow == (BaseType_t) 0 ) + { + /* Allocate the queue and storage area. Justification for MISRA + deviation as follows: pvPortMalloc() always ensures returned memory + blocks are aligned per the requirements of the MCU stack. In this case + pvPortMalloc() must return a pointer that is guaranteed to meet the + alignment requirements of the Queue_t structure - which in this case + is an int8_t *. Therefore, whenever the stack alignment requirements + are greater than or equal to the pointer to char requirements the cast + is safe. In other cases alignment requirements are not strict (one or + two bytes). */ + pxNewQueue = ( Queue_t * ) pvPortMalloc( sizeof( Queue_t ) + xQueueSizeInBytes ); /*lint !e9087 !e9079 see comment above. */ + } + else + { + pxNewQueue = NULL; + } if( pxNewQueue != NULL ) { diff --git a/components/freertos/freertos/stream_buffer.c b/components/freertos/freertos/stream_buffer.c index c60045f69..317be2355 100644 --- a/components/freertos/freertos/stream_buffer.c +++ b/components/freertos/freertos/stream_buffer.c @@ -242,8 +242,15 @@ static void prvInitialiseNewStreamBuffer( StreamBuffer_t * const pxStreamBuffer, this is a quirk of the implementation that means otherwise the free space would be reported as one byte smaller than would be logically expected. */ - xBufferSizeBytes++; - pucAllocatedMemory = ( uint8_t * ) pvPortMalloc( xBufferSizeBytes + sizeof( StreamBuffer_t ) ); /*lint !e9079 malloc() only returns void*. */ + if( xBufferSizeBytes < ( xBufferSizeBytes + 1 + sizeof( StreamBuffer_t ) ) ) + { + xBufferSizeBytes++; + pucAllocatedMemory = ( uint8_t * ) pvPortMalloc( xBufferSizeBytes + sizeof( StreamBuffer_t ) ); /*lint !e9079 malloc() only returns void*. */ + } + else + { + pucAllocatedMemory = NULL; + } if( pucAllocatedMemory != NULL ) { diff --git a/components/freertos/freertos/tasks.c b/components/freertos/freertos/tasks.c index c747cd74e..61e10e501 100644 --- a/components/freertos/freertos/tasks.c +++ b/components/freertos/freertos/tasks.c @@ -3044,6 +3044,19 @@ BaseType_t xReturn; { ( void ) uxListRemove( &( pxUnblockedTCB->xStateListItem ) ); prvAddTaskToReadyList( pxUnblockedTCB ); + #if( configUSE_TICKLESS_IDLE != 0 ) + { + /* If a task is blocked on a kernel object then xNextTaskUnblockTime + might be set to the blocked task's time out time. If the task is + unblocked for a reason other than a timeout xNextTaskUnblockTime is + normally left unchanged, because it is automatically reset to a new + value when the tick count equals xNextTaskUnblockTime. However if + tickless idling is used it might be more important to enter sleep mode + at the earliest possible time - so reset xNextTaskUnblockTime here to + ensure it is updated at the earliest possible time. */ + prvResetNextTaskUnblockTime(); + } + #endif } else { @@ -3068,20 +3081,6 @@ BaseType_t xReturn; xReturn = pdFALSE; } - #if( configUSE_TICKLESS_IDLE != 0 ) - { - /* If a task is blocked on a kernel object then xNextTaskUnblockTime - might be set to the blocked task's time out time. If the task is - unblocked for a reason other than a timeout xNextTaskUnblockTime is - normally left unchanged, because it is automatically reset to a new - value when the tick count equals xNextTaskUnblockTime. However if - tickless idling is used it might be more important to enter sleep mode - at the earliest possible time - so reset xNextTaskUnblockTime here to - ensure it is updated at the earliest possible time. */ - prvResetNextTaskUnblockTime(); - } - #endif - return xReturn; } /*-----------------------------------------------------------*/ diff --git a/components/libsodium/libsodium/test/default/sign.c b/components/libsodium/libsodium/test/default/sign.c index 814f96729..4faacf368 100644 --- a/components/libsodium/libsodium/test/default/sign.c +++ b/components/libsodium/libsodium/test/default/sign.c @@ -14,7 +14,7 @@ typedef struct TestData_ { const char *m; } TestData; -static TestData test_data[] = { +static const TestData test_data[] = { {{0x9d,0x61,0xb1,0x9d,0xef,0xfd,0x5a,0x60,0xba,0x84,0x4a,0xf4,0x92,0xec,0x2c,0xc4,0x44,0x49,0xc5,0x69,0x7b,0x32,0x69,0x19,0x70,0x3b,0xac,0x03,0x1c,0xae,0x7f,0x60,},{0xd7,0x5a,0x98,0x01,0x82,0xb1,0x0a,0xb7,0xd5,0x4b,0xfe,0xd3,0xc9,0x64,0x07,0x3a,0x0e,0xe1,0x72,0xf3,0xda,0xa6,0x23,0x25,0xaf,0x02,0x1a,0x68,0xf7,0x07,0x51,0x1a,},{0xe5,0x56,0x43,0x00,0xc3,0x60,0xac,0x72,0x90,0x86,0xe2,0xcc,0x80,0x6e,0x82,0x8a,0x84,0x87,0x7f,0x1e,0xb8,0xe5,0xd9,0x74,0xd8,0x73,0xe0,0x65,0x22,0x49,0x01,0x55,0x5f,0xb8,0x82,0x15,0x90,0xa3,0x3b,0xac,0xc6,0x1e,0x39,0x70,0x1c,0xf9,0xb4,0x6b,0xd2,0x5b,0xf5,0xf0,0x59,0x5b,0xbe,0x24,0x65,0x51,0x41,0x43,0x8e,0x7a,0x10,0x0b,},""}, {{0x4c,0xcd,0x08,0x9b,0x28,0xff,0x96,0xda,0x9d,0xb6,0xc3,0x46,0xec,0x11,0x4e,0x0f,0x5b,0x8a,0x31,0x9f,0x35,0xab,0xa6,0x24,0xda,0x8c,0xf6,0xed,0x4f,0xb8,0xa6,0xfb,},{0x3d,0x40,0x17,0xc3,0xe8,0x43,0x89,0x5a,0x92,0xb7,0x0a,0xa7,0x4d,0x1b,0x7e,0xbc,0x9c,0x98,0x2c,0xcf,0x2e,0xc4,0x96,0x8c,0xc0,0xcd,0x55,0xf1,0x2a,0xf4,0x66,0x0c,},{0x92,0xa0,0x09,0xa9,0xf0,0xd4,0xca,0xb8,0x72,0x0e,0x82,0x0b,0x5f,0x64,0x25,0x40,0xa2,0xb2,0x7b,0x54,0x16,0x50,0x3f,0x8f,0xb3,0x76,0x22,0x23,0xeb,0xdb,0x69,0xda,0x08,0x5a,0xc1,0xe4,0x3e,0x15,0x99,0x6e,0x45,0x8f,0x36,0x13,0xd0,0xf1,0x1d,0x8c,0x38,0x7b,0x2e,0xae,0xb4,0x30,0x2a,0xee,0xb0,0x0d,0x29,0x16,0x12,0xbb,0x0c,0x00,},"\x72"}, {{0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b,0xed,0xb7,0x44,0x2f,0x31,0xdc,0xb7,0xb1,0x66,0xd3,0x85,0x35,0x07,0x6f,0x09,0x4b,0x85,0xce,0x3a,0x2e,0x0b,0x44,0x58,0xf7,},{0xfc,0x51,0xcd,0x8e,0x62,0x18,0xa1,0xa3,0x8d,0xa4,0x7e,0xd0,0x02,0x30,0xf0,0x58,0x08,0x16,0xed,0x13,0xba,0x33,0x03,0xac,0x5d,0xeb,0x91,0x15,0x48,0x90,0x80,0x25,},{0x62,0x91,0xd6,0x57,0xde,0xec,0x24,0x02,0x48,0x27,0xe6,0x9c,0x3a,0xbe,0x01,0xa3,0x0c,0xe5,0x48,0xa2,0x84,0x74,0x3a,0x44,0x5e,0x36,0x80,0xd7,0xdb,0x5a,0xc3,0xac,0x18,0xff,0x9b,0x53,0x8d,0x16,0xf2,0x90,0xae,0x67,0xf7,0x60,0x98,0x4d,0xc6,0x59,0x4a,0x7c,0x15,0xe9,0x71,0x6e,0xd2,0x8d,0xc0,0x27,0xbe,0xce,0xea,0x1e,0xc4,0x0a,},"\xaf\x82"}, diff --git a/components/lwip/port/esp8266/netif/dhcp_state.c b/components/lwip/port/esp8266/netif/dhcp_state.c index 6ae00d700..8e2e3d9f3 100644 --- a/components/lwip/port/esp8266/netif/dhcp_state.c +++ b/components/lwip/port/esp8266/netif/dhcp_state.c @@ -31,7 +31,6 @@ static const char *interface_key[] = {"IF_STA", "IF_AP", "IF_ETH", "IF_TEST"}; _Static_assert(sizeof(interface_key) / sizeof(char*) == TCPIP_ADAPTER_IF_MAX, "Number interface keys differs from number of interfaces"); -typedef nvs_handle nvs_handle_t; bool dhcp_ip_addr_restore(void *netif) { nvs_handle_t nvs; diff --git a/components/mbedtls/CMakeLists.txt b/components/mbedtls/CMakeLists.txt index ea61a5217..b189fe05b 100644 --- a/components/mbedtls/CMakeLists.txt +++ b/components/mbedtls/CMakeLists.txt @@ -17,6 +17,20 @@ get_target_property(src_tls mbedtls SOURCES) list(REMOVE_ITEM src_tls net_sockets.c) set_property(TARGET mbedtls PROPERTY SOURCES ${src_tls}) +if(CONFIG_MBEDTLS_SSL_PROTO_GMTSSL1_1) +get_target_property(src_tls mbedtls SOURCES) +list(REMOVE_ITEM src_tls ssl_ciphersuites.c ssl_cli.c ssl_tls.c) +set_property(TARGET mbedtls PROPERTY SOURCES ${src_tls}) + +get_target_property(src_crypto mbedcrypto SOURCES) +list(REMOVE_ITEM src_crypto cipher_wrap.c ecdsa.c ecp.c ecp_curves.c oid.c pk_wrap.c) +set_property(TARGET mbedcrypto PROPERTY SOURCES ${src_crypto}) + +get_target_property(src_x509 mbedx509 SOURCES) +list(REMOVE_ITEM src_x509 x509_crt.c) +set_property(TARGET mbedx509 PROPERTY SOURCES ${src_x509}) +endif() + set(mbedtls_targets mbedtls mbedcrypto mbedx509) # Add port files to mbedtls targets diff --git a/components/mbedtls/Kconfig b/components/mbedtls/Kconfig index 0bec091bd..a59579fb9 100644 --- a/components/mbedtls/Kconfig +++ b/components/mbedtls/Kconfig @@ -328,6 +328,13 @@ menu "mbedTLS" depends on MBEDTLS_TLS_ENABLED default y + config MBEDTLS_SSL_PROTO_GMTSSL1_1 + bool "Support GM/T SSL 1.1 protocol" + depends on MBEDTLS_TLS_ENABLED + default n + help + Provisions for GM/T SSL 1.1 support + config MBEDTLS_SSL_PROTO_DTLS bool "Support DTLS protocol (all versions)" default n diff --git a/components/mbedtls/component.mk b/components/mbedtls/component.mk index 541009de6..6d49cd722 100644 --- a/components/mbedtls/component.mk +++ b/components/mbedtls/component.mk @@ -8,6 +8,20 @@ COMPONENT_SRCDIRS := mbedtls/library port port/$(IDF_TARGET) COMPONENT_OBJEXCLUDE := mbedtls/library/net_sockets.o +ifdef CONFIG_MBEDTLS_SSL_PROTO_GMTSSL1_1 +COMPONENT_OBJEXCLUDE += \ +mbedtls/library/ssl_ciphersuites.o \ +mbedtls/library/ecp.o \ +mbedtls/library/cipher_wrap.o \ +mbedtls/library/oid.o \ +mbedtls/library/ecp_curves.o \ +mbedtls/library/pk_wrap.o \ +mbedtls/library/ecdsa.o \ +mbedtls/library/x509_crt.o \ +mbedtls/library/ssl_tls.o \ +mbedtls/library/ssl_cli.o +endif + COMPONENT_SUBMODULES += mbedtls ifdef CONFIG_MBEDTLS_DYNAMIC_BUFFER diff --git a/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c b/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c index e78ea5afe..f64b09e35 100644 --- a/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c +++ b/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c @@ -23,6 +23,33 @@ static const char *TAG = "Dynamic Impl"; +static void esp_mbedtls_set_buf_state(unsigned char *buf, esp_mbedtls_ssl_buf_states state) +{ + struct esp_mbedtls_ssl_buf *temp = __containerof(buf, struct esp_mbedtls_ssl_buf, buf[0]); + temp->state = state; +} + +static esp_mbedtls_ssl_buf_states esp_mbedtls_get_buf_state(unsigned char *buf) +{ + struct esp_mbedtls_ssl_buf *temp = __containerof(buf, struct esp_mbedtls_ssl_buf, buf[0]); + return temp->state; +} + +void esp_mbedtls_free_buf(unsigned char *buf) +{ + struct esp_mbedtls_ssl_buf *temp = __containerof(buf, struct esp_mbedtls_ssl_buf, buf[0]); + ESP_LOGV(TAG, "free buffer @ %p", temp); + mbedtls_free(temp); +} + +static void esp_mbedtls_init_ssl_buf(struct esp_mbedtls_ssl_buf *buf, unsigned int len) +{ + if (buf) { + buf->state = ESP_MBEDTLS_SSL_BUF_CACHED; + buf->len = len; + } +} + static void esp_mbedtls_parse_record_header(mbedtls_ssl_context *ssl) { ssl->in_msgtype = ssl->in_hdr[0]; @@ -118,29 +145,30 @@ static void init_rx_buffer(mbedtls_ssl_context *ssl, unsigned char *buf) static int esp_mbedtls_alloc_tx_buf(mbedtls_ssl_context *ssl, int len) { - unsigned char *buf; + struct esp_mbedtls_ssl_buf *esp_buf; if (ssl->out_buf) { - mbedtls_free(ssl->out_buf); + esp_mbedtls_free_buf(ssl->out_buf); ssl->out_buf = NULL; } - buf = mbedtls_calloc(1, len); - if (!buf) { - ESP_LOGE(TAG, "alloc(%d bytes) failed", len); + esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + len); + if (!esp_buf) { + ESP_LOGE(TAG, "alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + len); return MBEDTLS_ERR_SSL_ALLOC_FAILED; } - ESP_LOGV(TAG, "add out buffer %d bytes @ %p", len, buf); + ESP_LOGV(TAG, "add out buffer %d bytes @ %p", len, esp_buf->buf); + esp_mbedtls_init_ssl_buf(esp_buf, len); /** * Mark the out_msg offset from ssl->out_buf. - * + * * In mbedtls, ssl->out_msg = ssl->out_buf + offset; */ ssl->out_msg = (unsigned char *)MBEDTLS_SSL_HEADER_LEN; - init_tx_buffer(ssl, buf); + init_tx_buffer(ssl, esp_buf->buf); return 0; } @@ -150,7 +178,7 @@ int esp_mbedtls_setup_tx_buffer(mbedtls_ssl_context *ssl) CHECK_OK(esp_mbedtls_alloc_tx_buf(ssl, TX_IDLE_BUFFER_SIZE)); /* mark the out buffer has no data cached */ - ssl->out_iv = NULL; + esp_mbedtls_set_buf_state(ssl->out_buf, ESP_MBEDTLS_SSL_BUF_NO_CACHED); return 0; } @@ -168,10 +196,7 @@ int esp_mbedtls_reset_add_tx_buffer(mbedtls_ssl_context *ssl) int esp_mbedtls_reset_free_tx_buffer(mbedtls_ssl_context *ssl) { - ESP_LOGV(TAG, "free out buffer @ %p", ssl->out_buf); - - mbedtls_free(ssl->out_buf); - + esp_mbedtls_free_buf(ssl->out_buf); init_tx_buffer(ssl, NULL); CHECK_OK(esp_mbedtls_setup_tx_buffer(ssl)); @@ -181,60 +206,57 @@ int esp_mbedtls_reset_free_tx_buffer(mbedtls_ssl_context *ssl) int esp_mbedtls_reset_add_rx_buffer(mbedtls_ssl_context *ssl) { - unsigned char *buf; + struct esp_mbedtls_ssl_buf *esp_buf; if (ssl->in_buf) { - mbedtls_free(ssl->in_buf); + esp_mbedtls_free_buf(ssl->in_buf); ssl->in_buf = NULL; } - buf = mbedtls_calloc(1, MBEDTLS_SSL_IN_BUFFER_LEN); - if (!buf) { - ESP_LOGE(TAG, "alloc(%d bytes) failed", MBEDTLS_SSL_IN_BUFFER_LEN); + esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + MBEDTLS_SSL_IN_BUFFER_LEN); + if (!esp_buf) { + ESP_LOGE(TAG, "alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + MBEDTLS_SSL_IN_BUFFER_LEN); return MBEDTLS_ERR_SSL_ALLOC_FAILED; } - ESP_LOGV(TAG, "add in buffer %d bytes @ %p", MBEDTLS_SSL_IN_BUFFER_LEN, buf); + ESP_LOGV(TAG, "add in buffer %d bytes @ %p", MBEDTLS_SSL_IN_BUFFER_LEN, esp_buf->buf); + esp_mbedtls_init_ssl_buf(esp_buf, MBEDTLS_SSL_IN_BUFFER_LEN); /** * Mark the in_msg offset from ssl->in_buf. - * + * * In mbedtls, ssl->in_msg = ssl->in_buf + offset; */ ssl->in_msg = (unsigned char *)MBEDTLS_SSL_HEADER_LEN; - init_rx_buffer(ssl, buf); + init_rx_buffer(ssl, esp_buf->buf); - return 0; + return 0; } void esp_mbedtls_reset_free_rx_buffer(mbedtls_ssl_context *ssl) { - ESP_LOGV(TAG, "free in buffer @ %p", ssl->in_buf); - - mbedtls_free(ssl->in_buf); - - init_rx_buffer(ssl, NULL); + esp_mbedtls_free_buf(ssl->in_buf); + init_rx_buffer(ssl, NULL); } int esp_mbedtls_add_tx_buffer(mbedtls_ssl_context *ssl, size_t buffer_len) { int ret = 0; int cached = 0; - unsigned char *buf; + struct esp_mbedtls_ssl_buf *esp_buf; unsigned char cache_buf[CACHE_BUFFER_SIZE]; ESP_LOGV(TAG, "--> add out"); if (ssl->out_buf) { - if (ssl->out_iv) { + if (esp_mbedtls_get_buf_state(ssl->out_buf) == ESP_MBEDTLS_SSL_BUF_CACHED) { ESP_LOGV(TAG, "out buffer is not empty"); ret = 0; goto exit; } else { memcpy(cache_buf, ssl->out_buf, CACHE_BUFFER_SIZE); - - mbedtls_free(ssl->out_buf); + esp_mbedtls_free_buf(ssl->out_buf); init_tx_buffer(ssl, NULL); cached = 1; } @@ -242,15 +264,17 @@ int esp_mbedtls_add_tx_buffer(mbedtls_ssl_context *ssl, size_t buffer_len) buffer_len = tx_buffer_len(ssl, buffer_len); - buf = mbedtls_calloc(1, buffer_len); - if (!buf) { - ESP_LOGE(TAG, "alloc(%d bytes) failed", buffer_len); + esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + buffer_len); + if (!esp_buf) { + ESP_LOGE(TAG, "alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + buffer_len); ret = MBEDTLS_ERR_SSL_ALLOC_FAILED; goto exit; } - ESP_LOGV(TAG, "add out buffer %d bytes @ %p", buffer_len, buf); - init_tx_buffer(ssl, buf); + ESP_LOGV(TAG, "add out buffer %d bytes @ %p", buffer_len, esp_buf->buf); + + esp_mbedtls_init_ssl_buf(esp_buf, buffer_len); + init_tx_buffer(ssl, esp_buf->buf); if (cached) { memcpy(ssl->out_ctr, cache_buf, COUNTER_SIZE); @@ -270,11 +294,11 @@ int esp_mbedtls_free_tx_buffer(mbedtls_ssl_context *ssl) { int ret = 0; unsigned char buf[CACHE_BUFFER_SIZE]; - unsigned char *pdata; + struct esp_mbedtls_ssl_buf *esp_buf; ESP_LOGV(TAG, "--> free out"); - if (!ssl->out_buf || (ssl->out_buf && !ssl->out_iv)) { + if (!ssl->out_buf || (ssl->out_buf && (esp_mbedtls_get_buf_state(ssl->out_buf) == ESP_MBEDTLS_SSL_BUF_NO_CACHED))) { ret = 0; goto exit; } @@ -282,22 +306,19 @@ int esp_mbedtls_free_tx_buffer(mbedtls_ssl_context *ssl) memcpy(buf, ssl->out_ctr, COUNTER_SIZE); memcpy(buf + COUNTER_SIZE, ssl->out_iv, CACHE_IV_SIZE); - ESP_LOGV(TAG, "free out buffer @ %p", ssl->out_buf); - - mbedtls_free(ssl->out_buf); - + esp_mbedtls_free_buf(ssl->out_buf); init_tx_buffer(ssl, NULL); - pdata = mbedtls_calloc(1, TX_IDLE_BUFFER_SIZE); - if (!pdata) { - ESP_LOGE(TAG, "alloc(%d bytes) failed", TX_IDLE_BUFFER_SIZE); + esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + TX_IDLE_BUFFER_SIZE); + if (!esp_buf) { + ESP_LOGE(TAG, "alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + TX_IDLE_BUFFER_SIZE); return MBEDTLS_ERR_SSL_ALLOC_FAILED; } - memcpy(pdata, buf, CACHE_BUFFER_SIZE); - init_tx_buffer(ssl, pdata); - ssl->out_iv = NULL; - + esp_mbedtls_init_ssl_buf(esp_buf, TX_IDLE_BUFFER_SIZE); + memcpy(esp_buf->buf, buf, CACHE_BUFFER_SIZE); + init_tx_buffer(ssl, esp_buf->buf); + esp_mbedtls_set_buf_state(ssl->out_buf, ESP_MBEDTLS_SSL_BUF_NO_CACHED); exit: ESP_LOGV(TAG, "<-- free out"); @@ -309,7 +330,7 @@ int esp_mbedtls_add_rx_buffer(mbedtls_ssl_context *ssl) int cached = 0; int ret = 0; int buffer_len; - unsigned char *buf; + struct esp_mbedtls_ssl_buf *esp_buf; unsigned char cache_buf[16]; unsigned char msg_head[5]; size_t in_msglen, in_left; @@ -317,15 +338,11 @@ int esp_mbedtls_add_rx_buffer(mbedtls_ssl_context *ssl) ESP_LOGV(TAG, "--> add rx"); if (ssl->in_buf) { - if (ssl->in_iv) { + if (esp_mbedtls_get_buf_state(ssl->in_buf) == ESP_MBEDTLS_SSL_BUF_CACHED) { ESP_LOGV(TAG, "in buffer is not empty"); ret = 0; goto exit; } else { - memcpy(cache_buf, ssl->in_buf, 16); - - mbedtls_free(ssl->in_buf); - init_rx_buffer(ssl, NULL); cached = 1; } } @@ -341,7 +358,7 @@ int esp_mbedtls_add_rx_buffer(mbedtls_ssl_context *ssl) } else { ESP_LOGE(TAG, "mbedtls_ssl_fetch_input error=-0x%x", -ret); } - + goto exit; } @@ -354,16 +371,23 @@ int esp_mbedtls_add_rx_buffer(mbedtls_ssl_context *ssl) ESP_LOGV(TAG, "message length is %d RX buffer length should be %d left is %d", (int)in_msglen, (int)buffer_len, (int)ssl->in_left); - buf = mbedtls_calloc(1, buffer_len); - if (!buf) { - ESP_LOGE(TAG, "alloc(%d bytes) failed", buffer_len); + if (cached) { + memcpy(cache_buf, ssl->in_buf, 16); + esp_mbedtls_free_buf(ssl->in_buf); + init_rx_buffer(ssl, NULL); + } + + esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + buffer_len); + if (!esp_buf) { + ESP_LOGE(TAG, "alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + buffer_len); ret = MBEDTLS_ERR_SSL_ALLOC_FAILED; goto exit; } - ESP_LOGV(TAG, "add in buffer %d bytes @ %p", buffer_len, buf); + ESP_LOGV(TAG, "add in buffer %d bytes @ %p", buffer_len, esp_buf->buf); - init_rx_buffer(ssl, buf); + esp_mbedtls_init_ssl_buf(esp_buf, buffer_len); + init_rx_buffer(ssl, esp_buf->buf); if (cached) { memcpy(ssl->in_ctr, cache_buf, 8); @@ -377,14 +401,14 @@ int esp_mbedtls_add_rx_buffer(mbedtls_ssl_context *ssl) exit: ESP_LOGV(TAG, "<-- add rx"); - return ret; + return ret; } int esp_mbedtls_free_rx_buffer(mbedtls_ssl_context *ssl) { int ret = 0; unsigned char buf[16]; - unsigned char *pdata; + struct esp_mbedtls_ssl_buf *esp_buf; ESP_LOGV(TAG, "--> free rx"); @@ -392,7 +416,7 @@ int esp_mbedtls_free_rx_buffer(mbedtls_ssl_context *ssl) * When have read multi messages once, can't free the input buffer directly. */ if (!ssl->in_buf || (ssl->in_hslen && (ssl->in_hslen < ssl->in_msglen)) || - (ssl->in_buf && !ssl->in_iv)) { + (ssl->in_buf && (esp_mbedtls_get_buf_state(ssl->in_buf) == ESP_MBEDTLS_SSL_BUF_NO_CACHED))) { ret = 0; goto exit; } @@ -407,23 +431,20 @@ int esp_mbedtls_free_rx_buffer(mbedtls_ssl_context *ssl) memcpy(buf, ssl->in_ctr, 8); memcpy(buf + 8, ssl->in_iv, 8); - ESP_LOGV(TAG, "free in buffer @ %p", ssl->out_buf); - - mbedtls_free(ssl->in_buf); - + esp_mbedtls_free_buf(ssl->in_buf); init_rx_buffer(ssl, NULL); - pdata = mbedtls_calloc(1, 16); - if (!pdata) { - ESP_LOGE(TAG, "alloc(%d bytes) failed", 16); + esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + 16); + if (!esp_buf) { + ESP_LOGE(TAG, "alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + 16); ret = MBEDTLS_ERR_SSL_ALLOC_FAILED; goto exit; } - memcpy(pdata, buf, 16); - init_rx_buffer(ssl, pdata); - ssl->in_iv = NULL; - + esp_mbedtls_init_ssl_buf(esp_buf, 16); + memcpy(esp_buf->buf, buf, 16); + init_rx_buffer(ssl, esp_buf->buf); + esp_mbedtls_set_buf_state(ssl->in_buf, ESP_MBEDTLS_SSL_BUF_NO_CACHED); exit: ESP_LOGV(TAG, "<-- free rx"); @@ -438,7 +459,7 @@ size_t esp_mbedtls_get_crt_size(mbedtls_x509_crt *cert, size_t *num) while (cert) { bytes += cert->raw.len; n++; - + cert = cert->next; } diff --git a/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.h b/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.h index 8f4bb144c..2c6b15c00 100644 --- a/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.h +++ b/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.h @@ -41,6 +41,21 @@ \ }) +typedef enum { + ESP_MBEDTLS_SSL_BUF_CACHED, + ESP_MBEDTLS_SSL_BUF_NO_CACHED, +} esp_mbedtls_ssl_buf_states; + +struct esp_mbedtls_ssl_buf { + esp_mbedtls_ssl_buf_states state; + unsigned int len; + unsigned char buf[]; +}; + +#define SSL_BUF_HEAD_OFFSET_SIZE offsetof(struct esp_mbedtls_ssl_buf, buf) + +void esp_mbedtls_free_buf(unsigned char *buf); + int esp_mbedtls_setup_tx_buffer(mbedtls_ssl_context *ssl); void esp_mbedtls_setup_rx_buffer(mbedtls_ssl_context *ssl); diff --git a/components/mbedtls/port/dynamic/esp_ssl_tls.c b/components/mbedtls/port/dynamic/esp_ssl_tls.c index 081e50af8..d8b4506b5 100644 --- a/components/mbedtls/port/dynamic/esp_ssl_tls.c +++ b/components/mbedtls/port/dynamic/esp_ssl_tls.c @@ -108,12 +108,12 @@ int __wrap_mbedtls_ssl_read(mbedtls_ssl_context *ssl, unsigned char *buf, size_t void __wrap_mbedtls_ssl_free(mbedtls_ssl_context *ssl) { if (ssl->out_buf) { - mbedtls_free(ssl->out_buf); + esp_mbedtls_free_buf(ssl->out_buf); ssl->out_buf = NULL; } if (ssl->in_buf) { - mbedtls_free(ssl->in_buf); + esp_mbedtls_free_buf(ssl->in_buf); ssl->in_buf = NULL; } diff --git a/components/nvs_flash/CMakeLists.txt b/components/nvs_flash/CMakeLists.txt index b36e6ba52..7ce2871c1 100644 --- a/components/nvs_flash/CMakeLists.txt +++ b/components/nvs_flash/CMakeLists.txt @@ -1,6 +1,51 @@ -set(COMPONENT_SRCDIRS "src") -set(COMPONENT_ADD_INCLUDEDIRS "include") +idf_build_get_property(target IDF_TARGET) -set(COMPONENT_PRIV_REQUIRES "spi_flash") +set(srcs "src/nvs_api.cpp" + "src/nvs_cxx_api.cpp" + "src/nvs_item_hash_list.cpp" + "src/nvs_page.cpp" + "src/nvs_pagemanager.cpp" + "src/nvs_storage.cpp" + "src/nvs_handle_simple.cpp" + "src/nvs_handle_locked.cpp" + "src/nvs_partition.cpp" + "src/nvs_partition_lookup.cpp" + "src/nvs_partition_manager.cpp" + "src/nvs_types.cpp") -register_component() +set(public_req spi_flash) + +set(include_dirs "include") + +idf_component_register(SRCS "${srcs}" + REQUIRES "${public_req}" + INCLUDE_DIRS "${include_dirs}") + +# If we use the linux target, we need to redirect the crc functions to the linux +if(${target} STREQUAL "linux") + if(CONFIG_NVS_ENCRYPTION) + # mbedtls isn't configured for building with linux or as mock target. It will draw in all kind of dependencies + message(FATAL_ERROR "NVS currently doesn't support encryption if built for Linux.") + endif() + idf_component_get_property(spi_flash_dir spi_flash COMPONENT_DIR) + target_include_directories(${COMPONENT_LIB} PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}/mock/int" + "${spi_flash_dir}/sim/stubs/freertos/include") + target_sources(${COMPONENT_LIB} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mock/int/crc.cpp") + target_compile_options(${COMPONENT_LIB} PUBLIC "-DLINUX_TARGET") +else() + # TODO: this is a workaround until IDF-2085 is fixed + idf_component_get_property(mbedtls_lib mbedtls COMPONENT_LIB) + target_link_libraries(${COMPONENT_LIB} PUBLIC ${mbedtls_lib}) +endif() + +if(CONFIG_NVS_ENCRYPTION) + target_sources(${COMPONENT_LIB} PRIVATE "src/nvs_encrypted_partition.cpp") + idf_component_get_property(mbedtls_lib mbedtls COMPONENT_LIB) + target_link_libraries(${COMPONENT_LIB} PUBLIC ${mbedtls_lib}) +endif() + +if(${target} STREQUAL "linux") + target_compile_options(${COMPONENT_LIB} PUBLIC --coverage) + target_link_libraries(${COMPONENT_LIB} PUBLIC --coverage) +endif() diff --git a/components/nvs_flash/README.rst b/components/nvs_flash/README.rst deleted file mode 100644 index f04ac592c..000000000 --- a/components/nvs_flash/README.rst +++ /dev/null @@ -1,234 +0,0 @@ -Non-volatile storage library -============================ - -Introduction ------------- - -Non-volatile storage (NVS) library is designed to store key-value pairs in flash. This sections introduces some concepts used by NVS. - -Underlying storage -^^^^^^^^^^^^^^^^^^ - -Currently NVS uses a portion of main flash memory through ``spi_flash_{read|write|erase}`` APIs. The library uses the all the partitions with ``data`` type and ``nvs`` subtype. The application can choose to use the partition with label ``nvs`` through ``nvs_open`` API or any of the other partition by specifying its name through ``nvs_open_from_part`` API. - -Future versions of this library may add other storage backends to keep data in another flash chip (SPI or I2C), RTC, FRAM, etc. - -.. note:: if an NVS partition is truncated (for example, when the partition table layout is changed), its contents should be erased. ESP-IDF build system provides a ``make erase_flash`` target to erase all contents of the flash chip. - -.. note:: NVS works best for storing many small values, rather than a few large values of type 'string' and 'blob'. If storing large blobs or strings is required, consider using the facilities provided by the FAT filesystem on top of the wear levelling library. - -Keys and values -^^^^^^^^^^^^^^^ - -NVS operates on key-value pairs. Keys are ASCII strings, maximum key length is currently 15 characters. Values can have one of the following types: - -- integer types: ``uint8_t``, ``int8_t``, ``uint16_t``, ``int16_t``, ``uint32_t``, ``int32_t``, ``uint64_t``, ``int64_t`` -- zero-terminated string -- variable length binary data (blob) - -.. note:: - String and blob values are currently limited to 1984 bytes. For strings, this includes the null terminator. - -Additional types, such as ``float`` and ``double`` may be added later. - -Keys are required to be unique. Writing a value for a key which already exists behaves as follows: - -- if the new value is of the same type as old one, value is updated -- if the new value has different data type, an error is returned - -Data type check is also performed when reading a value. An error is returned if data type of read operation doesn’t match the data type of the value. - -Namespaces -^^^^^^^^^^ - -To mitigate potential conflicts in key names between different components, NVS assigns each key-value pair to one of namespaces. Namespace names follow the same rules as key names, i.e. 15 character maximum length. Namespace name is specified in the ``nvs_open`` or ``nvs_open_from_part`` call. This call returns an opaque handle, which is used in subsequent calls to ``nvs_read_*``, ``nvs_write_*``, and ``nvs_commit`` functions. This way, handle is associated with a namespace, and key names will not collide with same names in other namespaces. -Please note that the namespaces with same name in different NVS partitions are considered as separate namespaces. - -Security, tampering, and robustness -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -NVS library doesn't implement tamper prevention measures. It is possible for anyone with physical access to the flash chip to alter, erase, or add key-value pairs. - -NVS is compatible with the ESP32 flash encryption system, and it can store key-value pairs in an encrypted form. Some metadata, like page state and write/erase flags of individual entries can not be encrypted as they are represented as bits of flash memory for efficient access and manipulation. Flash encryption can prevent some forms of modification: - -- replacing keys or values with arbitrary data -- changing data types of values - -The following forms of modification are still possible when flash encryption is used: - -- erasing a page completely, removing all key-value pairs which were stored in that page -- corrupting data in a page, which will cause the page to be erased automatically when such condition is detected -- rolling back the contents of flash memory to an earlier snapshot -- merging two snapshots of flash memory, rolling back some key-value pairs to an earlier state (although this is possible to mitigate with the current design — TODO) - -The library does try to recover from conditions when flash memory is in an inconsistent state. In particular, one should be able to power off the device at any point and time and then power it back on. This should not result in loss of data, expect for the new key-value pair if it was being written at the moment of power off. The library should also be able to initialize properly with any random data present in flash memory. - -Internals ---------- - -Log of key-value pairs -^^^^^^^^^^^^^^^^^^^^^^ - -NVS stores key-value pairs sequentially, with new key-value pairs being added at the end. When a value of any given key has to be updated, new key-value pair is added at the end of the log and old key-value pair is marked as erased. - -Pages and entries -^^^^^^^^^^^^^^^^^ - -NVS library uses two main entities in its operation: pages and entries. Page is a logical structure which stores a portion of the overall log. Logical page corresponds to one physical sector of flash memory. Pages which are in use have a *sequence number* associated with them. Sequence numbers impose an ordering on pages. Higher sequence numbers correspond to pages which were created later. Each page can be in one of the following states: - -Empty/uninitialized - Flash storage for the page is empty (all bytes are ``0xff``). Page isn't used to store any data at this point and doesn’t have a sequence number. - -Active - Flash storage is initialized, page header has been written to flash, page has a valid sequence number. Page has some empty entries and data can be written there. At most one page can be in this state at any given moment. - -Full - Flash storage is in a consistent state and is filled with key-value pairs. - Writing new key-value pairs into this page is not possible. It is still possible to mark some key-value pairs as erased. - -Erasing - Non-erased key-value pairs are being moved into another page so that the current page can be erased. This is a transient state, i.e. page should never stay in this state when any API call returns. In case of a sudden power off, move-and-erase process will be completed upon next power on. - -Corrupted - Page header contains invalid data, and further parsing of page data was canceled. Any items previously written into this page will not be accessible. Corresponding flash sector will not be erased immediately, and will be kept along with sectors in *uninitialized* state for later use. This may be useful for debugging. - -Mapping from flash sectors to logical pages doesn't have any particular order. Library will inspect sequence numbers of pages found in each flash sector and organize pages in a list based on these numbers. - -:: - - +--------+ +--------+ +--------+ +--------+ - | Page 1 | | Page 2 | | Page 3 | | Page 4 | - | Full +---> | Full +---> | Active | | Empty | <- states - | #11 | | #12 | | #14 | | | <- sequence numbers - +---+----+ +----+---+ +----+---+ +---+----+ - | | | | - | | | | - | | | | - +---v------+ +-----v----+ +------v---+ +------v---+ - | Sector 3 | | Sector 0 | | Sector 2 | | Sector 1 | <- physical sectors - +----------+ +----------+ +----------+ +----------+ - -Structure of a page -^^^^^^^^^^^^^^^^^^^ - -For now we assume that flash sector size is 4096 bytes and that ESP32 flash encryption hardware operates on 32-byte blocks. It is possible to introduce some settings configurable at compile-time (e.g. via menuconfig) to accommodate flash chips with different sector sizes (although it is not clear if other components in the system, e.g. SPI flash driver and SPI flash cache can support these other sizes). - -Page consists of three parts: header, entry state bitmap, and entries themselves. To be compatible with ESP32 flash encryption, entry size is 32 bytes. For integer types, entry holds one key-value pair. For strings and blobs, an entry holds part of key-value pair (more on that in the entry structure description). - -The following diagram illustrates page structure. Numbers in parentheses indicate size of each part in bytes. :: - - +-----------+--------------+-------------+-----------+ - | State (4) | Seq. no. (4) | Unused (20) | CRC32 (4) | Header (32) - +-----------+--------------+-------------+-----------+ - | Entry state bitmap (32) | - +----------------------------------------------------+ - | Entry 0 (32) | - +----------------------------------------------------+ - | Entry 1 (32) | - +----------------------------------------------------+ - / / - / / - +----------------------------------------------------+ - | Entry 125 (32) | - +----------------------------------------------------+ - -Page header and entry state bitmap are always written to flash unencrypted. Entries are encrypted if flash encryption feature of the ESP32 is used. - -Page state values are defined in such a way that changing state is possible by writing 0 into some of the bits. Therefore it not necessary to erase the page to change page state, unless that is a change to *erased* state. - -CRC32 value in header is calculated over the part which doesn't include state value (bytes 4 to 28). Unused part is currently filled with ``0xff`` bytes. Future versions of the library may store format version there. - -The following sections describe structure of entry state bitmap and entry itself. - -Entry and entry state bitmap -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Each entry can be in one of the following three states. Each state is represented with two bits in the entry state bitmap. Final four bits in the bitmap (256 - 2 * 126) are unused. - -Empty (2'b11) - Nothing is written into the specific entry yet. It is in an uninitialized state (all bytes ``0xff``). - -Written (2'b10) - A key-value pair (or part of key-value pair which spans multiple entries) has been written into the entry. - -Erased (2'b00) - A key-value pair in this entry has been discarded. Contents of this entry will not be parsed anymore. - - -Structure of entry -^^^^^^^^^^^^^^^^^^ - -For values of primitive types (currently integers from 1 to 8 bytes long), entry holds one key-value pair. For string and blob types, entry holds part of the whole key-value pair. In case when a key-value pair spans multiple entries, all entries are stored in the same page. - -:: - - +--------+----------+----------+---------+-----------+---------------+----------+ - | NS (1) | Type (1) | Span (1) | Rsv (1) | CRC32 (4) | Key (16) | Data (8) | - +--------+----------+----------+---------+-----------+---------------+----------+ - - +--------------------------------+ - +-> Fixed length: | Data (8) | - | +--------------------------------+ - Data format ---+ - | +----------+---------+-----------+ - +-> Variable length: | Size (2) | Rsv (2) | CRC32 (4) | - +----------+---------+-----------+ - - -Individual fields in entry structure have the following meanings: - -NS - Namespace index for this entry. See section on namespaces implementation for explanation of this value. - -Type - One byte indicating data type of value. See ``ItemType`` enumeration in ``nvs_types.h`` for possible values. - -Span - Number of entries used by this key-value pair. For integer types, this is equal to 1. For strings and blobs this depends on value length. - -Rsv - Unused field, should be ``0xff``. - -CRC32 - Checksum calculated over all the bytes in this entry, except for the CRC32 field itself. - -Key - Zero-terminated ASCII string containing key name. Maximum string length is 15 bytes, excluding zero terminator. - -Data - For integer types, this field contains the value itself. If the value itself is shorter than 8 bytes it is padded to the right, with unused bytes filled with ``0xff``. For string and blob values, these 8 bytes hold additional data about the value, described next: - -Size - (Only for strings and blobs.) Size, in bytes, of actual data. For strings, this includes zero terminator. - -CRC32 - (Only for strings and blobs.) Checksum calculated over all bytes of data. - -Variable length values (strings and blobs) are written into subsequent entries, 32 bytes per entry. `Span` field of the first entry indicates how many entries are used. - - -Namespaces -^^^^^^^^^^ - -As mentioned above, each key-value pair belongs to one of the namespaces. Namespaces identifiers (strings) are stored as keys of key-value pairs in namespace with index 0. Values corresponding to these keys are indexes of these namespaces. - -:: - - +-------------------------------------------+ - | NS=0 Type=uint8_t Key="wifi" Value=1 | Entry describing namespace "wifi" - +-------------------------------------------+ - | NS=1 Type=uint32_t Key="channel" Value=6 | Key "channel" in namespace "wifi" - +-------------------------------------------+ - | NS=0 Type=uint8_t Key="pwm" Value=2 | Entry describing namespace "pwm" - +-------------------------------------------+ - | NS=2 Type=uint16_t Key="channel" Value=20 | Key "channel" in namespace "pwm" - +-------------------------------------------+ - - -Item hash list -^^^^^^^^^^^^^^ - -To reduce the number of reads performed from flash memory, each member of Page class maintains a list of pairs: (item index; item hash). This list makes searches much quicker. Instead of iterating over all entries, reading them from flash one at a time, ``Page::findItem`` first performs search for item hash in the hash list. This gives the item index within the page, if such an item exists. Due to a hash collision it is possible that a different item will be found. This is handled by falling back to iteration over items in flash. - -Each node in hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace and key name. CRC32 is used for calculation, result is truncated to 24 bits. To reduce overhead of storing 32-bit entries in a linked list, list is implemented as a doubly-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and 32-bit count field. Minimal amount of extra RAM useage per page is therefore 128 bytes, maximum is 640 bytes. - diff --git a/components/nvs_flash/component.mk b/components/nvs_flash/component.mk index 20b0ec407..8b2fda4f5 100755 --- a/components/nvs_flash/component.mk +++ b/components/nvs_flash/component.mk @@ -6,4 +6,6 @@ COMPONENT_ADD_INCLUDEDIRS := include COMPONENT_SRCDIRS := src -CPPFLAGS += -DNVS_CRC_HEADER_FILE=\"rom/crc.h\" +ifndef CONFIG_NVS_ENCRYPTION +COMPONENT_OBJEXCLUDE := src/nvs_encr.o +endif diff --git a/components/nvs_flash/host_test/fixtures/test_fixtures.hpp b/components/nvs_flash/host_test/fixtures/test_fixtures.hpp new file mode 100644 index 000000000..b0915682f --- /dev/null +++ b/components/nvs_flash/host_test/fixtures/test_fixtures.hpp @@ -0,0 +1,436 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "nvs_partition.hpp" +#include "nvs.h" +#include "nvs_page.hpp" +#include "nvs_storage.hpp" +#include +#include + +#ifdef CONFIG_NVS_ENCRYPTION +#include "nvs_encrypted_partition.hpp" +#endif + +extern "C" { +#include "Mockesp_partition.h" +} + +struct FixtureException : std::exception { + FixtureException(const std::string& msg) : msg(msg) { } + + const char *what() { + return msg.c_str(); + } + + std::string msg; +}; + +class PartitionMock : public nvs::Partition { +public: + PartitionMock(uint32_t address, uint32_t size) + : partition(), address(address), size(size) + { + assert(size); + } + + const char *get_partition_name() override + { + return ""; + } + + esp_err_t read_raw(size_t src_offset, void* dst, size_t size) override + { + return esp_partition_read_raw(&partition, src_offset, dst, size); + } + + esp_err_t read(size_t src_offset, void* dst, size_t size) override + { + return esp_partition_read(&partition, src_offset, dst, size); + } + + esp_err_t write_raw(size_t dst_offset, const void* src, size_t size) override + { + return esp_partition_write_raw(&partition, dst_offset, src, size); + } + + esp_err_t write(size_t dst_offset, const void* src, size_t size) override + { + return esp_partition_write(&partition, dst_offset, src, size); + } + + esp_err_t erase_range(size_t dst_offset, size_t size) override + { + return esp_partition_erase_range(&partition, dst_offset, size); + } + + uint32_t get_address() override + { + return address; + } + + uint32_t get_size() override + { + return size; + } + + const esp_partition_t partition; + +private: + uint32_t address; + + uint32_t size; +}; + +#ifdef CONFIG_NVS_ENCRYPTION +struct EncryptedPartitionFixture { + EncryptedPartitionFixture(nvs_sec_cfg_t *cfg, + uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : esp_partition(), emu(start_sector + sector_size), + part(partition_name, &esp_partition) { + esp_partition.address = start_sector * SPI_FLASH_SEC_SIZE; + esp_partition.size = sector_size * SPI_FLASH_SEC_SIZE; + assert(part.init(cfg) == ESP_OK); + } + + ~EncryptedPartitionFixture() { } + + esp_partition_t esp_partition; + + SpiFlashEmulator emu; + + nvs::NVSEncryptedPartition part; +}; +#endif + +struct PartitionMockFixture { + PartitionMockFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : part_mock(start_sector * SPI_FLASH_SEC_SIZE, sector_size * SPI_FLASH_SEC_SIZE) { + std::fill_n(raw_header, sizeof(raw_header)/sizeof(raw_header[0]), UINT8_MAX); + + // This resets the mocks and prevents meeting accidental expectations from previous tests. + Mockesp_partition_Init(); + } + + ~PartitionMockFixture() { } + + uint8_t raw_header[512]; + + PartitionMock part_mock; +}; + +struct NVSPageFixture : public PartitionMockFixture { + NVSPageFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : PartitionMockFixture(start_sector, sector_size, partition_name), page() + { + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 32); + + for (int i = 0; i < 8; i++) { + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 512); + } + + if (page.load(&part_mock, start_sector) != ESP_OK) throw FixtureException("couldn't setup page"); + } + + nvs::Page page; +}; + +struct NVSValidPageFlashFixture : public PartitionMockFixture { + const static uint8_t NS_INDEX = 1; + + // valid header + uint8_t raw_header_valid [32]; + + // entry table with one entry + uint8_t raw_entry_table [32]; + + uint8_t ns_entry [32]; + + uint8_t value_entry [32]; + + NVSValidPageFlashFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : PartitionMockFixture(start_sector, sector_size, partition_name), + raw_header_valid {0xfe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x16, 0xdd, 0xdc}, + ns_entry {0x00, 0x01, 0x01, 0xff, 0x68, 0xc5, 0x3f, 0x0b, 't', 'e', 's', 't', '_', 'n', 's', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + value_entry {0x01, 0x01, 0x01, 0xff, 0x3d, 0xf3, 0x99, 0xe5, 't', 'e', 's', 't', '_', 'v', 'a', 'l', + 'u', 'e', '\0', '\0', '\0', '\0', '\0', '\0', 47, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + { + std::fill_n(raw_entry_table, sizeof(raw_entry_table)/sizeof(raw_entry_table[0]), 0); + raw_entry_table[0] = 0xfa; + + // read page header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header_valid, 32); + + // read entry table + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_entry_table, 32); + + // read next free entry's header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 4); + + // read namespace entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // read normal entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(value_entry, 32); + + // read normal entry second time during duplicated entry check + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(value_entry, 32); + } +}; + +struct NVSValidPageFixture : public NVSValidPageFlashFixture { + NVSValidPageFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : NVSValidPageFlashFixture(start_sector, sector_size, partition_name), page() + { + if (page.load(&part_mock, start_sector) != ESP_OK) throw FixtureException("couldn't setup page"); + } + + nvs::Page page; +}; + +struct NVSValidStorageFixture : public PartitionMockFixture { + const static uint8_t NS_INDEX = 1; + + uint8_t ns_entry [32]; + + uint8_t empty_entry [32]; + + NVSValidStorageFixture(uint32_t start_sector = 0, + uint32_t sector_size = 3, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : PartitionMockFixture(start_sector, sector_size, partition_name), + ns_entry {0x00, 0x01, 0x01, 0xff, 0x68, 0xc5, 0x3f, 0x0b, 't', 'e', 's', 't', '_', 'n', 's', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + empty_entry(), + storage(&part_mock) + { + std::fill_n(empty_entry, sizeof(empty_entry)/sizeof(empty_entry[0]), 0xFF); + + // entry table with one entry + uint8_t raw_entry_table [32]; + + uint8_t header_full_page [] = { + 0xfe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x16, 0xdd, 0xdc}; + + uint8_t header_second_page [] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + + uint8_t header_third_page [] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + + // entry_table with all elements deleted except the namespace entry written and the last entry free + std::fill_n(raw_entry_table, sizeof(raw_entry_table)/sizeof(raw_entry_table[0]), 0); + raw_entry_table[0] = 0x02; + raw_entry_table[31] = 0xFC; + + // read full page header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(header_full_page, 32); + + // read entry table + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_entry_table, 32); + + // reading entry table checks empty entry + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(empty_entry, 32); + + // read namespace entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // read last two pages' headers, which trigger an automatic full read each because each page is empty + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(header_second_page, 32); + for (int i = 0; i < 8; i++) { + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 512); + } + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(header_third_page, 32); + for (int i = 0; i < 8; i++) { + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 512); + } + + // read namespace entry in duplicated header item check of pagemanager::load + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // storage finally actually reads namespace + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // storage looks for blob index entries + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // Storage::eraseOrphanDataBlobs() also wants to take it's turn... + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + if (storage.init(start_sector, sector_size) != ESP_OK) throw FixtureException("couldn't setup page"); + } + + nvs::Storage storage; +}; + +struct NVSValidBlobPageFixture : public PartitionMockFixture { + const static uint8_t NS_INDEX = 1; + const static size_t BLOB_DATA_SIZE = 32; + + // valid header + uint8_t raw_header_valid [32]; + + // entry table with one entry + uint8_t raw_entry_table [32]; + + uint8_t ns_entry [32]; + + uint8_t blob_entry [32]; + uint8_t blob_data [BLOB_DATA_SIZE]; + uint8_t blob_index [32]; + + NVSValidBlobPageFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : PartitionMockFixture(start_sector, sector_size, partition_name), + raw_header_valid {0xfe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x16, 0xdd, 0xdc}, + ns_entry {0x00, 0x01, 0x01, 0xff, 0x68, 0xc5, 0x3f, 0x0b, 't', 'e', 's', 't', '_', 'n', 's', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + blob_entry {0x01, 0x42, 0x02, 0x00, 0xaa, 0xf3, 0x23, 0x87, 't', 'e', 's', 't', '_', 'b', 'l', 'o', + 'b', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 0x20, 0x00, 0xff, 0xff, 0xc6, 0x96, 0x86, 0xd9}, + blob_data {0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, + 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}, + blob_index {0x01, 0x48, 0x01, 0xff, 0x42, 0x6b, 0xdf, 0x66, 't', 'e', 's', 't', '_', 'b', 'l', 'o', + 'b', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0xff, 0xff}, + page() + { + std::fill_n(raw_entry_table, sizeof(raw_entry_table)/sizeof(raw_entry_table[0]), 0xFF); + raw_entry_table[0] = 0xaa; + + // read page header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header_valid, 32); + + // read entry table + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_entry_table, 32); + + // read next free entry's header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 4); + + // read namespace entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // read normal blob entry + index, not the data + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(blob_entry, 32); + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(blob_index, 32); + + // read normal entry second time during duplicated entry check + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(blob_entry, 32); + + if (page.load(&part_mock, start_sector) != ESP_OK) throw FixtureException("couldn't setup page"); + } + + nvs::Page page; +}; + +struct NVSFullPageFixture : public PartitionMockFixture { + const static uint8_t NS_INDEX = 1; + + // valid header + uint8_t raw_header_valid [32]; + + // entry table with one entry + uint8_t raw_entry_table [32]; + + uint8_t ns_entry [32]; + + uint8_t value_entry [32]; + + NVSFullPageFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME, + bool load = true) + : PartitionMockFixture(start_sector, sector_size, partition_name), + raw_header_valid {0xfc, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa3, 0x48, 0x9f, 0x38}, + ns_entry {0x00, 0x01, 0x01, 0xff, 0x68, 0xc5, 0x3f, 0x0b, 't', 'e', 's', 't', '_', 'n', 's', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + value_entry {0x01, 0x01, 0x01, 0xff, 0x3d, 0xf3, 0x99, 0xe5, 't', 'e', 's', 't', '_', 'v', 'a', 'l', + 'u', 'e', '\0', '\0', '\0', '\0', '\0', '\0', 47, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + page() + { + // entry_table with all elements deleted except the namespace entry written and the last entry free + std::fill_n(raw_entry_table, sizeof(raw_entry_table)/sizeof(raw_entry_table[0]), 0); + raw_entry_table[0] = 0x0a; + raw_entry_table[31] = 0xFC; + + // read page header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header_valid, 32); + + // read entry table + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_entry_table, 32); + + // no next free entry check, only one entry written + + // read namespace entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // read normal entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(value_entry, 32); + + // no duplicated entry check + + if (load) { + if (page.load(&part_mock, start_sector) != ESP_OK) throw FixtureException("couldn't setup page"); + } + } + + nvs::Page page; +}; diff --git a/components/nvs_flash/host_test/nvs_page_test/CMakeLists.txt b/components/nvs_flash/host_test/nvs_page_test/CMakeLists.txt new file mode 100644 index 000000000..cafc7c699 --- /dev/null +++ b/components/nvs_flash/host_test/nvs_page_test/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +set(COMPONENTS main) +idf_build_set_property(CONFIG_SPI_FLASH_MOCK 1) +idf_build_set_property(COMPILE_DEFINITIONS "-DNO_DEBUG_STORAGE" APPEND) +project(host_nvs_page_test) + +add_custom_command( + OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/build/coverage.info" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + COMMAND lcov --capture --directory . --output-file coverage.info + COMMENT "Create coverage report" + ) + +add_custom_command( + OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/build/coverage_report/" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/build/coverage.info" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + COMMAND genhtml coverage.info --output-directory coverage_report/ + COMMENT "Turn coverage report into html-based visualization" + ) + +add_custom_target(coverage + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + DEPENDS "coverage_report/" + ) diff --git a/components/nvs_flash/host_test/nvs_page_test/README.rst b/components/nvs_flash/host_test/nvs_page_test/README.rst new file mode 100644 index 000000000..c005b3f48 --- /dev/null +++ b/components/nvs_flash/host_test/nvs_page_test/README.rst @@ -0,0 +1,23 @@ +NVS Page Test for Host +====================== + +Build +----- + +First, make sure that the target is set to linux. +Run ``idf.py --preview set-target linux`` to be sure. +Then do a normal IDF build: ``idf.py build``. + +Run +--- + +IDF monitor doesn't work yet for Linux. +You have to run the app manually: ``./build/host_nvs_page_test.elf``. + +Coverage +--- + +To generate the coverage, run: ``idf.py coverage``. +Afterwards, you can view the coverage by opening ``build/coverage_report/index.html`` with your browser. +Note that you need to run the application at least once before generating the coverage information. +If you run it multiple times, the coverage information adds up. diff --git a/components/nvs_flash/host_test/nvs_page_test/main/CMakeLists.txt b/components/nvs_flash/host_test/nvs_page_test/main/CMakeLists.txt new file mode 100644 index 000000000..d2af1b6a6 --- /dev/null +++ b/components/nvs_flash/host_test/nvs_page_test/main/CMakeLists.txt @@ -0,0 +1,10 @@ +idf_component_register(SRCS "nvs_page_test.cpp" + INCLUDE_DIRS + "." + "${CMAKE_CURRENT_SOURCE_DIR}/../../fixtures" + "${CMAKE_CURRENT_SOURCE_DIR}/../../../test_nvs_host" + "${CMAKE_CURRENT_SOURCE_DIR}/../../../src" + REQUIRES cmock nvs_flash spi_flash) + +target_compile_options(${COMPONENT_LIB} PUBLIC --coverage) +target_link_libraries(${COMPONENT_LIB} --coverage) diff --git a/components/nvs_flash/host_test/nvs_page_test/main/nvs_page_test.cpp b/components/nvs_flash/host_test/nvs_page_test/main/nvs_page_test.cpp new file mode 100644 index 000000000..e5ca9ed08 --- /dev/null +++ b/components/nvs_flash/host_test/nvs_page_test/main/nvs_page_test.cpp @@ -0,0 +1,1000 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "unity.h" +#include "test_fixtures.hpp" + +extern "C" { +#include "Mockesp_partition.h" +} + +using namespace std; +using namespace nvs; + +void test_Page_load_reading_header_fails() +{ + PartitionMock mock(0, 4096); + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_ERR_INVALID_ARG); + Page page; + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, page.state()); + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_ARG, page.load(&mock, 0)); + + Mockesp_partition_Verify(); +} + +void test_Page_load_reading_data_fails() +{ + uint8_t header[64]; + std::fill_n(header, sizeof(header)/sizeof(header[0]), UINT8_MAX); + PartitionMock mock(0, 4096); + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(header, 32); + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_FAIL); + Page page; + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, page.state()); + TEST_ASSERT_EQUAL(ESP_FAIL, page.load(&mock, 0)); + + Mockesp_partition_Verify(); +} + +void test_Page_load__uninitialized_page_has_0xfe() +{ + PartitionMockFixture fix; + Page page; + + fix.raw_header[511] = 0xfe; + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(fix.raw_header, 32); + + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(fix.raw_header, 512); + + // Page::load() should return ESP_OK, but state has to be corrupt + TEST_ASSERT_EQUAL(ESP_OK, page.load(&fix.part_mock, 0)); + + TEST_ASSERT_EQUAL(Page::PageState::CORRUPT, page.state()); + + Mockesp_partition_Verify(); +} + +void test_Page_load__initialized_corrupt_header() +{ + PartitionMockFixture fix; + Page page; + + uint8_t raw_header_corrupt [] = {0xfe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, 0x16, 0xdd, 0xdc}; + + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header_corrupt, 32); + + // Page::load() should return ESP_OK, but state has to be corrupt + TEST_ASSERT_EQUAL(ESP_OK, page.load(&fix.part_mock, 0)); + + TEST_ASSERT_EQUAL(Page::PageState::CORRUPT, page.state()); + + Mockesp_partition_Verify(); +} + +void test_Page_load__corrupt_entry_table() +{ + PartitionMockFixture fix; + + // valid header + uint8_t raw_header_valid [32] = {0xfe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x16, 0xdd, 0xdc}; + + // entry table with one entry + uint8_t raw_entry_table [32]; + + uint8_t ns_entry [32] = {0x00, 0x01, 0x01, 0xff, 0x68, 0xc5, 0x3f, 0x0b, 't', 'e', 's', 't', '_', 'n', 's', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + + uint8_t raw_header[4] = {0xff, 0xff, 0xff, 0xff}; + std::fill_n(raw_entry_table, sizeof(raw_entry_table)/sizeof(raw_entry_table[0]), 0); + raw_entry_table[0] = 0xfa; + + // read page header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header_valid, 32); + + // read entry table + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_entry_table, 32); + + // read next free entry's header + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header, 4); + + // read namespace entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(ns_entry, 32); + + // we expect a raw word write from the partition in order to change the entry bits to erased (0) + esp_partition_write_raw_ExpectAndReturn(&fix.part_mock.partition, 32, nullptr, 4, ESP_OK); + esp_partition_write_raw_IgnoreArg_src(); + + // corrupt entry table as well as crc of corresponding item + raw_entry_table[0] = 0xf6; + + Page page; + + // Page::load() should return ESP_OK, but state has to be corrupt + TEST_ASSERT_EQUAL(ESP_OK, page.load(&fix.part_mock, 0)); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, page.state()); + TEST_ASSERT_EQUAL(1, page.getUsedEntryCount()); + + Mockesp_partition_Verify(); +} + +void test_Page_load_success() +{ + PartitionMockFixture fix; + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(fix.raw_header, 32); + for (int i = 0; i < 8; i++) { + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(fix.raw_header, 512); + } + Page page; + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, page.state()); + TEST_ASSERT_EQUAL(ESP_OK, page.load(&fix.part_mock, 0)); + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, page.state()); +} + +void test_Page_load_full_page() +{ + NVSFullPageFixture fix(0, 1, NVS_DEFAULT_PART_NAME, false); + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); + TEST_ASSERT_EQUAL(ESP_OK, fix.page.load(&fix.part_mock, 0)); + TEST_ASSERT_EQUAL(Page::PageState::FULL, fix.page.state()); +} +void test_Page_load__seq_number_0() +{ + NVSValidPageFixture fix; + + uint32_t seq_num; + fix.page.getSeqNumber(seq_num); + TEST_ASSERT_EQUAL(0, seq_num); +} + +void test_Page_erase__write_fail() +{ + NVSValidPageFixture fix; + + esp_partition_erase_range_ExpectAndReturn(&fix.part_mock.partition, 0, 4096, ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.erase()); + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); +} + +void test_Page_erase__success() +{ + NVSValidPageFixture fix; + + esp_partition_erase_range_ExpectAndReturn(&fix.part_mock.partition, 0, 4096, ESP_OK); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.erase()); + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); +} + +void test_Page_write__initialize_write_failure() +{ + PartitionMockFixture fix; + uint8_t write_data = 47; + + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(fix.raw_header, 32); + for (int i = 0; i < 8; i++) { + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(fix.raw_header, 512); + } + esp_partition_write_raw_ExpectAnyArgsAndReturn(ESP_FAIL); + + Page page; + + TEST_ASSERT_EQUAL(ESP_OK, page.load(&fix.part_mock, 0)); + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, page.state()); + + TEST_ASSERT_EQUAL(ESP_FAIL, page.writeItem(1, nvs::ItemType::U8, "test", &write_data, sizeof(write_data))); + TEST_ASSERT_EQUAL(Page::PageState::INVALID, page.state()); +} + +void test_Page_write__write_data_fails() +{ + NVSPageFixture fix; + uint8_t write_data = 47; + esp_partition_write_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_write_ExpectAnyArgsAndReturn(ESP_FAIL); + + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.writeItem(1, nvs::ItemType::U8, "test", &write_data, sizeof(write_data))); + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); +} + +void test_page_write__write_correct_entry_state() +{ + NVSPageFixture fix; + uint8_t write_data = 47; + uint8_t raw_result [4]; + std::fill_n(raw_result, sizeof(raw_result)/sizeof(raw_result[0]), UINT8_MAX); + // mark first entry as written + raw_result[0] = 0xfe; + + // initialize page + esp_partition_write_raw_ExpectAnyArgsAndReturn(ESP_OK); + + // write entry + esp_partition_write_ExpectAnyArgsAndReturn(ESP_OK); + + // write entry state + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 32, raw_result, 4, 4, ESP_OK); + + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.writeItem(1, nvs::ItemType::U8, "test_key", &write_data, sizeof(write_data))); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_write__write_correct_data() +{ + NVSPageFixture fix; + uint8_t write_data = 47; + uint8_t raw_result [32] = {0x01, 0x01, 0x01, 0xff, 0x98, 0x6f, 0x21, 0xfd, 't', 'e', 's', 't', '_', 'k', 'e', 'y', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 47, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + + // initialize page + esp_partition_write_raw_ExpectAnyArgsAndReturn(ESP_OK); + + // write entry + esp_partition_write_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 64, raw_result, 32, 32, ESP_OK); + + // write entry state + esp_partition_write_raw_ExpectAnyArgsAndReturn(ESP_OK); + + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.writeItem(1, nvs::ItemType::U8, "test_key", &write_data, sizeof(write_data))); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_readItem__read_entry_fails() +{ + NVSValidPageFixture fix; + + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + uint8_t read_value = 0; + + esp_partition_read_ExpectAnyArgsAndReturn(ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.readItem(NVSValidPageFixture::NS_INDEX, "test_value", read_value)); + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); + + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); +} + +void test_Page_readItem__read_corrupted_entry() +{ + NVSValidPageFixture fix; + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + uint8_t read_value = 0; + + // corrupting entry + fix.value_entry[0] = 0x0; + + // first read the entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // erasing entry by setting bit in entry table (0xfa -> 0xf2) + uint8_t raw_result [4] = {0xf2, 0x00, 0x00, 0x00}; + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 32, raw_result, 4, 4, ESP_OK); + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, fix.page.readItem(NVSValidPageFixture::NS_INDEX, "test_value", read_value)); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + TEST_ASSERT_EQUAL(1, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(123, fix.page.getErasedEntryCount()); +} + +void test_Page_readItem__read_corrupted_second_read_fail() +{ + NVSValidPageFixture fix; + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + uint8_t read_value = 0; + + // corrupting entry + fix.value_entry[0] = 0x0; + + // first read the entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.readItem(NVSValidPageFixture::NS_INDEX, "test_value", read_value)); + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); +} + +void test_Page_readItem__read_corrupted_erase_fail() +{ + NVSValidPageFixture fix; + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + uint8_t read_value = 0; + + // corrupting entry + fix.value_entry[0] = 0x0; + + // first read the entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // erasing entry by setting bit in entry table (0xfa -> 0xf2) + uint8_t raw_result [4] = {0xf2, 0x00, 0x00, 0x00}; + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 32, raw_result, 4, 4, ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.readItem(NVSValidPageFixture::NS_INDEX, "test_value", read_value)); + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); +} + +void test_Page_readItem__read_entry_suceeds() +{ + NVSValidPageFixture fix; + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + uint8_t read_value = 0; + + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.readItem(NVSValidPageFixture::NS_INDEX, "test_value", read_value)); + + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(47, read_value); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_readItem__blob_read_data_fails() +{ + NVSValidBlobPageFixture fix; + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + uint8_t chunk_start = 0; + uint8_t read_data [32]; + + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_entry, 32); + esp_partition_read_ExpectAnyArgsAndReturn(ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.readItem(NVSValidPageFixture::NS_INDEX, + ItemType::BLOB_DATA, + "test_blob", + read_data, + 32, + chunk_start)); + + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_readItem__corrupt_data_erase_failure() +{ + NVSValidBlobPageFixture fix; + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + uint8_t chunk_start = 0; + uint8_t read_data [32]; + + fix.blob_data[16] = 0xdf; + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_entry, 32); + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_FAIL); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.readItem(NVSValidPageFixture::NS_INDEX, + ItemType::BLOB_DATA, + "test_blob", + read_data, + 32, + chunk_start)); + + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_readItem__blob_corrupt_data() +{ + NVSValidBlobPageFixture fix; + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + uint8_t chunk_start = 0; + uint8_t read_data [32]; + + fix.blob_data[16] = 0xdf; + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_entry, 32); + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + // erasing entry by setting bit in entry table (0xfa -> 0xf2) + uint8_t raw_result [4] = {0xa2, 0xff, 0xff, 0xff}; + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 32, raw_result, 4, 4, ESP_OK); + + esp_partition_erase_range_ExpectAndReturn(&fix.part_mock.partition, 96, 64, ESP_OK); + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, fix.page.readItem(NVSValidPageFixture::NS_INDEX, + ItemType::BLOB_DATA, + "test_blob", + read_data, + 32, + chunk_start)); + + TEST_ASSERT_EQUAL(3, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(1, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_readItem__blob_read_entry_suceeds() +{ + NVSValidBlobPageFixture fix; + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + uint8_t chunk_start = 0; + uint8_t read_data [32]; + + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_entry, 32); + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.readItem(NVSValidPageFixture::NS_INDEX, + ItemType::BLOB_DATA, + "test_blob", + read_data, + 32, + chunk_start)); + + TEST_ASSERT_EQUAL_MEMORY(fix.blob_data, read_data, fix.BLOB_DATA_SIZE); + + // make sure nothing was erased, i.e. checksums matched + TEST_ASSERT_EQUAL(4, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(0, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_cmp__uninitialized() +{ + Page page; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_INVALID_STATE, page.cmpItem(uint8_t(1) , "test", 47)); +} + +void test_Page_cmp__item_not_found() +{ + NVSValidPageFixture fix; + + // no expectations here since comparison uses the item hash list + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, fix.page.cmpItem(uint8_t(1), "different", 47)); +} + +void test_Page_cmp__item_type_mismatch() +{ + NVSValidPageFixture fix; + + // read normal entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_TYPE_MISMATCH, fix.page.cmpItem(uint8_t(1), "test_value", int(47))); +} + +void test_Page_cmp__item_content_mismatch() +{ + NVSValidPageFixture fix; + + // read normal entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_CONTENT_DIFFERS, fix.page.cmpItem(uint8_t(1), "test_value", uint8_t(46))); +} + +void test_Page_cmp__item_content_match() +{ + NVSValidPageFixture fix; + + // read normal entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.cmpItem(NVSValidPageFixture::NS_INDEX, "test_value", uint8_t(47))); +} + +void test_Page_cmpItem__blob_data_mismatch() +{ + NVSValidBlobPageFixture fix; + + // read blob entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_entry, 32); + + // read blob data + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + + uint8_t blob_data_different [] = + {0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, + 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xee}; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_CONTENT_DIFFERS, + fix.page.cmpItem(uint8_t(1), + ItemType::BLOB_DATA, + "test_blob", + blob_data_different, + 32)); +} + +void test_Page_cmpItem__blob_data_match() +{ + NVSValidBlobPageFixture fix; + + // read blob entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_entry, 32); + + // read blob data + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.blob_data, fix.BLOB_DATA_SIZE); + + + uint8_t blob_data_same [] = + {0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, + 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}; + + TEST_ASSERT_EQUAL(ESP_OK, + fix.page.cmpItem(NVSValidPageFixture::NS_INDEX, + ItemType::BLOB_DATA, + "test_blob", + blob_data_same, + 32)); +} + +void test_Page_eraseItem__uninitialized() +{ + Page page; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, page.eraseItem(NVSValidPageFixture::NS_INDEX, "test_value")); +} + +void test_Page_eraseItem__key_not_found() +{ + NVSValidPageFixture fix; + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, fix.page.eraseItem(NVSValidPageFixture::NS_INDEX, "different")); + + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_eraseItem__write_fail() +{ + NVSValidPageFixture fix; + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + // first read the entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // erasing entry by setting bit in entry table (0xfa -> 0xf2) + uint8_t raw_result [4] = {0xf2, 0x00, 0x00, 0x00}; + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 32, raw_result, 4, 4, ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.eraseItem(NVSValidPageFixture::NS_INDEX, "test_value")); + + TEST_ASSERT_EQUAL(1, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(123, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); +} + +void test_Page_eraseItem__write_succeed() +{ + NVSValidPageFixture fix; + TEST_ASSERT_EQUAL(2, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(122, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); + + // first read the entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // Page::eraseEntryAndSpan() reads entry again + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + // erasing entry by setting bit in entry table (0xfa -> 0xf2) + uint8_t raw_result [4] = {0xf2, 0x00, 0x00, 0x00}; + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, 1, 32, raw_result, 4, 4, ESP_OK); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.eraseItem(NVSValidPageFixture::NS_INDEX, "test_value")); + + TEST_ASSERT_EQUAL(1, fix.page.getUsedEntryCount()); + TEST_ASSERT_EQUAL(123, fix.page.getErasedEntryCount()); + + TEST_ASSERT_EQUAL(Page::PageState::ACTIVE, fix.page.state()); +} + +void test_Page_findItem__uninitialized() +{ + Page page; + + size_t index = 0; + Item item; + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, + page.findItem(NVSValidPageFixture::NS_INDEX, nvs::ItemType::U8, "test_value", index, item)); +} + +void test_Page_find__wrong_ns() +{ + NVSValidPageFixture fix; + size_t index = 0; + Item item; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, + fix.page.findItem(NVSValidPageFixture::NS_INDEX + 1, nvs::ItemType::U8, "test_value", index, item)); +} + +void test_Page_find__wrong_type() +{ + NVSValidPageFixture fix; + size_t index = 0; + Item item; + + // read normal entry + esp_partition_read_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_ReturnArrayThruPtr_dst(fix.value_entry, 32); + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_TYPE_MISMATCH, + fix.page.findItem(NVSValidPageFixture::NS_INDEX, nvs::ItemType::I8, "test_value", index, item)); +} + +void test_Page_find__key_empty() +{ + NVSValidPageFixture fix; + size_t index = 0; + Item item; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, + fix.page.findItem(NVSValidPageFixture::NS_INDEX, nvs::ItemType::U8, "", index, item)); +} + +void test_Page_find__wrong_key() +{ + NVSValidPageFixture fix; + size_t index = 0; + Item item; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, + fix.page.findItem(NVSValidPageFixture::NS_INDEX, nvs::ItemType::U8, "different", index, item)); +} + +void test_Page_find__too_large_index() +{ + NVSValidPageFixture fix; + size_t index = 2; + Item item; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, + fix.page.findItem(NVSValidPageFixture::NS_INDEX, nvs::ItemType::U8, "test_value", index, item)); +} + +void test_Page_findItem__without_read() +{ + NVSValidPageFixture fix; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, + fix.page.findItem(NVSValidPageFixture::NS_INDEX, nvs::ItemType::U8, "different")); +} + +void test_Page_markFull__wrong_state() +{ + NVSPageFixture fix; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_INVALID_STATE, fix.page.markFull()); + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); +} + +void test_Page_markFull__success() +{ + NVSValidPageFixture fix; + Page::PageState expected_state = Page::PageState::FULL; + + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, sizeof(fix.part_mock.partition), 0, &expected_state, sizeof(expected_state), 4, ESP_OK); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.markFull()); + TEST_ASSERT_EQUAL(Page::PageState::FULL, fix.page.state()); +} + +void test_Page_markFull__write_fail() +{ + NVSValidPageFixture fix; + Page::PageState expected_state = Page::PageState::FREEING; + + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, sizeof(fix.part_mock.partition), 0, &expected_state, sizeof(expected_state), 4, ESP_FAIL); + + TEST_ASSERT_EQUAL(ESP_FAIL, fix.page.markFreeing()); + TEST_ASSERT_EQUAL(Page::PageState::INVALID, fix.page.state()); +} + +void test_Page_markFreeing__wrong_state() +{ + NVSPageFixture fix; + + TEST_ASSERT_EQUAL(ESP_ERR_NVS_INVALID_STATE, fix.page.markFreeing()); + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); +} + +void test_Page_markFreeing__success() +{ + NVSValidPageFixture fix; + Page::PageState expected_state = Page::PageState::FREEING; + + esp_partition_write_raw_ExpectWithArrayAndReturn(&fix.part_mock.partition, sizeof(fix.part_mock.partition), 0, &expected_state, sizeof(expected_state), 4, ESP_OK); + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.markFreeing()); + TEST_ASSERT_EQUAL(Page::PageState::FREEING, fix.page.state()); +} + +void test_Page_getVarDataTailroom__uninitialized_page() +{ + NVSPageFixture fix; + + TEST_ASSERT_EQUAL(Page::CHUNK_MAX_SIZE, fix.page.getVarDataTailroom()); +} + +void test_Page_getVarDataTailroom__success() +{ + NVSValidPageFixture fix; + + // blob data item, written namespace item, written normal item: 3 items + TEST_ASSERT_EQUAL((Page::ENTRY_COUNT - 3) * Page::ENTRY_SIZE, fix.page.getVarDataTailroom()); +} + +void test_Page_calcEntries__uninit() +{ + NVSPageFixture fix; + TEST_ASSERT_EQUAL(Page::PageState::UNINITIALIZED, fix.page.state()); + + nvs_stats_t nvsStats = {0, 0, 0, 0}; + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.calcEntries(nvsStats)); + TEST_ASSERT_EQUAL(0, nvsStats.used_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.free_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.total_entries); + TEST_ASSERT_EQUAL(0, nvsStats.namespace_count); +} + +void test_Page_calcEntries__corrupt() +{ + PartitionMockFixture fix; + Page page; + + uint8_t raw_header_corrupt [] = {0xfe, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, 0x16, 0xdd, 0xdc}; + + esp_partition_read_raw_ExpectAnyArgsAndReturn(ESP_OK); + esp_partition_read_raw_ReturnArrayThruPtr_dst(raw_header_corrupt, 32); + + // Page::load() should return ESP_OK, but state has to be corrupt + TEST_ASSERT_EQUAL(ESP_OK, page.load(&fix.part_mock, 0)); + + TEST_ASSERT_EQUAL(Page::PageState::CORRUPT, page.state()); + + nvs_stats_t nvsStats = {0, 0, 0, 0}; + + TEST_ASSERT_EQUAL(ESP_OK, page.calcEntries(nvsStats)); + TEST_ASSERT_EQUAL(0, nvsStats.used_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.free_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.total_entries); + TEST_ASSERT_EQUAL(0, nvsStats.namespace_count); +} + +void test_Page_calcEntries__active_wo_blob() +{ + NVSValidPageFixture fix; + + nvs_stats_t nvsStats = {0, 0, 0, 0}; + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.calcEntries(nvsStats)); + TEST_ASSERT_EQUAL(2, nvsStats.used_entries); + TEST_ASSERT_EQUAL(124, nvsStats.free_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.total_entries); + TEST_ASSERT_EQUAL(0, nvsStats.namespace_count); +} + +void test_Page_calcEntries__active_with_blob() +{ + NVSValidBlobPageFixture fix; + + nvs_stats_t nvsStats = {0, 0, 0, 0}; + + TEST_ASSERT_EQUAL(ESP_OK, fix.page.calcEntries(nvsStats)); + TEST_ASSERT_EQUAL(4, nvsStats.used_entries); + TEST_ASSERT_EQUAL(122, nvsStats.free_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.total_entries); + TEST_ASSERT_EQUAL(0, nvsStats.namespace_count); +} + +void test_Page_calcEntries__invalid() +{ + Page page; + + nvs_stats_t nvsStats = {0, 0, 0, 0}; + + TEST_ASSERT_EQUAL(Page::PageState::INVALID, page.state()); + + // total entries always get updated + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_STATE, page.calcEntries(nvsStats)); + TEST_ASSERT_EQUAL(0, nvsStats.used_entries); + TEST_ASSERT_EQUAL(0, nvsStats.free_entries); + TEST_ASSERT_EQUAL(Page::ENTRY_COUNT, nvsStats.total_entries); + TEST_ASSERT_EQUAL(0, nvsStats.namespace_count); +} + +int main(int argc, char **argv) +{ + UNITY_BEGIN(); + RUN_TEST(test_Page_load_reading_header_fails); + RUN_TEST(test_Page_load_reading_data_fails); + RUN_TEST(test_Page_load__uninitialized_page_has_0xfe); + RUN_TEST(test_Page_load__initialized_corrupt_header); + RUN_TEST(test_Page_load__corrupt_entry_table); + RUN_TEST(test_Page_load_success); + RUN_TEST(test_Page_load_full_page); + RUN_TEST(test_Page_load__seq_number_0); + RUN_TEST(test_Page_erase__write_fail); + RUN_TEST(test_Page_erase__success); + RUN_TEST(test_Page_write__initialize_write_failure); + RUN_TEST(test_Page_write__write_data_fails); + RUN_TEST(test_page_write__write_correct_entry_state); + RUN_TEST(test_Page_write__write_correct_data); + RUN_TEST(test_Page_readItem__read_entry_fails); + RUN_TEST(test_Page_readItem__read_corrupted_entry); + RUN_TEST(test_Page_readItem__read_corrupted_second_read_fail); + RUN_TEST(test_Page_readItem__read_corrupted_erase_fail); + RUN_TEST(test_Page_readItem__read_entry_suceeds); + RUN_TEST(test_Page_readItem__blob_read_data_fails); + RUN_TEST(test_Page_readItem__blob_corrupt_data); + RUN_TEST(test_Page_readItem__blob_read_entry_suceeds); + RUN_TEST(test_Page_cmp__uninitialized); + RUN_TEST(test_Page_cmp__item_not_found); + RUN_TEST(test_Page_cmp__item_type_mismatch); + RUN_TEST(test_Page_cmp__item_content_mismatch); + RUN_TEST(test_Page_cmp__item_content_match); + RUN_TEST(test_Page_cmpItem__blob_data_mismatch); + RUN_TEST(test_Page_cmpItem__blob_data_match); + RUN_TEST(test_Page_eraseItem__uninitialized); + RUN_TEST(test_Page_eraseItem__key_not_found); + RUN_TEST(test_Page_eraseItem__write_fail); + RUN_TEST(test_Page_readItem__corrupt_data_erase_failure); + RUN_TEST(test_Page_eraseItem__write_succeed); + RUN_TEST(test_Page_findItem__uninitialized); + RUN_TEST(test_Page_find__wrong_ns); + RUN_TEST(test_Page_find__wrong_type); + RUN_TEST(test_Page_find__key_empty); + RUN_TEST(test_Page_find__wrong_key); + RUN_TEST(test_Page_find__too_large_index); + RUN_TEST(test_Page_findItem__without_read); + RUN_TEST(test_Page_markFull__wrong_state); + RUN_TEST(test_Page_markFreeing__wrong_state); + RUN_TEST(test_Page_markFull__success); + RUN_TEST(test_Page_markFreeing__success); + RUN_TEST(test_Page_markFull__write_fail); + RUN_TEST(test_Page_getVarDataTailroom__uninitialized_page); + RUN_TEST(test_Page_getVarDataTailroom__success); + RUN_TEST(test_Page_calcEntries__uninit); + RUN_TEST(test_Page_calcEntries__corrupt); + RUN_TEST(test_Page_calcEntries__active_wo_blob); + RUN_TEST(test_Page_calcEntries__active_with_blob); + RUN_TEST(test_Page_calcEntries__invalid); + int failures = UNITY_END(); + return failures; +} diff --git a/components/nvs_flash/host_test/nvs_page_test/sdkconfig.defaults b/components/nvs_flash/host_test/nvs_page_test/sdkconfig.defaults new file mode 100644 index 000000000..a05773334 --- /dev/null +++ b/components/nvs_flash/host_test/nvs_page_test/sdkconfig.defaults @@ -0,0 +1,3 @@ +CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=n +CONFIG_IDF_TARGET="linux" +CONFIG_CXX_EXCEPTIONS=y diff --git a/components/nvs_flash/include/nvs.h b/components/nvs_flash/include/nvs.h index dfdd18c92..26183853b 100644 --- a/components/nvs_flash/include/nvs.h +++ b/components/nvs_flash/include/nvs.h @@ -17,6 +17,7 @@ #include #include #include +#include "esp_attr.h" #include "esp_err.h" #ifdef __cplusplus @@ -26,34 +27,92 @@ extern "C" { /** * Opaque pointer type representing non-volatile storage handle */ -typedef uint32_t nvs_handle; - -#define ESP_ERR_NVS_BASE 0x1100 /*!< Starting number of error codes */ -#define ESP_ERR_NVS_NOT_INITIALIZED (ESP_ERR_NVS_BASE + 0x01) /*!< The storage driver is not initialized */ -#define ESP_ERR_NVS_NOT_FOUND (ESP_ERR_NVS_BASE + 0x02) /*!< Id namespace doesn’t exist yet and mode is NVS_READONLY */ -#define ESP_ERR_NVS_TYPE_MISMATCH (ESP_ERR_NVS_BASE + 0x03) /*!< The type of set or get operation doesn't match the type of value stored in NVS */ -#define ESP_ERR_NVS_READ_ONLY (ESP_ERR_NVS_BASE + 0x04) /*!< Storage handle was opened as read only */ -#define ESP_ERR_NVS_NOT_ENOUGH_SPACE (ESP_ERR_NVS_BASE + 0x05) /*!< There is not enough space in the underlying storage to save the value */ -#define ESP_ERR_NVS_INVALID_NAME (ESP_ERR_NVS_BASE + 0x06) /*!< Namespace name doesn’t satisfy constraints */ -#define ESP_ERR_NVS_INVALID_HANDLE (ESP_ERR_NVS_BASE + 0x07) /*!< Handle has been closed or is NULL */ -#define ESP_ERR_NVS_REMOVE_FAILED (ESP_ERR_NVS_BASE + 0x08) /*!< The value wasn’t updated because flash write operation has failed. The value was written however, and update will be finished after re-initialization of nvs, provided that flash operation doesn’t fail again. */ -#define ESP_ERR_NVS_KEY_TOO_LONG (ESP_ERR_NVS_BASE + 0x09) /*!< Key name is too long */ -#define ESP_ERR_NVS_PAGE_FULL (ESP_ERR_NVS_BASE + 0x0a) /*!< Internal error; never returned by nvs_ API functions */ -#define ESP_ERR_NVS_INVALID_STATE (ESP_ERR_NVS_BASE + 0x0b) /*!< NVS is in an inconsistent state due to a previous error. Call nvs_flash_init and nvs_open again, then retry. */ -#define ESP_ERR_NVS_INVALID_LENGTH (ESP_ERR_NVS_BASE + 0x0c) /*!< String or blob length is not sufficient to store data */ -#define ESP_ERR_NVS_NO_FREE_PAGES (ESP_ERR_NVS_BASE + 0x0d) /*!< NVS partition doesn't contain any empty pages. This may happen if NVS partition was truncated. Erase the whole partition and call nvs_flash_init again. */ -#define ESP_ERR_NVS_VALUE_TOO_LONG (ESP_ERR_NVS_BASE + 0x0e) /*!< String or blob length is longer than supported by the implementation */ -#define ESP_ERR_NVS_PART_NOT_FOUND (ESP_ERR_NVS_BASE + 0x0f) /*!< Partition with specified name is not found in the partition table */ - -#define NVS_DEFAULT_PART_NAME "nvs" /*!< Default partition name of the NVS partition in the partition table */ +typedef uint32_t nvs_handle_t; + +/* + * Pre-IDF V4.0 uses nvs_handle, so leaving the original typedef here for compatibility. + */ +typedef nvs_handle_t nvs_handle IDF_DEPRECATED("Replace with nvs_handle_t"); + +#define ESP_ERR_NVS_BASE 0x1100 /*!< Starting number of error codes */ +#define ESP_ERR_NVS_NOT_INITIALIZED (ESP_ERR_NVS_BASE + 0x01) /*!< The storage driver is not initialized */ +#define ESP_ERR_NVS_NOT_FOUND (ESP_ERR_NVS_BASE + 0x02) /*!< Id namespace doesn’t exist yet and mode is NVS_READONLY */ +#define ESP_ERR_NVS_TYPE_MISMATCH (ESP_ERR_NVS_BASE + 0x03) /*!< The type of set or get operation doesn't match the type of value stored in NVS */ +#define ESP_ERR_NVS_READ_ONLY (ESP_ERR_NVS_BASE + 0x04) /*!< Storage handle was opened as read only */ +#define ESP_ERR_NVS_NOT_ENOUGH_SPACE (ESP_ERR_NVS_BASE + 0x05) /*!< There is not enough space in the underlying storage to save the value */ +#define ESP_ERR_NVS_INVALID_NAME (ESP_ERR_NVS_BASE + 0x06) /*!< Namespace name doesn’t satisfy constraints */ +#define ESP_ERR_NVS_INVALID_HANDLE (ESP_ERR_NVS_BASE + 0x07) /*!< Handle has been closed or is NULL */ +#define ESP_ERR_NVS_REMOVE_FAILED (ESP_ERR_NVS_BASE + 0x08) /*!< The value wasn’t updated because flash write operation has failed. The value was written however, and update will be finished after re-initialization of nvs, provided that flash operation doesn’t fail again. */ +#define ESP_ERR_NVS_KEY_TOO_LONG (ESP_ERR_NVS_BASE + 0x09) /*!< Key name is too long */ +#define ESP_ERR_NVS_PAGE_FULL (ESP_ERR_NVS_BASE + 0x0a) /*!< Internal error; never returned by nvs API functions */ +#define ESP_ERR_NVS_INVALID_STATE (ESP_ERR_NVS_BASE + 0x0b) /*!< NVS is in an inconsistent state due to a previous error. Call nvs_flash_init and nvs_open again, then retry. */ +#define ESP_ERR_NVS_INVALID_LENGTH (ESP_ERR_NVS_BASE + 0x0c) /*!< String or blob length is not sufficient to store data */ +#define ESP_ERR_NVS_NO_FREE_PAGES (ESP_ERR_NVS_BASE + 0x0d) /*!< NVS partition doesn't contain any empty pages. This may happen if NVS partition was truncated. Erase the whole partition and call nvs_flash_init again. */ +#define ESP_ERR_NVS_VALUE_TOO_LONG (ESP_ERR_NVS_BASE + 0x0e) /*!< String or blob length is longer than supported by the implementation */ +#define ESP_ERR_NVS_PART_NOT_FOUND (ESP_ERR_NVS_BASE + 0x0f) /*!< Partition with specified name is not found in the partition table */ + +#define ESP_ERR_NVS_NEW_VERSION_FOUND (ESP_ERR_NVS_BASE + 0x10) /*!< NVS partition contains data in new format and cannot be recognized by this version of code */ +#define ESP_ERR_NVS_XTS_ENCR_FAILED (ESP_ERR_NVS_BASE + 0x11) /*!< XTS encryption failed while writing NVS entry */ +#define ESP_ERR_NVS_XTS_DECR_FAILED (ESP_ERR_NVS_BASE + 0x12) /*!< XTS decryption failed while reading NVS entry */ +#define ESP_ERR_NVS_XTS_CFG_FAILED (ESP_ERR_NVS_BASE + 0x13) /*!< XTS configuration setting failed */ +#define ESP_ERR_NVS_XTS_CFG_NOT_FOUND (ESP_ERR_NVS_BASE + 0x14) /*!< XTS configuration not found */ +#define ESP_ERR_NVS_ENCR_NOT_SUPPORTED (ESP_ERR_NVS_BASE + 0x15) /*!< NVS encryption is not supported in this version */ +#define ESP_ERR_NVS_KEYS_NOT_INITIALIZED (ESP_ERR_NVS_BASE + 0x16) /*!< NVS key partition is uninitialized */ +#define ESP_ERR_NVS_CORRUPT_KEY_PART (ESP_ERR_NVS_BASE + 0x17) /*!< NVS key partition is corrupt */ +#define ESP_ERR_NVS_WRONG_ENCRYPTION (ESP_ERR_NVS_BASE + 0x19) /*!< NVS partition is marked as encrypted with generic flash encryption. This is forbidden since the NVS encryption works differently. */ + +#define ESP_ERR_NVS_CONTENT_DIFFERS (ESP_ERR_NVS_BASE + 0x18) /*!< Internal error; never returned by nvs API functions. NVS key is different in comparison */ + +#define NVS_DEFAULT_PART_NAME "nvs" /*!< Default partition name of the NVS partition in the partition table */ + +#define NVS_PART_NAME_MAX_SIZE 16 /*!< maximum length of partition name (excluding null terminator) */ +#define NVS_KEY_NAME_MAX_SIZE 16 /*!< Maximal length of NVS key name (including null terminator) */ + /** * @brief Mode of opening the non-volatile storage - * */ typedef enum { NVS_READONLY, /*!< Read only */ NVS_READWRITE /*!< Read and write */ -} nvs_open_mode; +} nvs_open_mode_t; + +/* + * Pre-IDF V4.0 uses nvs_open_mode, so leaving the original typedef here for compatibility. + */ +typedef nvs_open_mode_t nvs_open_mode IDF_DEPRECATED("Replace with nvs_open_mode_t"); + + +/** + * @brief Types of variables + * + */ +typedef enum { + NVS_TYPE_U8 = 0x01, /*!< Type uint8_t */ + NVS_TYPE_I8 = 0x11, /*!< Type int8_t */ + NVS_TYPE_U16 = 0x02, /*!< Type uint16_t */ + NVS_TYPE_I16 = 0x12, /*!< Type int16_t */ + NVS_TYPE_U32 = 0x04, /*!< Type uint32_t */ + NVS_TYPE_I32 = 0x14, /*!< Type int32_t */ + NVS_TYPE_U64 = 0x08, /*!< Type uint64_t */ + NVS_TYPE_I64 = 0x18, /*!< Type int64_t */ + NVS_TYPE_STR = 0x21, /*!< Type string */ + NVS_TYPE_BLOB = 0x42, /*!< Type blob */ + NVS_TYPE_ANY = 0xff /*!< Must be last */ +} nvs_type_t; + +/** + * @brief information about entry obtained from nvs_entry_info function + */ +typedef struct { + char namespace_name[16]; /*!< Namespace to which key-value belong */ + char key[16]; /*!< Key of stored key-value pair */ + nvs_type_t type; /*!< Type of stored key-value pair */ +} nvs_entry_info_t; + +/** + * Opaque pointer type representing iterator to nvs entries + */ +typedef struct nvs_opaque_iterator_t *nvs_iterator_t; /** * @brief Open non-volatile storage with a given namespace from the default NVS partition @@ -64,12 +123,10 @@ typedef enum { * The default NVS partition is the one that is labelled "nvs" in the partition * table. * - * @param[in] name Namespace name. Maximal length is determined by the - * underlying implementation, but is guaranteed to be - * at least 15 characters. Shouldn't be empty. + * @param[in] name Namespace name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. * @param[in] open_mode NVS_READWRITE or NVS_READONLY. If NVS_READONLY, will * open a handle for reading only. All write requests will - * be rejected for this handle. + * be rejected for this handle. * @param[out] out_handle If successful (return code is zero), handle will be * returned in this argument. * @@ -80,9 +137,10 @@ typedef enum { * - ESP_ERR_NVS_NOT_FOUND id namespace doesn't exist yet and * mode is NVS_READONLY * - ESP_ERR_NVS_INVALID_NAME if namespace name doesn't satisfy constraints + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures * - other error codes from the underlying storage driver */ -esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_handle); +esp_err_t nvs_open(const char* name, nvs_open_mode_t open_mode, nvs_handle_t *out_handle); /** * @brief Open non-volatile storage with a given namespace from specified partition @@ -92,12 +150,10 @@ esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_ha * with NVS using nvs_flash_init_partition() API. * * @param[in] part_name Label (name) of the partition of interest for object read/write/erase - * @param[in] name Namespace name. Maximal length is determined by the - * underlying implementation, but is guaranteed to be - * at least 15 characters. Shouldn't be empty. - * @param[in] open_mode NVS_READWRITE or NVS_READONLY. If NVS_READONLY, will - * open a handle for reading only. All write requests will - * be rejected for this handle. + * @param[in] name Namespace name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param[in] open_mode NVS_READWRITE or NVS_READONLY. If NVS_READONLY, will + * open a handle for reading only. All write requests will + * be rejected for this handle. * @param[out] out_handle If successful (return code is zero), handle will be * returned in this argument. * @@ -108,25 +164,99 @@ esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_ha * - ESP_ERR_NVS_NOT_FOUND id namespace doesn't exist yet and * mode is NVS_READONLY * - ESP_ERR_NVS_INVALID_NAME if namespace name doesn't satisfy constraints + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures * - other error codes from the underlying storage driver */ -esp_err_t nvs_open_from_partition(const char *part_name, const char* name, nvs_open_mode open_mode, nvs_handle *out_handle); +esp_err_t nvs_open_from_partition(const char *part_name, const char* name, nvs_open_mode_t open_mode, nvs_handle_t *out_handle); /**@{*/ /** - * @brief set value for given key + * @brief set int8_t value for given key * - * This family of functions set value for the key, given its name. Note that - * actual storage will not be updated until nvs_commit function is called. + * Set value for the key, given its name. Note that the actual storage will not be updated + * until \c nvs_commit is called. + * + * @param[in] handle Handle obtained from nvs_open function. + * Handles that were opened read only cannot be used. + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param[in] value The value to set. + * + * @return + * - ESP_OK if value was set successfully + * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL + * - ESP_ERR_NVS_READ_ONLY if storage handle was opened as read only + * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints + * - ESP_ERR_NVS_NOT_ENOUGH_SPACE if there is not enough space in the + * underlying storage to save the value + * - ESP_ERR_NVS_REMOVE_FAILED if the value wasn't updated because flash + * write operation has failed. The value was written however, and + * update will be finished after re-initialization of nvs, provided that + * flash operation doesn't fail again. + */ +esp_err_t nvs_set_i8 (nvs_handle_t handle, const char* key, int8_t value); + +/** + * @brief set uint8_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_u8 (nvs_handle_t handle, const char* key, uint8_t value); + +/** + * @brief set int16_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_i16 (nvs_handle_t handle, const char* key, int16_t value); + +/** + * @brief set uint16_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_u16 (nvs_handle_t handle, const char* key, uint16_t value); + +/** + * @brief set int32_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_i32 (nvs_handle_t handle, const char* key, int32_t value); + +/** + * @brief set uint32_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_u32 (nvs_handle_t handle, const char* key, uint32_t value); + +/** + * @brief set int64_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_i64 (nvs_handle_t handle, const char* key, int64_t value); + +/** + * @brief set uint64_t value for given key + * + * This function is the same as \c nvs_set_i8 except for the data type. + */ +esp_err_t nvs_set_u64 (nvs_handle_t handle, const char* key, uint64_t value); + +/** + * @brief set string for given key + * + * Set value for the key, given its name. Note that the actual storage will not be updated + * until \c nvs_commit is called. * * @param[in] handle Handle obtained from nvs_open function. * Handles that were opened read only cannot be used. - * @param[in] key Key name. Maximal length is determined by the underlying - * implementation, but is guaranteed to be at least - * 15 characters. Shouldn't be empty. + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. * @param[in] value The value to set. * For strings, the maximum length (including null character) is - * 1984 bytes. + * 4000 bytes, if there is one complete page free for writing. + * This decreases, however, if the free space is fragmented. * * @return * - ESP_OK if value was set successfully @@ -141,16 +271,8 @@ esp_err_t nvs_open_from_partition(const char *part_name, const char* name, nvs_o * flash operation doesn't fail again. * - ESP_ERR_NVS_VALUE_TOO_LONG if the string value is too long */ -esp_err_t nvs_set_i8 (nvs_handle handle, const char* key, int8_t value); -esp_err_t nvs_set_u8 (nvs_handle handle, const char* key, uint8_t value); -esp_err_t nvs_set_i16 (nvs_handle handle, const char* key, int16_t value); -esp_err_t nvs_set_u16 (nvs_handle handle, const char* key, uint16_t value); -esp_err_t nvs_set_i32 (nvs_handle handle, const char* key, int32_t value); -esp_err_t nvs_set_u32 (nvs_handle handle, const char* key, uint32_t value); -esp_err_t nvs_set_i64 (nvs_handle handle, const char* key, int64_t value); -esp_err_t nvs_set_u64 (nvs_handle handle, const char* key, uint64_t value); -esp_err_t nvs_set_str (nvs_handle handle, const char* key, const char* value); -/**@}*/ +esp_err_t nvs_set_str (nvs_handle_t handle, const char* key, const char* value); +/**@}*/ /** * @brief set variable length binary value for given key @@ -160,10 +282,11 @@ esp_err_t nvs_set_str (nvs_handle handle, const char* key, const char* value); * * @param[in] handle Handle obtained from nvs_open function. * Handles that were opened read only cannot be used. - * @param[in] key Key name. Maximal length is 15 characters. Shouldn't be empty. + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. * @param[in] value The value to set. * @param[in] length length of binary value to set, in bytes; Maximum length is - * 1984 bytes. + * 508000 bytes or (97.6% of the partition size - 4000) bytes + * whichever is lower. * * @return * - ESP_OK if value was set successfully @@ -178,20 +301,19 @@ esp_err_t nvs_set_str (nvs_handle handle, const char* key, const char* value); * flash operation doesn't fail again. * - ESP_ERR_NVS_VALUE_TOO_LONG if the value is too long */ -esp_err_t nvs_set_blob(nvs_handle handle, const char* key, const void* value, size_t length); +esp_err_t nvs_set_blob(nvs_handle_t handle, const char* key, const void* value, size_t length); /**@{*/ /** - * @brief get value for given key + * @brief get int8_t value for given key * - * These functions retrieve value for the key, given its name. If key does not + * These functions retrieve value for the key, given its name. If \c key does not * exist, or the requested variable type doesn't match the type which was used * when setting a value, an error is returned. * * In case of any error, out_value is not modified. * - * All functions expect out_value to be a pointer to an already allocated variable - * of the given type. + * \c out_value has to be a pointer to an already allocated variable of the given type. * * \code{c} * // Example of using nvs_get_i32: @@ -204,9 +326,7 @@ esp_err_t nvs_set_blob(nvs_handle handle, const char* key, const void* value, si * \endcode * * @param[in] handle Handle obtained from nvs_open function. - * @param[in] key Key name. Maximal length is determined by the underlying - * implementation, but is guaranteed to be at least - * 15 characters. Shouldn't be empty. + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. * @param out_value Pointer to the output value. * May be NULL for nvs_get_str and nvs_get_blob, in this * case required length will be returned in length argument. @@ -218,20 +338,63 @@ esp_err_t nvs_set_blob(nvs_handle handle, const char* key, const void* value, si * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints * - ESP_ERR_NVS_INVALID_LENGTH if length is not sufficient to store data */ -esp_err_t nvs_get_i8 (nvs_handle handle, const char* key, int8_t* out_value); -esp_err_t nvs_get_u8 (nvs_handle handle, const char* key, uint8_t* out_value); -esp_err_t nvs_get_i16 (nvs_handle handle, const char* key, int16_t* out_value); -esp_err_t nvs_get_u16 (nvs_handle handle, const char* key, uint16_t* out_value); -esp_err_t nvs_get_i32 (nvs_handle handle, const char* key, int32_t* out_value); -esp_err_t nvs_get_u32 (nvs_handle handle, const char* key, uint32_t* out_value); -esp_err_t nvs_get_i64 (nvs_handle handle, const char* key, int64_t* out_value); -esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value); -/**@}*/ +esp_err_t nvs_get_i8 (nvs_handle_t handle, const char* key, int8_t* out_value); + +/** + * @brief get uint8_t value for given key + * + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_u8 (nvs_handle_t handle, const char* key, uint8_t* out_value); + +/** + * @brief get int16_t value for given key + * + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_i16 (nvs_handle_t handle, const char* key, int16_t* out_value); /** - * @brief get value for given key + * @brief get uint16_t value for given key * - * These functions retrieve value for the key, given its name. If key does not + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_u16 (nvs_handle_t handle, const char* key, uint16_t* out_value); + +/** + * @brief get int32_t value for given key + * + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_i32 (nvs_handle_t handle, const char* key, int32_t* out_value); + +/** + * @brief get uint32_t value for given key + * + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_u32 (nvs_handle_t handle, const char* key, uint32_t* out_value); + +/** + * @brief get int64_t value for given key + * + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_i64 (nvs_handle_t handle, const char* key, int64_t* out_value); + +/** + * @brief get uint64_t value for given key + * + * This function is the same as \c nvs_get_i8 except for the data type. + */ +esp_err_t nvs_get_u64 (nvs_handle_t handle, const char* key, uint64_t* out_value); +/**@}*/ + +/**@{*/ +/** + * @brief get string value for given key + * + * These functions retrieve the data of an entry, given its key. If key does not * exist, or the requested variable type doesn't match the type which was used * when setting a value, an error is returned. * @@ -239,7 +402,7 @@ esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value); * * All functions expect out_value to be a pointer to an already allocated variable * of the given type. - * + * * nvs_get_str and nvs_get_blob functions support WinAPI-style length queries. * To get the size necessary to store the value, call nvs_get_str or nvs_get_blob * with zero out_value and non-zero pointer to length. Variable pointed to @@ -265,10 +428,8 @@ esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value); * \endcode * * @param[in] handle Handle obtained from nvs_open function. - * @param[in] key Key name. Maximal length is determined by the underlying - * implementation, but is guaranteed to be at least - * 15 characters. Shouldn't be empty. - * @param out_value Pointer to the output value. + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param[out] out_value Pointer to the output value. * May be NULL for nvs_get_str and nvs_get_blob, in this * case required length will be returned in length argument. * @param[inout] length A non-zero pointer to the variable holding the length of out_value. @@ -282,11 +443,16 @@ esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value); * - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints - * - ESP_ERR_NVS_INVALID_LENGTH if length is not sufficient to store data + * - ESP_ERR_NVS_INVALID_LENGTH if \c length is not sufficient to store data */ -/**@{*/ -esp_err_t nvs_get_str (nvs_handle handle, const char* key, char* out_value, size_t* length); -esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size_t* length); +esp_err_t nvs_get_str (nvs_handle_t handle, const char* key, char* out_value, size_t* length); + +/** + * @brief get blob value for given key + * + * This function behaves the same as \c nvs_get_str, except for the data type. + */ +esp_err_t nvs_get_blob(nvs_handle_t handle, const char* key, void* out_value, size_t* length); /**@}*/ /** @@ -297,9 +463,7 @@ esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size * @param[in] handle Storage handle obtained with nvs_open. * Handles that were opened read only cannot be used. * - * @param[in] key Key name. Maximal length is determined by the underlying - * implementation, but is guaranteed to be at least - * 15 characters. Shouldn't be empty. + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. * * @return * - ESP_OK if erase operation was successful @@ -308,7 +472,7 @@ esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size * - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist * - other error codes from the underlying storage driver */ -esp_err_t nvs_erase_key(nvs_handle handle, const char* key); +esp_err_t nvs_erase_key(nvs_handle_t handle, const char* key); /** * @brief Erase all key-value pairs in a namespace @@ -324,7 +488,7 @@ esp_err_t nvs_erase_key(nvs_handle handle, const char* key); * - ESP_ERR_NVS_READ_ONLY if handle was opened as read only * - other error codes from the underlying storage driver */ -esp_err_t nvs_erase_all(nvs_handle handle); +esp_err_t nvs_erase_all(nvs_handle_t handle); /** * @brief Write any pending changes to non-volatile storage @@ -341,7 +505,7 @@ esp_err_t nvs_erase_all(nvs_handle handle); * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL * - other error codes from the underlying storage driver */ -esp_err_t nvs_commit(nvs_handle handle); +esp_err_t nvs_commit(nvs_handle_t handle); /** * @brief Close the storage handle and free any allocated resources @@ -354,7 +518,154 @@ esp_err_t nvs_commit(nvs_handle handle); * * @param[in] handle Storage handle to close */ -void nvs_close(nvs_handle handle); +void nvs_close(nvs_handle_t handle); + +/** + * @note Info about storage space NVS. + */ +typedef struct { + size_t used_entries; /**< Amount of used entries. */ + size_t free_entries; /**< Amount of free entries. */ + size_t total_entries; /**< Amount all available entries. */ + size_t namespace_count; /**< Amount name space. */ +} nvs_stats_t; + +/** + * @brief Fill structure nvs_stats_t. It provides info about used memory the partition. + * + * This function calculates to runtime the number of used entries, free entries, total entries, + * and amount namespace in partition. + * + * \code{c} + * // Example of nvs_get_stats() to get the number of used entries and free entries: + * nvs_stats_t nvs_stats; + * nvs_get_stats(NULL, &nvs_stats); + * printf("Count: UsedEntries = (%d), FreeEntries = (%d), AllEntries = (%d)\n", + nvs_stats.used_entries, nvs_stats.free_entries, nvs_stats.total_entries); + * \endcode + * + * @param[in] part_name Partition name NVS in the partition table. + * If pass a NULL than will use NVS_DEFAULT_PART_NAME ("nvs"). + * + * @param[out] nvs_stats Returns filled structure nvs_states_t. + * It provides info about used memory the partition. + * + * + * @return + * - ESP_OK if the changes have been written successfully. + * Return param nvs_stats will be filled. + * - ESP_ERR_NVS_PART_NOT_FOUND if the partition with label "name" is not found. + * Return param nvs_stats will be filled 0. + * - ESP_ERR_NVS_NOT_INITIALIZED if the storage driver is not initialized. + * Return param nvs_stats will be filled 0. + * - ESP_ERR_INVALID_ARG if nvs_stats equal to NULL. + * - ESP_ERR_INVALID_STATE if there is page with the status of INVALID. + * Return param nvs_stats will be filled not with correct values because + * not all pages will be counted. Counting will be interrupted at the first INVALID page. + */ +esp_err_t nvs_get_stats(const char *part_name, nvs_stats_t *nvs_stats); + +/** + * @brief Calculate all entries in a namespace. + * + * An entry represents the smallest storage unit in NVS. + * Strings and blobs may occupy more than one entry. + * Note that to find out the total number of entries occupied by the namespace, + * add one to the returned value used_entries (if err is equal to ESP_OK). + * Because the name space entry takes one entry. + * + * \code{c} + * // Example of nvs_get_used_entry_count() to get amount of all key-value pairs in one namespace: + * nvs_handle_t handle; + * nvs_open("namespace1", NVS_READWRITE, &handle); + * ... + * size_t used_entries; + * size_t total_entries_namespace; + * if(nvs_get_used_entry_count(handle, &used_entries) == ESP_OK){ + * // the total number of entries occupied by the namespace + * total_entries_namespace = used_entries + 1; + * } + * \endcode + * + * @param[in] handle Handle obtained from nvs_open function. + * + * @param[out] used_entries Returns amount of used entries from a namespace. + * + * + * @return + * - ESP_OK if the changes have been written successfully. + * Return param used_entries will be filled valid value. + * - ESP_ERR_NVS_NOT_INITIALIZED if the storage driver is not initialized. + * Return param used_entries will be filled 0. + * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL. + * Return param used_entries will be filled 0. + * - ESP_ERR_INVALID_ARG if used_entries equal to NULL. + * - Other error codes from the underlying storage driver. + * Return param used_entries will be filled 0. + */ +esp_err_t nvs_get_used_entry_count(nvs_handle_t handle, size_t* used_entries); + +/** + * @brief Create an iterator to enumerate NVS entries based on one or more parameters + * + * \code{c} + * // Example of listing all the key-value pairs of any type under specified partition and namespace + * nvs_iterator_t it = nvs_entry_find(partition, namespace, NVS_TYPE_ANY); + * while (it != NULL) { + * nvs_entry_info_t info; + * nvs_entry_info(it, &info); + * it = nvs_entry_next(it); + * printf("key '%s', type '%d' \n", info.key, info.type); + * }; + * // Note: no need to release iterator obtained from nvs_entry_find function when + * // nvs_entry_find or nvs_entry_next function return NULL, indicating no other + * // element for specified criteria was found. + * } + * \endcode + * + * @param[in] part_name Partition name + * + * @param[in] namespace_name Set this value if looking for entries with + * a specific namespace. Pass NULL otherwise. + * + * @param[in] type One of nvs_type_t values. + * + * @return + * Iterator used to enumerate all the entries found, + * or NULL if no entry satisfying criteria was found. + * Iterator obtained through this function has to be released + * using nvs_release_iterator when not used any more. + */ +nvs_iterator_t nvs_entry_find(const char *part_name, const char *namespace_name, nvs_type_t type); + +/** + * @brief Returns next item matching the iterator criteria, NULL if no such item exists. + * + * Note that any copies of the iterator will be invalid after this call. + * + * @param[in] iterator Iterator obtained from nvs_entry_find function. Must be non-NULL. + * + * @return + * NULL if no entry was found, valid nvs_iterator_t otherwise. + */ +nvs_iterator_t nvs_entry_next(nvs_iterator_t iterator); + +/** + * @brief Fills nvs_entry_info_t structure with information about entry pointed to by the iterator. + * + * @param[in] iterator Iterator obtained from nvs_entry_find or nvs_entry_next function. Must be non-NULL. + * + * @param[out] out_info Structure to which entry information is copied. + */ +void nvs_entry_info(nvs_iterator_t iterator, nvs_entry_info_t *out_info); + +/** + * @brief Release iterator + * + * @param[in] iterator Release iterator obtained from nvs_entry_find function. NULL argument is allowed. + * + */ +void nvs_release_iterator(nvs_iterator_t iterator); #ifdef __cplusplus @@ -362,4 +673,3 @@ void nvs_close(nvs_handle handle); #endif #endif //ESP_NVS_H - diff --git a/components/nvs_flash/include/nvs_flash.h b/components/nvs_flash/include/nvs_flash.h index a7ef7f451..a5ad9ac28 100644 --- a/components/nvs_flash/include/nvs_flash.h +++ b/components/nvs_flash/include/nvs_flash.h @@ -19,6 +19,18 @@ extern "C" { #endif #include "nvs.h" +#include "esp_partition.h" + + +#define NVS_KEY_SIZE 32 // AES-256 + +/** + * @brief Key for encryption and decryption + */ +typedef struct { + uint8_t eky[NVS_KEY_SIZE]; /*!< XTS encryption and decryption key*/ + uint8_t tky[NVS_KEY_SIZE]; /*!< XTS tweak key */ +} nvs_sec_cfg_t; /** * @brief Initialize the default NVS partition. @@ -26,30 +38,62 @@ extern "C" { * This API initialises the default NVS partition. The default NVS partition * is the one that is labeled "nvs" in the partition table. * + * When "NVS_ENCRYPTION" is enabled in the menuconfig, this API enables + * the NVS encryption for the default NVS partition as follows + * 1. Read security configurations from the first NVS key + * partition listed in the partition table. (NVS key partition is + * any "data" type partition which has the subtype value set to "nvs_keys") + * 2. If the NVS key partiton obtained in the previous step is empty, + * generate and store new keys in that NVS key partiton. + * 3. Internally call "nvs_flash_secure_init()" with + * the security configurations obtained/generated in the previous steps. + * + * Post initialization NVS read/write APIs + * remain the same irrespective of NVS encryption. + * * @return * - ESP_OK if storage was successfully initialized. * - ESP_ERR_NVS_NO_FREE_PAGES if the NVS storage contains no empty pages * (which may happen if NVS partition was truncated) * - ESP_ERR_NOT_FOUND if no partition with label "nvs" is found in the partition table + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures * - one of the error codes from the underlying flash storage driver + * - error codes from nvs_flash_read_security_cfg API (when "NVS_ENCRYPTION" is enabled). + * - error codes from nvs_flash_generate_keys API (when "NVS_ENCRYPTION" is enabled). + * - error codes from nvs_flash_secure_init_partition API (when "NVS_ENCRYPTION" is enabled) . */ esp_err_t nvs_flash_init(void); /** * @brief Initialize NVS flash storage for the specified partition. * - * @param[in] partition_label Label of the partition. Note that internally a reference to - * passed value is kept and it should be accessible for future operations + * @param[in] partition_label Label of the partition. Must be no longer than 16 characters. * * @return * - ESP_OK if storage was successfully initialized. * - ESP_ERR_NVS_NO_FREE_PAGES if the NVS storage contains no empty pages * (which may happen if NVS partition was truncated) * - ESP_ERR_NOT_FOUND if specified partition is not found in the partition table + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures * - one of the error codes from the underlying flash storage driver */ esp_err_t nvs_flash_init_partition(const char *partition_label); +/** + * @brief Initialize NVS flash storage for the partition specified by partition pointer. + * + * @param[in] partition pointer to a partition obtained by the ESP partition API. + * + * @return + * - ESP_OK if storage was successfully initialized + * - ESP_ERR_NVS_NO_FREE_PAGES if the NVS storage contains no empty pages + * (which may happen if NVS partition was truncated) + * - ESP_ERR_INVALID_ARG in case partition is NULL + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures + * - one of the error codes from the underlying flash storage driver + */ +esp_err_t nvs_flash_init_partition_ptr(const esp_partition_t *partition); + /** * @brief Deinitialize NVS storage for the default NVS partition * @@ -76,29 +120,133 @@ esp_err_t nvs_flash_deinit_partition(const char* partition_label); /** * @brief Erase the default NVS partition * - * This function erases all contents of the default NVS partition (one with label "nvs") + * Erases all contents of the default NVS partition (one with label "nvs"). + * + * @note If the partition is initialized, this function first de-initializes it. Afterwards, the partition has to + * be initialized again to be used. * * @return * - ESP_OK on success * - ESP_ERR_NOT_FOUND if there is no NVS partition labeled "nvs" in the * partition table + * - different error in case de-initialization fails (shouldn't happen) */ esp_err_t nvs_flash_erase(void); /** * @brief Erase specified NVS partition * - * This function erases all contents of specified NVS partition + * Erase all content of a specified NVS partition * - * @param[in] part_name Name (label) of the partition to be erased + * @note If the partition is initialized, this function first de-initializes it. Afterwards, the partition has to + * be initialized again to be used. + * + * @param[in] part_name Name (label) of the partition which should be erased * * @return * - ESP_OK on success * - ESP_ERR_NOT_FOUND if there is no NVS partition with the specified name * in the partition table + * - different error in case de-initialization fails (shouldn't happen) */ esp_err_t nvs_flash_erase_partition(const char *part_name); +/** + * @brief Erase custom partition. + * + * Erase all content of specified custom partition. + * + * @note + * If the partition is initialized, this function first de-initializes it. + * Afterwards, the partition has to be initialized again to be used. + * + * @param[in] partition pointer to a partition obtained by the ESP partition API. + * + * @return + * - ESP_OK on success + * - ESP_ERR_NOT_FOUND if there is no partition with the specified + * parameters in the partition table + * - ESP_ERR_INVALID_ARG in case partition is NULL + * - one of the error codes from the underlying flash storage driver + */ +esp_err_t nvs_flash_erase_partition_ptr(const esp_partition_t *partition); + +/** + * @brief Initialize the default NVS partition. + * + * This API initialises the default NVS partition. The default NVS partition + * is the one that is labeled "nvs" in the partition table. + * + * @param[in] cfg Security configuration (keys) to be used for NVS encryption/decryption. + * If cfg is NULL, no encryption is used. + * + * @return + * - ESP_OK if storage was successfully initialized. + * - ESP_ERR_NVS_NO_FREE_PAGES if the NVS storage contains no empty pages + * (which may happen if NVS partition was truncated) + * - ESP_ERR_NOT_FOUND if no partition with label "nvs" is found in the partition table + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures + * - one of the error codes from the underlying flash storage driver + */ +esp_err_t nvs_flash_secure_init(nvs_sec_cfg_t* cfg); + +/** + * @brief Initialize NVS flash storage for the specified partition. + * + * @param[in] partition_label Label of the partition. Note that internally a reference to + * passed value is kept and it should be accessible for future operations + * + * @param[in] cfg Security configuration (keys) to be used for NVS encryption/decryption. + * If cfg is null, no encryption/decryption is used. + * @return + * - ESP_OK if storage was successfully initialized. + * - ESP_ERR_NVS_NO_FREE_PAGES if the NVS storage contains no empty pages + * (which may happen if NVS partition was truncated) + * - ESP_ERR_NOT_FOUND if specified partition is not found in the partition table + * - ESP_ERR_NO_MEM in case memory could not be allocated for the internal structures + * - one of the error codes from the underlying flash storage driver + */ +esp_err_t nvs_flash_secure_init_partition(const char *partition_label, nvs_sec_cfg_t* cfg); + +/** + * @brief Generate and store NVS keys in the provided esp partition + * + * @param[in] partition Pointer to partition structure obtained using + * esp_partition_find_first or esp_partition_get. + * Must be non-NULL. + * @param[out] cfg Pointer to nvs security configuration structure. + * Pointer must be non-NULL. + * Generated keys will be populated in this structure. + * + * + * @return + * -ESP_OK, if cfg was read successfully; + * -or error codes from esp_partition_write/erase APIs. + */ + +esp_err_t nvs_flash_generate_keys(const esp_partition_t* partition, nvs_sec_cfg_t* cfg); + + +/** + * @brief Read NVS security configuration from a partition. + * + * @param[in] partition Pointer to partition structure obtained using + * esp_partition_find_first or esp_partition_get. + * Must be non-NULL. + * @param[out] cfg Pointer to nvs security configuration structure. + * Pointer must be non-NULL. + * + * @note Provided parition is assumed to be marked 'encrypted'. + * + * @return + * -ESP_OK, if cfg was read successfully; + * -ESP_ERR_NVS_KEYS_NOT_INITIALIZED, if the partition is not yet written with keys. + * -ESP_ERR_NVS_CORRUPT_KEY_PART, if the partition containing keys is found to be corrupt + * -or error codes from esp_partition_read API. + */ + +esp_err_t nvs_flash_read_security_cfg(const esp_partition_t* partition, nvs_sec_cfg_t* cfg); + #ifdef __cplusplus } #endif diff --git a/components/nvs_flash/include/nvs_handle.hpp b/components/nvs_flash/include/nvs_handle.hpp new file mode 100644 index 000000000..287866fad --- /dev/null +++ b/components/nvs_flash/include/nvs_handle.hpp @@ -0,0 +1,280 @@ +#ifndef NVS_HANDLE_HPP_ +#define NVS_HANDLE_HPP_ + +#include +#include +#include + +#include "nvs.h" + +namespace nvs { + +/** + * The possible blob types. This is a helper definition for template functions. + */ +enum class ItemType : uint8_t { + U8 = NVS_TYPE_U8, + I8 = NVS_TYPE_I8, + U16 = NVS_TYPE_U16, + I16 = NVS_TYPE_I16, + U32 = NVS_TYPE_U32, + I32 = NVS_TYPE_I32, + U64 = NVS_TYPE_U64, + I64 = NVS_TYPE_I64, + SZ = NVS_TYPE_STR, + BLOB = 0x41, + BLOB_DATA = NVS_TYPE_BLOB, + BLOB_IDX = 0x48, + ANY = NVS_TYPE_ANY +}; + + +/** + * @brief A handle allowing nvs-entry related operations on the NVS. + * + * @note The scope of this handle may vary depending on the implementation, but normally would be the namespace of + * a particular partition. Outside that scope, nvs entries can't be accessed/altered. + */ +class NVSHandle { +public: + virtual ~NVSHandle() { } + + /** + * @brief set value for given key + * + * Sets value for key. Note that physical storage will not be updated until nvs_commit function is called. + * + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param[in] value The value to set. Allowed types are the ones declared in ItemType as well as enums. + * For strings, the maximum length (including null character) is + * 4000 bytes, if there is one complete page free for writing. + * This decreases, however, if the free space is fragmented. + * Note that enums loose their type information when stored in NVS. Ensure that the correct + * enum type is used during retrieval with \ref get_item! + * + * @return + * - ESP_OK if value was set successfully + * - ESP_ERR_NVS_READ_ONLY if storage handle was opened as read only + * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints + * - ESP_ERR_NVS_NOT_ENOUGH_SPACE if there is not enough space in the + * underlying storage to save the value + * - ESP_ERR_NVS_REMOVE_FAILED if the value wasn't updated because flash + * write operation has failed. The value was written however, and + * update will be finished after re-initialization of nvs, provided that + * flash operation doesn't fail again. + * - ESP_ERR_NVS_VALUE_TOO_LONG if the string value is too long + */ + template + esp_err_t set_item(const char *key, T value); + virtual + esp_err_t set_string(const char *key, const char* value) = 0; + + /** + * @brief get value for given key + * + * These functions retrieve value for the key, given its name. If key does not + * exist, or the requested variable type doesn't match the type which was used + * when setting a value, an error is returned. + * + * In case of any error, out_value is not modified. + * + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param value The output value. All integral types which are declared in ItemType as well as enums + * are allowed. Note however that enums lost their type information when stored in NVS. + * Ensure that the correct enum type is used during retrieval with \ref get_item! + * + * @return + * - ESP_OK if the value was retrieved successfully + * - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist + * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints + * - ESP_ERR_NVS_INVALID_LENGTH if length is not sufficient to store data + */ + template + esp_err_t get_item(const char *key, T &value); + + /** + * @brief set variable length binary value for given key + * + * This family of functions set value for the key, given its name. Note that + * actual storage will not be updated until nvs_commit function is called. + * + * @param[in] key Key name. Maximal length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param[in] blob The blob value to set. + * @param[in] len length of binary value to set, in bytes; Maximum length is + * 508000 bytes or (97.6% of the partition size - 4000) bytes + * whichever is lower. + * + * @return + * - ESP_OK if value was set successfully + * - ESP_ERR_NVS_READ_ONLY if storage handle was opened as read only + * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints + * - ESP_ERR_NVS_NOT_ENOUGH_SPACE if there is not enough space in the + * underlying storage to save the value + * - ESP_ERR_NVS_REMOVE_FAILED if the value wasn't updated because flash + * write operation has failed. The value was written however, and + * update will be finished after re-initialization of nvs, provided that + * flash operation doesn't fail again. + * - ESP_ERR_NVS_VALUE_TOO_LONG if the value is too long + * + * @note compare to \ref nvs_set_blob in nvs.h + */ + virtual esp_err_t set_blob(const char *key, const void* blob, size_t len) = 0; + + /** + * @brief get value for given key + * + * These functions retrieve the data of an entry, given its key. If key does not + * exist, or the requested variable type doesn't match the type which was used + * when setting a value, an error is returned. + * + * In case of any error, out_value is not modified. + * + * Both functions expect out_value to be a pointer to an already allocated variable + * of the given type. + * + * It is suggested that nvs_get/set_str is used for zero-terminated short C strings, and + * nvs_get/set_blob is used for arbitrary data structures and long C strings. + * + * @param[in] key Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param out_str/ Pointer to the output value. + * out_blob + * @param[inout] len The length of the output buffer pointed to by out_str/out_blob. + * Use \c get_item_size to query the size of the item beforehand. + * + * @return + * - ESP_OK if the value was retrieved successfully + * - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist + * - ESP_ERR_NVS_INVALID_NAME if key name doesn't satisfy constraints + * - ESP_ERR_NVS_INVALID_LENGTH if length is not sufficient to store data + */ + virtual esp_err_t get_string(const char *key, char* out_str, size_t len) = 0; + virtual esp_err_t get_blob(const char *key, void* out_blob, size_t len) = 0; + + /** + * @brief Look up the size of an entry's data. + * + * @param[in] datatype Data type to search for. + * @param[in] key Key name. Maximum length is (NVS_KEY_NAME_MAX_SIZE-1) characters. Shouldn't be empty. + * @param[out] size Size of the item, if it exists. + * For strings, this size includes the zero terminator. + * + * @return - ESP_OK if the item with specified type and key exists. Its size will be returned via \c size. + * - ESP_ERR_NVS_NOT_FOUND if an item with the requested key and type doesn't exist or any other + * error occurs. + */ + virtual esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) = 0; + + /** + * @brief Erases an entry. + */ + virtual esp_err_t erase_item(const char* key) = 0; + + /** + * Erases all entries in the scope of this handle. The scope may vary, depending on the implementation. + * + * @not If you want to erase the whole nvs flash (partition), refer to \ref + */ + virtual esp_err_t erase_all() = 0; + + /** + * Commits all changes done through this handle so far. + * Currently, NVS writes to storage right after the set and get functions, + * but this is not guaranteed. + */ + virtual esp_err_t commit() = 0; + + /** + * @brief Calculate all entries in the scope of the handle. + * + * @param[out] used_entries Returns amount of used entries from a namespace on success. + * + * + * @return + * - ESP_OK if the changes have been written successfully. + * Return param used_entries will be filled valid value. + * - ESP_ERR_NVS_NOT_INITIALIZED if the storage driver is not initialized. + * Return param used_entries will be filled 0. + * - ESP_ERR_INVALID_ARG if nvs_stats equal to NULL. + * - Other error codes from the underlying storage driver. + * Return param used_entries will be filled 0. + */ + virtual esp_err_t get_used_entry_count(size_t& usedEntries) = 0; + +protected: + virtual esp_err_t set_typed_item(ItemType datatype, const char *key, const void* data, size_t dataSize) = 0; + + virtual esp_err_t get_typed_item(ItemType datatype, const char *key, void* data, size_t dataSize) = 0; +}; + +/** + * @brief Opens non-volatile storage and returns a handle object. + * + * The handle is automatically closed on desctruction. The scope of the handle is the namespace ns_name + * in a particular partition partition_name. + * The parameters partition_name, ns_name and open_mode have the same meaning and restrictions as the parameters + * part_name, name and open_mode in \ref nvs_open_from_partition, respectively. + * + * @param err an optional pointer to an esp_err_t result of the open operation, having the same meaning as the return + * value in \ref nvs_open_from_partition: + * - ESP_OK if storage handle was opened successfully + * - ESP_ERR_NVS_NOT_INITIALIZED if the storage driver is not initialized + * - ESP_ERR_NVS_PART_NOT_FOUND if the partition with label "nvs" is not found + * - ESP_ERR_NVS_NOT_FOUND id namespace doesn't exist yet and + * mode is NVS_READONLY + * - ESP_ERR_NVS_INVALID_NAME if namespace name doesn't satisfy constraints + * - other error codes from the underlying storage driver + * + * @return shared pointer of an nvs handle on success, an empty shared pointer otherwise + */ +std::unique_ptr open_nvs_handle_from_partition(const char *partition_name, + const char *ns_name, + nvs_open_mode_t open_mode, + esp_err_t *err = nullptr); + +/** + * @brief This function does the same as \ref open_nvs_handle_from_partition but uses the default nvs partition + * instead of a partition_name parameter. + */ +std::unique_ptr open_nvs_handle(const char *ns_name, + nvs_open_mode_t open_mode, + esp_err_t *err = nullptr); + +// Helper functions for template usage +/** + * Help to translate all integral types into ItemType. + */ +template::value, void*>::type = nullptr> +constexpr ItemType itemTypeOf() +{ + return static_cast(((std::is_signed::value)?0x10:0x00) | sizeof(T)); +} + +/** + * Help to translate all enum types into integral ItemType. + */ +template::value, int>::type = 0> +constexpr ItemType itemTypeOf() +{ + return static_cast(((std::is_signed::value)?0x10:0x00) | sizeof(T)); +} + +template +constexpr ItemType itemTypeOf(const T&) +{ + return itemTypeOf(); +} + +// Template Implementations +template +esp_err_t NVSHandle::set_item(const char *key, T value) { + return set_typed_item(itemTypeOf(value), key, &value, sizeof(value)); +} + +template +esp_err_t NVSHandle::get_item(const char *key, T &value) { + return get_typed_item(itemTypeOf(value), key, &value, sizeof(value)); +} + +} // nvs + +#endif // NVS_HANDLE_HPP_ diff --git a/components/nvs_flash/test_nvs_host/crc.cpp b/components/nvs_flash/mock/int/crc.cpp similarity index 97% rename from components/nvs_flash/test_nvs_host/crc.cpp rename to components/nvs_flash/mock/int/crc.cpp index 4cbb9be9e..5098a1415 100644 --- a/components/nvs_flash/test_nvs_host/crc.cpp +++ b/components/nvs_flash/mock/int/crc.cpp @@ -31,7 +31,7 @@ static const unsigned int crc32_le_table[256] = { 0x4369e96aL, 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, - + 0xedb88320L, 0x9abfb3b6L, 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, 0x196c3671L, 0x6e6b06e7L, @@ -52,7 +52,7 @@ static const unsigned int crc32_le_table[256] = { -extern "C" unsigned int crc32_le(unsigned int crc, unsigned char const * buf,unsigned int len) +extern "C" uint32_t crc32_le(unsigned int crc, unsigned char const * buf,unsigned int len) { unsigned int i; crc = ~crc; @@ -61,4 +61,3 @@ extern "C" unsigned int crc32_le(unsigned int crc, unsigned char const * buf,uns } return ~crc; } - diff --git a/components/nvs_flash/test_nvs_host/crc.h b/components/nvs_flash/mock/int/crc.h similarity index 79% rename from components/nvs_flash/test_nvs_host/crc.h rename to components/nvs_flash/mock/int/crc.h index c752b3030..405295f8b 100644 --- a/components/nvs_flash/test_nvs_host/crc.h +++ b/components/nvs_flash/mock/int/crc.h @@ -20,6 +20,10 @@ extern "C" { #endif +/** + * Mock function to replace ESP ROM function used in IDF with a Linux implementation. + * Note: the name MUST have the prefix esp_rom_* since tools/ci/check_rom_apis.sh checks and complains otherwise. + */ uint32_t crc32_le(uint32_t crc, const uint8_t* buf, size_t len); #ifdef __cplusplus diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst index 11f0a2ea6..3a78a109d 100644 --- a/components/nvs_flash/nvs_partition_generator/README.rst +++ b/components/nvs_flash/nvs_partition_generator/README.rst @@ -1,76 +1,312 @@ NVS Partition Generator Utility =============================== +:link_to_translation:`zh_CN:[中文]` + Introduction ------------ -:component_file:`nvs_flash/nvs_partition_generator/nvs_partition_gen.py` utility is designed to help create a binary file, compatible with NVS architecture defined in :doc:`Non-Volatile Storage `, based on user provided key-value pairs in a CSV file. -Utility is ideally suited for generating a binary blob, containing data specific to ODM/OEM, which can be flashed externally at the time of device manufacturing. This helps manufacturers set unique value for various parameters for each device, e.g. serial number, while using same application firmware for all devices. - -CSV file format ---------------- - -Each row of the .csv file should have 4 parameters, separated by comma. Below is the description of each of these parameters: +The utility :component_file:`nvs_flash/nvs_partition_generator/nvs_partition_gen.py` creates a binary file based on key-value pairs provided in a CSV file. The binary file is compatible with NVS architecture defined in :doc:`Non-Volatile Storage `. +This utility is ideally suited for generating a binary blob, containing data specific to ODM/OEM, which can be flashed externally at the time of device manufacturing. This allows manufacturers to generate many instances of the same application firmware with customized parameters for each device, such as a serial number. -Key - Key of the data. Data can later be accessed from an application via this key. +Prerequisites +------------- +To use this utility in encryption mode, install the following packages: + - cryptography package -Type - Supported values are ``file``, ``data`` and ``namespace``. +All the required packages are included in `requirements.txt` in the root of the esp-idf directory. -Encoding - Supported values are: ``u8``, ``i8``, ``u16``, ``u32``, ``i32``, ``string``, ``hex2bin``, ``base64`` and ``binary``. This specifies how actual data values are encoded in the resultant binary file. Difference between ``string`` and ``binary`` encoding is that ``string`` data is terminated with a NULL character, whereas ``binary`` data is not. - - .. note:: For ``file`` type, only ``hex2bin``, ``base64``, ``string`` and ``binary`` is supported as of now. +CSV file format +--------------- -Value - Data value. +Each line of a .csv file should contain 4 parameters, separated by a comma. The table below provides the description for each of these parameters. -.. note:: Encoding and Value cells for ``namespace`` field type should be empty. Encoding and Value of ``namespace`` is fixed and isn't configurable. Any value in these cells are ignored. ++-----+-----------+----------------------------------------------------------------------+-----------------------------------------------------+ +| No. | Parameter | Description | Notes | ++=====+===========+======================================================================+=====================================================+ +| 1 | Key | Key of the data. The data can be accessed later from | | +| | | an application using this key. | | ++-----+-----------+----------------------------------------------------------------------+-----------------------------------------------------+ +| 2 | Type | Supported values are ``file``, ``data`` and ``namespace``. | | ++-----+-----------+----------------------------------------------------------------------+-----------------------------------------------------+ +| 3 | Encoding | Supported values are: ``u8``, ``i8``, ``u16``, ``i16``, ``u32``, | As of now, for the ``file`` type, | +| | | ``i32``, ``u64``, ``i64``, ``string``, ``hex2bin``, ``base64`` | only ``hex2bin``, ``base64``, ``string``, | +| | | and ``binary``. | and ``binary`` encoding is supported. | +| | | This specifies how actual data values are encoded in the | | +| | | resulting binary file. The difference between the ``string`` | | +| | | and ``binary`` encoding is that ``string`` data is terminated | | +| | | with a NULL character, whereas ``binary`` data is not. | | ++-----+-----------+----------------------------------------------------------------------+-----------------------------------------------------+ +| 4 | Value | Data value. | Encoding and Value cells for the ``namespace`` | +| | | | field type should be empty. Encoding and Value | +| | | | of ``namespace`` is fixed and is not configurable. | +| | | | Any values in these cells are ignored. | ++-----+-----------+----------------------------------------------------------------------+-----------------------------------------------------+ -.. note:: First row of the CSV file should always be column header and isn't configurable. +.. note:: The first line of the CSV file should always be the column header and it is not configurable. -Below is an example dump of such CSV file:: +Below is an example dump of such a CSV file:: key,type,encoding,value <-- column header namespace_name,namespace,, <-- First entry should be of type "namespace" key1,data,u8,1 key2,file,string,/path/to/file -.. note:: Make sure there are no spaces before and after ',' in CSV file. +.. note:: + + Make sure there are **no spaces**: + - before and after ',' + - at the end of each line in a CSV file + NVS Entry and Namespace association ----------------------------------- -When a new namespace entry is encountered in the CSV file, each follow-up entries will be part of that namespace, until next namespace entry is found, in which case all the follow-up entries will be part of the new namespace. +When a namespace entry is encountered in a CSV file, each following entry will be treated as part of that namespace until the next namespace entry is found. At this point, all the following entries will be treated as part of the new namespace. + +.. note:: First entry in a CSV file should always be a ``namespace`` entry. + + +Multipage Blob Support +---------------------- + +By default, binary blobs are allowed to span over multiple pages and are written in the format mentioned in Section :ref:`structure_of_entry`. +If you intend to use an older format, the utility provides an option to disable this feature. + + +Encryption Support +------------------- -.. note:: First entry in a CSV file should always be ``namespace`` entry. +The NVS Partition Generator utility also allows you to create an encrypted binary file. The utility uses the AES-XTS encryption. Please refer to :ref:`nvs_encryption` for more details. + + +Decryption Support +------------------- +This utility allows you to decrypt an encrypted NVS binary file. The utility uses an NVS binary file encrypted using AES-XTS encryption. Please refer to :ref:`nvs_encryption` for more details. Running the utility ------------------- -python nvs_partition_gen.py [-h] [--input INPUT] [--output OUTPUT] [--size SIZE] +**Usage**:: + + python nvs_partition_gen.py [-h] {generate,generate-key,encrypt,decrypt} ... + + Optional Arguments: + +-----+------------+----------------------------------------------------------------------+ + | No. | Parameter | Description | + +=====+============+======================================================================+ + | 1 | -h, --help | show this help message and exit | + +-----+------------+----------------------------------------------------------------------+ + + Commands: + Run nvs_partition_gen.py {command} -h for additional help + +-----+--------------+--------------------------------------------------------------------+ + | No. | Parameter | Description | + +=====+==============+====================================================================+ + | 1 | generate | Generate NVS partition | + +-----+--------------+--------------------------------------------------------------------+ + | 2 | generate-key | Generate keys for encryption | + +-----+--------------+--------------------------------------------------------------------+ + | 3 | encrypt | Generate NVS encrypted partition | + +-----+--------------+--------------------------------------------------------------------+ + | 4 | decrypt | Decrypt NVS encrypted partition | + +-----+--------------+--------------------------------------------------------------------+ + + +To generate NVS partition (Default): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py generate [-h] [--version {1,2}] [--outdir OUTDIR] + input output size + + Positional Arguments: + +--------------+----------------------------------------------------------------------+ + | Parameter | Description | + +==============+======================================================================+ + | input | Path to CSV file to parse | + +--------------+----------------------------------------------------------------------+ + | output | Path to output NVS binary file | + +--------------+----------------------------------------------------------------------+ + | size | Size of NVS partition in bytes (must be multiple of 4096) | + +--------------+----------------------------------------------------------------------+ -+------------------------+----------------------------------------------------------------------------------------------+ -| Arguments | Description | -+========================+==============================================================================================+ -| --input INPUT | Path to CSV file to parse. | -+------------------------+----------------------------------------------------------------------------------------------+ -| --output OUTPUT | Path to output generated binary file. | -+------------------------+----------------------------------------------------------------------------------------------+ -| --size SIZE | Size of NVS Partition in bytes (must be multiple of 4096) | -+------------------------+----------------------------------------------------------------------------------------------+ + Optional Arguments: + +-----------------+--------------------------------------------------------------------+ + | Parameter | Description | + +=================+====================================================================+ + | -h, --help | show this help message and exit | + +-----------------+--------------------------------------------------------------------+ + | --version {1,2} | Set multipage blob version. | + | | Version 1 - Multipage blob support disabled. | + | | Version 2 - Multipage blob support enabled. | + | | Default: Version 2 | + | | | + +-----------------+--------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created | + | | (Default: current directory) | + +-----------------+--------------------------------------------------------------------+ + + +You can run the utility to generate NVS partition using the command below: A sample CSV file is provided with the utility:: - python nvs_partition_gen.py --input sample.csv --output sample.bin --size 0x3000 + python nvs_partition_gen.py generate sample_singlepage_blob.csv sample.bin 0x3000 + + +To generate only encryption keys: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py generate-key [-h] [--keyfile KEYFILE] + [--outdir OUTDIR] + + Optional Arguments: + +--------------------+----------------------------------------------------------------------+ + | Parameter | Description | + +====================+======================================================================+ + | -h, --help | show this help message and exit | + +--------------------+----------------------------------------------------------------------+ + | --keyfile KEYFILE | Path to output encryption keys file | + +--------------------+----------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created. | + | | (Default: current directory) | + +--------------------+----------------------------------------------------------------------+ + +You can run the utility to generate only encryption keys using the command below:: + + python nvs_partition_gen.py generate-key + + +To generate encrypted NVS partition: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py encrypt [-h] [--version {1,2}] [--keygen] + [--keyfile KEYFILE] [--inputkey INPUTKEY] + [--outdir OUTDIR] + input output size + + Positional Arguments: + +--------------+----------------------------------------------------------------------+ + | Parameter | Description | + +==============+======================================================================+ + | input | Path to CSV file to parse | + +--------------+----------------------------------------------------------------------+ + | output | Path to output NVS binary file | + +--------------+----------------------------------------------------------------------+ + | size | Size of NVS partition in bytes (must be multiple of 4096) | + +--------------+----------------------------------------------------------------------+ + + + Optional Arguments: + +---------------------+--------------------------------------------------------------------+ + | Parameter | Description | + +=====================+====================================================================+ + | -h, --help | show this help message and exit | + | | | + +---------------------+--------------------------------------------------------------------+ + | --version {1,2} | Set multipage blob version. | + | | Version 1 - Multipage blob support disabled. | + | | Version 2 - Multipage blob support enabled. | + | | Default: Version 2 | + +---------------------+--------------------------------------------------------------------+ + | --keygen | Generates key for encrypting NVS partition | + +---------------------+--------------------------------------------------------------------+ + | --keyfile KEYFILE | Path to output encryption keys file | + +---------------------+--------------------------------------------------------------------+ + | --inputkey INPUTKEY | File having key for encrypting NVS partition | + +---------------------+--------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created | + | | (Default: current directory) | + +---------------------+--------------------------------------------------------------------+ + + +You can run the utility to encrypt NVS partition using the command below: +A sample CSV file is provided with the utility: + +- Encrypt by allowing the utility to generate encryption keys:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --keygen + +.. note:: Encryption key of the following format ``/keys/keys-.bin`` is created. + +- Encrypt by allowing the utility to generate encryption keys and store it in provided custom filename:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --keygen --keyfile sample_keys.bin + +.. note:: Encryption key of the following format ``/keys/sample_keys.bin`` is created. +.. note:: This newly created file having encryption keys in ``keys/`` directory is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. + +- Encrypt by providing the encryption keys as input binary file:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --inputkey sample_keys.bin + +To decrypt encrypted NVS partition: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py decrypt [-h] [--outdir OUTDIR] input key output + + Positional Arguments: + +--------------+----------------------------------------------------------------------+ + | Parameter | Description | + +==============+======================================================================+ + | input | Path to encrypted NVS partition file to parse | + +--------------+----------------------------------------------------------------------+ + | key | Path to file having keys for decryption | + +--------------+----------------------------------------------------------------------+ + | output | Path to output decrypted binary file | + +--------------+----------------------------------------------------------------------+ + + + Optional Arguments: + +---------------------+--------------------------------------------------------------------+ + | Parameter | Description | + +=====================+====================================================================+ + | -h, --help | show this help message and exit | + +---------------------+--------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created | + | | (Default: current directory) | + +---------------------+--------------------------------------------------------------------+ + + +You can run the utility to decrypt encrypted NVS partition using the command below:: + + python nvs_partition_gen.py decrypt sample_encr.bin sample_keys.bin sample_decr.bin + +You can also provide the format version number: + - Multipage Blob Support Disabled (Version 1) + - Multipage Blob Support Enabled (Version 2) + + +Multipage Blob Support Disabled (Version 1): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can run the utility in this format by setting the version parameter to 1, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py generate sample_singlepage_blob.csv sample.bin 0x3000 --version 1 + + +Multipage Blob Support Enabled (Version 2): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can run the utility in this format by setting the version parameter to 2, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py generate sample_multipage_blob.csv sample.bin 0x4000 --version 2 + .. note:: *Minimum NVS Partition Size needed is 0x3000 bytes.* .. note:: *When flashing the binary onto the device, make sure it is consistent with the application's sdkconfig.* + Caveats ------- -- Utility doesn't check for duplicate keys and will write data pertaining to both keys. User needs to make sure keys are distinct. -- Once a new page is created, no data will be written in the space left in previous page. Fields in the CSV file need to be ordered in such a way so as to optimize memory. +- Utility does not check for duplicate keys and will write data pertaining to both keys. You need to make sure that the keys are distinct. +- Once a new page is created, no data will be written in the space left on the previous page. Fields in the CSV file need to be ordered in such a way as to optimize memory. - 64-bit datatype is not yet supported. + diff --git a/components/nvs_flash/nvs_partition_generator/README_CN.rst b/components/nvs_flash/nvs_partition_generator/README_CN.rst new file mode 100644 index 000000000..24094e993 --- /dev/null +++ b/components/nvs_flash/nvs_partition_generator/README_CN.rst @@ -0,0 +1,304 @@ +NVS 分区生æˆç¨‹åº +=============================== + +:link_to_translation:`en:[English]` + +ä»‹ç» +------------ + +NVS 分区生æˆç¨‹åº (:component_file:`nvs_flash/nvs_partition_generator/nvs_partition_gen.py`) æ ¹æ® CSV 文件中的键值对生æˆäºŒè¿›åˆ¶æ–‡ä»¶ã€‚该二进制文件与 :doc:`éžæ˜“失性存储器 (NVS) ` 中定义的 NVS 结构兼容。NVS 分区生æˆç¨‹åºé€‚åˆç”¨äºŽç”ŸæˆäºŒè¿›åˆ¶æ•°æ®ï¼ˆBlob），其中包括设备生产时å¯ä»Žå¤–部烧录的 ODM/OEM æ•°æ®ã€‚这也使得生产制造商在使用åŒä¸€ä¸ªå›ºä»¶çš„基础上，通过自定义å‚数，如åºåˆ—å·ç­‰ï¼Œä¸ºæ¯ä¸ªè®¾å¤‡ç”Ÿæˆä¸åŒé…置。 + +准备工作 +------------- + +在加密模å¼ä¸‹ä½¿ç”¨è¯¥ç¨‹åºï¼Œéœ€å®‰è£…下列软件包： + - cryptography package + +根目录下的 `requirements.txt` 包å«å¿…需 python 包，请预先安装。 + + +CSV æ–‡ä»¶æ ¼å¼ +--------------- + +.csv 文件æ¯è¡Œéœ€åŒ…å«å››ä¸ªå‚数，以逗å·éš”开。具体å‚æ•°æè¿°è§ä¸‹è¡¨ï¼š + ++------+----------+--------------------------------------------------------------+-----------------------------------------------------------------+ +| åºå· | å‚æ•° | æè¿° | 说明 | ++======+==========+==============================================================+=================================================================+ +| 1 | Key | 主键，应用程åºå¯é€šè¿‡æŸ¥è¯¢æ­¤é”®æ¥èŽ·å–æ•°æ®ã€‚ | | ++------+----------+--------------------------------------------------------------+-----------------------------------------------------------------+ +| 2 | Type | æ”¯æŒ ``file``ã€``data`` å’Œ ``namespace``。 | | ++------+----------+--------------------------------------------------------------+-----------------------------------------------------------------+ +| 3 | Encoding | æ”¯æŒ ``u8``ã€``i8``ã€``u16``ã€``u32``〠| ``file`` | +| | | ``i32``ã€``string``ã€``hex2bin``ã€``base64`` å’Œ ``binary``。 | 类型当å‰ä»…æ”¯æŒ | +| | | 决定二进制 ``bin`` 文件中 value 被编ç æˆçš„类型。 | ``hex2bin``ã€``base64``〠| +| | | ``string`` å’Œ ``binary`` ç¼–ç çš„区别在于， | ``string`` å’Œ ``binary`` ç¼–ç ã€‚ | +| | | ``string`` æ•°æ®ä»¥ NULL 字符结尾，``binary`` æ•°æ®åˆ™ä¸æ˜¯ã€‚ | | ++------+----------+--------------------------------------------------------------+-----------------------------------------------------------------+ +| 4 | Value | Data value | ``namespace`` 字段的 ``encoding`` å’Œ ``value`` 应为空。 | +| | | | ``namespace`` çš„ ``encoding`` å’Œ ``value`` 为固定值，ä¸å¯è®¾ç½®ã€‚ | +| | | | 这些å•å…ƒæ ¼ä¸­çš„所有值都会被忽视。 | ++------+----------+--------------------------------------------------------------+-----------------------------------------------------------------+ + +.. note:: CSV 文件的第一行应为列标题，ä¸å¯è®¾ç½®ã€‚ + +此类 CSV 文件的 Dump 示例如下:: + + key,type,encoding,value <-- 列标题 + namespace_name,namespace,, <-- 第一个æ¡ç›®ä¸º "namespace" + key1,data,u8,1 + key2,file,string,/path/to/file + + +.. note:: + + 请确ä¿ï¼š + - é€—å· ',' å‰åŽæ— ç©ºæ ¼ï¼› + - CSV 文件æ¯è¡Œæœ«å°¾æ— ç©ºæ ¼ã€‚ + +NVS æ¡ç›®å’Œå‘½å空间 (namespace) +----------------------------------- + +如 CSV 文件中出现命å空间æ¡ç›®ï¼ŒåŽç»­æ¡ç›®å‡ä¼šè¢«è§†ä¸ºè¯¥å‘½å空间的一部分，直至找到下一个命å空间æ¡ç›®ã€‚找到新命å空间æ¡ç›®åŽï¼ŒåŽç»­æ‰€æœ‰æ¡ç›®éƒ½ä¼šè¢«è§†ä¸ºæ–°å‘½å空间的一部分。 + +.. note:: CSV 文件中第一个æ¡ç›®åº”始终为 ``namespace``。 + + +支æŒå¤šé¡µ Blob +---------------------- + +默认情况下，二进制 Blob å¯è·¨å¤šé¡µï¼Œæ ¼å¼å‚考 :ref:`structure_of_entry` 章节。如需使用旧版格å¼ï¼Œå¯åœ¨ç¨‹åºä¸­ç¦ç”¨è¯¥åŠŸèƒ½ã€‚ + + +支æŒåŠ å¯† +------------------- + +NVS 分区生æˆç¨‹åºè¿˜å¯ä½¿ç”¨ AES-XTS 加密生æˆäºŒè¿›åˆ¶åŠ å¯†æ–‡ä»¶ã€‚更多信æ¯è¯¦è§ :ref:`nvs_encryption`。 + +支æŒè§£å¯† +------------------- +如果 NVS 二进制文件采用了 AES-XTS 加密，该程åºè¿˜å¯å¯¹æ­¤ç±»æ–‡ä»¶è¿›è¡Œè§£å¯†ï¼Œæ›´å¤šä¿¡æ¯è¯¦è§ :ref:`nvs_encryption`。 + +è¿è¡Œç¨‹åº +------------------- + +**使用方法**:: + + python nvs_partition_gen.py [-h] {generate,generate-key,encrypt,decrypt} ... + +**å¯é€‰å‚æ•°**： + ++------+------------+----------------------+ +| åºå· | å‚æ•° | æè¿° | ++------+------------+----------------------+ +| 1 | -h, --help | 显示帮助信æ¯å¹¶é€€å‡º | ++------+------------+----------------------+ + +**命令**:: + + è¿è¡Œ nvs_partition_gen.py {command} -h æŸ¥çœ‹æ›´å¤šå¸®åŠ©ä¿¡æ¯ + ++------+--------------+---------------+ +| åºå· | å‚æ•° | æè¿° | ++------+--------------+---------------+ +| 1 | generate | ç”Ÿæˆ NVS 分区 | ++------+--------------+---------------+ +| 2 | generate-key | 生æˆåŠ å¯†å¯†é’¥ | ++------+--------------+---------------+ +| 3 | encrypt | 加密 NVS 分区 | ++------+--------------+---------------+ +| 4 | decrypt | 解密 NVS 分区 | ++------+--------------+---------------+ + + +ç”Ÿæˆ NVS 分区（默认模å¼ï¼‰ +---------------------------------- + +**使用方法**:: + + python nvs_partition_gen.py generate [-h] [--version {1,2}] [--outdir OUTDIR] + input output size + +**ä½ç½®å‚æ•°**： + ++--------+--------------------------------------------------+ +| å‚æ•° | æè¿° | ++--------+--------------------------------------------------+ +| input | 待解æžçš„ CSV 文件路径 | ++--------+--------------------------------------------------+ +| output | NVS 二进制文件的输出路径 | ++--------+--------------------------------------------------+ +| size | NVS 分区大å°ï¼ˆä»¥å­—节为å•ä½ï¼Œä¸”为 4096 çš„æ•´æ•°å€ï¼‰ | ++--------+--------------------------------------------------+ + +**å¯é€‰å‚æ•°**： + ++-----------------+------------------------------------------------+ +| å‚æ•° | æè¿° | ++-----------------+------------------------------------------------+ +| -h, --help | 显示帮助信æ¯å¹¶é€€å‡º | ++-----------------+------------------------------------------------+ +| --version {1,2} | - 设置多页 Blob 版本。 | +| | - 版本 1：ç¦ç”¨å¤šé¡µ Blobï¼› | +| | - 版本 2：å¯ç”¨å¤šé¡µ Blobï¼› | +| | - 默认版本：版本 2。 | ++-----------------+------------------------------------------------+ +| --outdir OUTDIR | 输出目录，用于存储创建的文件。（默认当å‰ç›®å½•ï¼‰ | ++-----------------+------------------------------------------------+ + +è¿è¡Œå¦‚下命令创建 NVS 分区，该程åºåŒæ—¶ä¼šæä¾› CSV 示例文件:: + + python nvs_partition_gen.py generate sample_singlepage_blob.csv sample.bin 0x3000 + +仅生æˆåŠ å¯†å¯†é’¥ +----------------------- + +**使用方法**:: + + python nvs_partition_gen.py generate-key [-h] [--keyfile KEYFILE] + [--outdir OUTDIR] + +**å¯é€‰å‚æ•°**： + ++-------------------+------------------------------------------------+ +| å‚æ•° | æè¿° | ++-------------------+------------------------------------------------+ +| -h, --help | 显示帮助信æ¯å¹¶é€€å‡º | ++-------------------+------------------------------------------------+ +| --keyfile KEYFILE | 加密密钥文件的输出路径 | ++-------------------+------------------------------------------------+ +| --outdir OUTDIR | 输出目录，用于存储创建的文件。（默认当å‰ç›®å½•ï¼‰ | ++-------------------+------------------------------------------------+ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ä»…生æˆåŠ å¯†å¯†é’¥:: + + python nvs_partition_gen.py generate-key + +ç”Ÿæˆ NVS 加密分区 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**使用方法**:: + + python nvs_partition_gen.py encrypt [-h] [--version {1,2}] [--keygen] + [--keyfile KEYFILE] [--inputkey INPUTKEY] + [--outdir OUTDIR] + input output size + +**ä½ç½®å‚æ•°**： + ++--------+--------------------------------------+ +| å‚æ•° | æè¿° | ++--------+--------------------------------------+ +| input | å¾…è§£æž CSV 文件的路径 | ++--------+--------------------------------------+ +| output | NVS 二进制文件的输出路径 | ++--------+--------------------------------------+ +| size | NVS åˆ†åŒºå¤§å° | +| | （以字节为å•ä½ï¼Œä¸”为 4096 çš„æ•´æ•°å€ï¼‰ | ++--------+--------------------------------------+ + +**å¯é€‰å‚æ•°**： + ++---------------------+------------------------------+ +| å‚æ•° | æè¿° | ++---------------------+------------------------------+ +| -h, --help | 显示帮助信æ¯å¹¶é€€å‡º | ++---------------------+------------------------------+ +| --version {1,2} | - 设置多页 Blob 版本。 | +| | - 版本 1：ç¦ç”¨å¤šé¡µ Blobï¼› | +| | - 版本 2：å¯ç”¨å¤šé¡µ Blobï¼› | +| | - 默认版本：版本 2。 | ++---------------------+------------------------------+ +| --keygen | ç”Ÿæˆ NVS 分区加密密钥 | ++---------------------+------------------------------+ +| --keyfile KEYFILE | 密钥文件的输出路径 | ++---------------------+------------------------------+ +| --inputkey INPUTKEY | å†…å« NVS 分区加密密钥的文件 | ++---------------------+------------------------------+ +| --outdir OUTDIR | 输出目录，用于存储创建的文件 | +| | （默认当å‰ç›®å½•ï¼‰ | ++---------------------+------------------------------+ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åŠ å¯† NVS 分区，该程åºåŒæ—¶ä¼šæ供一个 CSV 示例文件。 + +- 通过 NVS 分区生æˆç¨‹åºç”ŸæˆåŠ å¯†å¯†é’¥æ¥åŠ å¯†:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --keygen + +.. note:: 创建的加密密钥格å¼ä¸º ``/keys/keys-.bin``。 + +- 通过 NVS 分区生æˆç¨‹åºç”ŸæˆåŠ å¯†å¯†é’¥ï¼Œå¹¶å°†å¯†é’¥å­˜å‚¨äºŽè‡ªå®šä¹‰çš„文件中:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --keygen --keyfile sample_keys.bin + +.. note:: 创建的加密密钥格å¼ä¸º ``/keys/keys-.bin``。 +.. note:: 加密密钥存储于新建文件的 ``keys/`` 目录下，与 NVS 密钥分区结构兼容。更多信æ¯è¯·å‚考 :ref:`nvs_key_partition`。 + +- 将加密密钥用作二进制输入文件æ¥è¿›è¡ŒåŠ å¯†:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --inputkey sample_keys.bin + +解密 NVS 分区 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**使用方法**:: + + python nvs_partition_gen.py decrypt [-h] [--outdir OUTDIR] input key output + +**ä½ç½®å‚æ•°**： + ++--------+-------------------------------+ +| å‚æ•° | æè¿° | ++--------+-------------------------------+ +| input | 待解æžçš„ NVS 加密分区文件路径 | ++--------+-------------------------------+ +| key | å«æœ‰è§£å¯†å¯†é’¥çš„文件路径 | ++--------+-------------------------------+ +| output | 已解密的二进制文件输出路径 | ++--------+-------------------------------+ + +**å¯é€‰å‚æ•°**： + ++-----------------+------------------------------+ +| å‚æ•° | æè¿° | ++-----------------+------------------------------+ +| -h, --help | 显示帮助信æ¯å¹¶é€€å‡º | ++-----------------+------------------------------+ +| --outdir OUTDIR | 输出目录，用于存储创建的文件 | +| | （默认当å‰ç›®å½•ï¼‰ | ++-----------------+------------------------------+ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤è§£å¯†å·²åŠ å¯†çš„ NVS 分区:: + + python nvs_partition_gen.py decrypt sample_encr.bin sample_keys.bin sample_decr.bin + +您å¯ä»¥è‡ªå®šä¹‰æ ¼å¼ç‰ˆæœ¬å·ï¼š + +- 版本 1：ç¦ç”¨å¤šé¡µ Blob +- 版本 2：å¯ç”¨å¤šé¡µ Blob + +版本 1：ç¦ç”¨å¤šé¡µ Blob +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +如需ç¦ç”¨å¤šé¡µ Blob，请按照如下命令将版本å‚数设置为 1，以此格å¼è¿è¡Œåˆ†åŒºç”Ÿæˆç¨‹åºã€‚该程åºåŒæ—¶ä¼šæ供一个 CSV 示例文件:: + + python nvs_partition_gen.py generate sample_singlepage_blob.csv sample.bin 0x3000 --version 1 + +版本 2：å¯ç”¨å¤šé¡µ Blob +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +如需å¯ç”¨å¤šé¡µ Blob，请按照如下命令将版本å‚数设置为 2，以此格å¼è¿è¡Œåˆ†åŒºç”Ÿæˆç¨‹åºã€‚该程åºåŒæ—¶ä¼šæ供一个 CSV 示例文件:: + + python nvs_partition_gen.py generate sample_multipage_blob.csv sample.bin 0x4000 --version 2 + +.. note:: NVS 分区最å°ä¸º 0x3000 字节。 + +.. note:: 将二进制文件烧录至设备时，请确ä¿ä¸Žåº”用的 sdkconfig 设置一致。 + + +说明 +------- + +- 分区生æˆç¨‹åºä¸ä¼šå¯¹é‡å¤é”®è¿›è¡Œæ£€æŸ¥ï¼Œè€Œå°†æ•°æ®åŒæ—¶å†™å…¥è¿™ä¸¤ä¸ªé‡å¤é”®ä¸­ã€‚请注æ„ä¸è¦ä½¿ç”¨åŒå的键； +- 新页é¢åˆ›å»ºåŽï¼Œå‰ä¸€é¡µçš„空白处ä¸ä¼šå†å†™å…¥æ•°æ®ã€‚CSV 文件中的字段须按次åºæŽ’列以优化内存； +- æš‚ä¸æ”¯æŒ 64 ä½æ•°æ®ç±»åž‹ã€‚ diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index d5ddc55b0..890f5222b 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -19,32 +19,55 @@ # from __future__ import division, print_function -from builtins import int, range, bytes -from io import open -import sys + import argparse +import array import binascii +import codecs +import datetime +import distutils.dir_util +import os import random import struct -import os -import array -import csv +import sys import zlib -import codecs -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.backends import default_backend +from builtins import bytes, int, range +from io import open + +from future.moves.itertools import zip_longest -VERSION1_PRINT = "v1 - Multipage Blob Support Disabled" -VERSION2_PRINT = "v2 - Multipage Blob Support Enabled" +try: + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +except ImportError: + print('The cryptography package is not installed.' + 'Please refer to the Get Started section of the ESP-IDF Programming Guide for ' + 'setting up the required packages.') + raise + +VERSION1_PRINT = 'V1 - Multipage Blob Support Disabled' +VERSION2_PRINT = 'V2 - Multipage Blob Support Enabled' + + +def reverse_hexbytes(addr_tmp): + addr = [] + reversed_bytes = '' + for i in range(0, len(addr_tmp), 2): + addr.append(addr_tmp[i:i + 2]) + reversed_bytes = ''.join(reversed(addr)) + + return reversed_bytes """ Class for standard NVS page structure """ + + class Page(object): PAGE_PARAMS = { - "max_size": 4096, - "max_old_blob_size": 1984, - "max_new_blob_size": 4000, - "max_entries": 126 + 'max_size': 4096, + 'max_old_blob_size': 1984, + 'max_new_blob_size': 4000, + 'max_entries': 126 } # Item type codes @@ -54,6 +77,8 @@ class Page(object): I16 = 0x12 U32 = 0x04 I32 = 0x14 + U64 = 0x08 + I64 = 0x18 SZ = 0x21 BLOB = 0x41 BLOB_DATA = 0x42 @@ -68,25 +93,21 @@ class Page(object): CHUNK_ANY = 0xFF ACTIVE = 0xFFFFFFFE FULL = 0xFFFFFFFC - VERSION1=0xFF - VERSION2=0xFE + VERSION1 = 0xFF + VERSION2 = 0xFE - def __init__(self, page_num, is_rsrv_page=False): + def __init__(self, page_num, version, is_rsrv_page=False): self.entry_num = 0 - self.is_encrypt = False - self.encr_key = None self.bitmap_array = array.array('B') - self.version = Page.VERSION2 - self.page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"] + self.version = version + self.page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS['max_size'] if not is_rsrv_page: self.bitmap_array = self.create_bitmap_array() - self.set_header(page_num) - - def set_header(self, page_num): - global page_header + self.set_header(page_num, version) + def set_header(self, page_num, version): # set page state to active - page_header= bytearray(b'\xff') *32 + page_header = bytearray(b'\xff') * 32 page_state_active_seq = Page.ACTIVE struct.pack_into(' 2: if not addr_len % 2: addr_tmp = addr - tweak_tmp = self.reverse_hexbytes(addr_tmp) - tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) else: addr_tmp = init_tweak_val + addr - tweak_tmp = self.reverse_hexbytes(addr_tmp) - tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) + tweak_tmp = reverse_hexbytes(addr_tmp) + tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) else: tweak_val = addr + (init_tweak_val * (tweak_len_needed - len(addr))) @@ -207,11 +212,10 @@ def encrypt_data(self, data_input, no_of_entries, nvs_obj): return encr_data_to_write - def write_entry_to_buf(self, data, entrycount,nvs_obj): encr_data = bytearray() - if self.is_encrypt: + if nvs_obj.encrypt: encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj) encr_data[0:len(encr_data_ret)] = encr_data_ret data = encr_data @@ -226,7 +230,6 @@ def write_entry_to_buf(self, data, entrycount,nvs_obj): self.write_bitmaparray() self.entry_num += 1 - def set_crc_header(self, entry_struct): crc_data = bytearray(b'28') crc_data[0:4] = entry_struct[0:4] @@ -236,7 +239,6 @@ def set_crc_header(self, entry_struct): struct.pack_into('=0, "Page overflow!!" + tailroom = (Page.PAGE_PARAMS['max_entries'] - self.entry_num - 1) * Page.SINGLE_ENTRY_SIZE + assert tailroom >= 0, 'Page overflow!!' # Split the binary data into two and store a chunk of available size onto curr page if tailroom < remaining_size: @@ -266,7 +268,7 @@ def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, # Calculate no. of entries data chunk will require datachunk_rounded_size = (chunk_size + 31) & ~31 datachunk_entry_count = datachunk_rounded_size // 32 - datachunk_total_entry_count = datachunk_entry_count + 1 # +1 for the entry header + datachunk_total_entry_count = datachunk_entry_count + 1 # +1 for the entry header # Set Span entry_struct[2] = datachunk_total_entry_count @@ -276,7 +278,7 @@ def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, entry_struct[3] = chunk_index # Set data chunk - data_chunk = data[offset:offset + chunk_size] + data_chunk = data[offset:offset + chunk_size] # Compute CRC of data chunk struct.pack_into(' Page.PAGE_PARAMS["max_old_blob_size"]: - raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION1_PRINT,key)) - - if version == Page.VERSION2: - if encoding == "string": - if datalen > Page.PAGE_PARAMS["max_new_blob_size"]: - raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION2_PRINT,key)) + if datalen > Page.PAGE_PARAMS['max_old_blob_size']: + if self.version == Page.VERSION1: + raise InputError(' Input File: Size (%d) exceeds max allowed length `%s` bytes for key `%s`.' + % (datalen, Page.PAGE_PARAMS['max_old_blob_size'], key)) + else: + if encoding == 'string': + raise InputError(' Input File: Size (%d) exceeds max allowed length `%s` bytes for key `%s`.' + % (datalen, Page.PAGE_PARAMS['max_old_blob_size'], key)) # Calculate no. of entries data will require rounded_size = (datalen + 31) & ~31 data_entry_count = rounded_size // 32 - total_entry_count = data_entry_count + 1 # +1 for the entry header + total_entry_count = data_entry_count + 1 # +1 for the entry header # Check if page is already full and new page is needed to be created right away - if version == Page.VERSION1: - if encoding in ["string", "hex2bin", "binary", "base64"]: - if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]: - raise PageFullError() - else: - if encoding == "string": - if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]: - raise PageFullError() + if self.entry_num >= Page.PAGE_PARAMS['max_entries']: + raise PageFullError() + elif (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS['max_entries']: + if not (self.version == Page.VERSION2 and encoding in ['hex2bin', 'binary', 'base64']): + raise PageFullError() # Entry header - entry_struct = bytearray(b'\xff')*32 + entry_struct = bytearray(b'\xff') * 32 # Set Namespace Index entry_struct[0] = ns_index # Set Span - if version == Page.VERSION2: - if encoding == "string": + if self.version == Page.VERSION2: + if encoding == 'string': entry_struct[2] = data_entry_count + 1 # Set Chunk Index chunk_index = Page.CHUNK_ANY @@ -408,51 +401,58 @@ def write_varlen_data(self, key, data, encoding, ns_index,nvs_obj): entry_struct[8:8 + len(key)] = key.encode() # set Type - if encoding == "string": + if encoding == 'string': entry_struct[1] = Page.SZ - elif encoding in ["hex2bin", "binary", "base64"]: + elif encoding in ['hex2bin', 'binary', 'base64']: entry_struct[1] = Page.BLOB - if version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]): - entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\ - datalen,total_entry_count, encoding, nvs_obj) + if self.version == Page.VERSION2 and (encoding in ['hex2bin', 'binary', 'base64']): + entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data, + datalen,total_entry_count, encoding, nvs_obj) else: self.write_single_page_entry(entry_struct, data, datalen, data_entry_count, nvs_obj) - - """ Low-level function to write data of primitive type into page buffer. """ def write_primitive_data(self, key, data, encoding, ns_index,nvs_obj): # Check if entry exceeds max number of entries allowed per page - if self.entry_num >= Page.PAGE_PARAMS["max_entries"]: + if self.entry_num >= Page.PAGE_PARAMS['max_entries']: raise PageFullError() - entry_struct = bytearray(b'\xff')*32 - entry_struct[0] = ns_index # namespace index - entry_struct[2] = 0x01 # Span + entry_struct = bytearray(b'\xff') * 32 + entry_struct[0] = ns_index # namespace index + entry_struct[2] = 0x01 # Span chunk_index = Page.CHUNK_ANY entry_struct[3] = chunk_index # write key - key_array = b'\x00' *16 + key_array = b'\x00' * 16 entry_struct[8:24] = key_array entry_struct[8:8 + len(key)] = key.encode() - if encoding == "u8": + if encoding == 'u8': entry_struct[1] = Page.U8 struct.pack_into('/ + :param outdir: Target output dir to store files + :param filepath: Path of target file + ''' + bin_ext = '.bin' + # Expand if tilde(~) provided in path + outdir = os.path.expanduser(outdir) + + if filepath: + key_file_name, ext = os.path.splitext(filepath) + if not ext: + filepath = key_file_name + bin_ext + elif bin_ext not in ext: + sys.exit('Error: `%s`. Only `%s` extension allowed.' % (filepath, bin_ext)) + + # Create dir if does not exist + if not (os.path.isdir(outdir)): + distutils.dir_util.mkpath(outdir) + + filedir, filename = os.path.split(filepath) + filedir = os.path.join(outdir,filedir,'') + if filedir and not os.path.isdir(filedir): + distutils.dir_util.mkpath(filedir) + + if os.path.isabs(filepath): + if not outdir == os.getcwd(): + print('\nWarning: `%s` \n\t==> absolute path given so outdir is ignored for this file.' % filepath) + # Set to empty as outdir is ignored here + outdir = '' + + # Set full path - outdir + filename + filepath = os.path.join(outdir, '') + filepath + + return outdir, filepath + + +def encrypt(args): + ''' + Generate encrypted NVS Partition + :param args: Command line arguments given + ''' + key = None + bin_ext = '.bin' + + check_size(args.size) + if (args.keygen is False) and (not args.inputkey): + sys.exit('Error. --keygen or --inputkey argument needed.') + elif args.keygen and args.inputkey: + sys.exit('Error. --keygen and --inputkey both are not allowed.') + elif not args.keygen and args.keyfile: + print('\nWarning:','--inputkey argument is given. --keyfile argument will be ignored...') + + if args.inputkey: + # Check if key file has .bin extension + filename, ext = os.path.splitext(args.inputkey) + if bin_ext not in ext: + sys.exit('Error: `%s`. Only `%s` extension allowed.' % (args.inputkey, bin_ext)) + key = bytearray() + with open(args.inputkey, 'rb') as key_f: + key = key_f.read(64) + + # Generate encrypted NVS Partition + generate(args, is_encr_enabled=True, encr_key=key) + + +def decrypt_data(data_input, decr_key, page_num, entry_no, entry_size): + ''' + Decrypt NVS data entry + ''' + page_max_size = 4096 + first_entry_offset = 64 + init_tweak_val = '0' + tweak_len_needed = 32 # in hex + tweak_tmp = '' + + data_input = binascii.hexlify(data_input) + rel_addr = page_num * page_max_size + first_entry_offset + + # Set tweak value + offset = entry_no * entry_size + addr = hex(rel_addr + offset)[2:] + addr_len = len(addr) + if addr_len > 2: + if not addr_len % 2: + addr_tmp = addr + else: + addr_tmp = init_tweak_val + addr + tweak_tmp = reverse_hexbytes(addr_tmp) + tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) + else: + tweak_val = addr + (init_tweak_val * (tweak_len_needed - len(addr))) + + if type(data_input) == bytes: + data_input = data_input.decode() + + # Decrypt 32 bytes of data using AES-XTS decryption + backend = default_backend() + plain_text = codecs.decode(data_input, 'hex') + tweak = codecs.decode(tweak_val, 'hex') + cipher = Cipher(algorithms.AES(decr_key), modes.XTS(tweak), backend=backend) + decryptor = cipher.decryptor() + decrypted_data = decryptor.update(plain_text) + + return decrypted_data + + +def decrypt(args): + ''' + Decrypt encrypted NVS Partition + :param args: Command line arguments given + ''' + bin_ext = '.bin' + nvs_read_bytes = 32 + decrypted_entry_no = 0 + file_entry_no = 0 + page_num = 0 + page_max_size = 4096 + start_entry_offset = 0 + empty_data_entry = bytearray(b'\xff') * nvs_read_bytes + + # Check if key file has .bin extension + input_files = [args.input, args.key, args.output] + for filepath in input_files: + filename, ext = os.path.splitext(filepath) + if bin_ext not in ext: + sys.exit('Error: `%s`. Only `%s` extension allowed.' % (filepath, bin_ext)) + with open(args.key,'rb') as decr_key_file: + decr_key = decr_key_file.read(64) + + args.outdir, args.output = set_target_filepath(args.outdir, args.output) + + output_buf = bytearray(b'\xff') + + with open(args.input, 'rb') as input_file, open(args.output,'wb') as output_file: + while True: + if file_entry_no == 128: + decrypted_entry_no = 0 + file_entry_no = 0 + page_num += 1 + data_entry = input_file.read(nvs_read_bytes) + if not data_entry: + break + if data_entry != empty_data_entry and file_entry_no not in [0,1]: + data_entry = decrypt_data(data_entry, decr_key, page_num, decrypted_entry_no, nvs_read_bytes) + decrypted_entry_no += 1 + write_entry_no = ((page_num * page_max_size) + file_entry_no) + start_idx = start_entry_offset + (write_entry_no * nvs_read_bytes) + end_idx = nvs_read_bytes + output_buf[start_idx:end_idx] = data_entry + file_entry_no += 1 + start_entry_offset += nvs_read_bytes + output_file.write(output_buf) + + print('\nCreated NVS decrypted binary: ===>', args.output) + + +def generate_key(args): + ''' + Generate encryption keys + :param args: Command line arguments given + ''' + page_max_size = 4096 + keys_dir = 'keys' + output_keyfile = None + bin_ext = '.bin' + + if not args.keyfile: + timestamp = datetime.datetime.now().strftime('%m-%d_%H-%M') + args.keyfile = 'keys-' + timestamp + bin_ext + + keys_outdir = os.path.join(args.outdir,keys_dir, '') + # Create keys/ dir in if does not exist + if not (os.path.isdir(keys_outdir)): + distutils.dir_util.mkpath(keys_outdir) + keys_outdir, output_keyfile = set_target_filepath(keys_outdir, args.keyfile) + + key = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip() + encr_key_bytes = codecs.decode(key, 'hex') + key_len = len(encr_key_bytes) + + keys_buf = bytearray(b'\xff') * page_max_size + keys_buf[0:key_len] = encr_key_bytes + crc_data = keys_buf[0:key_len] + crc_data = bytes(crc_data) + crc = zlib.crc32(crc_data, 0xFFFFFFFF) + struct.pack_into(' ', output_keyfile) + + return key + + +def generate(args, is_encr_enabled=False, encr_key=None): + ''' + Generate NVS Partition + :param args: Command line arguments given + :param is_encr_enabled: Encryption enabled/disabled + :param encr_key: Key to encrypt NVS partition + ''' + is_dir_new = False + bin_ext = '.bin' + + input_size = check_size(args.size) + if args.version == 1: + args.version = Page.VERSION1 + elif args.version == 2: + args.version = Page.VERSION2 + + # Check if key file has .bin extension + filename, ext = os.path.splitext(args.output) + if bin_ext not in ext: + sys.exit('Error: `%s`. Only `.bin` extension allowed.' % args.output) + args.outdir, args.output = set_target_filepath(args.outdir, args.output) + + if is_encr_enabled and not encr_key: + encr_key = generate_key(args) + + input_file = open(args.input, 'rt', encoding='utf8') + output_file = open(args.output, 'wb') + + with open(args.input, 'rt', encoding='utf8') as input_file,\ + open(args.output, 'wb') as output_file,\ + nvs_open(output_file, input_size, args.version, is_encrypt=is_encr_enabled, key=encr_key) as nvs_obj: + + if nvs_obj.version == Page.VERSION1: + version_set = VERSION1_PRINT + else: + version_set = VERSION2_PRINT -def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, key_file=None, version_no=None): - """ Wrapper to generate nvs partition binary + print('\nCreating NVS binary with version:', version_set) - :param input_filename: Name of input file containing data - :param output_filename: Name of output file to store generated binary - :param input_part_size: Size of partition in bytes (must be multiple of 4096) - :param is_key_gen: Enable encryption key generation in encryption mode - :param encrypt_mode: Enable/Disable encryption mode - :param key_file: Input file having encryption keys in encryption mode - :param version_no: Format Version number - :return: None - """ + line = input_file.readline().strip() - global key_input, key_len_needed + # Comments are skipped + while line.startswith('#'): + line = input_file.readline().strip() + if not isinstance(line, str): + line = line.encode('utf-8') - key_len_needed = 64 - key_input = bytearray() + header = line.split(',') - if key_gen: - key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip() - elif key_file: - with open(key_file, 'rb') as key_f: - key_input = key_f.read(64) + while True: + line = input_file.readline().strip() + if not isinstance(line, str): + line = line.encode('utf-8') - if all(arg is not None for arg in [input_filename, output_filename, input_size]): - input_file = open(input_filename, 'rt', encoding='utf8') - output_file = open(output_filename, 'wb') + value = line.split(',') + if len(value) == 1 and '' in value: + break - with nvs_open(output_file, input_size) as nvs_obj: - reader = csv.DictReader(input_file, delimiter=',') - for row in reader: - try: - write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"]) - except (InputError) as e: - print(e) - input_file.close() - output_file.close() - sys.exit(-2) - - input_file.close() - output_file.close() - - - if key_gen: - keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"] - key_bytes = bytearray() - if len(key_input) == key_len_needed: - key_bytes = key_input - else: - key_bytes = codecs.decode(key_input, 'hex') - key_len = len(key_bytes) - keys_page_buf[0:key_len] = key_bytes - crc_data = keys_page_buf[0:key_len] - crc_data = bytes(crc_data) - crc = zlib.crc32(crc_data, 0xFFFFFFFF) - struct.pack_into(' 15: + raise InputError('Length of key `{}` should be <= 15 characters.'.format(data['key'])) + write_entry(nvs_obj, data['key'], data['type'], data['encoding'], data['value']) + except InputError as e: + print(e) + filedir, filename = os.path.split(args.output) + if filename: + print('\nWarning: NVS binary not created...') + os.remove(args.output) + if is_dir_new and not filedir == os.getcwd(): + print('\nWarning: Output dir not created...') + os.rmdir(filedir) + sys.exit(-2) + + print('\nCreated NVS binary: ===>', args.output) def main(): - parser = argparse.ArgumentParser(description="ESP32 NVS partition generation utility") - nvs_part_gen_group = parser.add_argument_group('To generate NVS partition') - nvs_part_gen_group.add_argument( - "--input", - help="Path to CSV file to parse.", - default=None) - - nvs_part_gen_group.add_argument( - "--output", - help='Path to output converted binary file.', - default=None) - - nvs_part_gen_group.add_argument( - "--size", - help='Size of NVS Partition in bytes (must be multiple of 4096)') - + parser = argparse.ArgumentParser(description='\nESP NVS partition generation utility', formatter_class=argparse.RawTextHelpFormatter) + subparser = parser.add_subparsers(title='Commands', + dest='command', + help='\nRun nvs_partition_gen.py {command} -h for additional help\n\n') + + parser_gen = subparser.add_parser('generate', + help='Generate NVS partition', + formatter_class=argparse.RawTextHelpFormatter) + parser_gen.set_defaults(func=generate) + parser_gen.add_argument('input', + default=None, + help='Path to CSV file to parse') + parser_gen.add_argument('output', + default=None, + help='Path to output NVS binary file') + parser_gen.add_argument('size', + default=None, + help='Size of NVS partition in bytes\ + \n(must be multiple of 4096)') + parser_gen.add_argument('--version', + choices=[1,2], + default=2, + type=int, + help='''Set multipage blob version.\ + \nVersion 1 - Multipage blob support disabled.\ + \nVersion 2 - Multipage blob support enabled.\ + \nDefault: Version 2''') + parser_gen.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created\ + \n(Default: current directory)') + parser_gen_key = subparser.add_parser('generate-key', + help='Generate keys for encryption', + formatter_class=argparse.RawTextHelpFormatter) + parser_gen_key.set_defaults(func=generate_key) + parser_gen_key.add_argument('--keyfile', + default=None, + help='Path to output encryption keys file') + parser_gen_key.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created.\ + \n(Default: current directory)') + parser_encr = subparser.add_parser('encrypt', + help='Generate NVS encrypted partition', + formatter_class=argparse.RawTextHelpFormatter) + parser_encr.set_defaults(func=encrypt) + parser_encr.add_argument('input', + default=None, + help='Path to CSV file to parse') + parser_encr.add_argument('output', + default=None, + help='Path to output NVS binary file') + parser_encr.add_argument('size', + default=None, + help='Size of NVS partition in bytes\ + \n(must be multiple of 4096)') + parser_encr.add_argument('--version', + choices=[1,2], + default=2, + type=int, + help='''Set multipage blob version.\ + \nVersion 1 - Multipage blob support disabled.\ + \nVersion 2 - Multipage blob support enabled.\ + \nDefault: Version 2''') + parser_encr.add_argument('--keygen', + action='store_true', + default=False, + help='Generates key for encrypting NVS partition') + parser_encr.add_argument('--keyfile', + default=None, + help='Path to output encryption keys file') + parser_encr.add_argument('--inputkey', + default=None, + help='File having key for encrypting NVS partition') + parser_encr.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created.\ + \n(Default: current directory)') + parser_decr = subparser.add_parser('decrypt', + help='Decrypt NVS encrypted partition', + formatter_class=argparse.RawTextHelpFormatter) + parser_decr.set_defaults(func=decrypt) + parser_decr.add_argument('input', + default=None, + help='Path to encrypted NVS partition file to parse') + parser_decr.add_argument('key', + default=None, + help='Path to file having keys for decryption') + parser_decr.add_argument('output', + default=None, + help='Path to output decrypted binary file') + parser_decr.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created.\ + \n(Default: current directory)') args = parser.parse_args() - input_filename = args.input - output_filename = args.output - part_size = args.size - version_no = 'v1' - is_key_gen = 'false' - is_encrypt_data = 'false' - key_file = None - - print_arg_str = "Invalid.\nTo generate nvs partition binary --input, --output and --size arguments are mandatory." - - check_input_args(input_filename,output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no, print_arg_str) - nvs_part_gen(input_filename, output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no) + args.func(args) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/components/nvs_flash/nvs_partition_generator/sample_multipage_blob.csv b/components/nvs_flash/nvs_partition_generator/sample_multipage_blob.csv index 384ac6919..e9546cb41 100644 --- a/components/nvs_flash/nvs_partition_generator/sample_multipage_blob.csv +++ b/components/nvs_flash/nvs_partition_generator/sample_multipage_blob.csv @@ -1,3 +1,4 @@ +# Sample csv file key,type,encoding,value dummyNamespace,namespace,, dummyU8Key,data,u8,127 diff --git a/components/nvs_flash/nvs_partition_generator/sample_singlepage_blob.csv b/components/nvs_flash/nvs_partition_generator/sample_singlepage_blob.csv index c99f513cf..10d3cc635 100644 --- a/components/nvs_flash/nvs_partition_generator/sample_singlepage_blob.csv +++ b/components/nvs_flash/nvs_partition_generator/sample_singlepage_blob.csv @@ -1,3 +1,4 @@ +# Sample csv file key,type,encoding,value dummyNamespace,namespace,, dummyU8Key,data,u8,127 diff --git a/components/nvs_flash/src/compressed_enum_table.hpp b/components/nvs_flash/src/compressed_enum_table.hpp index 319d86a45..dcf9d0999 100644 --- a/components/nvs_flash/src/compressed_enum_table.hpp +++ b/components/nvs_flash/src/compressed_enum_table.hpp @@ -35,7 +35,7 @@ class CompressedEnumTable Tenum get(size_t index) const { - assert(index >= 0 && index < Nitems); + assert(index < Nitems); size_t wordIndex = index / ITEMS_PER_WORD; size_t offset = (index % ITEMS_PER_WORD) * Nbits; @@ -44,7 +44,7 @@ class CompressedEnumTable void set(size_t index, Tenum val) { - assert(index >= 0 && index < Nitems); + assert(index < Nitems); size_t wordIndex = index / ITEMS_PER_WORD; size_t offset = (index % ITEMS_PER_WORD) * Nbits; diff --git a/components/nvs_flash/src/intrusive_list.h b/components/nvs_flash/src/intrusive_list.h index fc92442cd..bb580502e 100644 --- a/components/nvs_flash/src/intrusive_list.h +++ b/components/nvs_flash/src/intrusive_list.h @@ -15,6 +15,7 @@ #define intrusive_list_h #include +#include template class intrusive_list; @@ -229,8 +230,7 @@ class intrusive_list { return mSize == 0; } - - + void clear() { while (mFirst) { @@ -238,6 +238,16 @@ class intrusive_list } } + void clearAndFreeNodes() + { + while (mFirst) { + auto tmp = mFirst; + erase(mFirst); + delete tmp; + } + } + + protected: T* mFirst = nullptr; T* mLast = nullptr; diff --git a/components/nvs_flash/src/nvs_api.cpp b/components/nvs_flash/src/nvs_api.cpp index 15f76d1dd..8d0848c48 100644 --- a/components/nvs_flash/src/nvs_api.cpp +++ b/components/nvs_flash/src/nvs_api.cpp @@ -11,64 +11,64 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "sdkconfig.h" #include "nvs.hpp" #include "nvs_flash.h" #include "nvs_storage.hpp" #include "intrusive_list.h" #include "nvs_platform.hpp" +#include "nvs_partition_manager.hpp" #include "esp_partition.h" -#include "sdkconfig.h" +#include +#include "nvs_handle_simple.hpp" +#include "esp_err.h" + +#ifdef LINUX_TARGET +#include "crc.h" +#define ESP_LOGD(...) +#else // LINUX_TARGET +#include -#ifdef ESP_PLATFORM // Uncomment this line to force output from this module // #define LOG_LOCAL_LEVEL ESP_LOG_DEBUG #include "esp_log.h" static const char* TAG = "nvs"; -#else -#define ESP_LOGD(...) -#endif +#endif // ! LINUX_TARGET -class HandleEntry : public intrusive_list_node -{ - static uint32_t s_nvs_next_handle; +class NVSHandleEntry : public intrusive_list_node { public: - HandleEntry() {} + NVSHandleEntry(nvs::NVSHandleSimple *handle, const char* part_name) + : nvs_handle(handle), + mHandle(++s_nvs_next_handle), + handle_part_name(part_name) { } - HandleEntry(bool readOnly, uint8_t nsIndex, nvs::Storage* StoragePtr) : - mHandle(++s_nvs_next_handle), // Begin the handle value with 1 - mReadOnly(readOnly), - mNsIndex(nsIndex), - mStoragePtr(StoragePtr) - { + ~NVSHandleEntry() { + delete nvs_handle; } - nvs_handle mHandle; - uint8_t mReadOnly; - uint8_t mNsIndex; - nvs::Storage* mStoragePtr; + nvs::NVSHandleSimple *nvs_handle; + nvs_handle_t mHandle; + const char* handle_part_name; +private: + static uint32_t s_nvs_next_handle; }; -#ifdef ESP_PLATFORM -SemaphoreHandle_t nvs::Lock::mSemaphore = NULL; -#endif +uint32_t NVSHandleEntry::s_nvs_next_handle; + +extern "C" void nvs_dump(const char *partName); + +#ifndef LINUX_TARGET +SemaphoreHandle_t nvs::Lock::mSemaphore = nullptr; +#endif // ! LINUX_TARGET using namespace std; using namespace nvs; -static intrusive_list s_nvs_handles; -uint32_t HandleEntry::s_nvs_next_handle; -static intrusive_list s_nvs_storage_list; +static intrusive_list s_nvs_handles; static nvs::Storage* lookup_storage_from_name(const char *name) { - auto it = find_if(begin(s_nvs_storage_list), end(s_nvs_storage_list), [=](Storage& e) -> bool { - return (strcmp(e.getPartName(), name) == 0); - }); - - if (it == end(s_nvs_storage_list)) { - return NULL; - } - return it; + return NVSPartitionManager::get_instance()->lookup_storage_from_name(name); } extern "C" void nvs_dump(const char *partName) @@ -77,359 +77,415 @@ extern "C" void nvs_dump(const char *partName) nvs::Storage* pStorage; pStorage = lookup_storage_from_name(partName); - if (pStorage == NULL) { + if (pStorage == nullptr) { return; } pStorage->debugDump(); - return; } -extern "C" esp_err_t nvs_flash_init_custom(const char *partName, uint32_t baseSector, uint32_t sectorCount) +static esp_err_t close_handles_and_deinit(const char* part_name) { - ESP_LOGD(TAG, "nvs_flash_init_custom partition=%s start=%d count=%d", partName, baseSector, sectorCount); - nvs::Storage* new_storage = NULL; - nvs::Storage* storage = lookup_storage_from_name(partName); - if (storage == NULL) { - new_storage = new nvs::Storage((const char *)partName); - storage = new_storage; - } + auto belongs_to_part = [=](NVSHandleEntry& e) -> bool { + return strncmp(e.nvs_handle->get_partition_name(), part_name, NVS_PART_NAME_MAX_SIZE) == 0; + }; - esp_err_t err = storage->init(baseSector, sectorCount); - if (new_storage != NULL) { - if (err == ESP_OK) { - s_nvs_storage_list.push_back(new_storage); - } else { - delete new_storage; - } + auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), belongs_to_part); + + while (it != end(s_nvs_handles)) { + s_nvs_handles.erase(it); + it = find_if(begin(s_nvs_handles), end(s_nvs_handles), belongs_to_part); } - return err; + + // Deinit partition + return NVSPartitionManager::get_instance()->deinit_partition(part_name); } -#ifdef ESP_PLATFORM -extern "C" esp_err_t nvs_flash_init_partition(const char *part_name) +extern "C" esp_err_t nvs_flash_init_partition_ptr(const esp_partition_t *partition) { Lock::init(); Lock lock; - nvs::Storage* mStorage; - mStorage = lookup_storage_from_name(part_name); - if (mStorage) { - return ESP_OK; + if (partition == nullptr) { + return ESP_ERR_INVALID_ARG; } - const esp_partition_t* partition = esp_partition_find_first( - ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, part_name); - if (partition == NULL) { - return ESP_ERR_NOT_FOUND; + NVSPartition *part = new (std::nothrow) NVSPartition(partition); + if (part == nullptr) { + return ESP_ERR_NO_MEM; } - return nvs_flash_init_custom(part_name, partition->address / SPI_FLASH_SEC_SIZE, + esp_err_t init_res = NVSPartitionManager::get_instance()->init_custom(part, + partition->address / SPI_FLASH_SEC_SIZE, partition->size / SPI_FLASH_SEC_SIZE); -} -extern "C" esp_err_t nvs_flash_init(void) -{ - return nvs_flash_init_partition(NVS_DEFAULT_PART_NAME); + if (init_res != ESP_OK) { + delete part; + } + + return init_res; } -extern "C" esp_err_t nvs_flash_deinit_partition(const char* partition_name) +#ifndef LINUX_TARGET +extern "C" esp_err_t nvs_flash_init_partition(const char *part_name) { Lock::init(); Lock lock; - nvs::Storage* storage = lookup_storage_from_name(partition_name); - if (!storage) { - return ESP_ERR_NVS_NOT_INITIALIZED; + return NVSPartitionManager::get_instance()->init_partition(part_name); +} + +extern "C" esp_err_t nvs_flash_init(void) +{ +#ifdef CONFIG_NVS_ENCRYPTION + esp_err_t ret = ESP_FAIL; + const esp_partition_t *key_part = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS_KEYS, NULL); + if (key_part == NULL) { + ESP_LOGE(TAG, "CONFIG_NVS_ENCRYPTION is enabled, but no partition with subtype nvs_keys found in the partition table."); + return ret; } - /* Clean up handles related to the storage being deinitialized */ - auto it = s_nvs_handles.begin(); - auto next = it; - while(it != s_nvs_handles.end()) { - next++; - if (it->mStoragePtr == storage) { - ESP_LOGD(TAG, "Deleting handle %d (ns=%d) related to partition \"%s\" (missing call to nvs_close?)", - it->mHandle, it->mNsIndex, partition_name); - s_nvs_handles.erase(it); - delete static_cast(it); + nvs_sec_cfg_t cfg = {}; + ret = nvs_flash_read_security_cfg(key_part, &cfg); + if (ret == ESP_ERR_NVS_KEYS_NOT_INITIALIZED) { + ESP_LOGI(TAG, "NVS key partition empty, generating keys"); + ret = nvs_flash_generate_keys(key_part, &cfg); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to generate keys: [0x%02X] (%s)", ret, esp_err_to_name(ret)); + return ret; } - it = next; + } else if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to read NVS security cfg: [0x%02X] (%s)", ret, esp_err_to_name(ret)); + return ret; } - /* Finally delete the storage itself */ - s_nvs_storage_list.erase(storage); - delete storage; + ret = nvs_flash_secure_init_partition(NVS_DEFAULT_PART_NAME, &cfg); + if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_LOGE(TAG, "Failed to initialize NVS partition: [0x%02X] (%s)", ret, esp_err_to_name(ret)); + return ret; + } + ESP_LOGI(TAG, "NVS partition \"%s\" is encrypted.", NVS_DEFAULT_PART_NAME); + return ret; +#else // CONFIG_NVS_ENCRYPTION + return nvs_flash_init_partition(NVS_DEFAULT_PART_NAME); +#endif +} - return ESP_OK; +#ifdef CONFIG_NVS_ENCRYPTION +extern "C" esp_err_t nvs_flash_secure_init_partition(const char *part_name, nvs_sec_cfg_t* cfg) +{ + Lock::init(); + Lock lock; + + return NVSPartitionManager::get_instance()->secure_init_partition(part_name, cfg); } -extern "C" esp_err_t nvs_flash_deinit(void) +extern "C" esp_err_t nvs_flash_secure_init(nvs_sec_cfg_t* cfg) { - return nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME); + return nvs_flash_secure_init_partition(NVS_DEFAULT_PART_NAME, cfg); } +#endif extern "C" esp_err_t nvs_flash_erase_partition(const char *part_name) { + Lock::init(); + Lock lock; + + // if the partition is initialized, uninitialize it first + if (NVSPartitionManager::get_instance()->lookup_storage_from_name(part_name)) { + esp_err_t err = close_handles_and_deinit(part_name); + + // only hypothetical/future case, deinit_partition() only fails if partition is uninitialized + if (err != ESP_OK) { + return err; + } + } + const esp_partition_t* partition = esp_partition_find_first( ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, part_name); - if (partition == NULL) { + if (partition == nullptr) { return ESP_ERR_NOT_FOUND; } return esp_partition_erase_range(partition, 0, partition->size); } -extern "C" esp_err_t nvs_flash_erase() +extern "C" esp_err_t nvs_flash_erase_partition_ptr(const esp_partition_t *partition) +{ + Lock::init(); + Lock lock; + + if (partition == nullptr) { + return ESP_ERR_INVALID_ARG; + } + + // if the partition is initialized, uninitialize it first + if (NVSPartitionManager::get_instance()->lookup_storage_from_name(partition->label)) { + const esp_err_t err = close_handles_and_deinit(partition->label); + + // only hypothetical/future case, deinit_partition() only fails if partition is uninitialized + if (err != ESP_OK) { + return err; + } + } + + return esp_partition_erase_range(partition, 0, partition->size); +} + +extern "C" esp_err_t nvs_flash_erase(void) { return nvs_flash_erase_partition(NVS_DEFAULT_PART_NAME); } -#endif +#endif // ! LINUX_TARGET -static esp_err_t nvs_find_ns_handle(nvs_handle handle, HandleEntry& entry) +extern "C" esp_err_t nvs_flash_deinit_partition(const char* partition_name) { - auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](HandleEntry& e) -> bool { - return e.mHandle == handle; + Lock::init(); + Lock lock; + + return close_handles_and_deinit(partition_name); +} + +extern "C" esp_err_t nvs_flash_deinit(void) +{ + return nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME); +} + +static esp_err_t nvs_find_ns_handle(nvs_handle_t c_handle, NVSHandleSimple** handle) +{ + auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](NVSHandleEntry& e) -> bool { + return e.mHandle == c_handle; }); if (it == end(s_nvs_handles)) { return ESP_ERR_NVS_INVALID_HANDLE; } - entry = *it; + *handle = it->nvs_handle; return ESP_OK; } -extern "C" esp_err_t nvs_open_from_partition(const char *part_name, const char* name, nvs_open_mode open_mode, nvs_handle *out_handle) +extern "C" esp_err_t nvs_open_from_partition(const char *part_name, const char* name, nvs_open_mode_t open_mode, nvs_handle_t *out_handle) { Lock lock; ESP_LOGD(TAG, "%s %s %d", __func__, name, open_mode); - uint8_t nsIndex; - nvs::Storage* sHandle; - - sHandle = lookup_storage_from_name(part_name); - if (sHandle == NULL) { - return ESP_ERR_NVS_PART_NOT_FOUND; - } - esp_err_t err = sHandle->createOrOpenNamespace(name, open_mode == NVS_READWRITE, nsIndex); - if (err != ESP_OK) { - return err; + NVSHandleSimple *handle; + esp_err_t result = NVSPartitionManager::get_instance()->open_handle(part_name, name, open_mode, &handle); + if (result == ESP_OK) { + NVSHandleEntry *entry = new (std::nothrow) NVSHandleEntry(handle, part_name); + if (entry) { + s_nvs_handles.push_back(entry); + *out_handle = entry->mHandle; + } else { + delete handle; + return ESP_ERR_NO_MEM; + } } - HandleEntry *handle_entry = new HandleEntry(open_mode==NVS_READONLY, nsIndex, sHandle); - s_nvs_handles.push_back(handle_entry); - - *out_handle = handle_entry->mHandle; - - return ESP_OK; + return result; } -extern "C" esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_handle) +extern "C" esp_err_t nvs_open(const char* name, nvs_open_mode_t open_mode, nvs_handle_t *out_handle) { - if (s_nvs_storage_list.size() == 0) { - return ESP_ERR_NVS_NOT_INITIALIZED; - } - return nvs_open_from_partition(NVS_DEFAULT_PART_NAME, name, open_mode, out_handle); } -extern "C" void nvs_close(nvs_handle handle) +extern "C" void nvs_close(nvs_handle_t handle) { Lock lock; ESP_LOGD(TAG, "%s %d", __func__, handle); - auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](HandleEntry& e) -> bool { + auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](NVSHandleEntry& e) -> bool { return e.mHandle == handle; }); if (it == end(s_nvs_handles)) { return; } s_nvs_handles.erase(it); - delete static_cast(it); + delete static_cast(it); } -extern "C" esp_err_t nvs_erase_key(nvs_handle handle, const char* key) +extern "C" esp_err_t nvs_erase_key(nvs_handle_t c_handle, const char* key) { Lock lock; ESP_LOGD(TAG, "%s %s\r\n", __func__, key); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } - if (entry.mReadOnly) { - return ESP_ERR_NVS_READ_ONLY; - } - return entry.mStoragePtr->eraseItem(entry.mNsIndex, key); + + return handle->erase_item(key); } -extern "C" esp_err_t nvs_erase_all(nvs_handle handle) +extern "C" esp_err_t nvs_erase_all(nvs_handle_t c_handle) { Lock lock; ESP_LOGD(TAG, "%s\r\n", __func__); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } - if (entry.mReadOnly) { - return ESP_ERR_NVS_READ_ONLY; - } - return entry.mStoragePtr->eraseNamespace(entry.mNsIndex); + + return handle->erase_all(); } template -static esp_err_t nvs_set(nvs_handle handle, const char* key, T value) +static esp_err_t nvs_set(nvs_handle_t c_handle, const char* key, T value) { Lock lock; ESP_LOGD(TAG, "%s %s %d %d", __func__, key, sizeof(T), (uint32_t) value); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } - if (entry.mReadOnly) { - return ESP_ERR_NVS_READ_ONLY; - } - return entry.mStoragePtr->writeItem(entry.mNsIndex, key, value); + + return handle->set_item(key, value); } -extern "C" esp_err_t nvs_set_i8 (nvs_handle handle, const char* key, int8_t value) +extern "C" esp_err_t nvs_set_i8 (nvs_handle_t handle, const char* key, int8_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_u8 (nvs_handle handle, const char* key, uint8_t value) +extern "C" esp_err_t nvs_set_u8 (nvs_handle_t handle, const char* key, uint8_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_i16 (nvs_handle handle, const char* key, int16_t value) +extern "C" esp_err_t nvs_set_i16 (nvs_handle_t handle, const char* key, int16_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_u16 (nvs_handle handle, const char* key, uint16_t value) +extern "C" esp_err_t nvs_set_u16 (nvs_handle_t handle, const char* key, uint16_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_i32 (nvs_handle handle, const char* key, int32_t value) +extern "C" esp_err_t nvs_set_i32 (nvs_handle_t handle, const char* key, int32_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_u32 (nvs_handle handle, const char* key, uint32_t value) +extern "C" esp_err_t nvs_set_u32 (nvs_handle_t handle, const char* key, uint32_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_i64 (nvs_handle handle, const char* key, int64_t value) +extern "C" esp_err_t nvs_set_i64 (nvs_handle_t handle, const char* key, int64_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_set_u64 (nvs_handle handle, const char* key, uint64_t value) +extern "C" esp_err_t nvs_set_u64 (nvs_handle_t handle, const char* key, uint64_t value) { return nvs_set(handle, key, value); } -extern "C" esp_err_t nvs_commit(nvs_handle handle) +extern "C" esp_err_t nvs_commit(nvs_handle_t c_handle) { Lock lock; // no-op for now, to be used when intermediate cache is added - HandleEntry entry; - return nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); + if (err != ESP_OK) { + return err; + } + return handle->commit(); } -extern "C" esp_err_t nvs_set_str(nvs_handle handle, const char* key, const char* value) +extern "C" esp_err_t nvs_set_str(nvs_handle_t c_handle, const char* key, const char* value) { Lock lock; ESP_LOGD(TAG, "%s %s %s", __func__, key, value); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } - return entry.mStoragePtr->writeItem(entry.mNsIndex, nvs::ItemType::SZ, key, value, strlen(value) + 1); + return handle->set_string(key, value); } -extern "C" esp_err_t nvs_set_blob(nvs_handle handle, const char* key, const void* value, size_t length) +extern "C" esp_err_t nvs_set_blob(nvs_handle_t c_handle, const char* key, const void* value, size_t length) { Lock lock; ESP_LOGD(TAG, "%s %s %d", __func__, key, length); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } - return entry.mStoragePtr->writeItem(entry.mNsIndex, nvs::ItemType::BLOB, key, value, length); + return handle->set_blob(key, value, length); } template -static esp_err_t nvs_get(nvs_handle handle, const char* key, T* out_value) +static esp_err_t nvs_get(nvs_handle_t c_handle, const char* key, T* out_value) { Lock lock; ESP_LOGD(TAG, "%s %s %d", __func__, key, sizeof(T)); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } - return entry.mStoragePtr->readItem(entry.mNsIndex, key, *out_value); + return handle->get_item(key, *out_value); } -extern "C" esp_err_t nvs_get_i8 (nvs_handle handle, const char* key, int8_t* out_value) +extern "C" esp_err_t nvs_get_i8 (nvs_handle_t c_handle, const char* key, int8_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_u8 (nvs_handle handle, const char* key, uint8_t* out_value) +extern "C" esp_err_t nvs_get_u8 (nvs_handle_t c_handle, const char* key, uint8_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_i16 (nvs_handle handle, const char* key, int16_t* out_value) +extern "C" esp_err_t nvs_get_i16 (nvs_handle_t c_handle, const char* key, int16_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_u16 (nvs_handle handle, const char* key, uint16_t* out_value) +extern "C" esp_err_t nvs_get_u16 (nvs_handle_t c_handle, const char* key, uint16_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_i32 (nvs_handle handle, const char* key, int32_t* out_value) +extern "C" esp_err_t nvs_get_i32 (nvs_handle_t c_handle, const char* key, int32_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_u32 (nvs_handle handle, const char* key, uint32_t* out_value) +extern "C" esp_err_t nvs_get_u32 (nvs_handle_t c_handle, const char* key, uint32_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_i64 (nvs_handle handle, const char* key, int64_t* out_value) +extern "C" esp_err_t nvs_get_i64 (nvs_handle_t c_handle, const char* key, int64_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -extern "C" esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value) +extern "C" esp_err_t nvs_get_u64 (nvs_handle_t c_handle, const char* key, uint64_t* out_value) { - return nvs_get(handle, key, out_value); + return nvs_get(c_handle, key, out_value); } -static esp_err_t nvs_get_str_or_blob(nvs_handle handle, nvs::ItemType type, const char* key, void* out_value, size_t* length) +static esp_err_t nvs_get_str_or_blob(nvs_handle_t c_handle, nvs::ItemType type, const char* key, void* out_value, size_t* length) { Lock lock; ESP_LOGD(TAG, "%s %s", __func__, key); - HandleEntry entry; - auto err = nvs_find_ns_handle(handle, entry); + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); if (err != ESP_OK) { return err; } size_t dataSize; - err = entry.mStoragePtr->getItemDataSize(entry.mNsIndex, type, key, dataSize); + err = handle->get_item_size(type, key, dataSize); if (err != ESP_OK) { return err; } @@ -445,16 +501,254 @@ static esp_err_t nvs_get_str_or_blob(nvs_handle handle, nvs::ItemType type, cons } *length = dataSize; - return entry.mStoragePtr->readItem(entry.mNsIndex, type, key, out_value, dataSize); + return handle->get_typed_item(type, key, out_value, dataSize); } -extern "C" esp_err_t nvs_get_str(nvs_handle handle, const char* key, char* out_value, size_t* length) +extern "C" esp_err_t nvs_get_str(nvs_handle_t c_handle, const char* key, char* out_value, size_t* length) { - return nvs_get_str_or_blob(handle, nvs::ItemType::SZ, key, out_value, length); + return nvs_get_str_or_blob(c_handle, nvs::ItemType::SZ, key, out_value, length); } -extern "C" esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size_t* length) +extern "C" esp_err_t nvs_get_blob(nvs_handle_t c_handle, const char* key, void* out_value, size_t* length) { - return nvs_get_str_or_blob(handle, nvs::ItemType::BLOB, key, out_value, length); + return nvs_get_str_or_blob(c_handle, nvs::ItemType::BLOB, key, out_value, length); } +extern "C" esp_err_t nvs_get_stats(const char* part_name, nvs_stats_t* nvs_stats) +{ + Lock lock; + nvs::Storage* pStorage; + + if (nvs_stats == nullptr) { + return ESP_ERR_INVALID_ARG; + } + nvs_stats->used_entries = 0; + nvs_stats->free_entries = 0; + nvs_stats->total_entries = 0; + nvs_stats->namespace_count = 0; + + pStorage = lookup_storage_from_name((part_name == nullptr) ? NVS_DEFAULT_PART_NAME : part_name); + if (pStorage == nullptr) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + + if(!pStorage->isValid()){ + return ESP_ERR_NVS_INVALID_STATE; + } + + return pStorage->fillStats(*nvs_stats); +} + +extern "C" esp_err_t nvs_get_used_entry_count(nvs_handle_t c_handle, size_t* used_entries) +{ + Lock lock; + if(used_entries == nullptr){ + return ESP_ERR_INVALID_ARG; + } + *used_entries = 0; + + NVSHandleSimple *handle; + auto err = nvs_find_ns_handle(c_handle, &handle); + if (err != ESP_OK) { + return err; + } + + size_t used_entry_count; + err = handle->get_used_entry_count(used_entry_count); + if(err == ESP_OK){ + *used_entries = used_entry_count; + } + return err; +} + +#if (defined CONFIG_NVS_ENCRYPTION) && (!defined LINUX_TARGET) + +extern "C" esp_err_t nvs_flash_generate_keys(const esp_partition_t* partition, nvs_sec_cfg_t* cfg) +{ + auto err = esp_partition_erase_range(partition, 0, partition->size); + if(err != ESP_OK) { + return err; + } + + for(uint8_t cnt = 0; cnt < NVS_KEY_SIZE; cnt++) { + /* Adjacent 16-byte blocks should be different */ + if (((cnt / 16) & 1) == 0) { + cfg->eky[cnt] = 0xff; + cfg->tky[cnt] = 0xee; + } else { + cfg->eky[cnt] = 0x99; + cfg->tky[cnt] = 0x88; + } + } + + /** + * Write key configuration without encryption engine (using raw partition write APIs). + * But the read is decrypted through flash encryption engine. This allows unique NVS encryption configuration, + * as flash encryption key is randomly generated per device. + */ + err = esp_partition_write(partition, 0, cfg->eky, NVS_KEY_SIZE); + if(err != ESP_OK) { + return err; + } + + /* Write without encryption, see note above */ + err = esp_partition_write(partition, NVS_KEY_SIZE, cfg->tky, NVS_KEY_SIZE); + if(err != ESP_OK) { + return err; + } + + err = esp_partition_read(partition, 0, cfg->eky, NVS_KEY_SIZE); + if(err != ESP_OK) { + return err; + } + + err = esp_partition_read(partition, NVS_KEY_SIZE, cfg->tky, NVS_KEY_SIZE); + if(err != ESP_OK) { + return err; + } + + uint32_t crc_calc = crc32_le(0xffffffff, cfg->eky, NVS_KEY_SIZE); + crc_calc = crc32_le(crc_calc, cfg->tky, NVS_KEY_SIZE); + + uint8_t crc_wr[16]; + memset(crc_wr, 0xff, sizeof(crc_wr)); + memcpy(crc_wr, &crc_calc, 4); + + err = esp_partition_write(partition, 2 * NVS_KEY_SIZE, crc_wr, sizeof(crc_wr)); + if(err != ESP_OK) { + return err; + } + + return ESP_OK; + +} + +extern "C" esp_err_t nvs_flash_read_security_cfg(const esp_partition_t* partition, nvs_sec_cfg_t* cfg) +{ + uint8_t eky_raw[NVS_KEY_SIZE], tky_raw[NVS_KEY_SIZE]; + uint32_t crc_raw, crc_read, crc_calc; + + auto check_if_initialized = [](uint8_t* eky, uint8_t* tky, uint32_t crc) { + uint8_t cnt = 0; + while(cnt < NVS_KEY_SIZE && eky[cnt] == 0xff && tky[cnt] == 0xff) cnt++; + + if(cnt == NVS_KEY_SIZE && crc == 0xffffffff) { + return false; + } + return true; + }; + + auto err = esp_partition_read(partition, 0, eky_raw, NVS_KEY_SIZE); + if(err != ESP_OK) { + return err; + } + + err = esp_partition_read(partition, NVS_KEY_SIZE, tky_raw, NVS_KEY_SIZE); + if(err != ESP_OK) { + return err; + } + + err = esp_partition_read(partition, 2 * NVS_KEY_SIZE, &crc_raw, 4); + if(err != ESP_OK) { + return err; + } + + if(!check_if_initialized(eky_raw, tky_raw, crc_raw)) { + /* This is an uninitialized key partition*/ + return ESP_ERR_NVS_KEYS_NOT_INITIALIZED; + } + + err = esp_partition_read(partition, 0, cfg->eky, NVS_KEY_SIZE); + + if(err != ESP_OK) { + return err; + } + + err = esp_partition_read(partition, NVS_KEY_SIZE, cfg->tky, NVS_KEY_SIZE); + + if(err != ESP_OK) { + return err; + } + + err = esp_partition_read(partition, 2 * NVS_KEY_SIZE, &crc_read, 4); + + if(err != ESP_OK) { + return err; + } + + crc_calc = crc32_le(0xffffffff, cfg->eky, NVS_KEY_SIZE); + crc_calc = crc32_le(crc_calc, cfg->tky, NVS_KEY_SIZE); + + if(crc_calc != crc_read) { + if(!check_if_initialized(cfg->eky, cfg->tky, crc_read)) { + /* This is an uninitialized key partition*/ + return ESP_ERR_NVS_KEYS_NOT_INITIALIZED; + } + return ESP_ERR_NVS_CORRUPT_KEY_PART; + } + + return ESP_OK; +} + +#endif + +static nvs_iterator_t create_iterator(nvs::Storage *storage, nvs_type_t type) +{ + nvs_iterator_t it = (nvs_iterator_t)calloc(1, sizeof(nvs_opaque_iterator_t)); + if (it == nullptr) { + return nullptr; + } + + it->storage = storage; + it->type = type; + + return it; +} + +extern "C" nvs_iterator_t nvs_entry_find(const char *part_name, const char *namespace_name, nvs_type_t type) +{ + Lock lock; + nvs::Storage *pStorage; + + pStorage = lookup_storage_from_name(part_name); + if (pStorage == nullptr) { + return nullptr; + } + + nvs_iterator_t it = create_iterator(pStorage, type); + if (it == nullptr) { + return nullptr; + } + + bool entryFound = pStorage->findEntry(it, namespace_name); + if (!entryFound) { + free(it); + return nullptr; + } + + return it; +} + +extern "C" nvs_iterator_t nvs_entry_next(nvs_iterator_t it) +{ + Lock lock; + assert(it); + + bool entryFound = it->storage->nextEntry(it); + if (!entryFound) { + free(it); + return nullptr; + } + + return it; +} + +extern "C" void nvs_entry_info(nvs_iterator_t it, nvs_entry_info_t *out_info) +{ + *out_info = it->entry_info; +} + +extern "C" void nvs_release_iterator(nvs_iterator_t it) +{ + free(it); +} diff --git a/components/nvs_flash/src/nvs_cxx_api.cpp b/components/nvs_flash/src/nvs_cxx_api.cpp new file mode 100644 index 000000000..65fb1128b --- /dev/null +++ b/components/nvs_flash/src/nvs_cxx_api.cpp @@ -0,0 +1,68 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "nvs_partition_manager.hpp" +#include "nvs_handle.hpp" +#include "nvs_handle_simple.hpp" +#include "nvs_handle_locked.hpp" +#include "nvs_platform.hpp" + +namespace nvs { + +std::unique_ptr open_nvs_handle_from_partition(const char *partition_name, + const char *ns_name, + nvs_open_mode_t open_mode, + esp_err_t *err) +{ + if (partition_name == nullptr || ns_name == nullptr) { + if (err) { + *err = ESP_ERR_INVALID_ARG; + } + return nullptr; + } + + Lock lock; + + NVSHandleSimple *handle_simple; + esp_err_t result = nvs::NVSPartitionManager::get_instance()-> + open_handle(partition_name, ns_name, open_mode, &handle_simple); + + if (err) { + *err = result; + } + + if (result != ESP_OK) { + return nullptr; + } + + NVSHandleLocked *locked_handle = new (nothrow) NVSHandleLocked(handle_simple); + + if (!locked_handle) { + if (err) { + *err = ESP_ERR_NO_MEM; + } + delete handle_simple; + return nullptr; + } + + return std::unique_ptr(locked_handle); +} + +std::unique_ptr open_nvs_handle(const char *ns_name, + nvs_open_mode_t open_mode, + esp_err_t *err) +{ + return open_nvs_handle_from_partition(NVS_DEFAULT_PART_NAME, ns_name, open_mode, err); +} + +} // namespace nvs diff --git a/components/nvs_flash/src/nvs_encrypted_partition.cpp b/components/nvs_flash/src/nvs_encrypted_partition.cpp new file mode 100644 index 000000000..26e8a3314 --- /dev/null +++ b/components/nvs_flash/src/nvs_encrypted_partition.cpp @@ -0,0 +1,121 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "nvs_encrypted_partition.hpp" +#include "nvs_types.hpp" + +namespace nvs { + +NVSEncryptedPartition::NVSEncryptedPartition(const esp_partition_t *partition) + : NVSPartition(partition) { } + +esp_err_t NVSEncryptedPartition::init(nvs_sec_cfg_t* cfg) +{ + uint8_t* eky = reinterpret_cast(cfg); + + mbedtls_aes_xts_init(&mEctxt); + mbedtls_aes_xts_init(&mDctxt); + + if (mbedtls_aes_xts_setkey_enc(&mEctxt, eky, 2 * NVS_KEY_SIZE * 8) != 0) { + return ESP_ERR_NVS_XTS_CFG_FAILED; + } + + if (mbedtls_aes_xts_setkey_dec(&mDctxt, eky, 2 * NVS_KEY_SIZE * 8) != 0) { + return ESP_ERR_NVS_XTS_CFG_FAILED; + } + + return ESP_OK; +} + +esp_err_t NVSEncryptedPartition::read(size_t src_offset, void* dst, size_t size) +{ + /** Currently upper layer of NVS reads entries one by one even for variable size + * multi-entry data types. So length should always be equal to size of an entry.*/ + if (size != sizeof(Item)) return ESP_ERR_INVALID_SIZE; + + // read data + esp_err_t read_result = esp_partition_read(mESPPartition, src_offset, dst, size); + if (read_result != ESP_OK) { + return read_result; + } + + // decrypt data + //sector num required as an arr by mbedtls. Should have been just uint64/32. + uint8_t data_unit[16]; + + uint32_t relAddr = src_offset; + + memset(data_unit, 0, sizeof(data_unit)); + + memcpy(data_unit, &relAddr, sizeof(relAddr)); + + uint8_t *destination = reinterpret_cast(dst); + + if (mbedtls_aes_crypt_xts(&mDctxt, MBEDTLS_AES_DECRYPT, size, data_unit, destination, destination) != 0) { + return ESP_ERR_NVS_XTS_DECR_FAILED; + } + + return ESP_OK; +} + +esp_err_t NVSEncryptedPartition::write(size_t addr, const void* src, size_t size) +{ + if (size % ESP_ENCRYPT_BLOCK_SIZE != 0) return ESP_ERR_INVALID_SIZE; + + // copy data to buffer for encryption + uint8_t* buf = new (std::nothrow) uint8_t [size]; + + if (!buf) return ESP_ERR_NO_MEM; + + memcpy(buf, src, size); + + // encrypt data + uint8_t entrySize = sizeof(Item); + + //sector num required as an arr by mbedtls. Should have been just uint64/32. + uint8_t data_unit[16]; + + /* Use relative address instead of absolute address (relocatable), so that host-generated + * encrypted nvs images can be used*/ + uint32_t relAddr = addr; + + memset(data_unit, 0, sizeof(data_unit)); + + for(uint8_t entry = 0; entry < (size/entrySize); entry++) + { + uint32_t offset = entry * entrySize; + uint32_t *addr_loc = (uint32_t*) &data_unit[0]; + + *addr_loc = relAddr + offset; + if (mbedtls_aes_crypt_xts(&mEctxt, + MBEDTLS_AES_ENCRYPT, + entrySize, + data_unit, + buf + offset, + buf + offset) != 0) { + delete buf; + return ESP_ERR_NVS_XTS_ENCR_FAILED; + } + } + + // write data + esp_err_t result = esp_partition_write(mESPPartition, addr, buf, size); + + delete buf; + + return result; +} + +} // nvs diff --git a/components/nvs_flash/src/nvs_encrypted_partition.hpp b/components/nvs_flash/src/nvs_encrypted_partition.hpp new file mode 100644 index 000000000..76c0607a0 --- /dev/null +++ b/components/nvs_flash/src/nvs_encrypted_partition.hpp @@ -0,0 +1,43 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NVS_ENCRYPTED_PARTITION_HPP_ +#define NVS_ENCRYPTED_PARTITION_HPP_ + +#include "mbedtls/aes.h" +#include "nvs_flash.h" +#include "nvs_partition.hpp" + +namespace nvs { + +class NVSEncryptedPartition : public NVSPartition { +public: + NVSEncryptedPartition(const esp_partition_t *partition); + + virtual ~NVSEncryptedPartition() { } + + esp_err_t init(nvs_sec_cfg_t* cfg); + + esp_err_t read(size_t src_offset, void* dst, size_t size) override; + + esp_err_t write(size_t dst_offset, const void* src, size_t size) override; + +protected: + mbedtls_aes_xts_context mEctxt; + mbedtls_aes_xts_context mDctxt; +}; + +} // nvs + +#endif // NVS_ENCRYPTED_PARTITION_HPP_ diff --git a/components/nvs_flash/src/nvs_handle_locked.cpp b/components/nvs_flash/src/nvs_handle_locked.cpp new file mode 100644 index 000000000..89e5cbbca --- /dev/null +++ b/components/nvs_flash/src/nvs_handle_locked.cpp @@ -0,0 +1,82 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "nvs_handle_locked.hpp" + +namespace nvs { + +NVSHandleLocked::NVSHandleLocked(NVSHandleSimple *handle) : handle(handle) { + Lock::init(); +} + +NVSHandleLocked::~NVSHandleLocked() { + Lock lock; + delete handle; +} + +esp_err_t NVSHandleLocked::set_string(const char *key, const char* str) { + Lock lock; + return handle->set_string(key, str); +} + +esp_err_t NVSHandleLocked::set_blob(const char *key, const void* blob, size_t len) { + Lock lock; + return handle->set_blob(key, blob, len); +} + +esp_err_t NVSHandleLocked::get_string(const char *key, char* out_str, size_t len) { + Lock lock; + return handle->get_string(key, out_str, len); +} + +esp_err_t NVSHandleLocked::get_blob(const char *key, void* out_blob, size_t len) { + Lock lock; + return handle->get_blob(key, out_blob, len); +} + +esp_err_t NVSHandleLocked::get_item_size(ItemType datatype, const char *key, size_t &size) { + Lock lock; + return handle->get_item_size(datatype, key, size); +} + +esp_err_t NVSHandleLocked::erase_item(const char* key) { + Lock lock; + return handle->erase_item(key); +} + +esp_err_t NVSHandleLocked::erase_all() { + Lock lock; + return handle->erase_all(); +} + +esp_err_t NVSHandleLocked::commit() { + Lock lock; + return handle->commit(); +} + +esp_err_t NVSHandleLocked::get_used_entry_count(size_t& usedEntries) { + Lock lock; + return handle->get_used_entry_count(usedEntries); +} + +esp_err_t NVSHandleLocked::set_typed_item(ItemType datatype, const char *key, const void* data, size_t dataSize) { + Lock lock; + return handle->set_typed_item(datatype, key, data, dataSize); +} + +esp_err_t NVSHandleLocked::get_typed_item(ItemType datatype, const char *key, void* data, size_t dataSize) { + Lock lock; + return handle->get_typed_item(datatype, key, data, dataSize); +} + +} // namespace nvs diff --git a/components/nvs_flash/src/nvs_handle_locked.hpp b/components/nvs_flash/src/nvs_handle_locked.hpp new file mode 100644 index 000000000..39d514c7a --- /dev/null +++ b/components/nvs_flash/src/nvs_handle_locked.hpp @@ -0,0 +1,66 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef NVS_HANDLE_LOCKED_HPP_ +#define NVS_HANDLE_LOCKED_HPP_ + +#include "nvs_handle_simple.hpp" + +namespace nvs { + +/** + * @brief A class which behaves the same as NVSHandleSimple, except that all public member functions are locked. + * + * This class follows the decorator design pattern. The reason why we don't want locks in NVSHandleSimple is that + * NVSHandleSimple can also be used by the C-API which locks its public functions already. + * Thus, we avoid double-locking. + * + * @note this class becomes responsible for its internal NVSHandleSimple object, i.e. it deletes the handle object on + * destruction + */ +class NVSHandleLocked : public NVSHandle { +public: + NVSHandleLocked(NVSHandleSimple *handle); + + virtual ~NVSHandleLocked(); + + esp_err_t set_string(const char *key, const char* str) override; + + esp_err_t set_blob(const char *key, const void* blob, size_t len) override; + + esp_err_t get_string(const char *key, char* out_str, size_t len) override; + + esp_err_t get_blob(const char *key, void* out_blob, size_t len) override; + + esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override; + + esp_err_t erase_item(const char* key) override; + + esp_err_t erase_all() override; + + esp_err_t commit() override; + + esp_err_t get_used_entry_count(size_t& usedEntries) override; + +protected: + esp_err_t set_typed_item(ItemType datatype, const char *key, const void* data, size_t dataSize) override; + + esp_err_t get_typed_item(ItemType datatype, const char *key, void* data, size_t dataSize) override; + +private: + NVSHandleSimple *handle; +}; + +} // namespace nvs + +#endif // NVS_HANDLE_LOCKED_HPP_ diff --git a/components/nvs_flash/src/nvs_handle_simple.cpp b/components/nvs_flash/src/nvs_handle_simple.cpp new file mode 100644 index 000000000..348e197b7 --- /dev/null +++ b/components/nvs_flash/src/nvs_handle_simple.cpp @@ -0,0 +1,137 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "nvs_handle.hpp" +#include "nvs_partition_manager.hpp" + +namespace nvs { + +NVSHandleSimple::~NVSHandleSimple() { + NVSPartitionManager::get_instance()->close_handle(this); +} + +esp_err_t NVSHandleSimple::set_typed_item(ItemType datatype, const char *key, const void* data, size_t dataSize) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + if (mReadOnly) return ESP_ERR_NVS_READ_ONLY; + + return mStoragePtr->writeItem(mNsIndex, datatype, key, data, dataSize); +} + +esp_err_t NVSHandleSimple::get_typed_item(ItemType datatype, const char *key, void* data, size_t dataSize) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + + return mStoragePtr->readItem(mNsIndex, datatype, key, data, dataSize); +} + +esp_err_t NVSHandleSimple::set_string(const char *key, const char* str) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + if (mReadOnly) return ESP_ERR_NVS_READ_ONLY; + + return mStoragePtr->writeItem(mNsIndex, nvs::ItemType::SZ, key, str, strlen(str) + 1); +} + +esp_err_t NVSHandleSimple::set_blob(const char *key, const void* blob, size_t len) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + if (mReadOnly) return ESP_ERR_NVS_READ_ONLY; + + return mStoragePtr->writeItem(mNsIndex, nvs::ItemType::BLOB, key, blob, len); +} + +esp_err_t NVSHandleSimple::get_string(const char *key, char* out_str, size_t len) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + + return mStoragePtr->readItem(mNsIndex, nvs::ItemType::SZ, key, out_str, len); +} + +esp_err_t NVSHandleSimple::get_blob(const char *key, void* out_blob, size_t len) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + + return mStoragePtr->readItem(mNsIndex, nvs::ItemType::BLOB, key, out_blob, len); +} + +esp_err_t NVSHandleSimple::get_item_size(ItemType datatype, const char *key, size_t &size) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + + return mStoragePtr->getItemDataSize(mNsIndex, datatype, key, size); +} + +esp_err_t NVSHandleSimple::erase_item(const char* key) +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + if (mReadOnly) return ESP_ERR_NVS_READ_ONLY; + + return mStoragePtr->eraseItem(mNsIndex, key); +} + +esp_err_t NVSHandleSimple::erase_all() +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + if (mReadOnly) return ESP_ERR_NVS_READ_ONLY; + + return mStoragePtr->eraseNamespace(mNsIndex); +} + +esp_err_t NVSHandleSimple::commit() +{ + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + + return ESP_OK; +} + +esp_err_t NVSHandleSimple::get_used_entry_count(size_t& used_entries) +{ + used_entries = 0; + + if (!valid) return ESP_ERR_NVS_INVALID_HANDLE; + + size_t used_entry_count; + esp_err_t err = mStoragePtr->calcEntriesInNamespace(mNsIndex, used_entry_count); + if(err == ESP_OK){ + used_entries = used_entry_count; + } + return err; +} + +void NVSHandleSimple::debugDump() { + return mStoragePtr->debugDump(); +} + +esp_err_t NVSHandleSimple::fillStats(nvs_stats_t& nvsStats) { + return mStoragePtr->fillStats(nvsStats); +} + +esp_err_t NVSHandleSimple::calcEntriesInNamespace(size_t& usedEntries) { + return mStoragePtr->calcEntriesInNamespace(mNsIndex, usedEntries); +} + +bool NVSHandleSimple::findEntry(nvs_opaque_iterator_t* it, const char* name) { + return mStoragePtr->findEntry(it, name); +} + +bool NVSHandleSimple::nextEntry(nvs_opaque_iterator_t* it) { + return mStoragePtr->nextEntry(it); +} + +const char *NVSHandleSimple::get_partition_name() const { + return mStoragePtr->getPartName(); +} + +} diff --git a/components/nvs_flash/src/nvs_handle_simple.hpp b/components/nvs_flash/src/nvs_handle_simple.hpp new file mode 100644 index 000000000..0a20aa4e8 --- /dev/null +++ b/components/nvs_flash/src/nvs_handle_simple.hpp @@ -0,0 +1,107 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef NVS_HANDLE_SIMPLE_HPP_ +#define NVS_HANDLE_SIMPLE_HPP_ + +#include "intrusive_list.h" +#include "nvs_storage.hpp" +#include "nvs_platform.hpp" + +#include "nvs_handle.hpp" + +namespace nvs { + +/** + * @brief This class implements NVSHandle according to the ESP32's flash and partitioning scheme. + * + * It is used by both the C API and the C++ API. The main responsibility is to check whether the handle is valid + * and in the right read/write mode and then forward the calls to the storage object. + * + * For more details about the general member functions, see nvs_handle.hpp. + */ +class NVSHandleSimple : public intrusive_list_node, public NVSHandle { + friend class NVSPartitionManager; +public: + NVSHandleSimple(bool readOnly, uint8_t nsIndex, Storage *StoragePtr) : + mStoragePtr(StoragePtr), + mNsIndex(nsIndex), + mReadOnly(readOnly), + valid(1) + { } + + ~NVSHandleSimple(); + + esp_err_t set_typed_item(ItemType datatype, const char *key, const void *data, size_t dataSize) override; + + esp_err_t get_typed_item(ItemType datatype, const char *key, void *data, size_t dataSize) override; + + esp_err_t set_string(const char *key, const char *str) override; + + esp_err_t set_blob(const char *key, const void *blob, size_t len) override; + + esp_err_t get_string(const char *key, char *out_str, size_t len) override; + + esp_err_t get_blob(const char *key, void *out_blob, size_t len) override; + + esp_err_t get_item_size(ItemType datatype, const char *key, size_t &size) override; + + esp_err_t erase_item(const char *key) override; + + esp_err_t erase_all() override; + + esp_err_t commit() override; + + esp_err_t get_used_entry_count(size_t &usedEntries) override; + + esp_err_t getItemDataSize(ItemType datatype, const char *key, size_t &dataSize); + + void debugDump(); + + esp_err_t fillStats(nvs_stats_t &nvsStats); + + esp_err_t calcEntriesInNamespace(size_t &usedEntries); + + bool findEntry(nvs_opaque_iterator_t *it, const char *name); + + bool nextEntry(nvs_opaque_iterator_t *it); + + const char *get_partition_name() const; + +private: + /** + * The underlying storage's object. + */ + Storage *mStoragePtr; + + /** + * Numeric representation of the namespace as it is saved in flash (see README.rst for further details). + */ + uint8_t mNsIndex; + + /** + * Whether this handle is marked as read-only or read-write. + * 0 indicates read-only, any other value read-write. + */ + uint8_t mReadOnly; + + /** + * Indicates the validity of this handle. + * Upon opening, a handle is valid. It becomes invalid if the underlying storage is de-initialized. + */ + uint8_t valid; +}; + +} // nvs + +#endif // NVS_HANDLE_SIMPLE_HPP_ diff --git a/components/nvs_flash/src/nvs_item_hash_list.cpp b/components/nvs_flash/src/nvs_item_hash_list.cpp index 845dd0910..21bf8b315 100644 --- a/components/nvs_flash/src/nvs_item_hash_list.cpp +++ b/components/nvs_flash/src/nvs_item_hash_list.cpp @@ -20,7 +20,7 @@ namespace nvs HashList::HashList() { } - + void HashList::clear() { for (auto it = mBlockList.begin(); it != mBlockList.end();) { @@ -30,7 +30,7 @@ void HashList::clear() delete static_cast(tmp); } } - + HashList::~HashList() { clear(); @@ -42,7 +42,7 @@ HashList::HashListBlock::HashListBlock() "cache block size calculation incorrect"); } -void HashList::insert(const Item& item, size_t index) +esp_err_t HashList::insert(const Item& item, size_t index) { const uint32_t hash_24 = item.calculateCrc32WithoutValue() & 0xffffff; // add entry to the end of last block if possible @@ -50,29 +50,41 @@ void HashList::insert(const Item& item, size_t index) auto& block = mBlockList.back(); if (block.mCount < HashListBlock::ENTRY_COUNT) { block.mNodes[block.mCount++] = HashListNode(hash_24, index); - return; + return ESP_OK; } } // if the above failed, create a new block and add entry to it - HashListBlock* newBlock = new HashListBlock; + HashListBlock* newBlock = new (std::nothrow) HashListBlock; + + if (!newBlock) return ESP_ERR_NO_MEM; + mBlockList.push_back(newBlock); newBlock->mNodes[0] = HashListNode(hash_24, index); newBlock->mCount++; + + return ESP_OK; } -void HashList::erase(size_t index, bool itemShouldExist) +bool HashList::erase(size_t index) { for (auto it = mBlockList.begin(); it != mBlockList.end();) { bool haveEntries = false; + bool foundIndex = false; for (size_t i = 0; i < it->mCount; ++i) { if (it->mNodes[i].mIndex == index) { it->mNodes[i].mIndex = 0xff; - return; + foundIndex = true; + /* found the item and removed it */ } if (it->mNodes[i].mIndex != 0xff) { haveEntries = true; } + if (haveEntries && foundIndex) { + /* item was found, and HashListBlock still has some items */ + return true; + } } + /* no items left in HashListBlock, can remove */ if (!haveEntries) { auto tmp = it; ++it; @@ -81,10 +93,14 @@ void HashList::erase(size_t index, bool itemShouldExist) } else { ++it; } + if (foundIndex) { + /* item was found and empty HashListBlock was removed */ + return true; + } } - if (itemShouldExist) { - assert(false && "item should have been present in cache"); - } + + // item hasn't been present in cache"); + return false; } size_t HashList::find(size_t start, const Item& item) diff --git a/components/nvs_flash/src/nvs_item_hash_list.hpp b/components/nvs_flash/src/nvs_item_hash_list.hpp index e759cd818..e724c4f02 100644 --- a/components/nvs_flash/src/nvs_item_hash_list.hpp +++ b/components/nvs_flash/src/nvs_item_hash_list.hpp @@ -27,16 +27,16 @@ class HashList public: HashList(); ~HashList(); - - void insert(const Item& item, size_t index); - void erase(const size_t index, bool itemShouldExist=true); + + esp_err_t insert(const Item& item, size_t index); + bool erase(const size_t index); size_t find(size_t start, const Item& item); void clear(); - + private: HashList(const HashList& other); const HashList& operator= (const HashList& rhs); - + protected: struct HashListNode { diff --git a/components/nvs_flash/src/nvs_page.cpp b/components/nvs_flash/src/nvs_page.cpp index acbae5358..0be9960f5 100644 --- a/components/nvs_flash/src/nvs_page.cpp +++ b/components/nvs_flash/src/nvs_page.cpp @@ -12,13 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "nvs_page.hpp" -#include +#if defined(LINUX_TARGET) +#include "crc.h" +#else +#include +#endif #include #include namespace nvs { +Page::Page() : mPartition(nullptr) { } + uint32_t Page::Header::calculateCrc32() { return crc32_le(0xffffffff, @@ -26,14 +32,19 @@ uint32_t Page::Header::calculateCrc32() offsetof(Header, mCrc32) - offsetof(Header, mSeqNumber)); } -esp_err_t Page::load(uint32_t sectorNumber) +esp_err_t Page::load(Partition *partition, uint32_t sectorNumber) { + if (partition == nullptr) { + return ESP_ERR_INVALID_ARG; + } + + mPartition = partition; mBaseAddress = sectorNumber * SEC_SIZE; mUsedEntryCount = 0; mErasedEntryCount = 0; Header header; - auto rc = spi_flash_read(mBaseAddress, &header, sizeof(header)); + auto rc = mPartition->read_raw(mBaseAddress, &header, sizeof(header)); if (rc != ESP_OK) { mState = PageState::INVALID; return rc; @@ -42,24 +53,35 @@ esp_err_t Page::load(uint32_t sectorNumber) mState = header.mState; // check if the whole page is really empty // reading the whole page takes ~40 times less than erasing it - uint32_t line[8]; - for (uint32_t i = 0; i < SPI_FLASH_SEC_SIZE; i += sizeof(line)) { - rc = spi_flash_read(mBaseAddress + i, line, sizeof(line)); + const int BLOCK_SIZE = 128; + uint32_t* block = new (std::nothrow) uint32_t[BLOCK_SIZE]; + + if (!block) return ESP_ERR_NO_MEM; + + for (uint32_t i = 0; i < SPI_FLASH_SEC_SIZE; i += 4 * BLOCK_SIZE) { + rc = mPartition->read_raw(mBaseAddress + i, block, 4 * BLOCK_SIZE); if (rc != ESP_OK) { mState = PageState::INVALID; + delete[] block; return rc; } - if (std::any_of(line, line + 4, [](uint32_t val) -> bool { return val != 0xffffffff; })) { + if (std::any_of(block, block + BLOCK_SIZE, [](uint32_t val) -> bool { return val != 0xffffffff; })) { // page isn't as empty after all, mark it as corrupted mState = PageState::CORRUPT; break; } } + delete[] block; } else if (header.mCrc32 != header.calculateCrc32()) { header.mState = PageState::CORRUPT; } else { mState = header.mState; mSeqNumber = header.mSeqNumber; + if(header.mVersion < NVS_VERSION) { + return ESP_ERR_NVS_NEW_VERSION_FOUND; + } else { + mVersion = header.mVersion; + } } switch (mState) { @@ -82,13 +104,16 @@ esp_err_t Page::load(uint32_t sectorNumber) esp_err_t Page::writeEntry(const Item& item) { - auto rc = spi_flash_write(getEntryAddress(mNextFreeEntry), &item, sizeof(item)); - if (rc != ESP_OK) { + esp_err_t err; + + err = mPartition->write(getEntryAddress(mNextFreeEntry), &item, sizeof(item)); + + if (err != ESP_OK) { mState = PageState::INVALID; - return rc; + return err; } - auto err = alterEntryState(mNextFreeEntry, EntryState::WRITTEN); + err = alterEntryState(mNextFreeEntry, EntryState::WRITTEN); if (err != ESP_OK) { return err; } @@ -102,17 +127,18 @@ esp_err_t Page::writeEntry(const Item& item) return ESP_OK; } - + esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) { assert(size % ENTRY_SIZE == 0); assert(mNextFreeEntry != INVALID_ENTRY); assert(mFirstUsedEntry != INVALID_ENTRY); const uint16_t count = size / ENTRY_SIZE; - + const uint8_t* buf = data; - -#ifdef ESP_PLATFORM + +#if !defined LINUX_TARGET + // TODO: check whether still necessary with esp_partition* API /* On the ESP32, data can come from DROM, which is not accessible by spi_flash_write * function. To work around this, we copy the data to heap if it came from DROM. * Hopefully this won't happen very often in practice. For data from DRAM, we should @@ -127,13 +153,15 @@ esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) } memcpy((void*)buf, data, size); } -#endif //ESP_PLATFORM - auto rc = spi_flash_write(getEntryAddress(mNextFreeEntry), buf, size); -#ifdef ESP_PLATFORM +#endif // ! LINUX_TARGET + + auto rc = mPartition->write(getEntryAddress(mNextFreeEntry), buf, size); + +#if !defined LINUX_TARGET if (buf != data) { free((void*)buf); } -#endif //ESP_PLATFORM +#endif // ! LINUX_TARGET if (rc != ESP_OK) { mState = PageState::INVALID; return rc; @@ -147,15 +175,15 @@ esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) return ESP_OK; } -esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) +esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx) { Item item; esp_err_t err; - + if (mState == PageState::INVALID) { return ESP_ERR_NVS_INVALID_STATE; } - + if (mState == PageState::UNINITIALIZED) { err = initialize(); if (err != ESP_OK) { @@ -171,21 +199,22 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c if (keySize > Item::MAX_KEY_LENGTH) { return ESP_ERR_NVS_KEY_TOO_LONG; } - - if (dataSize > Page::BLOB_MAX_SIZE) { + + if (dataSize > Page::CHUNK_MAX_SIZE) { return ESP_ERR_NVS_VALUE_TOO_LONG; } size_t totalSize = ENTRY_SIZE; size_t entriesCount = 1; - if (datatype == ItemType::SZ || datatype == ItemType::BLOB) { + if (isVariableLengthType(datatype)) { size_t roundedSize = (dataSize + ENTRY_SIZE - 1) & ~(ENTRY_SIZE - 1); totalSize += roundedSize; entriesCount += roundedSize / ENTRY_SIZE; } // primitive types should fit into one entry - assert(totalSize == ENTRY_SIZE || datatype == ItemType::BLOB || datatype == ItemType::SZ); + assert(totalSize == ENTRY_SIZE || + isVariableLengthType(datatype)); if (mNextFreeEntry == INVALID_ENTRY || mNextFreeEntry + entriesCount > ENTRY_COUNT) { // page will not fit this amount of data @@ -194,10 +223,14 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c // write first item size_t span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE; - item = Item(nsIndex, datatype, span, key); - mHashList.insert(item, mNextFreeEntry); + item = Item(nsIndex, datatype, span, key, chunkIdx); + err = mHashList.insert(item, mNextFreeEntry); - if (datatype != ItemType::SZ && datatype != ItemType::BLOB) { + if (err != ESP_OK) { + return err; + } + + if (!isVariableLengthType(datatype)) { memcpy(item.data, data, dataSize); item.crc32 = item.calculateCrc32(); err = writeEntry(item); @@ -208,7 +241,7 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c const uint8_t* src = reinterpret_cast(data); item.varLength.dataCrc32 = Item::calculateCrc32(src, dataSize); item.varLength.dataSize = dataSize; - item.varLength.reserved2 = 0xffff; + item.varLength.reserved = 0xffff; item.crc32 = item.calculateCrc32(); err = writeEntry(item); if (err != ESP_OK) { @@ -222,36 +255,36 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return err; } } - + size_t tail = dataSize - left; if (tail > 0) { - std::fill_n(item.rawData, ENTRY_SIZE / 4, 0xffffffff); + std::fill_n(item.rawData, ENTRY_SIZE, 0xff); memcpy(item.rawData, static_cast(data) + left, tail); err = writeEntry(item); if (err != ESP_OK) { return err; } } - + } return ESP_OK; } -esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize) +esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; Item item; - + if (mState == PageState::INVALID) { return ESP_ERR_NVS_INVALID_STATE; } - - esp_err_t rc = findItem(nsIndex, datatype, key, index, item); + + esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); if (rc != ESP_OK) { return rc; } - if (datatype != ItemType::SZ && datatype != ItemType::BLOB) { + if (!isVariableLengthType(datatype)) { if (dataSize != getAlignmentForType(datatype)) { return ESP_ERR_NVS_TYPE_MISMATCH; } @@ -288,28 +321,81 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo return ESP_OK; } -esp_err_t Page::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key) +esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) +{ + size_t index = 0; + Item item; + + if (mState == PageState::INVALID) { + return ESP_ERR_NVS_INVALID_STATE; + } + + esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); + if (rc != ESP_OK) { + return rc; + } + + if (!isVariableLengthType(datatype)) { + if (dataSize != getAlignmentForType(datatype)) { + return ESP_ERR_NVS_TYPE_MISMATCH; + } + + if (memcmp(data, item.data, dataSize)) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + return ESP_OK; + } + + if (dataSize < static_cast(item.varLength.dataSize)) { + return ESP_ERR_NVS_INVALID_LENGTH; + } + + const uint8_t* dst = reinterpret_cast(data); + size_t left = item.varLength.dataSize; + for (size_t i = index + 1; i < index + item.span; ++i) { + Item ditem; + rc = readEntry(i, ditem); + if (rc != ESP_OK) { + return rc; + } + size_t willCopy = ENTRY_SIZE; + willCopy = (left < willCopy)?left:willCopy; + if (memcmp(dst, ditem.rawData, willCopy)) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + left -= willCopy; + dst += willCopy; + } + if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { + return ESP_ERR_NVS_NOT_FOUND; + } + + return ESP_OK; +} + +esp_err_t Page::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; Item item; - esp_err_t rc = findItem(nsIndex, datatype, key, index, item); + esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); if (rc != ESP_OK) { return rc; } return eraseEntryAndSpan(index); } -esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key) +esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; Item item; - return findItem(nsIndex, datatype, key, index, item); + return findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); } esp_err_t Page::eraseEntryAndSpan(size_t index) { + uint32_t seq_num; + getSeqNumber(seq_num); auto state = mEntryTable.get(index); - assert(state == EntryState::WRITTEN || state == EntryState::EMPTY); size_t span = 1; if (state == EntryState::WRITTEN) { @@ -319,7 +405,7 @@ esp_err_t Page::eraseEntryAndSpan(size_t index) return rc; } if (item.calculateCrc32() != item.crc32) { - mHashList.erase(index, false); + mHashList.erase(index); rc = alterEntryState(index, EntryState::ERASED); --mUsedEntryCount; ++mErasedEntryCount; @@ -406,7 +492,11 @@ esp_err_t Page::copyItems(Page& other) return err; } - other.mHashList.insert(entry, other.mNextFreeEntry); + err = other.mHashList.insert(entry, other.mNextFreeEntry); + if (err != ESP_OK) { + return err; + } + err = other.writeEntry(entry); if (err != ESP_OK) { return err; @@ -435,7 +525,7 @@ esp_err_t Page::mLoadEntryTable() if (mState == PageState::ACTIVE || mState == PageState::FULL || mState == PageState::FREEING) { - auto rc = spi_flash_read(mBaseAddress + ENTRY_TABLE_OFFSET, mEntryTable.data(), + auto rc = mPartition->read_raw(mBaseAddress + ENTRY_TABLE_OFFSET, mEntryTable.data(), mEntryTable.byteSize()); if (rc != ESP_OK) { mState = PageState::INVALID; @@ -474,7 +564,7 @@ esp_err_t Page::mLoadEntryTable() while (mNextFreeEntry < ENTRY_COUNT) { uint32_t entryAddress = getEntryAddress(mNextFreeEntry); uint32_t header; - auto rc = spi_flash_read(entryAddress, &header, sizeof(header)); + auto rc = mPartition->read_raw(entryAddress, &header, sizeof(header)); if (rc != ESP_OK) { mState = PageState::INVALID; return rc; @@ -512,6 +602,16 @@ esp_err_t Page::mLoadEntryTable() continue; } + if (mEntryTable.get(i) == static_cast(0x1)) { + lastItemIndex = INVALID_ENTRY; + auto err = eraseEntryAndSpan(i); + if (err != ESP_OK) { + mState = PageState::INVALID; + return err; + } + continue; + } + lastItemIndex = i; auto err = readEntry(i, item); @@ -519,7 +619,7 @@ esp_err_t Page::mLoadEntryTable() mState = PageState::INVALID; return err; } - + if (item.crc32 != item.calculateCrc32()) { err = eraseEntryAndSpan(i); if (err != ESP_OK) { @@ -529,12 +629,16 @@ esp_err_t Page::mLoadEntryTable() continue; } - mHashList.insert(item, i); + err = mHashList.insert(item, i); + if (err != ESP_OK) { + mState = PageState::INVALID; + return err; + } // search for potential duplicate item size_t duplicateIndex = mHashList.find(0, item); - - if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) { + + if (isVariableLengthType(item.datatype)) { span = item.span; bool needErase = false; for (size_t j = i; j < i + span; ++j) { @@ -549,7 +653,11 @@ esp_err_t Page::mLoadEntryTable() continue; } } - + + /* Note that logic for duplicate detections works fine even + * when old-format blob is present along with new-format blob-index + * for same key on active page. Since datatype is not used in hash calculation, + * old-format blob will be removed.*/ if (duplicateIndex < i) { eraseEntryAndSpan(duplicateIndex); } @@ -592,12 +700,18 @@ esp_err_t Page::mLoadEntryTable() } continue; } + assert(item.span > 0); - mHashList.insert(item, i); + err = mHashList.insert(item, i); + if (err != ESP_OK) { + mState = PageState::INVALID; + return err; + } + size_t span = item.span; - if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) { + if (isVariableLengthType(item.datatype)) { for (size_t j = i + 1; j < i + span; ++j) { if (mEntryTable.get(j) != EntryState::WRITTEN) { eraseEntryAndSpan(i); @@ -622,9 +736,10 @@ esp_err_t Page::initialize() Header header; header.mState = mState; header.mSeqNumber = mSeqNumber; + header.mVersion = mVersion; header.mCrc32 = header.calculateCrc32(); - auto rc = spi_flash_write(mBaseAddress, &header, sizeof(header)); + auto rc = mPartition->write_raw(mBaseAddress, &header, sizeof(header)); if (rc != ESP_OK) { mState = PageState::INVALID; return rc; @@ -641,7 +756,7 @@ esp_err_t Page::alterEntryState(size_t index, EntryState state) mEntryTable.set(index, state); size_t wordToWrite = mEntryTable.getWordIndex(index); uint32_t word = mEntryTable.data()[wordToWrite]; - auto rc = spi_flash_write(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast(wordToWrite) * 4, + auto rc = mPartition->write_raw(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast(wordToWrite) * 4, &word, sizeof(word)); if (rc != ESP_OK) { mState = PageState::INVALID; @@ -665,7 +780,7 @@ esp_err_t Page::alterEntryRangeState(size_t begin, size_t end, EntryState state) } if (nextWordIndex != wordIndex) { uint32_t word = mEntryTable.data()[wordIndex]; - auto rc = spi_flash_write(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast(wordIndex) * 4, + auto rc = mPartition->write_raw(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast(wordIndex) * 4, &word, 4); if (rc != ESP_OK) { return rc; @@ -679,7 +794,7 @@ esp_err_t Page::alterEntryRangeState(size_t begin, size_t end, EntryState state) esp_err_t Page::alterPageState(PageState state) { uint32_t state_val = static_cast(state); - auto rc = spi_flash_write(mBaseAddress, &state_val, sizeof(state)); + auto rc = mPartition->write_raw(mBaseAddress, &state_val, sizeof(state)); if (rc != ESP_OK) { mState = PageState::INVALID; return rc; @@ -690,19 +805,19 @@ esp_err_t Page::alterPageState(PageState state) esp_err_t Page::readEntry(size_t index, Item& dst) const { - auto rc = spi_flash_read(getEntryAddress(index), &dst, sizeof(dst)); + auto rc = mPartition->read(getEntryAddress(index), &dst, sizeof(dst)); if (rc != ESP_OK) { return rc; } return ESP_OK; } -esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item) +esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item, uint8_t chunkIdx, VerOffset chunkStart) { if (mState == PageState::CORRUPT || mState == PageState::INVALID || mState == PageState::UNINITIALIZED) { return ESP_ERR_NVS_NOT_FOUND; } - + size_t findBeginIndex = itemIndex; if (findBeginIndex >= ENTRY_COUNT) { return ESP_ERR_NVS_NOT_FOUND; @@ -719,7 +834,7 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si } if (nsIndex != NS_ANY && datatype != ItemType::ANY && key != NULL) { - size_t cachedIndex = mHashList.find(start, Item(nsIndex, datatype, 0, key)); + size_t cachedIndex = mHashList.find(start, Item(nsIndex, datatype, 0, key, chunkIdx)); if (cachedIndex < ENTRY_COUNT) { start = cachedIndex; } else { @@ -750,7 +865,7 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si continue; } - if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) { + if (isVariableLengthType(item.datatype)) { next = i + item.span; } @@ -761,8 +876,31 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si if (key != nullptr && strncmp(key, item.key, Item::MAX_KEY_LENGTH) != 0) { continue; } + /* For blob data, chunkIndex should match*/ + if (chunkIdx != CHUNK_ANY + && datatype == ItemType::BLOB_DATA + && item.chunkIndex != chunkIdx) { + continue; + } + /* Blob-index will match the with blob data. + * Skip data chunks when searching for blob index*/ + if (datatype == ItemType::BLOB_IDX + && item.chunkIndex != CHUNK_ANY) { + continue; + } + /* Match the version for blob-index*/ + if (datatype == ItemType::BLOB_IDX + && chunkStart != VerOffset::VER_ANY + && item.blobIndex.chunkStart != chunkStart) { + continue; + } + if (datatype != ItemType::ANY && item.datatype != datatype) { + if (key == nullptr && nsIndex == NS_ANY && chunkIdx == CHUNK_ANY) { + continue; // continue for bruteforce search on blob indices. + } + itemIndex = i; return ESP_ERR_NVS_TYPE_MISMATCH; } @@ -793,10 +931,18 @@ esp_err_t Page::setSeqNumber(uint32_t seqNumber) return ESP_OK; } +esp_err_t Page::setVersion(uint8_t ver) +{ + if (mState != PageState::UNINITIALIZED) { + return ESP_ERR_NVS_INVALID_STATE; + } + mVersion = ver; + return ESP_OK; +} + esp_err_t Page::erase() { - auto sector = mBaseAddress / SPI_FLASH_SEC_SIZE; - auto rc = spi_flash_erase_sector(sector); + auto rc = mPartition->erase_range(mBaseAddress, SPI_FLASH_SEC_SIZE); if (rc != ESP_OK) { mState = PageState::INVALID; return rc; @@ -825,28 +971,39 @@ esp_err_t Page::markFull() } return alterPageState(PageState::FULL); } - + +size_t Page::getVarDataTailroom() const +{ + if (mState == PageState::UNINITIALIZED) { + return CHUNK_MAX_SIZE; + } else if (mState == PageState::FULL) { + return 0; + } + /* Skip one entry for blob data item precessing the data */ + return ((mNextFreeEntry < (ENTRY_COUNT-1)) ? ((ENTRY_COUNT - mNextFreeEntry - 1) * ENTRY_SIZE): 0); +} + const char* Page::pageStateToName(PageState ps) { switch (ps) { case PageState::CORRUPT: return "CORRUPT"; - + case PageState::ACTIVE: return "ACTIVE"; - + case PageState::FREEING: return "FREEING"; - + case PageState::FULL: return "FULL"; - + case PageState::INVALID: return "INVALID"; - + case PageState::UNINITIALIZED: return "UNINITIALIZED"; - + default: assert(0 && "invalid state value"); return ""; @@ -868,7 +1025,7 @@ void Page::debugDump() const Item item; readEntry(i, item); if (skip == 0) { - printf("W ns=%2u type=%2u span=%3u key=\"%s\" len=%d\n", item.nsIndex, static_cast(item.datatype), item.span, item.key, (item.span != 1)?((int)item.varLength.dataSize):-1); + printf("W ns=%2u type=%2u span=%3u key=\"%s\" chunkIdx=%d len=%d\n", item.nsIndex, static_cast(item.datatype), item.span, item.key, item.chunkIndex, (item.span != 1)?((int)item.varLength.dataSize):-1); if (item.span > 0 && item.span <= ENTRY_COUNT - i) { skip = item.span - 1; } else { @@ -882,4 +1039,33 @@ void Page::debugDump() const } } +esp_err_t Page::calcEntries(nvs_stats_t &nvsStats) +{ + assert(mState != PageState::FREEING); + + nvsStats.total_entries += ENTRY_COUNT; + + switch (mState) { + case PageState::UNINITIALIZED: + case PageState::CORRUPT: + nvsStats.free_entries += ENTRY_COUNT; + break; + + case PageState::FULL: + case PageState::ACTIVE: + nvsStats.used_entries += mUsedEntryCount; + nvsStats.free_entries += ENTRY_COUNT - mUsedEntryCount; // it's equivalent free + erase entries. + break; + + case PageState::INVALID: + return ESP_ERR_INVALID_STATE; + break; + + default: + assert(false && "Unhandled state"); + break; + } + return ESP_OK; +} + } // namespace nvs diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 413da4589..5857f1ffe 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -24,6 +24,7 @@ #include "compressed_enum_table.hpp" #include "intrusive_list.h" #include "nvs_item_hash_list.hpp" +#include "partition.hpp" namespace nvs { @@ -45,12 +46,16 @@ class Page : public intrusive_list_node static const size_t ENTRY_SIZE = 32; static const size_t ENTRY_COUNT = 126; static const uint32_t INVALID_ENTRY = 0xffffffff; - - static const size_t BLOB_MAX_SIZE = ENTRY_SIZE * (ENTRY_COUNT / 2 - 1); + + static const size_t CHUNK_MAX_SIZE = ENTRY_SIZE * (ENTRY_COUNT - 1); static const uint8_t NS_INDEX = 0; static const uint8_t NS_ANY = 255; + static const uint8_t CHUNK_ANY = Item::CHUNK_ANY; + + static const uint8_t NVS_VERSION = 0xfe; // Decrement to upgrade + enum class PageState : uint32_t { // All bits set, default state after flash erase. Page has not been initialized yet. UNINITIALIZED = 0xffffffff, @@ -73,26 +78,32 @@ class Page : public intrusive_list_node INVALID = 0 }; + Page(); + PageState state() const { return mState; } - esp_err_t load(uint32_t sectorNumber); + esp_err_t load(Partition *partition, uint32_t sectorNumber); esp_err_t getSeqNumber(uint32_t& seqNumber) const; esp_err_t setSeqNumber(uint32_t seqNumber); - esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize); + esp_err_t setVersion(uint8_t version); - esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize); + esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY); - esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key); + esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key); + esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item); + esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); + + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); + + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); template esp_err_t writeItem(uint8_t nsIndex, const char* key, const T& value) @@ -106,6 +117,12 @@ class Page : public intrusive_list_node return readItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value)); } + template + esp_err_t cmpItem(uint8_t nsIndex, const char* key, const T& value) + { + return cmpItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value)); + } + template esp_err_t eraseItem(uint8_t nsIndex, const char* key) { @@ -121,7 +138,7 @@ class Page : public intrusive_list_node { return mErasedEntryCount; } - + size_t getVarDataTailroom() const ; esp_err_t markFull(); @@ -133,6 +150,8 @@ class Page : public intrusive_list_node void debugDump() const; + esp_err_t calcEntries(nvs_stats_t &nvsStats); + protected: class Header @@ -140,12 +159,13 @@ class Page : public intrusive_list_node public: Header() { - std::fill_n(mReserved, sizeof(mReserved)/sizeof(mReserved[0]), UINT32_MAX); + std::fill_n(mReserved, sizeof(mReserved)/sizeof(mReserved[0]), UINT8_MAX); } PageState mState; // page state uint32_t mSeqNumber; // sequence number of this page - uint32_t mReserved[5]; // unused, must be 0xffffffff + uint8_t mVersion; // nvs format version + uint8_t mReserved[19]; // unused, must be 0xff uint32_t mCrc32; // crc of everything except mState uint32_t calculateCrc32(); @@ -171,7 +191,7 @@ class Page : public intrusive_list_node esp_err_t readEntry(size_t index, Item& dst) const; esp_err_t writeEntry(const Item& item); - + esp_err_t writeEntryData(const uint8_t* data, size_t size); esp_err_t eraseEntryAndSpan(size_t index); @@ -188,7 +208,7 @@ class Page : public intrusive_list_node assert(entry < ENTRY_COUNT); return mBaseAddress + ENTRY_DATA_OFFSET + static_cast(entry) * ENTRY_SIZE; } - + static const char* pageStateToName(PageState ps); @@ -196,6 +216,7 @@ class Page : public intrusive_list_node uint32_t mBaseAddress = 0; PageState mState = PageState::INVALID; uint32_t mSeqNumber = UINT32_MAX; + uint8_t mVersion = NVS_VERSION; typedef CompressedEnumTable TEntryTable; TEntryTable mEntryTable; size_t mNextFreeEntry = INVALID_ENTRY; @@ -203,8 +224,13 @@ class Page : public intrusive_list_node uint16_t mUsedEntryCount = 0; uint16_t mErasedEntryCount = 0; + /** + * This hash list stores hashes of namespace index, key, and ChunkIndex for quick lookup when searching items. + */ HashList mHashList; + Partition *mPartition; + static const uint32_t HEADER_OFFSET = 0; static const uint32_t ENTRY_TABLE_OFFSET = HEADER_OFFSET + 32; static const uint32_t ENTRY_DATA_OFFSET = ENTRY_TABLE_OFFSET + 32; diff --git a/components/nvs_flash/src/nvs_pagemanager.cpp b/components/nvs_flash/src/nvs_pagemanager.cpp index 31240d98b..88a830bd0 100644 --- a/components/nvs_flash/src/nvs_pagemanager.cpp +++ b/components/nvs_flash/src/nvs_pagemanager.cpp @@ -15,16 +15,22 @@ namespace nvs { -esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) +esp_err_t PageManager::load(Partition *partition, uint32_t baseSector, uint32_t sectorCount) { + if (partition == nullptr) { + return ESP_ERR_INVALID_ARG; + } + mBaseSector = baseSector; mPageCount = sectorCount; mPageList.clear(); mFreePageList.clear(); - mPages.reset(new Page[sectorCount]); + mPages.reset(new (nothrow) Page[sectorCount]); + + if (!mPages) return ESP_ERR_NO_MEM; for (uint32_t i = 0; i < sectorCount; ++i) { - auto err = mPages[i].load(baseSector + i); + auto err = mPages[i].load(partition, baseSector + i); if (err != ESP_OK) { return err; } @@ -66,13 +72,26 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) if (lastItemIndex != SIZE_MAX) { auto last = PageManager::TPageListIterator(&lastPage); - for (auto it = begin(); it != last; ++it) { + TPageListIterator it; + + for (it = begin(); it != last; ++it) { if ((it->state() != Page::PageState::FREEING) && - (it->eraseItem(item.nsIndex, item.datatype, item.key) == ESP_OK)) { + (it->eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex) == ESP_OK)) { break; } } + if ((it == last) && (item.datatype == ItemType::BLOB_IDX)) { + /* Rare case in which the blob was stored using old format, but power went just after writing + * blob index during modification. Loop again and delete the old version blob*/ + for (it = begin(); it != last; ++it) { + + if ((it->state() != Page::PageState::FREEING) && + (it->eraseItem(item.nsIndex, ItemType::BLOB, item.key, item.chunkIndex) == ESP_OK)) { + break; + } + } + } } // check if power went out while page was being freed @@ -111,7 +130,7 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) } // partition should have at least one free page - if (mFreePageList.size() == 0) { + if (mFreePageList.empty()) { return ESP_ERR_NVS_NO_FREE_PAGES; } @@ -200,4 +219,26 @@ esp_err_t PageManager::activatePage() return ESP_OK; } +esp_err_t PageManager::fillStats(nvs_stats_t& nvsStats) +{ + nvsStats.used_entries = 0; + nvsStats.free_entries = 0; + nvsStats.total_entries = 0; + esp_err_t err = ESP_OK; + + // list of used pages + for (auto p = mPageList.begin(); p != mPageList.end(); ++p) { + err = p->calcEntries(nvsStats); + if (err != ESP_OK) { + return err; + } + } + + // free pages + nvsStats.total_entries += mFreePageList.size() * Page::ENTRY_COUNT; + nvsStats.free_entries += mFreePageList.size() * Page::ENTRY_COUNT; + + return err; +} + } // namespace nvs diff --git a/components/nvs_flash/src/nvs_pagemanager.hpp b/components/nvs_flash/src/nvs_pagemanager.hpp index 10c545f0f..4a1c19f7e 100644 --- a/components/nvs_flash/src/nvs_pagemanager.hpp +++ b/components/nvs_flash/src/nvs_pagemanager.hpp @@ -18,7 +18,7 @@ #include #include "nvs_types.hpp" #include "nvs_page.hpp" -#include "nvs_pagemanager.hpp" +#include "partition.hpp" #include "intrusive_list.h" namespace nvs @@ -31,7 +31,7 @@ class PageManager PageManager() {} - esp_err_t load(uint32_t baseSector, uint32_t sectorCount); + esp_err_t load(Partition *partition, uint32_t baseSector, uint32_t sectorCount); TPageListIterator begin() { @@ -48,8 +48,19 @@ class PageManager return mPageList.back(); } + uint32_t getPageCount() { + return mPageCount; + } + esp_err_t requestNewPage(); + esp_err_t fillStats(nvs_stats_t& nvsStats); + + uint32_t getBaseSector() + { + return mBaseSector; + } + protected: friend class Iterator; diff --git a/components/nvs_flash/src/nvs_partition.cpp b/components/nvs_flash/src/nvs_partition.cpp new file mode 100644 index 000000000..9ee8d378c --- /dev/null +++ b/components/nvs_flash/src/nvs_partition.cpp @@ -0,0 +1,77 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "nvs_partition.hpp" + +namespace nvs { + +NVSPartition::NVSPartition(const esp_partition_t* partition) + : mESPPartition(partition) +{ + // ensure the class is in a valid state + if (partition == nullptr) { + std::abort(); + } +} + +const char *NVSPartition::get_partition_name() +{ + return mESPPartition->label; +} + +esp_err_t NVSPartition::read_raw(size_t src_offset, void* dst, size_t size) +{ + return esp_partition_read(mESPPartition, src_offset, dst, size); +} + +esp_err_t NVSPartition::read(size_t src_offset, void* dst, size_t size) +{ + if (size % ESP_ENCRYPT_BLOCK_SIZE != 0) { + return ESP_ERR_INVALID_ARG; + } + + return esp_partition_read(mESPPartition, src_offset, dst, size); +} + +esp_err_t NVSPartition::write_raw(size_t dst_offset, const void* src, size_t size) +{ + return esp_partition_write(mESPPartition, dst_offset, src, size); +} + +esp_err_t NVSPartition::write(size_t dst_offset, const void* src, size_t size) +{ + if (size % ESP_ENCRYPT_BLOCK_SIZE != 0) { + return ESP_ERR_INVALID_ARG; + } + + return esp_partition_write(mESPPartition, dst_offset, src, size); +} + +esp_err_t NVSPartition::erase_range(size_t dst_offset, size_t size) +{ + return esp_partition_erase_range(mESPPartition, dst_offset, size); +} + +uint32_t NVSPartition::get_address() +{ + return mESPPartition->address; +} + +uint32_t NVSPartition::get_size() +{ + return mESPPartition->size; +} + +} // nvs diff --git a/components/nvs_flash/src/nvs_partition.hpp b/components/nvs_flash/src/nvs_partition.hpp new file mode 100644 index 000000000..d5259cdb5 --- /dev/null +++ b/components/nvs_flash/src/nvs_partition.hpp @@ -0,0 +1,115 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ESP_PARTITION_HPP_ +#define ESP_PARTITION_HPP_ + +#include "esp_partition.h" +#include "intrusive_list.h" +#include "partition.hpp" + +#define ESP_ENCRYPT_BLOCK_SIZE 16 + +#define PART_NAME_MAX_SIZE 16 /*!< maximum length of partition name (excluding null terminator) */ + +namespace nvs { + +/** + * Implementation of Partition for NVS. + * + * It is implemented as an intrusive_list_node to easily store instances of it. NVSStorage and NVSPage take pointer + * references of this class to abstract their partition operations. + */ +class NVSPartition : public Partition, public intrusive_list_node { +public: + /** + * Copy partition_name to mPartitionName and initialize mESPPartition. + * + * @param partition_name the name of the partition as in the partition table, must be non-NULL! + * @param partition an already initialized partition structure + */ + NVSPartition(const esp_partition_t* partition); + + /** + * No need to de-initialize mESPPartition here, if you used esp_partition_find_first. + * Otherwise, the user is responsible for de-initializing it. + */ + virtual ~NVSPartition() { } + + const char *get_partition_name() override; + + /** + * Look into \c esp_partition_read for more details. + * + * @return + * - ESP_OK on success + * - other error codes from the esp_partition API + */ + esp_err_t read_raw(size_t src_offset, void* dst, size_t size) override; + + /** + * Look into \c esp_partition_read for more details. + * + * @return + * - ESP_OK on success + * - ESP_ERR_INVALID_ARG if size isn't a multiple of ESP_ENCRYPT_BLOCK_SIZE + * - other error codes from the esp_partition API + */ + esp_err_t read(size_t src_offset, void* dst, size_t size) override; + + /** + * Look into \c esp_partition_write for more details. + * + * @return + * - ESP_OK on success + * - error codes from the esp_partition API + */ + esp_err_t write_raw(size_t dst_offset, const void* src, size_t size) override; + + /** + * Look into \c esp_partition_write for more details. + * + * @return + * - ESP_OK on success + * - ESP_ERR_INVALID_ARG if size isn't a multiple of ESP_ENCRYPT_BLOCK_SIZE + * - other error codes from the esp_partition API + */ + esp_err_t write(size_t dst_offset, const void* src, size_t size) override; + + /** + * Look into \c esp_partition_erase_range for more details. + * + * @return + * - ESP_OK on success + * - error codes from the esp_partition API + */ + esp_err_t erase_range(size_t dst_offset, size_t size) override; + + /** + * @return the base address of the partition. + */ + uint32_t get_address() override; + + /** + * @return the size of the partition in bytes. + */ + uint32_t get_size() override; + +protected: + const esp_partition_t* mESPPartition; +}; + +} // nvs + +#endif // ESP_PARTITION_HPP_ diff --git a/components/nvs_flash/src/nvs_partition_lookup.cpp b/components/nvs_flash/src/nvs_partition_lookup.cpp new file mode 100644 index 000000000..e9d27256f --- /dev/null +++ b/components/nvs_flash/src/nvs_partition_lookup.cpp @@ -0,0 +1,69 @@ +#include "esp_partition.h" +#include "nvs_partition_lookup.hpp" + +#ifdef CONFIG_NVS_ENCRYPTION +#include "nvs_encrypted_partition.hpp" +#endif // CONFIG_NVS_ENCRYPTION + +namespace nvs { + +namespace partition_lookup { + +esp_err_t lookup_nvs_partition(const char* label, NVSPartition **p) +{ + const esp_partition_t* esp_partition = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, label); + + if (esp_partition == nullptr) { + return ESP_ERR_NOT_FOUND; + } + + if (esp_partition->encrypted) { + return ESP_ERR_NVS_WRONG_ENCRYPTION; + } + + NVSPartition *partition = new (std::nothrow) NVSPartition(esp_partition); + if (partition == nullptr) { + return ESP_ERR_NO_MEM; + } + + *p = partition; + + return ESP_OK; +} + +#ifdef CONFIG_NVS_ENCRYPTION +esp_err_t lookup_nvs_encrypted_partition(const char* label, nvs_sec_cfg_t* cfg, NVSPartition **p) +{ + const esp_partition_t* esp_partition = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, label); + + if (esp_partition == nullptr) { + return ESP_ERR_NOT_FOUND; + } + + if (esp_partition->encrypted) { + return ESP_ERR_NVS_WRONG_ENCRYPTION; + } + + NVSEncryptedPartition *enc_p = new (std::nothrow) NVSEncryptedPartition(esp_partition); + if (enc_p == nullptr) { + return ESP_ERR_NO_MEM; + } + + esp_err_t result = enc_p->init(cfg); + if (result != ESP_OK) { + delete enc_p; + return result; + } + + *p = enc_p; + + return ESP_OK; +} + +#endif // CONFIG_NVS_ENCRYPTION + +} // partition_lookup + +} // nvs diff --git a/components/nvs_flash/src/nvs_partition_lookup.hpp b/components/nvs_flash/src/nvs_partition_lookup.hpp new file mode 100644 index 000000000..62a8e8b6f --- /dev/null +++ b/components/nvs_flash/src/nvs_partition_lookup.hpp @@ -0,0 +1,22 @@ +#include "esp_err.h" +#include "nvs_partition.hpp" +#include "nvs_flash.h" + +#ifndef NVS_PARTITION_LOOKUP_HPP_ +#define NVS_PARTITION_LOOKUP_HPP_ + +namespace nvs { + +namespace partition_lookup { + +esp_err_t lookup_nvs_partition(const char* label, NVSPartition **p); + +#ifdef CONFIG_NVS_ENCRYPTION +esp_err_t lookup_nvs_encrypted_partition(const char* label, nvs_sec_cfg_t* cfg, NVSPartition **p); +#endif // CONFIG_NVS_ENCRYPTION + +} // partition_lookup + +} // nvs + +#endif // NVS_PARTITION_LOOKUP_HPP_ diff --git a/components/nvs_flash/src/nvs_partition_manager.cpp b/components/nvs_flash/src/nvs_partition_manager.cpp new file mode 100644 index 000000000..b95db7e81 --- /dev/null +++ b/components/nvs_flash/src/nvs_partition_manager.cpp @@ -0,0 +1,244 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "esp_partition.h" +#include "nvs_partition_manager.hpp" +#include "nvs_partition_lookup.hpp" + +#ifdef CONFIG_NVS_ENCRYPTION +#include "nvs_encrypted_partition.hpp" +#endif // CONFIG_NVS_ENCRYPTION + +namespace nvs { + +NVSPartitionManager* NVSPartitionManager::instance = nullptr; + +NVSPartitionManager* NVSPartitionManager::get_instance() +{ + if (!instance) { + instance = new (std::nothrow) NVSPartitionManager(); + } + + return instance; +} + +#ifdef ESP_PLATFORM +esp_err_t NVSPartitionManager::init_partition(const char *partition_label) +{ + if (strlen(partition_label) > NVS_PART_NAME_MAX_SIZE) { + return ESP_ERR_INVALID_ARG; + } + + uint32_t size; + Storage* mStorage; + + mStorage = lookup_storage_from_name(partition_label); + if (mStorage) { + return ESP_OK; + } + + assert(SPI_FLASH_SEC_SIZE != 0); + + NVSPartition *p = nullptr; + esp_err_t result = partition_lookup::lookup_nvs_partition(partition_label, &p); + + if (result != ESP_OK) { + goto error; + } + + size = p->get_size(); + + result = init_custom(p, 0, size / SPI_FLASH_SEC_SIZE); + if (result != ESP_OK) { + goto error; + } + + nvs_partition_list.push_back(p); + + return ESP_OK; + +error: + delete p; + return result; +} +#endif // ESP_PLATFORM + +esp_err_t NVSPartitionManager::init_custom(Partition *partition, uint32_t baseSector, uint32_t sectorCount) +{ + Storage* new_storage = nullptr; + Storage* storage = lookup_storage_from_name(partition->get_partition_name()); + if (storage == nullptr) { + new_storage = new (std::nothrow) Storage(partition); + + if (new_storage == nullptr) { + return ESP_ERR_NO_MEM; + } + + storage = new_storage; + } else { + // if storage was initialized already, we don't need partition and hence delete it + for (auto it = nvs_partition_list.begin(); it != nvs_partition_list.end(); ++it) { + if (partition == it) { + nvs_partition_list.erase(it); + delete partition; + break; + } + } + } + + esp_err_t err = storage->init(baseSector, sectorCount); + if (new_storage != nullptr) { + if (err == ESP_OK) { + nvs_storage_list.push_back(new_storage); + } else { + delete new_storage; + } + } + return err; +} + +#ifdef CONFIG_NVS_ENCRYPTION +#ifdef ESP_PLATFORM +esp_err_t NVSPartitionManager::secure_init_partition(const char *part_name, nvs_sec_cfg_t* cfg) +{ + if (strlen(part_name) > NVS_PART_NAME_MAX_SIZE) { + return ESP_ERR_INVALID_ARG; + } + + Storage* mStorage; + + mStorage = lookup_storage_from_name(part_name); + if (mStorage != nullptr) { + return ESP_OK; + } + + NVSPartition *p; + esp_err_t result; + if (cfg != nullptr) { + result = partition_lookup::lookup_nvs_encrypted_partition(part_name, cfg, &p); + } else { + result = partition_lookup::lookup_nvs_partition(part_name, &p); + } + + if (result != ESP_OK) { + return result; + } + + uint32_t size = p->get_size(); + + result = init_custom(p, 0, size / SPI_FLASH_SEC_SIZE); + if (result != ESP_OK) { + delete p; + return result; + } + + nvs_partition_list.push_back(p); + + return ESP_OK; +} +#endif // ESP_PLATFORM +#endif // CONFIG_NVS_ENCRYPTION + +esp_err_t NVSPartitionManager::deinit_partition(const char *partition_label) +{ + Storage* storage = lookup_storage_from_name(partition_label); + if (!storage) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + + /* Clean up handles related to the storage being deinitialized */ + for (auto it = nvs_handles.begin(); it != nvs_handles.end(); ++it) { + if (it->mStoragePtr == storage) { + it->valid = false; + nvs_handles.erase(it); + } + } + + /* Finally delete the storage and its partition */ + nvs_storage_list.erase(storage); + delete storage; + + for (auto it = nvs_partition_list.begin(); it != nvs_partition_list.end(); ++it) { + if (strcmp(it->get_partition_name(), partition_label) == 0) { + NVSPartition *p = it; + nvs_partition_list.erase(it); + delete p; + break; + } + } + + return ESP_OK; +} + +esp_err_t NVSPartitionManager::open_handle(const char *part_name, + const char *ns_name, + nvs_open_mode_t open_mode, + NVSHandleSimple** handle) +{ + uint8_t nsIndex; + Storage* sHandle; + + if (nvs_storage_list.empty()) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + + sHandle = lookup_storage_from_name(part_name); + if (sHandle == nullptr) { + return ESP_ERR_NVS_PART_NOT_FOUND; + } + + esp_err_t err = sHandle->createOrOpenNamespace(ns_name, open_mode == NVS_READWRITE, nsIndex); + if (err != ESP_OK) { + return err; + } + + *handle = new (std::nothrow) NVSHandleSimple(open_mode==NVS_READONLY, nsIndex, sHandle); + + if (handle == nullptr) { + return ESP_ERR_NO_MEM; + } + + nvs_handles.push_back(*handle); + + return ESP_OK; +} + +esp_err_t NVSPartitionManager::close_handle(NVSHandleSimple* handle) { + for (auto it = nvs_handles.begin(); it != nvs_handles.end(); ++it) { + if (it == intrusive_list::iterator(handle)) { + nvs_handles.erase(it); + return ESP_OK; + } + } + + return ESP_ERR_NVS_INVALID_HANDLE; +} + +size_t NVSPartitionManager::open_handles_size() +{ + return nvs_handles.size(); +} + +Storage* NVSPartitionManager::lookup_storage_from_name(const char* name) +{ + auto it = find_if(begin(nvs_storage_list), end(nvs_storage_list), [=](Storage& e) -> bool { + return (strcmp(e.getPartName(), name) == 0); + }); + + if (it == end(nvs_storage_list)) { + return nullptr; + } + return it; +} + +} // nvs diff --git a/components/nvs_flash/src/nvs_partition_manager.hpp b/components/nvs_flash/src/nvs_partition_manager.hpp new file mode 100644 index 000000000..885787de4 --- /dev/null +++ b/components/nvs_flash/src/nvs_partition_manager.hpp @@ -0,0 +1,62 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef NVS_PARTITION_MANAGER_HPP_ +#define NVS_PARTITION_MANAGER_HPP_ + +#include "nvs_handle_simple.hpp" +#include "nvs_storage.hpp" +#include "nvs_partition.hpp" +#include "nvs_flash.h" + +namespace nvs { + +class NVSPartitionManager { +public: + virtual ~NVSPartitionManager() { } + + static NVSPartitionManager* get_instance(); + + esp_err_t init_partition(const char *partition_label); + + esp_err_t init_custom(Partition *partition, uint32_t baseSector, uint32_t sectorCount); + +#ifdef CONFIG_NVS_ENCRYPTION + esp_err_t secure_init_partition(const char *part_name, nvs_sec_cfg_t* cfg); +#endif + + esp_err_t deinit_partition(const char *partition_label); + + Storage* lookup_storage_from_name(const char* name); + + esp_err_t open_handle(const char *part_name, const char *ns_name, nvs_open_mode_t open_mode, NVSHandleSimple** handle); + + esp_err_t close_handle(NVSHandleSimple* handle); + + size_t open_handles_size(); + +protected: + NVSPartitionManager() { } + + static NVSPartitionManager* instance; + + intrusive_list nvs_handles; + + intrusive_list nvs_storage_list; + + intrusive_list nvs_partition_list; +}; + +} // nvs + +#endif // NVS_PARTITION_MANAGER_HPP_ diff --git a/components/nvs_flash/src/nvs_platform.hpp b/components/nvs_flash/src/nvs_platform.hpp index 0973c4875..5c6b5b8b2 100644 --- a/components/nvs_flash/src/nvs_platform.hpp +++ b/components/nvs_flash/src/nvs_platform.hpp @@ -11,11 +11,23 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#ifndef nvs_platform_h -#define nvs_platform_h +#pragma once +#ifdef LINUX_TARGET +namespace nvs +{ +class Lock +{ +public: + Lock() { } + ~Lock() { } + static void init() {} + static void uninit() {} +}; +} // namespace nvs + +#else // LINUX_TARGET -#ifdef ESP_PLATFORM #include "freertos/FreeRTOS.h" #include "freertos/semphr.h" @@ -63,19 +75,4 @@ class Lock }; } // namespace nvs -#else // ESP_PLATFORM -namespace nvs -{ -class Lock -{ -public: - Lock() { } - ~Lock() { } - static void init() {} - static void uninit() {} -}; -} // namespace nvs -#endif // ESP_PLATFORM - - -#endif /* nvs_platform_h */ +#endif // LINUX_TARGET diff --git a/components/nvs_flash/src/nvs_storage.cpp b/components/nvs_flash/src/nvs_storage.cpp index f8da28fa2..52952b7a3 100644 --- a/components/nvs_flash/src/nvs_storage.cpp +++ b/components/nvs_flash/src/nvs_storage.cpp @@ -14,9 +14,15 @@ #include "nvs_storage.hpp" #ifndef ESP_PLATFORM +// We need NO_DEBUG_STORAGE here since the integration tests on the host add some debug code. +// The unit tests, however, don't want debug code since they check the behavior via data in/output and disturb +// the order of calling mocked functions. +#ifndef NO_DEBUG_STORAGE #include #include +#define DEBUG_STORAGE #endif +#endif // !ESP_PLATFORM namespace nvs { @@ -28,17 +34,69 @@ Storage::~Storage() void Storage::clearNamespaces() { - for (auto it = std::begin(mNamespaces); it != std::end(mNamespaces); ) { - auto tmp = it; - ++it; - mNamespaces.erase(tmp); - delete static_cast(tmp); + mNamespaces.clearAndFreeNodes(); +} + +esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList) +{ + for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + Page& p = *it; + size_t itemIndex = 0; + Item item; + + /* If the power went off just after writing a blob index, the duplicate detection + * logic in pagemanager will remove the earlier index. So we should never find a + * duplicate index at this point */ + + while (p.findItem(Page::NS_ANY, ItemType::BLOB_IDX, nullptr, itemIndex, item) == ESP_OK) { + BlobIndexNode* entry = new (std::nothrow) BlobIndexNode; + + if (!entry) return ESP_ERR_NO_MEM; + + item.getKey(entry->key, sizeof(entry->key)); + entry->nsIndex = item.nsIndex; + entry->chunkStart = item.blobIndex.chunkStart; + entry->chunkCount = item.blobIndex.chunkCount; + + blobIdxList.push_back(entry); + itemIndex += item.span; + } + } + + return ESP_OK; +} + +void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) +{ + for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + Page& p = *it; + size_t itemIndex = 0; + Item item; + /* Chunks with same and with chunkIndex in the following ranges + * belong to same family. + * 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks + * 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks + */ + while (p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { + + auto iter = std::find_if(blobIdxList.begin(), + blobIdxList.end(), + [=] (const BlobIndexNode& e) -> bool + {return (strncmp(item.key, e.key, sizeof(e.key) - 1) == 0) + && (item.nsIndex == e.nsIndex) + && (item.chunkIndex >= static_cast (e.chunkStart)) + && (item.chunkIndex < static_cast (e.chunkStart) + e.chunkCount);}); + if (iter == std::end(blobIdxList)) { + p.eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex); + } + itemIndex += item.span; + } } } esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) { - auto err = mPageManager.load(baseSector, sectorCount); + auto err = mPageManager.load(mPartition, baseSector, sectorCount); if (err != ESP_OK) { mState = StorageState::INVALID; return err; @@ -52,8 +110,14 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) size_t itemIndex = 0; Item item; while (p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) { - NamespaceEntry* entry = new NamespaceEntry; - item.getKey(entry->mName, sizeof(entry->mName) - 1); + NamespaceEntry* entry = new (std::nothrow) NamespaceEntry; + + if (!entry) { + mState = StorageState::INVALID; + return ESP_ERR_NO_MEM; + } + + item.getKey(entry->mName, sizeof(entry->mName)); item.getValue(entry->mIndex); mNamespaces.push_back(entry); mNamespaceUsage.set(entry->mIndex, true); @@ -63,7 +127,22 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) mNamespaceUsage.set(0, true); mNamespaceUsage.set(255, true); mState = StorageState::ACTIVE; -#ifndef ESP_PLATFORM + + // Populate list of multi-page index entries. + TBlobIndexList blobIdxList; + err = populateBlobIndices(blobIdxList); + if (err != ESP_OK) { + mState = StorageState::INVALID; + return ESP_ERR_NO_MEM; + } + + // Remove the entries for which there is no parent multi-page index. + eraseOrphanDataBlobs(blobIdxList); + + // Purge the blob index list + blobIdxList.clearAndFreeNodes(); + +#ifdef DEBUG_STORAGE debugCheck(); #endif return ESP_OK; @@ -74,11 +153,11 @@ bool Storage::isValid() const return mState == StorageState::ACTIVE; } -esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item) +esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart) { for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { size_t itemIndex = 0; - auto err = it->findItem(nsIndex, datatype, key, itemIndex, item); + auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart); if (err == ESP_OK) { page = it; return ESP_OK; @@ -87,6 +166,109 @@ esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, return ESP_ERR_NVS_NOT_FOUND; } +esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const void* data, size_t dataSize, VerOffset chunkStart) +{ + uint8_t chunkCount = 0; + TUsedPageList usedPages; + size_t remainingSize = dataSize; + size_t offset = 0; + esp_err_t err = ESP_OK; + + /* Check how much maximum data can be accommodated**/ + uint32_t max_pages = mPageManager.getPageCount() - 1; + + if(max_pages > (Page::CHUNK_ANY-1)/2) { + max_pages = (Page::CHUNK_ANY-1)/2; + } + + if (dataSize > max_pages * Page::CHUNK_MAX_SIZE) { + return ESP_ERR_NVS_VALUE_TOO_LONG; + } + + do { + Page& page = getCurrentPage(); + size_t tailroom = page.getVarDataTailroom(); + size_t chunkSize = 0; + if (chunkCount == 0U && ((tailroom < dataSize) || (tailroom == 0 && dataSize == 0)) && tailroom < Page::CHUNK_MAX_SIZE/10) { + /** This is the first chunk and tailroom is too small ***/ + if (page.state() != Page::PageState::FULL) { + err = page.markFull(); + if (err != ESP_OK) { + return err; + } + } + err = mPageManager.requestNewPage(); + if (err != ESP_OK) { + return err; + } else if(getCurrentPage().getVarDataTailroom() == tailroom) { + /* We got the same page or we are not improving.*/ + return ESP_ERR_NVS_NOT_ENOUGH_SPACE; + } else { + continue; + } + } else if (!tailroom) { + err = ESP_ERR_NVS_NOT_ENOUGH_SPACE; + break; + } + + /* Split the blob into two and store the chunk of available size onto the current page */ + assert(tailroom != 0); + chunkSize = (remainingSize > tailroom)? tailroom : remainingSize; + remainingSize -= chunkSize; + + err = page.writeItem(nsIndex, ItemType::BLOB_DATA, key, + static_cast (data) + offset, chunkSize, static_cast (chunkStart) + chunkCount); + chunkCount++; + assert(err != ESP_ERR_NVS_PAGE_FULL); + if (err != ESP_OK) { + break; + } else { + UsedPageNode* node = new (std::nothrow) UsedPageNode(); + if (!node) { + err = ESP_ERR_NO_MEM; + break; + } + node->mPage = &page; + usedPages.push_back(node); + if (remainingSize || (tailroom - chunkSize) < Page::ENTRY_SIZE) { + if (page.state() != Page::PageState::FULL) { + err = page.markFull(); + if (err != ESP_OK) { + break; + } + } + err = mPageManager.requestNewPage(); + if (err != ESP_OK) { + break; + } + } + } + offset += chunkSize; + if (!remainingSize) { + /* All pages are stored. Now store the index.*/ + Item item; + std::fill_n(item.data, sizeof(item.data), 0xff); + item.blobIndex.dataSize = dataSize; + item.blobIndex.chunkCount = chunkCount; + item.blobIndex.chunkStart = chunkStart; + + err = getCurrentPage().writeItem(nsIndex, ItemType::BLOB_IDX, key, item.data, sizeof(item.data)); + assert(err != ESP_ERR_NVS_PAGE_FULL); + break; + } + } while (1); + + if (err != ESP_OK) { + /* Anything failed, then we should erase all the written chunks*/ + int ii=0; + for (auto it = std::begin(usedPages); it != std::end(usedPages); it++) { + it->mPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, ii++); + } + } + usedPages.clearAndFreeNodes(); + return err; +} + esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { if (mState != StorageState::ACTIVE) { @@ -95,40 +277,109 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key Page* findPage = nullptr; Item item; - auto err = findItem(nsIndex, datatype, key, findPage, item); + + esp_err_t err; + if (datatype == ItemType::BLOB) { + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + } else { + err = findItem(nsIndex, datatype, key, findPage, item); + } + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { return err; } - Page& page = getCurrentPage(); - err = page.writeItem(nsIndex, datatype, key, data, dataSize); - if (err == ESP_ERR_NVS_PAGE_FULL) { - if (page.state() != Page::PageState::FULL) { - err = page.markFull(); - if (err != ESP_OK) { - return err; + if (datatype == ItemType::BLOB) { + VerOffset prevStart, nextStart; + prevStart = nextStart = VerOffset::VER_0_OFFSET; + if (findPage) { + // Do a sanity check that the item in question is actually being modified. + // If it isn't, it is cheaper to purposefully not write out new data. + // since it may invoke an erasure of flash. + if (cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) { + return ESP_OK; } + + if (findPage->state() == Page::PageState::UNINITIALIZED || + findPage->state() == Page::PageState::INVALID) { + ESP_ERROR_CHECK(findItem(nsIndex, datatype, key, findPage, item)); + } + /* Get the version of the previous index with same */ + prevStart = item.blobIndex.chunkStart; + assert(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET); + + /* Toggle the version by changing the offset */ + nextStart + = (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET; } - err = mPageManager.requestNewPage(); - if (err != ESP_OK) { - return err; - } + /* Write the blob with new version*/ + err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart); - err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize); if (err == ESP_ERR_NVS_PAGE_FULL) { return ESP_ERR_NVS_NOT_ENOUGH_SPACE; } if (err != ESP_OK) { return err; } - } else if (err != ESP_OK) { - return err; + + if (findPage) { + /* Erase the blob with earlier version*/ + err = eraseMultiPageBlob(nsIndex, key, prevStart); + + if (err == ESP_ERR_FLASH_OP_FAIL) { + return ESP_ERR_NVS_REMOVE_FAILED; + } + if (err != ESP_OK) { + return err; + } + + findPage = nullptr; + } else { + /* Support for earlier versions where BLOBS were stored without index */ + err = findItem(nsIndex, datatype, key, findPage, item); + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } + } + } else { + // Do a sanity check that the item in question is actually being modified. + // If it isn't, it is cheaper to purposefully not write out new data. + // since it may invoke an erasure of flash. + if (findPage != nullptr && + findPage->cmpItem(nsIndex, datatype, key, data, dataSize) == ESP_OK) { + return ESP_OK; + } + + Page& page = getCurrentPage(); + err = page.writeItem(nsIndex, datatype, key, data, dataSize); + if (err == ESP_ERR_NVS_PAGE_FULL) { + if (page.state() != Page::PageState::FULL) { + err = page.markFull(); + if (err != ESP_OK) { + return err; + } + } + err = mPageManager.requestNewPage(); + if (err != ESP_OK) { + return err; + } + + err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize); + if (err == ESP_ERR_NVS_PAGE_FULL) { + return ESP_ERR_NVS_NOT_ENOUGH_SPACE; + } + if (err != ESP_OK) { + return err; + } + } else if (err != ESP_OK) { + return err; + } } if (findPage) { if (findPage->state() == Page::PageState::UNINITIALIZED || findPage->state() == Page::PageState::INVALID) { - ESP_ERROR_CHECK( findItem(nsIndex, datatype, key, findPage, item) ); + ESP_ERROR_CHECK(findItem(nsIndex, datatype, key, findPage, item)); } err = findPage->eraseItem(nsIndex, datatype, key); if (err == ESP_ERR_FLASH_OP_FAIL) { @@ -138,7 +389,7 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key return err; } } -#ifndef ESP_PLATFORM +#ifdef DEBUG_STORAGE debugCheck(); #endif return ESP_OK; @@ -168,6 +419,11 @@ esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uin return ESP_ERR_NVS_NOT_ENOUGH_SPACE; } + NamespaceEntry* entry = new (std::nothrow) NamespaceEntry; + if (!entry) { + return ESP_ERR_NO_MEM; + } + auto err = writeItem(Page::NS_INDEX, ItemType::U8, nsName, &ns, sizeof(ns)); if (err != ESP_OK) { return err; @@ -175,7 +431,6 @@ esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uin mNamespaceUsage.set(ns, true); nsIndex = ns; - NamespaceEntry* entry = new NamespaceEntry; entry->mIndex = ns; strncpy(entry->mName, nsName, sizeof(entry->mName) - 1); entry->mName[sizeof(entry->mName) - 1] = 0; @@ -187,6 +442,91 @@ esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uin return ESP_OK; } +esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* data, size_t dataSize) +{ + Item item; + Page* findPage = nullptr; + + /* First read the blob index */ + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + if (err != ESP_OK) { + return err; + } + + uint8_t chunkCount = item.blobIndex.chunkCount; + VerOffset chunkStart = item.blobIndex.chunkStart; + size_t readSize = item.blobIndex.dataSize; + size_t offset = 0; + + assert(dataSize == readSize); + + /* Now read corresponding chunks */ + for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + if (err == ESP_ERR_NVS_NOT_FOUND) { + break; + } + return err; + } + err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + return err; + } + assert(static_cast (chunkStart) + chunkNum == item.chunkIndex); + offset += item.varLength.dataSize; + } + if (err == ESP_OK) { + assert(offset == dataSize); + } + if (err == ESP_ERR_NVS_NOT_FOUND) { + eraseMultiPageBlob(nsIndex, key); // cleanup if a chunk is not found + } + return err; +} + +esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void* data, size_t dataSize) +{ + Item item; + Page* findPage = nullptr; + + /* First read the blob index */ + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + if (err != ESP_OK) { + return err; + } + + uint8_t chunkCount = item.blobIndex.chunkCount; + VerOffset chunkStart = item.blobIndex.chunkStart; + size_t readSize = item.blobIndex.dataSize; + size_t offset = 0; + + if (dataSize != readSize) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + + /* Now read corresponding chunks */ + for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + if (err == ESP_ERR_NVS_NOT_FOUND) { + break; + } + return err; + } + err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + return err; + } + assert(static_cast (chunkStart) + chunkNum == item.chunkIndex); + offset += item.varLength.dataSize; + } + if (err == ESP_OK) { + assert(offset == dataSize); + } + return err; +} + esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize) { if (mState != StorageState::ACTIVE) { @@ -195,12 +535,64 @@ esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, Item item; Page* findPage = nullptr; + if (datatype == ItemType::BLOB) { + auto err = readMultiPageBlob(nsIndex, key, data, dataSize); + if (err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } // else check if the blob is stored with earlier version format without index + } + auto err = findItem(nsIndex, datatype, key, findPage, item); if (err != ESP_OK) { return err; } - return findPage->readItem(nsIndex, datatype, key, data, dataSize); + +} + +esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart) +{ + if (mState != StorageState::ACTIVE) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + Item item; + Page* findPage = nullptr; + + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); + if (err != ESP_OK) { + return err; + } + /* Erase the index first and make children blobs orphan*/ + err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); + if (err != ESP_OK) { + return err; + } + + uint8_t chunkCount = item.blobIndex.chunkCount; + + if (chunkStart == VerOffset::VER_ANY) { + chunkStart = item.blobIndex.chunkStart; + } else { + assert(chunkStart == item.blobIndex.chunkStart); + } + + /* Now erase corresponding chunks*/ + for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } else if (err == ESP_ERR_NVS_NOT_FOUND) { + continue; // Keep erasing other chunks + } + err = findPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + return err; + } + + } + + return ESP_OK; } esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key) @@ -209,6 +601,10 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key return ESP_ERR_NVS_NOT_INITIALIZED; } + if (datatype == ItemType::BLOB) { + return eraseMultiPageBlob(nsIndex, key); + } + Item item; Page* findPage = nullptr; auto err = findItem(nsIndex, datatype, key, findPage, item); @@ -216,6 +612,10 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key return err; } + if (item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) { + return eraseMultiPageBlob(nsIndex, key); + } + return findPage->eraseItem(nsIndex, datatype, key); } @@ -250,7 +650,15 @@ esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const cha Page* findPage = nullptr; auto err = findItem(nsIndex, datatype, key, findPage, item); if (err != ESP_OK) { - return err; + if (datatype != ItemType::BLOB) { + return err; + } + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + if (err != ESP_OK) { + return err; + } + dataSize = item.blobIndex.dataSize; + return ESP_OK; } dataSize = item.varLength.dataSize; @@ -264,18 +672,18 @@ void Storage::debugDump() } } -#ifndef ESP_PLATFORM +#ifdef DEBUG_STORAGE void Storage::debugCheck() { std::map keys; - + for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { size_t itemIndex = 0; size_t usedCount = 0; Item item; while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { std::stringstream keyrepr; - keyrepr << static_cast(item.nsIndex) << "_" << static_cast(item.datatype) << "_" << item.key; + keyrepr << static_cast(item.nsIndex) << "_" << static_cast(item.datatype) << "_" << item.key <<"_"<(item.chunkIndex); std::string keystr = keyrepr.str(); if (keys.find(keystr) != std::end(keys)) { printf("Duplicate key: %s\n", keystr.c_str()); @@ -289,6 +697,104 @@ void Storage::debugCheck() assert(usedCount == p->getUsedEntryCount()); } } -#endif //ESP_PLATFORM +#endif //DEBUG_STORAGE + +esp_err_t Storage::fillStats(nvs_stats_t& nvsStats) +{ + nvsStats.namespace_count = mNamespaces.size(); + return mPageManager.fillStats(nvsStats); +} + +esp_err_t Storage::calcEntriesInNamespace(uint8_t nsIndex, size_t& usedEntries) +{ + usedEntries = 0; + + if (mState != StorageState::ACTIVE) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + + for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + size_t itemIndex = 0; + Item item; + while (true) { + auto err = it->findItem(nsIndex, ItemType::ANY, nullptr, itemIndex, item); + if (err == ESP_ERR_NVS_NOT_FOUND) { + break; + } + else if (err != ESP_OK) { + return err; + } + usedEntries += item.span; + itemIndex += item.span; + if (itemIndex >= it->ENTRY_COUNT) break; + } + } + return ESP_OK; +} + +void Storage::fillEntryInfo(Item &item, nvs_entry_info_t &info) +{ + info.type = static_cast(item.datatype); + strncpy(info.key, item.key, sizeof(info.key)); + + for (auto &name : mNamespaces) { + if(item.nsIndex == name.mIndex) { + strncpy(info.namespace_name, name.mName, sizeof(info.namespace_name)); + break; + } + } +} + +bool Storage::findEntry(nvs_opaque_iterator_t* it, const char* namespace_name) +{ + it->entryIndex = 0; + it->nsIndex = Page::NS_ANY; + it->page = mPageManager.begin(); + + if (namespace_name != nullptr) { + if(createOrOpenNamespace(namespace_name, false, it->nsIndex) != ESP_OK) { + return false; + } + } + + return nextEntry(it); +} + +inline bool isIterableItem(Item& item) +{ + return (item.nsIndex != 0 && + item.datatype != ItemType::BLOB && + item.datatype != ItemType::BLOB_IDX); +} + +inline bool isMultipageBlob(Item& item) +{ + return (item.datatype == ItemType::BLOB_DATA && + !(item.chunkIndex == static_cast(VerOffset::VER_0_OFFSET) + || item.chunkIndex == static_cast(VerOffset::VER_1_OFFSET))); +} + +bool Storage::nextEntry(nvs_opaque_iterator_t* it) +{ + Item item; + esp_err_t err; + + for (auto page = it->page; page != mPageManager.end(); ++page) { + do { + err = page->findItem(it->nsIndex, (ItemType)it->type, nullptr, it->entryIndex, item); + it->entryIndex += item.span; + if(err == ESP_OK && isIterableItem(item) && !isMultipageBlob(item)) { + fillEntryInfo(item, it->entry_info); + it->page = page; + return true; + } + } while (err != ESP_ERR_NVS_NOT_FOUND); + + it->entryIndex = 0; + } + + return false; +} + } diff --git a/components/nvs_flash/src/nvs_storage.hpp b/components/nvs_flash/src/nvs_storage.hpp index 3c0e0c85a..fd6eb1cc9 100644 --- a/components/nvs_flash/src/nvs_storage.hpp +++ b/components/nvs_flash/src/nvs_storage.hpp @@ -15,12 +15,12 @@ #define nvs_storage_hpp #include -#include #include #include "nvs.hpp" #include "nvs_types.hpp" #include "nvs_page.hpp" #include "nvs_pagemanager.hpp" +#include "partition.hpp" //extern void dumpBytes(const uint8_t* data, size_t count); @@ -42,10 +42,30 @@ class Storage : public intrusive_list_node typedef intrusive_list TNamespaces; + struct UsedPageNode: public intrusive_list_node { + public: Page* mPage; + }; + + typedef intrusive_list TUsedPageList; + + struct BlobIndexNode: public intrusive_list_node { + public: + char key[Item::MAX_KEY_LENGTH + 1]; + uint8_t nsIndex; + uint8_t chunkCount; + VerOffset chunkStart; + }; + + typedef intrusive_list TBlobIndexList; + public: ~Storage(); - Storage(const char *pName = NVS_DEFAULT_PART_NAME) : mPartitionName(pName) { }; + Storage(Partition *partition) : mPartition(partition) { + if (partition == nullptr) { + abort(); + } + }; esp_err_t init(uint32_t baseSector, uint32_t sectorCount); @@ -77,18 +97,43 @@ class Storage : public intrusive_list_node { return eraseItem(nsIndex, ItemType::ANY, key); } - + esp_err_t eraseNamespace(uint8_t nsIndex); + const Partition *getPart() const + { + return mPartition; + } + const char *getPartName() const { - return mPartitionName; + return mPartition->get_partition_name(); + } + + uint32_t getBaseSector() + { + return mPageManager.getBaseSector(); } + esp_err_t writeMultiPageBlob(uint8_t nsIndex, const char* key, const void* data, size_t dataSize, VerOffset chunkStart); + + esp_err_t readMultiPageBlob(uint8_t nsIndex, const char* key, void* data, size_t dataSize); + + esp_err_t cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void* data, size_t dataSize); + + esp_err_t eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart = VerOffset::VER_ANY); + void debugDump(); - + void debugCheck(); + esp_err_t fillStats(nvs_stats_t& nvsStats); + + esp_err_t calcEntriesInNamespace(uint8_t nsIndex, size_t& usedEntries); + + bool findEntry(nvs_opaque_iterator_t*, const char* name); + + bool nextEntry(nvs_opaque_iterator_t* it); protected: @@ -99,10 +144,16 @@ class Storage : public intrusive_list_node void clearNamespaces(); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item); + esp_err_t populateBlobIndices(TBlobIndexList&); + + void eraseOrphanDataBlobs(TBlobIndexList&); + + void fillEntryInfo(Item &item, nvs_entry_info_t &info); + + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); protected: - const char *mPartitionName; + Partition *mPartition; size_t mPageCount; PageManager mPageManager; TNamespaces mNamespaces; @@ -112,6 +163,14 @@ class Storage : public intrusive_list_node } // namespace nvs - +struct nvs_opaque_iterator_t +{ + nvs_type_t type; + uint8_t nsIndex; + size_t entryIndex; + nvs::Storage *storage; + intrusive_list::iterator page; + nvs_entry_info_t entry_info; +}; #endif /* nvs_storage_hpp */ diff --git a/components/nvs_flash/src/nvs_test_api.h b/components/nvs_flash/src/nvs_test_api.h index 3cf3e7fa1..8ba2c4faa 100644 --- a/components/nvs_flash/src/nvs_test_api.h +++ b/components/nvs_flash/src/nvs_test_api.h @@ -17,22 +17,38 @@ #ifdef __cplusplus extern "C" { #endif - + #include "nvs_flash.h" /** * @brief Initialize NVS flash storage with custom flash sector layout * * @note This API is intended to be used in unit tests. - * + * * @param partName Partition name of the NVS partition as per partition table * @param baseSector Flash sector (units of 4096 bytes) offset to start NVS - * @param sectorCount Length (in flash sectors) of NVS region. + * @param sectorCount Length (in flash sectors) of NVS region. NVS partition must be at least 3 sectors long. * @return ESP_OK if flash was successfully initialized */ esp_err_t nvs_flash_init_custom(const char *partName, uint32_t baseSector, uint32_t sectorCount); +#ifdef CONFIG_NVS_ENCRYPTION +/** + * @brief Initialize NVS flash storage with custom flash sector layout + * + * @note This API is intended to be used in unit tests. + * + * @param partName Partition name of the NVS partition as per partition table + * @param baseSector Flash sector (units of 4096 bytes) offset to start NVS + * @param sectorCount Length (in flash sectors) of NVS region. + NVS partition must be at least 3 sectors long. + * @param[in] cfg Security configuration (keys) to be used for NVS encryption/decryption. + * If cfg is null, no encryption/decryption is used. + * @return ESP_OK if flash was successfully initialized + */ +esp_err_t nvs_flash_secure_init_custom(const char *partName, uint32_t baseSector, uint32_t sectorCount, nvs_sec_cfg_t* cfg); +#endif /** * @brief Dump contents of NVS storage to stdout diff --git a/components/nvs_flash/src/nvs_types.cpp b/components/nvs_flash/src/nvs_types.cpp index 8a1a42fb2..127e24b88 100644 --- a/components/nvs_flash/src/nvs_types.cpp +++ b/components/nvs_flash/src/nvs_types.cpp @@ -13,7 +13,11 @@ // limitations under the License. #include "nvs_types.hpp" -#include +#if defined(LINUX_TARGET) +#include "crc.h" +#else +#include +#endif namespace nvs { @@ -35,6 +39,7 @@ uint32_t Item::calculateCrc32WithoutValue() const result = crc32_le(result, p + offsetof(Item, nsIndex), offsetof(Item, datatype) - offsetof(Item, nsIndex)); result = crc32_le(result, p + offsetof(Item, key), sizeof(key)); + result = crc32_le(result, p + offsetof(Item, chunkIndex), sizeof(chunkIndex)); return result; } diff --git a/components/nvs_flash/src/nvs_types.hpp b/components/nvs_flash/src/nvs_types.hpp index 5306744b5..d05d4d94b 100644 --- a/components/nvs_flash/src/nvs_types.hpp +++ b/components/nvs_flash/src/nvs_types.hpp @@ -15,41 +15,36 @@ #define nvs_types_h #include -#include #include #include #include #include "nvs.h" +#include "nvs_handle.hpp" #include "compressed_enum_table.hpp" +#include "string.h" +using namespace std; namespace nvs { -enum class ItemType : uint8_t { - U8 = 0x01, - I8 = 0x11, - U16 = 0x02, - I16 = 0x12, - U32 = 0x04, - I32 = 0x14, - U64 = 0x08, - I64 = 0x18, - SZ = 0x21, - BLOB = 0x41, - ANY = 0xff +/** + * Used to recognize transient states of a blob. Once a blob is modified, new chunks with the new data are written + * with a new version. The version is saved in the highest bit of Item::chunkIndex as well as in + * Item::blobIndex::chunkStart. + * If a chunk is modified and hence re-written, the version swaps: 0x0 -> 0x80 or 0x80 -> 0x0. + */ +enum class VerOffset: uint8_t { + VER_0_OFFSET = 0x0, + VER_1_OFFSET = 0x80, + VER_ANY = 0xff, }; -template::value, void*>::type = nullptr> -constexpr ItemType itemTypeOf() +inline bool isVariableLengthType(ItemType type) { - return static_cast(((std::is_signed::value)?0x10:0x00) | sizeof(T)); -} - -template -constexpr ItemType itemTypeOf(const T&) -{ - return itemTypeOf(); + return (type == ItemType::BLOB || + type == ItemType::SZ || + type == ItemType::BLOB_DATA); } class Item @@ -60,15 +55,21 @@ class Item uint8_t nsIndex; ItemType datatype; uint8_t span; - uint8_t reserved; + uint8_t chunkIndex; uint32_t crc32; - char key[16]; + char key[NVS_KEY_NAME_MAX_SIZE]; union { struct { uint16_t dataSize; - uint16_t reserved2; + uint16_t reserved; uint32_t dataCrc32; } varLength; + struct { + uint32_t dataSize; + uint8_t chunkCount; // Number of children data blobs. + VerOffset chunkStart; // Offset from which the chunkIndex for children blobs starts + uint16_t reserved; + } blobIndex; uint8_t data[8]; }; }; @@ -77,8 +78,12 @@ class Item static const size_t MAX_KEY_LENGTH = sizeof(key) - 1; - Item(uint8_t nsIndex, ItemType datatype, uint8_t span, const char* key_) - : nsIndex(nsIndex), datatype(datatype), span(span), reserved(0xff) + // 0xff cannot be used as a valid chunkIndex for blob datatype. + static const uint8_t CHUNK_ANY = 0xff; + + + Item(uint8_t nsIndex, ItemType datatype, uint8_t span, const char* key_, uint8_t chunkIdx = CHUNK_ANY) + : nsIndex(nsIndex), datatype(datatype), span(span), chunkIndex(chunkIdx) { std::fill_n(reinterpret_cast(key), sizeof(key) / 4, 0xffffffff); std::fill_n(reinterpret_cast(data), sizeof(data) / 4, 0xffffffff); @@ -100,7 +105,8 @@ class Item void getKey(char* dst, size_t dstSize) { - strncpy(dst, key, (dstSize @@ -113,6 +119,4 @@ class Item } // namespace nvs - - #endif /* nvs_types_h */ diff --git a/components/nvs_flash/src/partition.hpp b/components/nvs_flash/src/partition.hpp new file mode 100644 index 000000000..4cbfb6d9d --- /dev/null +++ b/components/nvs_flash/src/partition.hpp @@ -0,0 +1,59 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PARTITION_HPP_ +#define PARTITION_HPP_ + +#include "esp_err.h" + +namespace nvs { + +/** + * @brief Abstract interface for partition related operations, currently in NVS. + * + * It resembles the main operations according to esp_partition.h. + */ +class Partition { +public: + virtual ~Partition() { } + + /** + * Return the partition name as in the partition table. + */ + virtual const char *get_partition_name() = 0; + + virtual esp_err_t read_raw(size_t src_offset, void* dst, size_t size) = 0; + + virtual esp_err_t read(size_t src_offset, void* dst, size_t size) = 0; + + virtual esp_err_t write_raw(size_t dst_offset, const void* src, size_t size) = 0; + + virtual esp_err_t write(size_t dst_offset, const void* src, size_t size) = 0; + + virtual esp_err_t erase_range(size_t dst_offset, size_t size) = 0; + + /** + * Return the address of the beginning of the partition. + */ + virtual uint32_t get_address() = 0; + + /** + * Return the partition size in bytes. + */ + virtual uint32_t get_size() = 0; +}; + +} // nvs + +#endif // PARTITION_HPP_ diff --git a/components/nvs_flash/test/CMakeLists.txt b/components/nvs_flash/test/CMakeLists.txt new file mode 100644 index 000000000..f69580398 --- /dev/null +++ b/components/nvs_flash/test/CMakeLists.txt @@ -0,0 +1,4 @@ +idf_component_register(SRC_DIRS "." + PRIV_INCLUDE_DIRS "." + PRIV_REQUIRES cmock test_utils nvs_flash bootloader_support + EMBED_TXTFILES encryption_keys.bin partition_encrypted.bin sample.bin) diff --git a/components/nvs_flash/test/component.mk b/components/nvs_flash/test/component.mk index 5dd172bdb..a0ea7a406 100644 --- a/components/nvs_flash/test/component.mk +++ b/components/nvs_flash/test/component.mk @@ -3,3 +3,4 @@ # COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive +COMPONENT_EMBED_TXTFILES += encryption_keys.bin partition_encrypted.bin sample.bin diff --git a/components/nvs_flash/test/encryption_keys.bin b/components/nvs_flash/test/encryption_keys.bin new file mode 100644 index 000000000..9ef4439d8 --- /dev/null +++ b/components/nvs_flash/test/encryption_keys.bin @@ -0,0 +1 @@ +"""""""""""""""""""""""""""""""",ïÏ<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/components/nvs_flash/test/partition_encrypted.bin b/components/nvs_flash/test/partition_encrypted.bin new file mode 100644 index 000000000..93c900ff4 Binary files /dev/null and b/components/nvs_flash/test/partition_encrypted.bin differ diff --git a/components/nvs_flash/test/sample.bin b/components/nvs_flash/test/sample.bin new file mode 100644 index 000000000..fe69eca4f --- /dev/null +++ b/components/nvs_flash/test/sample.bin @@ -0,0 +1 @@ +start0000000000000000000000start0123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef0000000000000000end00000000000000000000000000end \ No newline at end of file diff --git a/components/nvs_flash/test/test_nvs.c b/components/nvs_flash/test/test_nvs.c index ed0884a7d..7e172f5ac 100644 --- a/components/nvs_flash/test/test_nvs.c +++ b/components/nvs_flash/test/test_nvs.c @@ -9,26 +9,75 @@ #include "esp_partition.h" #include "esp_log.h" #include +#include "esp_system.h" + +#ifdef CONFIG_NVS_ENCRYPTION +#include "mbedtls/aes.h" +#endif static const char* TAG = "test_nvs"; +TEST_CASE("Partition name no longer than 16 characters", "[nvs]") +{ + const char *TOO_LONG_NAME = "0123456789abcdefg"; + + TEST_ESP_ERR(ESP_ERR_INVALID_ARG, nvs_flash_init_partition(TOO_LONG_NAME)); + + nvs_flash_deinit_partition(TOO_LONG_NAME); // just in case +} + +TEST_CASE("flash erase deinitializes initialized partition", "[nvs]") +{ + nvs_handle_t handle; + esp_err_t err = nvs_flash_init(); + if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) { + nvs_flash_erase(); + err = nvs_flash_init(); + } + TEST_ESP_OK( err ); + + TEST_ESP_OK(nvs_flash_init()); + TEST_ESP_OK(nvs_open("uninit_ns", NVS_READWRITE, &handle)); + nvs_close(handle); + TEST_ESP_OK(nvs_flash_erase()); + + // exptected: no partition is initialized since nvs_flash_erase() deinitialized the partition again + TEST_ESP_ERR(ESP_ERR_NVS_NOT_INITIALIZED, nvs_open("uninit_ns", NVS_READWRITE, &handle)); + + // just to be sure it's deinitialized in case of error and not affecting other tests + nvs_flash_deinit(); +} + +// test could have different output on host tests +TEST_CASE("nvs deinit with open handle", "[nvs]") +{ + nvs_handle_t handle_1; + esp_err_t err = nvs_flash_init(); + if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_LOGW(TAG, "nvs_flash_init failed (0x%x), erasing partition and retrying", err); + ESP_ERROR_CHECK(nvs_flash_erase()); + err = nvs_flash_init(); + } + ESP_ERROR_CHECK( err ); + + TEST_ESP_OK(nvs_open("deinit_ns", NVS_READWRITE, &handle_1)); + nvs_flash_deinit(); +} + TEST_CASE("various nvs tests", "[nvs]") { - nvs_handle handle_1; + nvs_handle_t handle_1; esp_err_t err = nvs_flash_init(); - if (err == ESP_ERR_NVS_NO_FREE_PAGES) { + if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) { ESP_LOGW(TAG, "nvs_flash_init failed (0x%x), erasing partition and retrying", err); - const esp_partition_t* nvs_partition = esp_partition_find_first( - ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL); - assert(nvs_partition && "partition table must have an NVS partition"); - ESP_ERROR_CHECK( esp_partition_erase_range(nvs_partition, 0, nvs_partition->size) ); + ESP_ERROR_CHECK(nvs_flash_erase()); err = nvs_flash_init(); } ESP_ERROR_CHECK( err ); - TEST_ESP_ERR(nvs_open("test_namespace1", NVS_READONLY, &handle_1), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(ESP_ERR_NVS_NOT_FOUND, nvs_open("test_namespace1", NVS_READONLY, &handle_1)); - TEST_ESP_ERR(nvs_set_i32(handle_1, "foo", 0x12345678), ESP_ERR_NVS_INVALID_HANDLE); + TEST_ESP_ERR(ESP_ERR_NVS_INVALID_HANDLE, nvs_set_i32(handle_1, "foo", 0x12345678)); nvs_close(handle_1); TEST_ESP_OK(nvs_open("test_namespace2", NVS_READWRITE, &handle_1)); @@ -36,7 +85,7 @@ TEST_CASE("various nvs tests", "[nvs]") TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x12345678)); TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x23456789)); - nvs_handle handle_2; + nvs_handle_t handle_2; TEST_ESP_OK(nvs_open("test_namespace3", NVS_READWRITE, &handle_2)); TEST_ESP_OK(nvs_erase_all(handle_2)); TEST_ESP_OK(nvs_set_i32(handle_2, "foo", 0x3456789a)); @@ -65,3 +114,430 @@ TEST_CASE("various nvs tests", "[nvs]") nvs_close(handle_2); } + +TEST_CASE("calculate used and free space", "[nvs]") +{ + TEST_ESP_ERR(ESP_ERR_INVALID_ARG, nvs_get_stats(NULL, NULL)); + nvs_stats_t stat1; + nvs_stats_t stat2; + TEST_ESP_ERR(ESP_ERR_NVS_NOT_INITIALIZED, nvs_get_stats(NULL, &stat1)); + TEST_ASSERT_TRUE(stat1.free_entries == 0); + TEST_ASSERT_TRUE(stat1.namespace_count == 0); + TEST_ASSERT_TRUE(stat1.total_entries == 0); + TEST_ASSERT_TRUE(stat1.used_entries == 0); + + nvs_handle_t handle = 0; + size_t h_count_entries; + TEST_ESP_ERR(ESP_ERR_NVS_INVALID_HANDLE, nvs_get_used_entry_count(handle, &h_count_entries)); + TEST_ASSERT_TRUE(h_count_entries == 0); + + esp_err_t err = nvs_flash_init(); + if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_LOGW(TAG, "nvs_flash_init failed (0x%x), erasing partition and retrying", err); + const esp_partition_t* nvs_partition = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL); + assert(nvs_partition && "partition table must have an NVS partition"); + ESP_ERROR_CHECK(nvs_flash_erase()); + err = nvs_flash_init(); + } + ESP_ERROR_CHECK( err ); + + // erase if have any namespace + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + if(stat1.namespace_count != 0) { + TEST_ESP_OK(nvs_flash_deinit()); + TEST_ESP_OK(nvs_flash_erase()); + TEST_ESP_OK(nvs_flash_init()); + } + + // after erase. empty partition + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + TEST_ASSERT_TRUE(stat1.free_entries != 0); + TEST_ASSERT_TRUE(stat1.namespace_count == 0); + TEST_ASSERT_TRUE(stat1.total_entries != 0); + TEST_ASSERT_TRUE(stat1.used_entries == 0); + + // create namespace test_k1 + nvs_handle_t handle_1; + TEST_ESP_OK(nvs_open("test_k1", NVS_READWRITE, &handle_1)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + TEST_ASSERT_TRUE(stat2.free_entries + 1 == stat1.free_entries); + TEST_ASSERT_TRUE(stat2.namespace_count == 1); + TEST_ASSERT_TRUE(stat2.total_entries == stat1.total_entries); + TEST_ASSERT_TRUE(stat2.used_entries == 1); + + // create pair key-value com + TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x12345678)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + TEST_ASSERT_TRUE(stat1.free_entries + 1 == stat2.free_entries); + TEST_ASSERT_TRUE(stat1.namespace_count == 1); + TEST_ASSERT_TRUE(stat1.total_entries == stat2.total_entries); + TEST_ASSERT_TRUE(stat1.used_entries == 2); + + // change value in com + TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x01234567)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + TEST_ASSERT_TRUE(stat2.free_entries == stat1.free_entries); + TEST_ASSERT_TRUE(stat2.namespace_count == 1); + TEST_ASSERT_TRUE(stat2.total_entries != 0); + TEST_ASSERT_TRUE(stat2.used_entries == 2); + + // create pair key-value ru + TEST_ESP_OK(nvs_set_i32(handle_1, "ru", 0x00FF00FF)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + TEST_ASSERT_TRUE(stat1.free_entries + 1 == stat2.free_entries); + TEST_ASSERT_TRUE(stat1.namespace_count == 1); + TEST_ASSERT_TRUE(stat1.total_entries != 0); + TEST_ASSERT_TRUE(stat1.used_entries == 3); + + // amount valid pair in namespace 1 + size_t h1_count_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_1, &h1_count_entries)); + TEST_ASSERT_TRUE(h1_count_entries == 2); + + nvs_handle_t handle_2; + // create namespace test_k2 + TEST_ESP_OK(nvs_open("test_k2", NVS_READWRITE, &handle_2)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + TEST_ASSERT_TRUE(stat2.free_entries + 1 == stat1.free_entries); + TEST_ASSERT_TRUE(stat2.namespace_count == 2); + TEST_ASSERT_TRUE(stat2.total_entries == stat1.total_entries); + TEST_ASSERT_TRUE(stat2.used_entries == 4); + + // create pair key-value + TEST_ESP_OK(nvs_set_i32(handle_2, "su1", 0x00000001)); + TEST_ESP_OK(nvs_set_i32(handle_2, "su2", 0x00000002)); + TEST_ESP_OK(nvs_set_i32(handle_2, "sus", 0x00000003)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + TEST_ASSERT_TRUE(stat1.free_entries + 3 == stat2.free_entries); + TEST_ASSERT_TRUE(stat1.namespace_count == 2); + TEST_ASSERT_TRUE(stat1.total_entries == stat2.total_entries); + TEST_ASSERT_TRUE(stat1.used_entries == 7); + + TEST_ASSERT_TRUE(stat1.total_entries == (stat1.used_entries + stat1.free_entries)); + + // amount valid pair in namespace 2 + size_t h2_count_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_2, &h2_count_entries)); + TEST_ASSERT_TRUE(h2_count_entries == 3); + + TEST_ASSERT_TRUE(stat1.used_entries == (h1_count_entries + h2_count_entries + stat1.namespace_count)); + + nvs_close(handle_1); + nvs_close(handle_2); + + size_t temp = h2_count_entries; + TEST_ESP_ERR(ESP_ERR_NVS_INVALID_HANDLE, nvs_get_used_entry_count(handle_1, &h2_count_entries)); + TEST_ASSERT_TRUE(h2_count_entries == 0); + h2_count_entries = temp; + TEST_ESP_ERR(ESP_ERR_INVALID_ARG, nvs_get_used_entry_count(handle_1, NULL)); + + nvs_handle_t handle_3; + // create namespace test_k3 + TEST_ESP_OK(nvs_open("test_k3", NVS_READWRITE, &handle_3)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + TEST_ASSERT_TRUE(stat2.free_entries + 1 == stat1.free_entries); + TEST_ASSERT_TRUE(stat2.namespace_count == 3); + TEST_ASSERT_TRUE(stat2.total_entries == stat1.total_entries); + TEST_ASSERT_TRUE(stat2.used_entries == 8); + + // create pair blobs + uint32_t blob[12]; + TEST_ESP_OK(nvs_set_blob(handle_3, "bl1", &blob, sizeof(blob))); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + TEST_ASSERT_TRUE(stat1.free_entries + 4 == stat2.free_entries); + TEST_ASSERT_TRUE(stat1.namespace_count == 3); + TEST_ASSERT_TRUE(stat1.total_entries == stat2.total_entries); + TEST_ASSERT_TRUE(stat1.used_entries == 12); + + // amount valid pair in namespace 2 + size_t h3_count_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &h3_count_entries)); + TEST_ASSERT_TRUE(h3_count_entries == 4); + + TEST_ASSERT_TRUE(stat1.used_entries == (h1_count_entries + h2_count_entries + h3_count_entries + stat1.namespace_count)); + + nvs_close(handle_3); + + TEST_ESP_OK(nvs_flash_deinit()); + TEST_ESP_OK(nvs_flash_erase()); +} + +TEST_CASE("check for memory leaks in nvs_set_blob", "[nvs]") +{ + esp_err_t err = nvs_flash_init(); + if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_ERROR_CHECK(nvs_flash_erase()); + err = nvs_flash_init(); + } + TEST_ESP_OK( err ); + + for (int i = 0; i < 500; ++i) { + nvs_handle_t my_handle; + uint8_t key[20] = {0}; + + TEST_ESP_OK( nvs_open("leak_check_ns", NVS_READWRITE, &my_handle) ); + TEST_ESP_OK( nvs_set_blob(my_handle, "key", key, sizeof(key)) ); + TEST_ESP_OK( nvs_commit(my_handle) ); + nvs_close(my_handle); + printf("%d\n", esp_get_free_heap_size()); + } + + nvs_flash_deinit(); + printf("%d\n", esp_get_free_heap_size()); + /* heap leaks will be checked in unity_platform.c */ +} + +#ifdef CONFIG_NVS_ENCRYPTION +TEST_CASE("check underlying xts code for 32-byte size sector encryption", "[nvs]") +{ + uint8_t eky_hex[2 * NVS_KEY_SIZE] = { 0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11, + 0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11, + 0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11, + 0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11, + /* Tweak key below*/ + 0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22, + 0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22, + 0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22, + 0x22,0x22,0x22,0x22,0x22,0x22,0x22,0x22 }; + + uint8_t ba_hex[16] = { 0x33,0x33,0x33,0x33,0x33,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 }; + + uint8_t ptxt_hex[32] = { 0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44, + 0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44, + 0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44, + 0x44,0x44,0x44,0x44,0x44,0x44,0x44,0x44 }; + + + uint8_t ctxt_hex[32] = { 0xe6,0x22,0x33,0x4f,0x18,0x4b,0xbc,0xe1, + 0x29,0xa2,0x5b,0x2a,0xc7,0x6b,0x3d,0x92, + 0xab,0xf9,0x8e,0x22,0xdf,0x5b,0xdd,0x15, + 0xaf,0x47,0x1f,0x3d,0xb8,0x94,0x6a,0x85 }; + + mbedtls_aes_xts_context ectx[1]; + mbedtls_aes_xts_context dctx[1]; + + mbedtls_aes_xts_init(ectx); + mbedtls_aes_xts_init(dctx); + + TEST_ASSERT_TRUE(!mbedtls_aes_xts_setkey_enc(ectx, eky_hex, 2 * NVS_KEY_SIZE * 8)); + TEST_ASSERT_TRUE(!mbedtls_aes_xts_setkey_enc(dctx, eky_hex, 2 * NVS_KEY_SIZE * 8)); + + TEST_ASSERT_TRUE(!mbedtls_aes_crypt_xts(ectx, MBEDTLS_AES_ENCRYPT, 32, ba_hex, ptxt_hex, ptxt_hex)); + + TEST_ASSERT_TRUE(!memcmp(ptxt_hex, ctxt_hex, 32)); +} + +TEST_CASE("Check nvs key partition APIs (read and generate keys)", "[nvs]") +{ + nvs_sec_cfg_t cfg, cfg2; + + const esp_partition_t* key_part = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS_KEYS, NULL); + + if (!esp_flash_encryption_enabled()) { + TEST_IGNORE_MESSAGE("flash encryption disabled, skipping nvs_key partition related tests"); + } + + TEST_ESP_OK(esp_partition_erase_range(key_part, 0, key_part->size)); + TEST_ESP_ERR(ESP_ERR_NVS_KEYS_NOT_INITIALIZED, nvs_flash_read_security_cfg(key_part, &cfg)); + + TEST_ESP_OK(nvs_flash_generate_keys(key_part, &cfg)); + + TEST_ESP_OK(nvs_flash_read_security_cfg(key_part, &cfg2)); + + TEST_ASSERT_TRUE(!memcmp(&cfg, &cfg2, sizeof(nvs_sec_cfg_t))); +} + + +TEST_CASE("test nvs apis with encryption enabled", "[nvs]") +{ + if (!esp_flash_encryption_enabled()) { + TEST_IGNORE_MESSAGE("flash encryption disabled, skipping nvs_api tests with encryption enabled"); + } + const esp_partition_t* key_part = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS_KEYS, NULL); + + assert(key_part && "partition table must have an NVS Key partition"); + + const esp_partition_t* nvs_partition = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL); + assert(nvs_partition && "partition table must have an NVS partition"); + + ESP_ERROR_CHECK( esp_partition_erase_range(key_part, 0, key_part->size) ); + + bool done = false; + + do { + ESP_ERROR_CHECK( esp_partition_erase_range(nvs_partition, 0, nvs_partition->size) ); + + nvs_sec_cfg_t cfg; + esp_err_t err = nvs_flash_read_security_cfg(key_part, &cfg); + + if(err == ESP_ERR_NVS_KEYS_NOT_INITIALIZED) { + uint8_t value[4096] = {[0 ... 4095] = 0xff}; + TEST_ESP_OK(esp_partition_write(key_part, 0, value, sizeof(value))); + + TEST_ESP_ERR(ESP_ERR_NVS_KEYS_NOT_INITIALIZED, nvs_flash_read_security_cfg(key_part, &cfg)); + + TEST_ESP_OK(nvs_flash_generate_keys(key_part, &cfg)); + } else { + /* Second time key_partition exists already*/ + ESP_ERROR_CHECK(err); + done = true; + } + TEST_ESP_OK(nvs_flash_secure_init(&cfg)); + + nvs_handle_t handle_1; + + TEST_ESP_ERR(ESP_ERR_NVS_NOT_FOUND, nvs_open("namespace1", NVS_READONLY, &handle_1)); + + + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle_1)); + + TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x12345678)); + TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x23456789)); + + nvs_handle_t handle_2; + TEST_ESP_OK(nvs_open("namespace2", NVS_READWRITE, &handle_2)); + TEST_ESP_OK(nvs_set_i32(handle_2, "foo", 0x3456789a)); + const char* str = "value 0123456789abcdef0123456789abcdef"; + TEST_ESP_OK(nvs_set_str(handle_2, "key", str)); + + int32_t v1; + TEST_ESP_OK(nvs_get_i32(handle_1, "foo", &v1)); + TEST_ASSERT_TRUE(0x23456789 == v1); + + int32_t v2; + TEST_ESP_OK(nvs_get_i32(handle_2, "foo", &v2)); + TEST_ASSERT_TRUE(0x3456789a == v2); + + char buf[strlen(str) + 1]; + size_t buf_len = sizeof(buf); + + size_t buf_len_needed; + TEST_ESP_OK(nvs_get_str(handle_2, "key", NULL, &buf_len_needed)); + TEST_ASSERT_TRUE(buf_len_needed == buf_len); + + size_t buf_len_short = buf_len - 1; + TEST_ESP_ERR(ESP_ERR_NVS_INVALID_LENGTH, nvs_get_str(handle_2, "key", buf, &buf_len_short)); + TEST_ASSERT_TRUE(buf_len_short == buf_len); + + size_t buf_len_long = buf_len + 1; + TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len_long)); + TEST_ASSERT_TRUE(buf_len_long == buf_len); + + TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len)); + + TEST_ASSERT_TRUE(0 == strcmp(buf, str)); + + nvs_close(handle_1); + nvs_close(handle_2); + + TEST_ESP_OK(nvs_flash_deinit()); + } while(!done); +} + +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled", "[nvs_part_gen]") +{ + + if (!esp_flash_encryption_enabled()) { + TEST_IGNORE_MESSAGE("flash encryption disabled, skipping nvs_api tests with encryption enabled"); + } + + nvs_handle_t handle; + nvs_sec_cfg_t xts_cfg; + + extern const char nvs_key_start[] asm("_binary_encryption_keys_bin_start"); + extern const char nvs_key_end[] asm("_binary_encryption_keys_bin_end"); + + extern const char nvs_data_start[] asm("_binary_partition_encrypted_bin_start"); + + extern const char sample_bin_start[] asm("_binary_sample_bin_start"); + + const esp_partition_t* key_part = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS_KEYS, NULL); + + const esp_partition_t* nvs_part = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL); + + assert(key_part && "partition table must have a KEY partition"); + TEST_ASSERT_TRUE((nvs_key_end - nvs_key_start - 1) == SPI_FLASH_SEC_SIZE); + + assert(nvs_part && "partition table must have an NVS partition"); + printf("\n nvs_part size:%d\n", nvs_part->size); + + ESP_ERROR_CHECK(esp_partition_erase_range(key_part, 0, key_part->size)); + ESP_ERROR_CHECK( esp_partition_erase_range(nvs_part, 0, nvs_part->size) ); + + for (int i = 0; i < key_part->size; i+= SPI_FLASH_SEC_SIZE) { + ESP_ERROR_CHECK( esp_partition_write(key_part, i, nvs_key_start + i, SPI_FLASH_SEC_SIZE) ); + } + + for (int i = 0; i < nvs_part->size; i+= SPI_FLASH_SEC_SIZE) { + ESP_ERROR_CHECK( esp_partition_write(nvs_part, i, nvs_data_start + i, SPI_FLASH_SEC_SIZE) ); + } + + esp_err_t err = nvs_flash_read_security_cfg(key_part, &xts_cfg); + ESP_ERROR_CHECK(err); + + TEST_ESP_OK(nvs_flash_secure_init(&xts_cfg)); + + TEST_ESP_OK(nvs_open("dummyNamespace", NVS_READONLY, &handle)); + uint8_t u8v; + TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v)); + TEST_ASSERT_TRUE(u8v == 127); + int8_t i8v; + TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v)); + TEST_ASSERT_TRUE(i8v == -128); + uint16_t u16v; + TEST_ESP_OK( nvs_get_u16(handle, "dummyU16Key", &u16v)); + TEST_ASSERT_TRUE(u16v == 32768); + uint32_t u32v; + TEST_ESP_OK( nvs_get_u32(handle, "dummyU32Key", &u32v)); + TEST_ASSERT_TRUE(u32v == 4294967295); + int32_t i32v; + TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v)); + TEST_ASSERT_TRUE(i32v == -2147483648); + + char buf[64] = {0}; + size_t buflen = 64; + TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen)); + TEST_ASSERT_TRUE(strncmp(buf, "0A:0B:0C:0D:0E:0F", buflen) == 0); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + buflen = 64; + TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); + TEST_ASSERT_TRUE(memcmp(buf, hexdata, buflen) == 0); + + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; + buflen = 64; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + TEST_ASSERT_TRUE(memcmp(buf, base64data, buflen) == 0); + + uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}; + buflen = 64; + TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen)); + TEST_ASSERT_TRUE(memcmp(buf, hexfiledata, buflen) == 0); + + uint8_t base64filedata[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xab, 0xcd, 0xef}; + buflen = 64; + TEST_ESP_OK( nvs_get_blob(handle, "base64FileKey", buf, &buflen)); + TEST_ASSERT_TRUE(memcmp(buf, base64filedata, buflen) == 0); + + uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0"; + buflen = 64; + TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen)); + TEST_ASSERT_TRUE(memcmp(buf, strfiledata, buflen) == 0); + + char bin_data[5120]; + size_t bin_len = sizeof(bin_data); + TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); + TEST_ASSERT_TRUE(memcmp(bin_data, sample_bin_start, bin_len) == 0); + + nvs_close(handle); + TEST_ESP_OK(nvs_flash_deinit()); + +} +#endif diff --git a/components/nvs_flash/test_nvs_host/Makefile b/components/nvs_flash/test_nvs_host/Makefile index f1f5b52f2..5031de6a8 100644 --- a/components/nvs_flash/test_nvs_host/Makefile +++ b/components/nvs_flash/test_nvs_host/Makefile @@ -3,6 +3,7 @@ all: $(TEST_PROGRAM) SOURCE_FILES = \ esp_error_check_stub.cpp \ + ../mock/int/crc.cpp \ $(addprefix ../src/, \ nvs_types.cpp \ nvs_api.cpp \ @@ -10,37 +11,61 @@ SOURCE_FILES = \ nvs_pagemanager.cpp \ nvs_storage.cpp \ nvs_item_hash_list.cpp \ + nvs_handle_simple.cpp \ + nvs_handle_locked.cpp \ + nvs_partition_manager.cpp \ + nvs_partition.cpp \ + nvs_encrypted_partition.cpp \ + nvs_cxx_api.cpp \ ) \ spi_flash_emulation.cpp \ test_compressed_enum_table.cpp \ test_spi_flash_emulation.cpp \ test_intrusive_list.cpp \ test_nvs.cpp \ - crc.cpp \ + test_partition_manager.cpp \ + test_nvs_handle.cpp \ + test_nvs_storage.cpp \ + test_nvs_partition.cpp \ + test_nvs_cxx_api.cpp \ + test_nvs_initialization.cpp \ main.cpp -CPPFLAGS += -I../include -I../src -I./ -I../../esp32/include -I ../../spi_flash/include -I ../../../tools/catch -fprofile-arcs -ftest-coverage -CFLAGS += -fprofile-arcs -ftest-coverage -CXXFLAGS += -std=c++11 -Wall -Werror +ifeq ($(shell $(CC) -v 2>&1 | grep -c "clang version"), 1) +COMPILER := clang +else +COMPILER := gcc +endif + +CPPFLAGS += -I../include -I../src -I../mock/int -I./ -I../../esp_common/include -I../../esp32/include -I ../../mbedtls/mbedtls/include -I ../../spi_flash/include -I ../../hal/include -I ../../xtensa/include -I ../../../tools/catch -fprofile-arcs -ftest-coverage -g2 -ggdb +CFLAGS += -fprofile-arcs -ftest-coverage -DLINUX_TARGET +CXXFLAGS += -std=c++11 -Wall -Werror -DLINUX_TARGET LDFLAGS += -lstdc++ -Wall -fprofile-arcs -ftest-coverage +ifeq ($(COMPILER),clang) +CFLAGS += -fsanitize=address +CXXFLAGS += -fsanitize=address +LDFLAGS += -fsanitize=address +endif + OBJ_FILES = $(SOURCE_FILES:.cpp=.o) COVERAGE_FILES = $(OBJ_FILES:.o=.gc*) $(OBJ_FILES): %.o: %.cpp -$(TEST_PROGRAM): $(OBJ_FILES) - g++ $(LDFLAGS) -o $(TEST_PROGRAM) $(OBJ_FILES) +$(TEST_PROGRAM): clean-coverage $(OBJ_FILES) + $(MAKE) -C ../../mbedtls/mbedtls/ lib + g++ $(LDFLAGS) -o $(TEST_PROGRAM) $(OBJ_FILES) ../../mbedtls/mbedtls/library/libmbedcrypto.a $(OUTPUT_DIR): mkdir -p $(OUTPUT_DIR) test: $(TEST_PROGRAM) - ./$(TEST_PROGRAM) + ./$(TEST_PROGRAM) -d yes exclude:[long] long-test: $(TEST_PROGRAM) - ./$(TEST_PROGRAM) [list],[enumtable],[spi_flash_emu],[nvs],[long] + ./$(TEST_PROGRAM) -d yes $(COVERAGE_FILES): $(TEST_PROGRAM) long-test @@ -52,10 +77,26 @@ coverage_report: coverage.info genhtml coverage.info --output-directory coverage_report @echo "Coverage report is in coverage_report/index.html" -clean: - rm -f $(OBJ_FILES) $(TEST_PROGRAM) +clean-coverage: rm -f $(COVERAGE_FILES) *.gcov rm -rf coverage_report/ rm -f coverage.info -.PHONY: clean all test +clean: clean-coverage + $(MAKE) -C ../../mbedtls/mbedtls/ clean + rm -f $(OBJ_FILES) $(TEST_PROGRAM) + rm -f ../nvs_partition_generator/partition_single_page.bin + rm -f ../nvs_partition_generator/partition_multipage_blob.bin + rm -f ../nvs_partition_generator/partition_encrypted.bin + rm -f ../nvs_partition_generator/partition_encrypted_using_keygen.bin + rm -f ../nvs_partition_generator/partition_encrypted_using_keyfile.bin + rm -f ../nvs_partition_generator/partition_decrypted.bin + rm -f ../nvs_partition_generator/partition_encoded.bin + rm -f ../nvs_partition_generator/Test-1-partition-encrypted.bin + rm -f ../nvs_partition_generator/Test-1-partition.bin + rm -f ../../../tools/mass_mfg/samples/sample_values_multipage_blob_created.csv + rm -f ../../../tools/mass_mfg/samples/sample_values_singlepage_blob_created.csv + + + +.PHONY: clean clean-coverage all test long-test diff --git a/components/nvs_flash/test_nvs_host/README.md b/components/nvs_flash/test_nvs_host/README.md new file mode 100644 index 000000000..bad5b08cd --- /dev/null +++ b/components/nvs_flash/test_nvs_host/README.md @@ -0,0 +1,22 @@ +# Build + +```bash +make -j 6 +``` + +# Run +* Run particular test case: +```bash +./test_nvs "" + +``` +* Run all quick tests: +```bash +./test_nvs -d yes exclude:[long] +``` + +* Run all tests (takes several hours) +```bash +./test_nvs -d yes +``` + diff --git a/components/nvs_flash/test_nvs_host/esp_error_check_stub.cpp b/components/nvs_flash/test_nvs_host/esp_error_check_stub.cpp index 9cff4af31..34d8704e7 100644 --- a/components/nvs_flash/test_nvs_host/esp_error_check_stub.cpp +++ b/components/nvs_flash/test_nvs_host/esp_error_check_stub.cpp @@ -1,9 +1,14 @@ #include "catch.hpp" #include "esp_err.h" +#include "sdkconfig.h" void _esp_error_check_failed(esp_err_t rc, const char *file, int line, const char *function, const char *expression) { - printf("ESP_ERROR_CHECK failed: esp_err_t 0x%x at %p\n", rc, __builtin_return_address(0)); + printf("ESP_ERROR_CHECK failed: esp_err_t 0x%x", rc); +#ifdef CONFIG_ESP_ERR_TO_NAME_LOOKUP + printf(" (%s)", esp_err_to_name(rc)); +#endif //CONFIG_ESP_ERR_TO_NAME_LOOKUP + printf(" at %p\n", __builtin_return_address(0)); printf("file: \"%s\" line %d\nfunc: %s\nexpression: %s\n", file, line, function, expression); abort(); } diff --git a/components/nvs_flash/test_nvs_host/sdkconfig.h b/components/nvs_flash/test_nvs_host/sdkconfig.h index e69de29bb..a38e0a10d 100644 --- a/components/nvs_flash/test_nvs_host/sdkconfig.h +++ b/components/nvs_flash/test_nvs_host/sdkconfig.h @@ -0,0 +1,3 @@ +#define CONFIG_NVS_ENCRYPTION 1 +//currently use the legacy implementation, since the stubs for new HAL are not done yet +#define CONFIG_SPI_FLASH_USE_LEGACY_IMPL 1 diff --git a/components/nvs_flash/test_nvs_host/spi_flash_emulation.cpp b/components/nvs_flash/test_nvs_host/spi_flash_emulation.cpp index 914efc145..e5b9ae071 100644 --- a/components/nvs_flash/test_nvs_host/spi_flash_emulation.cpp +++ b/components/nvs_flash/test_nvs_host/spi_flash_emulation.cpp @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "esp_spi_flash.h" +#include "esp_partition.h" #include "spi_flash_emulation.h" @@ -22,39 +22,82 @@ void spi_flash_emulator_set(SpiFlashEmulator* e) s_emulator = e; } -esp_err_t spi_flash_erase_sector(size_t sec) +esp_err_t esp_partition_erase_range(const esp_partition_t* partition, + size_t offset, size_t size) { if (!s_emulator) { return ESP_ERR_FLASH_OP_TIMEOUT; } - if (!s_emulator->erase(sec)) { + if (size % SPI_FLASH_SEC_SIZE != 0) { + return ESP_ERR_INVALID_SIZE; + } + + if (offset % SPI_FLASH_SEC_SIZE != 0) { + return ESP_ERR_INVALID_ARG; + } + + size_t start_sector = offset / SPI_FLASH_SEC_SIZE; + size_t num_sectors = size / SPI_FLASH_SEC_SIZE; + for (size_t sector = start_sector; sector < (start_sector + num_sectors); sector++) { + if (!s_emulator->erase(sector)) { + return ESP_ERR_FLASH_OP_FAIL; + } + } + + return ESP_OK; +} + +esp_err_t esp_partition_read(const esp_partition_t* partition, + size_t src_offset, void* dst, size_t size) +{ + if (!s_emulator) { + return ESP_ERR_FLASH_OP_TIMEOUT; + } + + if (!s_emulator->read(reinterpret_cast(dst), src_offset, size)) { return ESP_ERR_FLASH_OP_FAIL; } return ESP_OK; } -esp_err_t spi_flash_write(size_t des_addr, const void *src_addr, size_t size) +esp_err_t esp_partition_read(const esp_partition_t* partition, + size_t src_offset, void* dst, size_t size) { if (!s_emulator) { return ESP_ERR_FLASH_OP_TIMEOUT; } - if (!s_emulator->write(des_addr, reinterpret_cast(src_addr), size)) { + if (!s_emulator->read(reinterpret_cast(dst), src_offset, size)) { return ESP_ERR_FLASH_OP_FAIL; } return ESP_OK; } -esp_err_t spi_flash_read(size_t src_addr, void *des_addr, size_t size) +esp_err_t esp_partition_write(const esp_partition_t* partition, + size_t dst_offset, const void* src, size_t size) { if (!s_emulator) { return ESP_ERR_FLASH_OP_TIMEOUT; } - if (!s_emulator->read(reinterpret_cast(des_addr), src_addr, size)) { + if (!s_emulator->write(dst_offset, reinterpret_cast(src), size)) { + return ESP_ERR_FLASH_OP_FAIL; + } + + return ESP_OK; +} + +esp_err_t esp_partition_write(const esp_partition_t* partition, + size_t dst_offset, const void* src, size_t size) +{ + if (!s_emulator) { + return ESP_ERR_FLASH_OP_TIMEOUT; + } + + if (!s_emulator->write(dst_offset, reinterpret_cast(src), size)) { return ESP_ERR_FLASH_OP_FAIL; } @@ -71,10 +114,11 @@ static size_t blockEraseTime = 37142; static size_t timeInterp(uint32_t bytes, size_t* lut) { + const int lut_size = sizeof(readTimes)/sizeof(readTimes[0]); int lz = __builtin_clz(bytes / 4); int log_size = 32 - lz; size_t x2 = 1 << (log_size + 2); - size_t y2 = lut[log_size]; + size_t y2 = lut[std::min(log_size, lut_size - 1)]; size_t x1 = 1 << (log_size + 1); size_t y1 = lut[log_size - 1]; return (bytes - x1) * (y2 - y1) / (x2 - x1) + y1; @@ -94,4 +138,3 @@ size_t SpiFlashEmulator::getEraseOpTime() { return blockEraseTime; } - diff --git a/components/nvs_flash/test_nvs_host/spi_flash_emulation.h b/components/nvs_flash/test_nvs_host/spi_flash_emulation.h index d8099322d..f2aef6a01 100644 --- a/components/nvs_flash/test_nvs_host/spi_flash_emulation.h +++ b/components/nvs_flash/test_nvs_host/spi_flash_emulation.h @@ -36,6 +36,16 @@ class SpiFlashEmulator SpiFlashEmulator(size_t sectorCount) : mUpperSectorBound(sectorCount) { mData.resize(sectorCount * SPI_FLASH_SEC_SIZE / 4, 0xffffffff); + mEraseCnt.resize(sectorCount); + spi_flash_emulator_set(this); + } + + SpiFlashEmulator(const char *filename) + { + load(filename); + // Atleast one page should be free, hence we create mData of size of 2 sectors. + mData.resize(mData.size() + SPI_FLASH_SEC_SIZE / 4, 0xffffffff); + mUpperSectorBound = mData.size() * 4 / SPI_FLASH_SEC_SIZE; spi_flash_emulator_set(this); } @@ -67,13 +77,13 @@ class SpiFlashEmulator WARN("invalid flash operation detected: erase sector=" << sectorNumber); return false; } - + if (dstAddr % 4 != 0 || size % 4 != 0 || dstAddr + size > mData.size() * 4) { return false; } - + for (size_t i = 0; i < size / 4; ++i) { if (mFailCountdown != SIZE_MAX && mFailCountdown-- == 0) { return false; @@ -102,12 +112,12 @@ class SpiFlashEmulator if (offset > mData.size()) { return false; } - + if (sectorNumber < mLowerSectorBound || sectorNumber >= mUpperSectorBound) { WARN("invalid flash operation detected: erase sector=" << sectorNumber); return false; } - + if (mFailCountdown != SIZE_MAX && mFailCountdown-- == 0) { return false; } @@ -115,10 +125,11 @@ class SpiFlashEmulator std::fill_n(begin(mData) + offset, SPI_FLASH_SEC_SIZE / 4, 0xffffffff); ++mEraseOps; + mEraseCnt[sectorNumber]++; mTotalTime += getEraseOpTime(); return true; } - + void randomize(uint32_t seed) { std::random_device rd; @@ -141,7 +152,7 @@ class SpiFlashEmulator { return reinterpret_cast(mData.data()); } - + void load(const char* filename) { FILE* f = fopen(filename, "rb"); @@ -154,7 +165,7 @@ class SpiFlashEmulator assert(s == static_cast(size / SPI_FLASH_SEC_SIZE)); fclose(f); } - + void save(const char* filename) { FILE* f = fopen(filename, "wb"); @@ -198,16 +209,20 @@ class SpiFlashEmulator { return mTotalTime; } - + void setBounds(uint32_t lowerSector, uint32_t upperSector) { mLowerSectorBound = lowerSector; mUpperSectorBound = upperSector; } - + void failAfter(uint32_t count) { mFailCountdown = count; } + size_t getSectorEraseCount(uint32_t sector) const { + return mEraseCnt[sector]; + } + protected: static size_t getReadOpTime(uint32_t bytes); static size_t getWriteOpTime(uint32_t bytes); @@ -215,6 +230,7 @@ class SpiFlashEmulator std::vector mData; + std::vector mEraseCnt; mutable size_t mReadOps = 0; mutable size_t mWriteOps = 0; @@ -224,7 +240,7 @@ class SpiFlashEmulator mutable size_t mTotalTime = 0; size_t mLowerSectorBound = 0; size_t mUpperSectorBound = 0; - + size_t mFailCountdown = SIZE_MAX; }; diff --git a/components/nvs_flash/test_nvs_host/test_compressed_enum_table.cpp b/components/nvs_flash/test_nvs_host/test_compressed_enum_table.cpp index 225a11c27..a404545c7 100644 --- a/components/nvs_flash/test_nvs_host/test_compressed_enum_table.cpp +++ b/components/nvs_flash/test_nvs_host/test_compressed_enum_table.cpp @@ -53,4 +53,4 @@ TEST_CASE("test if CompressedEnumTable works as expected", "[enumtable]") CHECK(table.data()[0] == 0x93909249); -} \ No newline at end of file +} diff --git a/components/nvs_flash/test_nvs_host/test_fixtures.hpp b/components/nvs_flash/test_nvs_host/test_fixtures.hpp new file mode 100644 index 000000000..473f1da97 --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_fixtures.hpp @@ -0,0 +1,149 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "nvs_partition.hpp" +#include "nvs_encrypted_partition.hpp" +#include "spi_flash_emulation.h" +#include "nvs.h" + +class PartitionEmulation : public nvs::Partition { +public: + PartitionEmulation(SpiFlashEmulator *spi_flash_emulator, + uint32_t address, + uint32_t size, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : partition_name(partition_name), flash_emu(spi_flash_emulator), address(address), size(size) + { + assert(partition_name); + assert(flash_emu); + assert(size); + } + + const char *get_partition_name() override + { + return partition_name; + } + + esp_err_t read_raw(size_t src_offset, void* dst, size_t size) override + { + if (!flash_emu->read(reinterpret_cast(dst), src_offset, size)) { + return ESP_ERR_FLASH_OP_FAIL; + } + + return ESP_OK; + } + + esp_err_t read(size_t src_offset, void* dst, size_t size) override + { + if (!flash_emu->read(reinterpret_cast(dst), src_offset, size)) { + return ESP_ERR_FLASH_OP_FAIL; + } + + return ESP_OK; + } + + esp_err_t write_raw(size_t dst_offset, const void* src, size_t size) override + { + if (!flash_emu->write(dst_offset, reinterpret_cast(src), size)) { + return ESP_ERR_FLASH_OP_FAIL; + } + + return ESP_OK; + } + + esp_err_t write(size_t dst_offset, const void* src, size_t size) override + { + if (!flash_emu->write(dst_offset, reinterpret_cast(src), size)) { + return ESP_ERR_FLASH_OP_FAIL; + } + + return ESP_OK; + } + + esp_err_t erase_range(size_t dst_offset, size_t size) override + { + if (size % SPI_FLASH_SEC_SIZE != 0) { + return ESP_ERR_INVALID_SIZE; + } + + if (dst_offset % SPI_FLASH_SEC_SIZE != 0) { + return ESP_ERR_INVALID_ARG; + } + + size_t start_sector = dst_offset / SPI_FLASH_SEC_SIZE; + size_t num_sectors = size / SPI_FLASH_SEC_SIZE; + for (size_t sector = start_sector; sector < (start_sector + num_sectors); sector++) { + if (!flash_emu->erase(sector)) { + return ESP_ERR_FLASH_OP_FAIL; + } + } + + return ESP_OK; + } + + uint32_t get_address() override + { + return address; + } + + uint32_t get_size() override + { + return size; + } + +private: + const char *partition_name; + + SpiFlashEmulator *flash_emu; + + uint32_t address; + + uint32_t size; +}; + +struct PartitionEmulationFixture { + PartitionEmulationFixture(uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : emu(start_sector + sector_size), + part(&emu, start_sector * SPI_FLASH_SEC_SIZE, sector_size * SPI_FLASH_SEC_SIZE, partition_name) { + } + + ~PartitionEmulationFixture() { } + + SpiFlashEmulator emu; + + PartitionEmulation part; +}; + +struct EncryptedPartitionFixture { + EncryptedPartitionFixture(nvs_sec_cfg_t *cfg, + uint32_t start_sector = 0, + uint32_t sector_size = 1, + const char *partition_name = NVS_DEFAULT_PART_NAME) + : esp_partition(), emu(start_sector + sector_size), + part(&esp_partition) { + esp_partition.address = start_sector * SPI_FLASH_SEC_SIZE; + esp_partition.size = sector_size * SPI_FLASH_SEC_SIZE; + strncpy(esp_partition.label, partition_name, PART_NAME_MAX_SIZE); + assert(part.init(cfg) == ESP_OK); + } + + ~EncryptedPartitionFixture() { } + + esp_partition_t esp_partition; + + SpiFlashEmulator emu; + + nvs::NVSEncryptedPartition part; +}; diff --git a/components/nvs_flash/test_nvs_host/test_intrusive_list.cpp b/components/nvs_flash/test_nvs_host/test_intrusive_list.cpp index 979438a0b..dea63f3c3 100644 --- a/components/nvs_flash/test_nvs_host/test_intrusive_list.cpp +++ b/components/nvs_flash/test_nvs_host/test_intrusive_list.cpp @@ -197,8 +197,8 @@ TEST_CASE("can erase all items in the list using clear method", "[list]") TestNode n4("four", 4); TestNode n5("five", 5); TestNode n6("six", 6); - - + + list.push_back(&n1); list.push_back(&n2); list.insert(++list.begin(), &n3); @@ -207,7 +207,6 @@ TEST_CASE("can erase all items in the list using clear method", "[list]") list.insert(list.begin(), &n6); list.clear(); - + REQUIRE(std::begin(list) == std::end(list)); } - diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp index c23f5d661..b7a2110a2 100644 --- a/components/nvs_flash/test_nvs_host/test_nvs.cpp +++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp @@ -14,9 +14,21 @@ #include "catch.hpp" #include "nvs.hpp" #include "nvs_test_api.h" +#include "sdkconfig.h" #include "spi_flash_emulation.h" +#include "nvs_partition_manager.hpp" +#include "nvs_partition.hpp" +#include "mbedtls/aes.h" #include #include +#include +#include +#include +#include +#include +#include + +#include "test_fixtures.hpp" #define TEST_ESP_ERR(rc, res) CHECK((rc) == (res)) #define TEST_ESP_OK(rc) CHECK((rc) == ESP_OK) @@ -45,7 +57,7 @@ TEST_CASE("crc32 behaves as expected", "[nvs]") item1.datatype = ItemType::I32; item1.nsIndex = 1; item1.crc32 = 0; - item1.reserved = 0xff; + item1.chunkIndex = 0xff; fill_n(item1.key, sizeof(item1.key), 0xbb); fill_n(item1.data, sizeof(item1.data), 0xaa); @@ -69,20 +81,20 @@ TEST_CASE("crc32 behaves as expected", "[nvs]") CHECK(crc32_1 != item2.calculateCrc32()); } -TEST_CASE("starting with empty flash, page is in uninitialized state", "[nvs]") +TEST_CASE("Page starting with empty flash is in uninitialized state", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; CHECK(page.state() == Page::PageState::INVALID); - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); CHECK(page.state() == Page::PageState::UNINITIALIZED); } -TEST_CASE("can distinguish namespaces", "[nvs]") +TEST_CASE("Page can distinguish namespaces", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); int32_t val1 = 0x12345678; CHECK(page.writeItem(1, ItemType::I32, "intval1", &val1, sizeof(val1)) == ESP_OK); int32_t val2 = 0x23456789; @@ -94,32 +106,32 @@ TEST_CASE("can distinguish namespaces", "[nvs]") } -TEST_CASE("reading with different type causes type mismatch error", "[nvs]") +TEST_CASE("Page reading with different type causes type mismatch error", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); int32_t val = 0x12345678; CHECK(page.writeItem(1, ItemType::I32, "intval1", &val, sizeof(val)) == ESP_OK); CHECK(page.readItem(1, ItemType::U32, "intval1", &val, sizeof(val)) == ESP_ERR_NVS_TYPE_MISMATCH); } -TEST_CASE("when page is erased, it's state becomes UNITIALIZED", "[nvs]") +TEST_CASE("Page when erased, it's state becomes UNITIALIZED", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); int32_t val = 0x12345678; CHECK(page.writeItem(1, ItemType::I32, "intval1", &val, sizeof(val)) == ESP_OK); CHECK(page.erase() == ESP_OK); CHECK(page.state() == Page::PageState::UNINITIALIZED); } -TEST_CASE("when writing and erasing, used/erased counts are updated correctly", "[nvs]") +TEST_CASE("Page when writing and erasing, used/erased counts are updated correctly", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); CHECK(page.getUsedEntryCount() == 0); CHECK(page.getErasedEntryCount() == 0); uint32_t foo1 = 0; @@ -146,11 +158,11 @@ TEST_CASE("when writing and erasing, used/erased counts are updated correctly", CHECK(page.getErasedEntryCount() == Page::ENTRY_COUNT - 1); } -TEST_CASE("when page is full, adding an element fails", "[nvs]") +TEST_CASE("Page when page is full, adding an element fails", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); for (size_t i = 0; i < Page::ENTRY_COUNT; ++i) { char name[16]; snprintf(name, sizeof(name), "i%ld", (long int)i); @@ -159,30 +171,30 @@ TEST_CASE("when page is full, adding an element fails", "[nvs]") CHECK(page.writeItem(1, "foo", 64UL) == ESP_ERR_NVS_PAGE_FULL); } -TEST_CASE("page maintains its seq number") +TEST_CASE("Page maintains its seq number") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; { Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); CHECK(page.setSeqNumber(123) == ESP_OK); int32_t val = 42; CHECK(page.writeItem(1, ItemType::I32, "dummy", &val, sizeof(val)) == ESP_OK); } { Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); uint32_t seqno; CHECK(page.getSeqNumber(seqno) == ESP_OK); CHECK(seqno == 123); } } -TEST_CASE("can write and read variable length data", "[nvs]") +TEST_CASE("Page can write and read variable length data", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - CHECK(page.load(0) == ESP_OK); + CHECK(page.load(&f.part, 0) == ESP_OK); const char str[] = "foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234"; size_t len = strlen(str); CHECK(page.writeItem(1, "stuff1", 42) == ESP_OK); @@ -212,11 +224,11 @@ TEST_CASE("can write and read variable length data", "[nvs]") CHECK(memcmp(buf, str, strlen(str)) == 0); } -TEST_CASE("different key names are distinguished even if the pointer is the same", "[nvs]") +TEST_CASE("Page different key names are distinguished even if the pointer is the same", "[nvs]") { - SpiFlashEmulator emu(1); + PartitionEmulationFixture f; Page page; - TEST_ESP_OK(page.load(0)); + TEST_ESP_OK(page.load(&f.part, 0)); TEST_ESP_OK(page.writeItem(1, "i1", 1)); TEST_ESP_OK(page.writeItem(1, "i2", 2)); int32_t value; @@ -233,9 +245,9 @@ TEST_CASE("different key names are distinguished even if the pointer is the same TEST_CASE("Page validates key size", "[nvs]") { - SpiFlashEmulator emu(4); + PartitionEmulationFixture f(0, 4); Page page; - TEST_ESP_OK(page.load(0)); + TEST_ESP_OK(page.load(&f.part, 0)); // 16-character key fails TEST_ESP_ERR(page.writeItem(1, "0123456789123456", 1), ESP_ERR_NVS_KEY_TOO_LONG); // 15-character key is okay @@ -244,56 +256,98 @@ TEST_CASE("Page validates key size", "[nvs]") TEST_CASE("Page validates blob size", "[nvs]") { - SpiFlashEmulator emu(4); + PartitionEmulationFixture f(0, 4); Page page; - TEST_ESP_OK(page.load(0)); + TEST_ESP_OK(page.load(&f.part, 0)); - char buf[2048] = { 0 }; + char buf[4096] = { 0 }; // There are two potential errors here: // - not enough space in the page (because one value has been written already) // - value is too long // Check that the second one is actually returned. TEST_ESP_ERR(page.writeItem(1, ItemType::BLOB, "2", buf, Page::ENTRY_COUNT * Page::ENTRY_SIZE), ESP_ERR_NVS_VALUE_TOO_LONG); // Should fail as well - TEST_ESP_ERR(page.writeItem(1, ItemType::BLOB, "2", buf, Page::BLOB_MAX_SIZE + 1), ESP_ERR_NVS_VALUE_TOO_LONG); - TEST_ESP_OK(page.writeItem(1, ItemType::BLOB, "2", buf, Page::BLOB_MAX_SIZE)); + TEST_ESP_ERR(page.writeItem(1, ItemType::BLOB, "2", buf, Page::CHUNK_MAX_SIZE + 1), ESP_ERR_NVS_VALUE_TOO_LONG); + TEST_ESP_OK(page.writeItem(1, ItemType::BLOB, "2", buf, Page::CHUNK_MAX_SIZE)); } TEST_CASE("Page handles invalid CRC of variable length items", "[nvs][cur]") { - SpiFlashEmulator emu(4); + PartitionEmulationFixture f(0, 4); { Page page; - TEST_ESP_OK(page.load(0)); + TEST_ESP_OK(page.load(&f.part, 0)); char buf[128] = {0}; TEST_ESP_OK(page.writeItem(1, ItemType::BLOB, "1", buf, sizeof(buf))); } // corrupt header of the item (64 is the offset of the first item in page) uint32_t overwrite_buf = 0; - emu.write(64, &overwrite_buf, 4); + f.emu.write(64, &overwrite_buf, 4); // load page again { Page page; - TEST_ESP_OK(page.load(0)); + TEST_ESP_OK(page.load(&f.part, 0)); + } +} + +class HashListTestHelper : public HashList +{ + public: + size_t getBlockCount() + { + return mBlockList.size(); + } +}; + +TEST_CASE("HashList is cleaned up as soon as items are erased", "[nvs]") +{ + HashListTestHelper hashlist; + // Add items + const size_t count = 128; + for (size_t i = 0; i < count; ++i) { + char key[16]; + snprintf(key, sizeof(key), "i%ld", (long int)i); + Item item(1, ItemType::U32, 1, key); + hashlist.insert(item, i); + } + INFO("Added " << count << " items, " << hashlist.getBlockCount() << " blocks"); + // Remove them in reverse order + for (size_t i = count; i > 0; --i) { + // Make sure that the element existed before it's erased + CHECK(hashlist.erase(i - 1) == true); + } + CHECK(hashlist.getBlockCount() == 0); + // Add again + for (size_t i = 0; i < count; ++i) { + char key[16]; + snprintf(key, sizeof(key), "i%ld", (long int)i); + Item item(1, ItemType::U32, 1, key); + hashlist.insert(item, i); + } + INFO("Added " << count << " items, " << hashlist.getBlockCount() << " blocks"); + // Remove them in the same order + for (size_t i = 0; i < count; ++i) { + CHECK(hashlist.erase(i) == true); } + CHECK(hashlist.getBlockCount() == 0); } TEST_CASE("can init PageManager in empty flash", "[nvs]") { - SpiFlashEmulator emu(4); + PartitionEmulationFixture f(0, 4); PageManager pm; - CHECK(pm.load(0, 4) == ESP_OK); + CHECK(pm.load(&f.part, 0, 4) == ESP_OK); } TEST_CASE("PageManager adds page in the correct order", "[nvs]") { const size_t pageCount = 8; - SpiFlashEmulator emu(pageCount); + PartitionEmulationFixture f(0, pageCount); uint32_t pageNo[pageCount] = { -1U, 50, 11, -1U, 23, 22, 24, 49}; for (uint32_t i = 0; i < pageCount; ++i) { Page p; - p.load(i); + p.load(&f.part, i); if (pageNo[i] != -1U) { p.setSeqNumber(pageNo[i]); p.writeItem(1, "foo", 10U); @@ -301,7 +355,7 @@ TEST_CASE("PageManager adds page in the correct order", "[nvs]") } PageManager pageManager; - CHECK(pageManager.load(0, pageCount) == ESP_OK); + CHECK(pageManager.load(&f.part, 0, pageCount) == ESP_OK); uint32_t lastSeqNo = 0; for (auto it = std::begin(pageManager); it != std::end(pageManager); ++it) { @@ -313,76 +367,77 @@ TEST_CASE("PageManager adds page in the correct order", "[nvs]") TEST_CASE("can init storage in empty flash", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); + cout << "before check" << endl; CHECK(storage.init(4, 4) == ESP_OK); - s_perf << "Time to init empty storage (4 sectors): " << emu.getTotalTime() << " us" << std::endl; + s_perf << "Time to init empty storage (4 sectors): " << f.emu.getTotalTime() << " us" << std::endl; } TEST_CASE("storage doesn't add duplicates within one page", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); int bar = 0; - CHECK(storage.writeItem(1, "bar", bar) == ESP_OK); - CHECK(storage.writeItem(1, "bar", bar) == ESP_OK); + CHECK(storage.writeItem(1, "bar", ++bar) == ESP_OK); + CHECK(storage.writeItem(1, "bar", ++bar) == ESP_OK); Page page; - page.load(4); + page.load(&f.part, 4); CHECK(page.getUsedEntryCount() == 1); CHECK(page.getErasedEntryCount() == 1); } TEST_CASE("can write one item a thousand times", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); for (size_t i = 0; i < Page::ENTRY_COUNT * 4 * 2; ++i) { REQUIRE(storage.writeItem(1, "i", static_cast(i)) == ESP_OK); } - s_perf << "Time to write one item a thousand times: " << emu.getTotalTime() << " us (" << emu.getEraseOps() << " " << emu.getWriteOps() << " " << emu.getReadOps() << " " << emu.getWriteBytes() << " " << emu.getReadBytes() << ")" << std::endl; + s_perf << "Time to write one item a thousand times: " << f.emu.getTotalTime() << " us (" << f.emu.getEraseOps() << " " << f.emu.getWriteOps() << " " << f.emu.getReadOps() << " " << f.emu.getWriteBytes() << " " << f.emu.getReadBytes() << ")" << std::endl; } TEST_CASE("storage doesn't add duplicates within multiple pages", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); int bar = 0; - CHECK(storage.writeItem(1, "bar", bar) == ESP_OK); + CHECK(storage.writeItem(1, "bar", ++bar) == ESP_OK); for (size_t i = 0; i < Page::ENTRY_COUNT; ++i) { - CHECK(storage.writeItem(1, "foo", static_cast(bar)) == ESP_OK); + CHECK(storage.writeItem(1, "foo", static_cast(++bar)) == ESP_OK); } - CHECK(storage.writeItem(1, "bar", bar) == ESP_OK); + CHECK(storage.writeItem(1, "bar", ++bar) == ESP_OK); Page page; - page.load(4); + page.load(&f.part, 4); CHECK(page.findItem(1, itemTypeOf(), "bar") == ESP_ERR_NVS_NOT_FOUND); - page.load(5); + page.load(&f.part, 5); CHECK(page.findItem(1, itemTypeOf(), "bar") == ESP_OK); } TEST_CASE("storage can find items on second page if first is not fully written and has cached search data", "[nvs]") { - SpiFlashEmulator emu(3); - Storage storage; + PartitionEmulationFixture f(0, 3); + Storage storage(&f.part); CHECK(storage.init(0, 3) == ESP_OK); int bar = 0; - uint8_t bigdata[Page::BLOB_MAX_SIZE] = {0}; + uint8_t bigdata[(Page::CHUNK_MAX_SIZE - Page::ENTRY_SIZE)/2] = {0}; // write one big chunk of data ESP_ERROR_CHECK(storage.writeItem(0, ItemType::BLOB, "1", bigdata, sizeof(bigdata))); // write another big chunk of data ESP_ERROR_CHECK(storage.writeItem(0, ItemType::BLOB, "2", bigdata, sizeof(bigdata))); - + // write third one; it will not fit into the first page ESP_ERROR_CHECK(storage.writeItem(0, ItemType::BLOB, "3", bigdata, sizeof(bigdata))); - + size_t size; ESP_ERROR_CHECK(storage.getItemDataSize(0, ItemType::BLOB, "1", size)); CHECK(size == sizeof(bigdata)); @@ -393,9 +448,9 @@ TEST_CASE("storage can find items on second page if first is not fully written a TEST_CASE("can write and read variable length data lots of times", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); const char str[] = "foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234"; char buf[sizeof(str) + 16]; @@ -413,16 +468,16 @@ TEST_CASE("can write and read variable length data lots of times", "[nvs]") CHECK(storage.readItem(1, ItemType::SZ, "foobaar", buf, sizeof(buf)) == ESP_OK); CHECK(memcmp(buf, str, strlen(str) + 1) == 0); } - s_perf << "Time to write one string and one integer a thousand times: " << emu.getTotalTime() << " us (" << emu.getEraseOps() << " " << emu.getWriteOps() << " " << emu.getReadOps() << " " << emu.getWriteBytes() << " " << emu.getReadBytes() << ")" << std::endl; + s_perf << "Time to write one string and one integer a thousand times: " << f.emu.getTotalTime() << " us (" << f.emu.getEraseOps() << " " << f.emu.getWriteOps() << " " << f.emu.getReadOps() << " " << f.emu.getWriteBytes() << " " << f.emu.getReadBytes() << ")" << std::endl; } TEST_CASE("can get length of variable length data", "[nvs]") { - SpiFlashEmulator emu(8); - emu.randomize(200); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + f.emu.randomize(200); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); const char str[] = "foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234"; size_t len = strlen(str); @@ -439,24 +494,24 @@ TEST_CASE("can get length of variable length data", "[nvs]") TEST_CASE("can create namespaces", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); uint8_t nsi; CHECK(storage.createOrOpenNamespace("wifi", false, nsi) == ESP_ERR_NVS_NOT_FOUND); CHECK(storage.createOrOpenNamespace("wifi", true, nsi) == ESP_OK); Page page; - page.load(4); + page.load(&f.part, 4); CHECK(page.findItem(Page::NS_INDEX, ItemType::U8, "wifi") == ESP_OK); } TEST_CASE("storage may become full", "[nvs]") { - SpiFlashEmulator emu(8); - Storage storage; - emu.setBounds(4, 8); + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); + f.emu.setBounds(4, 8); CHECK(storage.init(4, 4) == ESP_OK); for (size_t i = 0; i < Page::ENTRY_COUNT * 3; ++i) { char name[Item::MAX_KEY_LENGTH + 1]; @@ -468,19 +523,48 @@ TEST_CASE("storage may become full", "[nvs]") TEST_CASE("can modify an item on a page which will be erased", "[nvs]") { - SpiFlashEmulator emu(2); - Storage storage; + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); CHECK(storage.init(0, 2) == ESP_OK); for (size_t i = 0; i < Page::ENTRY_COUNT * 3 + 1; ++i) { REQUIRE(storage.writeItem(1, "foo", 42U) == ESP_OK); } } +TEST_CASE("erase operations are distributed among sectors", "[nvs]") +{ + const size_t sectors = 6; + PartitionEmulationFixture f(0, sectors); + Storage storage(&f.part); + CHECK(storage.init(0, sectors) == ESP_OK); + + /* Fill some part of storage with static values */ + const size_t static_sectors = 2; + for (size_t i = 0; i < static_sectors * Page::ENTRY_COUNT; ++i) { + char name[Item::MAX_KEY_LENGTH]; + snprintf(name, sizeof(name), "static%d", (int) i); + REQUIRE(storage.writeItem(1, name, i) == ESP_OK); + } + + /* Now perform many write operations */ + const size_t write_ops = 2000; + for (size_t i = 0; i < write_ops; ++i) { + REQUIRE(storage.writeItem(1, "value", i) == ESP_OK); + } + + /* Check that erase counts are distributed between the remaining sectors */ + const size_t max_erase_cnt = write_ops / Page::ENTRY_COUNT / (sectors - static_sectors) + 1; + for (size_t i = 0; i < sectors; ++i) { + auto erase_cnt = f.emu.getSectorEraseCount(i); + INFO("Sector " << i << " erased " << erase_cnt); + CHECK(erase_cnt <= max_erase_cnt); + } +} TEST_CASE("can erase items", "[nvs]") { - SpiFlashEmulator emu(3); - Storage storage; + PartitionEmulationFixture f(0, 8); + Storage storage(&f.part); CHECK(storage.init(0, 3) == ESP_OK); for (size_t i = 0; i < Page::ENTRY_COUNT * 2 - 3; ++i) { char name[Item::MAX_KEY_LENGTH + 1]; @@ -498,22 +582,82 @@ TEST_CASE("can erase items", "[nvs]") CHECK(storage.readItem(3, "key00222", val) == ESP_ERR_NVS_NOT_FOUND); } +TEST_CASE("namespace name is deep copy", "[nvs]") +{ + char ns_name[16]; + strcpy(ns_name, "const_name"); + + nvs_handle_t handle_1; + nvs_handle_t handle_2; + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN); + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + TEST_ESP_OK(nvs_open("const_name", NVS_READWRITE, &handle_1)); + strcpy(ns_name, "just_kidding"); + + CHECK(nvs_open("just_kidding", NVS_READONLY, &handle_2) == ESP_ERR_NVS_NOT_FOUND); + + nvs_close(handle_1); + nvs_close(handle_2); + + nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME); +} + +TEST_CASE("readonly handle fails on writing", "[nvs]") +{ + PartitionEmulationFixture f(0, 10); + const char* str = "value 0123456789abcdef0123456789abcdef"; + const uint8_t blob[8] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7}; + + nvs_handle_t handle_1; + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + // first, creating namespace... + TEST_ESP_OK(nvs_open("ro_ns", NVS_READWRITE, &handle_1)); + nvs_close(handle_1); + + TEST_ESP_OK(nvs_open("ro_ns", NVS_READONLY, &handle_1)); + TEST_ESP_ERR(nvs_set_i32(handle_1, "key", 47), ESP_ERR_NVS_READ_ONLY); + TEST_ESP_ERR(nvs_set_str(handle_1, "key", str), ESP_ERR_NVS_READ_ONLY); + TEST_ESP_ERR(nvs_set_blob(handle_1, "key", blob, 8), ESP_ERR_NVS_READ_ONLY); + + nvs_close(handle_1); + + // without deinit it affects "nvs api tests" + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); +} + TEST_CASE("nvs api tests", "[nvs]") { - SpiFlashEmulator emu(10); - emu.randomize(100); - - nvs_handle handle_1; + PartitionEmulationFixture f(0, 10); + f.emu.randomize(100); + + nvs_handle_t handle_1; const uint32_t NVS_FLASH_SECTOR = 6; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); TEST_ESP_ERR(nvs_open("namespace1", NVS_READWRITE, &handle_1), ESP_ERR_NVS_NOT_INITIALIZED); for (uint16_t i = NVS_FLASH_SECTOR; i init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); TEST_ESP_ERR(nvs_open("namespace1", NVS_READONLY, &handle_1), ESP_ERR_NVS_NOT_FOUND); @@ -524,7 +668,7 @@ TEST_CASE("nvs api tests", "[nvs]") TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x12345678)); TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x23456789)); - nvs_handle handle_2; + nvs_handle_t handle_2; TEST_ESP_OK(nvs_open("namespace2", NVS_READWRITE, &handle_2)); TEST_ESP_OK(nvs_set_i32(handle_2, "foo", 0x3456789a)); const char* str = "value 0123456789abcdef0123456789abcdef"; @@ -544,11 +688,11 @@ TEST_CASE("nvs api tests", "[nvs]") size_t buf_len_needed; TEST_ESP_OK(nvs_get_str(handle_2, "key", NULL, &buf_len_needed)); CHECK(buf_len_needed == buf_len); - + size_t buf_len_short = buf_len - 1; TEST_ESP_ERR(ESP_ERR_NVS_INVALID_LENGTH, nvs_get_str(handle_2, "key", buf, &buf_len_short)); CHECK(buf_len_short == buf_len); - + size_t buf_len_long = buf_len + 1; TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len_long)); CHECK(buf_len_long == buf_len); @@ -556,51 +700,294 @@ TEST_CASE("nvs api tests", "[nvs]") TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len)); CHECK(0 == strcmp(buf, str)); + nvs_close(handle_1); + nvs_close(handle_2); + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); +} + +TEST_CASE("deinit partition doesn't affect other partition's open handles", "[nvs]") +{ + const char *OTHER_PARTITION_NAME = "other_part"; + PartitionEmulationFixture f(0, 10); + PartitionEmulationFixture f_other(0, 10, OTHER_PARTITION_NAME); + const char* str = "value 0123456789abcdef0123456789abcdef"; + const uint8_t blob[8] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7}; + + nvs_handle_t handle_1; + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + f_other.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f_other.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + TEST_ESP_OK(nvs_open_from_partition(OTHER_PARTITION_NAME, "ns", NVS_READWRITE, &handle_1)); + + // Deinitializing must not interfere with the open handle from the other partition. + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); + + TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x3456789a)); + nvs_close(handle_1); + + TEST_ESP_OK(nvs_flash_deinit_partition(OTHER_PARTITION_NAME)); +} + +TEST_CASE("nvs iterators tests", "[nvs]") +{ + PartitionEmulationFixture f(0, 5); + + const uint32_t NVS_FLASH_SECTOR = 0; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 5; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + for (uint16_t i = NVS_FLASH_SECTOR; i < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; ++i) { + f.emu.erase(i); + } + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_iterator_t it; + nvs_entry_info_t info; + nvs_handle handle_1; + nvs_handle handle_2; + const uint32_t blob = 0x11223344; + const char *name_1 = "namespace1"; + const char *name_2 = "namespace2"; + TEST_ESP_OK(nvs_open(name_1, NVS_READWRITE, &handle_1)); + TEST_ESP_OK(nvs_open(name_2, NVS_READWRITE, &handle_2)); + + TEST_ESP_OK(nvs_set_i8(handle_1, "value1", -11)); + TEST_ESP_OK(nvs_set_u8(handle_1, "value2", 11)); + TEST_ESP_OK(nvs_set_i16(handle_1, "value3", 1234)); + TEST_ESP_OK(nvs_set_u16(handle_1, "value4", -1234)); + TEST_ESP_OK(nvs_set_i32(handle_1, "value5", -222)); + TEST_ESP_OK(nvs_set_i32(handle_1, "value6", -222)); + TEST_ESP_OK(nvs_set_i32(handle_1, "value7", -222)); + TEST_ESP_OK(nvs_set_u32(handle_1, "value8", 222)); + TEST_ESP_OK(nvs_set_u32(handle_1, "value9", 222)); + TEST_ESP_OK(nvs_set_str(handle_1, "value10", "foo")); + TEST_ESP_OK(nvs_set_blob(handle_1, "value11", &blob, sizeof(blob))); + TEST_ESP_OK(nvs_set_i32(handle_2, "value1", -111)); + TEST_ESP_OK(nvs_set_i32(handle_2, "value2", -111)); + TEST_ESP_OK(nvs_set_i64(handle_2, "value3", -555)); + TEST_ESP_OK(nvs_set_u64(handle_2, "value4", 555)); + + auto entry_count = [](const char *part, const char *name, nvs_type_t type)-> int { + int count; + nvs_iterator_t it = nvs_entry_find(part, name, type); + for (count = 0; it != nullptr; count++) { + it = nvs_entry_next(it); + } + return count; + }; + + SECTION("Number of entries found for specified namespace and type is correct") + { + CHECK(nvs_entry_find("", NULL, NVS_TYPE_ANY) == NULL); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, NULL, NVS_TYPE_ANY) == 15); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, name_1, NVS_TYPE_ANY) == 11); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, name_1, NVS_TYPE_I32) == 3); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, NULL, NVS_TYPE_I32) == 5); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, NULL, NVS_TYPE_U64) == 1); + } + + SECTION("New entry is not created when existing key-value pair is set") + { + CHECK(entry_count(NVS_DEFAULT_PART_NAME, name_2, NVS_TYPE_ANY) == 4); + TEST_ESP_OK(nvs_set_i32(handle_2, "value1", -222)); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, name_2, NVS_TYPE_ANY) == 4); + } + + SECTION("Number of entries found decrease when entry is erased") + { + CHECK(entry_count(NVS_DEFAULT_PART_NAME, NULL, NVS_TYPE_U64) == 1); + TEST_ESP_OK(nvs_erase_key(handle_2, "value4")); + CHECK(entry_count(NVS_DEFAULT_PART_NAME, "", NVS_TYPE_U64) == 0); + } + + SECTION("All fields of nvs_entry_info_t structure are correct") + { + it = nvs_entry_find(NVS_DEFAULT_PART_NAME, name_1, NVS_TYPE_I32); + CHECK(it != nullptr); + string key = "value5"; + do { + nvs_entry_info(it, &info); + + CHECK(string(name_1) == info.namespace_name); + CHECK(key == info.key); + CHECK(info.type == NVS_TYPE_I32); + + it = nvs_entry_next(it); + key[5]++; + } while (it != NULL); + nvs_release_iterator(it); + } + + SECTION("Entry info is not affected by subsequent erase") + { + nvs_entry_info_t info_after_erase; + + it = nvs_entry_find(NVS_DEFAULT_PART_NAME, name_1, NVS_TYPE_ANY); + nvs_entry_info(it, &info); + TEST_ESP_OK(nvs_erase_key(handle_1, "value1")); + nvs_entry_info(it, &info_after_erase); + CHECK(memcmp(&info, &info_after_erase, sizeof(info)) == 0); + nvs_release_iterator(it); + } + + SECTION("Entry info is not affected by subsequent set") + { + nvs_entry_info_t info_after_set; + + it = nvs_entry_find(NVS_DEFAULT_PART_NAME, name_1, NVS_TYPE_ANY); + nvs_entry_info(it, &info); + TEST_ESP_OK(nvs_set_u8(handle_1, info.key, 44)); + nvs_entry_info(it, &info_after_set); + CHECK(memcmp(&info, &info_after_set, sizeof(info)) == 0); + nvs_release_iterator(it); + } + + + SECTION("Iterating over multiple pages works correctly") + { + nvs_handle handle_3; + const char *name_3 = "namespace3"; + const int entries_created = 250; + + TEST_ESP_OK(nvs_open(name_3, NVS_READWRITE, &handle_3)); + for (size_t i = 0; i < entries_created; i++) { + TEST_ESP_OK(nvs_set_u8(handle_3, to_string(i).c_str(), 123)); + } + + int entries_found = 0; + it = nvs_entry_find(NVS_DEFAULT_PART_NAME, name_3, NVS_TYPE_ANY); + while(it != nullptr) { + entries_found++; + it = nvs_entry_next(it); + } + CHECK(entries_created == entries_found); + + nvs_release_iterator(it); + nvs_close(handle_3); + } + + SECTION("Iterating over multi-page blob works correctly") + { + nvs_handle handle_3; + const char *name_3 = "namespace3"; + const uint8_t multipage_blob[4096 * 2] = { 0 }; + const int NUMBER_OF_ENTRIES_PER_PAGE = 125; + size_t occupied_entries; + + TEST_ESP_OK(nvs_open(name_3, NVS_READWRITE, &handle_3)); + nvs_set_blob(handle_3, "blob", multipage_blob, sizeof(multipage_blob)); + TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &occupied_entries)); + CHECK(occupied_entries > NUMBER_OF_ENTRIES_PER_PAGE * 2); + + CHECK(entry_count(NVS_DEFAULT_PART_NAME, name_3, NVS_TYPE_BLOB) == 1); + + nvs_close(handle_3); + } + + nvs_close(handle_1); + nvs_close(handle_2); + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); +} + +TEST_CASE("Iterator with not matching type iterates correctly", "[nvs]") +{ + PartitionEmulationFixture f(0, 5); + nvs_iterator_t it; + nvs_handle_t my_handle; + const char* NAMESPACE = "test_ns_4"; + + const uint32_t NVS_FLASH_SECTOR = 0; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 5; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + for (uint16_t i = NVS_FLASH_SECTOR; i < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; ++i) { + f.emu.erase(i); + } + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + // writing string to namespace (a type which spans multiple entries) + TEST_ESP_OK(nvs_open(NAMESPACE, NVS_READWRITE, &my_handle)); + TEST_ESP_OK(nvs_set_str(my_handle, "test-string", "InitString0")); + TEST_ESP_OK(nvs_commit(my_handle)); + nvs_close(my_handle); + + it = nvs_entry_find(NVS_DEFAULT_PART_NAME, NAMESPACE, NVS_TYPE_I32); + CHECK(it == NULL); + + // re-init to trigger cleaning up of broken items -> a corrupted string will be erased + nvs_flash_deinit(); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + it = nvs_entry_find(NVS_DEFAULT_PART_NAME, NAMESPACE, NVS_TYPE_STR); + CHECK(it != NULL); + nvs_release_iterator(it); + + // without deinit it affects "nvs api tests" + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } TEST_CASE("wifi test", "[nvs]") { - SpiFlashEmulator emu(10); - emu.randomize(10); - - + PartitionEmulationFixture f(0, 10); + f.emu.randomize(10); + + const uint32_t NVS_FLASH_SECTOR = 5; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); - TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - - nvs_handle misc_handle; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_handle_t misc_handle; TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &misc_handle)); char log[33]; size_t log_size = sizeof(log); TEST_ESP_ERR(nvs_get_str(misc_handle, "log", log, &log_size), ESP_ERR_NVS_NOT_FOUND); strcpy(log, "foobarbazfizzz"); TEST_ESP_OK(nvs_set_str(misc_handle, "log", log)); - - nvs_handle net80211_handle; + + nvs_handle_t net80211_handle; TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &net80211_handle)); - + uint8_t opmode = 2; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.opmode", &opmode), ESP_ERR_NVS_NOT_FOUND); - + TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.opmode", opmode)); - + uint8_t country = 0; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &opmode), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", opmode)); - + char ssid[36]; size_t size = sizeof(ssid); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND); strcpy(ssid, "my android AP"); TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.ssid", ssid, size)); - + char mac[6]; size = sizeof(mac); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND); memset(mac, 0xab, 6); TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.mac", mac, size)); - + uint8_t authmode = 1; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.authmode", authmode)); @@ -616,11 +1003,11 @@ TEST_CASE("wifi test", "[nvs]") TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND); memset(pmk, 1, size); TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pmk", pmk, size)); - + uint8_t chan = 1; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.chan", &chan), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.chan", chan)); - + uint8_t autoconn = 1; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "auto.conn", &autoconn), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "auto.conn", autoconn)); @@ -638,43 +1025,43 @@ TEST_CASE("wifi test", "[nvs]") uint8_t phym = 3; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phym", &phym), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phym", phym)); - + uint8_t phybw = 2; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phybw", &phybw), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phybw", phybw)); - + char apsw[2]; size = sizeof(apsw); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apsw", apsw, &size), ESP_ERR_NVS_NOT_FOUND); memset(apsw, 0x2, size); TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apsw", apsw, size)); - + char apinfo[700]; size = sizeof(apinfo); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apinfo", apinfo, &size), ESP_ERR_NVS_NOT_FOUND); memset(apinfo, 0, size); TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apinfo", apinfo, size)); - + size = sizeof(ssid); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND); strcpy(ssid, "ESP_A2F340"); TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.ssid", ssid, size)); - + size = sizeof(mac); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND); memset(mac, 0xac, 6); TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.mac", mac, size)); - + size = sizeof(pswd); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.passwd", pswd, &size), ESP_ERR_NVS_NOT_FOUND); strcpy(pswd, ""); TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.passwd", pswd, size)); - + size = sizeof(pmk); TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND); memset(pmk, 1, size); TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.pmk", pmk, size)); - + chan = 6; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.chan", &chan), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.chan", chan)); @@ -682,63 +1069,133 @@ TEST_CASE("wifi test", "[nvs]") authmode = 0; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.authmode", authmode)); - + uint8_t hidden = 0; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.hidden", &hidden), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.hidden", hidden)); - + uint8_t max_conn = 4; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.max.conn", &max_conn), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.max.conn", max_conn)); - + uint8_t bcn_interval = 2; TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bcn_interval", &bcn_interval), ESP_ERR_NVS_NOT_FOUND); TEST_ESP_OK(nvs_set_u8(net80211_handle, "bcn_interval", bcn_interval)); - - s_perf << "Time to simulate nvs init with wifi libs: " << emu.getTotalTime() << " us (" << emu.getEraseOps() << "E " << emu.getWriteOps() << "W " << emu.getReadOps() << "R " << emu.getWriteBytes() << "Wb " << emu.getReadBytes() << "Rb)" << std::endl; + s_perf << "Time to simulate nvs init with wifi libs: " << f.emu.getTotalTime() << " us (" << f.emu.getEraseOps() << "E " << f.emu.getWriteOps() << "W " << f.emu.getReadOps() << "R " << f.emu.getWriteBytes() << "Wb " << f.emu.getReadBytes() << "Rb)" << std::endl; + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } +TEST_CASE("writing the identical content does not write or erase", "[nvs]") +{ + PartitionEmulationFixture f(0, 20); + + const uint32_t NVS_FLASH_SECTOR = 5; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 10; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_handle misc_handle; + TEST_ESP_OK(nvs_open("test", NVS_READWRITE, &misc_handle)); + + // Test writing a u8 twice, then changing it + nvs_set_u8(misc_handle, "test_u8", 8); + f.emu.clearStats(); + nvs_set_u8(misc_handle, "test_u8", 8); + CHECK(f.emu.getWriteOps() == 0); + CHECK(f.emu.getEraseOps() == 0); + CHECK(f.emu.getReadOps() != 0); + f.emu.clearStats(); + nvs_set_u8(misc_handle, "test_u8", 9); + CHECK(f.emu.getWriteOps() != 0); + CHECK(f.emu.getReadOps() != 0); + + // Test writing a string twice, then changing it + static const char *test[2] = {"Hello world.", "Hello world!"}; + nvs_set_str(misc_handle, "test_str", test[0]); + f.emu.clearStats(); + nvs_set_str(misc_handle, "test_str", test[0]); + CHECK(f.emu.getWriteOps() == 0); + CHECK(f.emu.getEraseOps() == 0); + CHECK(f.emu.getReadOps() != 0); + f.emu.clearStats(); + nvs_set_str(misc_handle, "test_str", test[1]); + CHECK(f.emu.getWriteOps() != 0); + CHECK(f.emu.getReadOps() != 0); + + // Test writing a multi-page blob, then changing it + uint8_t blob[Page::CHUNK_MAX_SIZE * 3] = {0}; + memset(blob, 1, sizeof(blob)); + nvs_set_blob(misc_handle, "test_blob", blob, sizeof(blob)); + f.emu.clearStats(); + nvs_set_blob(misc_handle, "test_blob", blob, sizeof(blob)); + CHECK(f.emu.getWriteOps() == 0); + CHECK(f.emu.getEraseOps() == 0); + CHECK(f.emu.getReadOps() != 0); + blob[sizeof(blob) - 1]++; + f.emu.clearStats(); + nvs_set_blob(misc_handle, "test_blob", blob, sizeof(blob)); + CHECK(f.emu.getWriteOps() != 0); + CHECK(f.emu.getReadOps() != 0); + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); +} TEST_CASE("can init storage from flash with random contents", "[nvs]") { - SpiFlashEmulator emu(10); - emu.randomize(42); - - nvs_handle handle; + PartitionEmulationFixture f(0, 10); + f.emu.randomize(42); + + nvs_handle_t handle; const uint32_t NVS_FLASH_SECTOR = 5; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); - TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &handle)); - + uint8_t opmode = 2; if (nvs_get_u8(handle, "wifi.opmode", &opmode) != ESP_OK) { TEST_ESP_OK(nvs_set_u8(handle, "wifi.opmode", opmode)); } + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } -TEST_CASE("nvs api tests, starting with random data in flash", "[nvs][.][long]") +TEST_CASE("nvs api tests, starting with random data in flash", "[nvs][long]") { - for (size_t count = 0; count < 10000; ++count) { - SpiFlashEmulator emu(10); - emu.randomize(static_cast(count)); - + const size_t testIters = 3000; + int lastPercent = -1; + for (size_t count = 0; count < testIters; ++count) { + int percentDone = (int) (count * 100 / testIters); + if (percentDone != lastPercent) { + lastPercent = percentDone; + printf("%d%%\n", percentDone); + } + PartitionEmulationFixture f(0, 10); + f.emu.randomize(static_cast(count)); + const uint32_t NVS_FLASH_SECTOR = 6; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); - - TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - - nvs_handle handle_1; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_handle_t handle_1; TEST_ESP_ERR(nvs_open("namespace1", NVS_READONLY, &handle_1), ESP_ERR_NVS_NOT_FOUND); - + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle_1)); TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x12345678)); for (size_t i = 0; i < 500; ++i) { - nvs_handle handle_2; + nvs_handle_t handle_2; TEST_ESP_OK(nvs_open("namespace2", NVS_READWRITE, &handle_2)); TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x23456789 % (i + 1))); TEST_ESP_OK(nvs_set_i32(handle_2, "foo", static_cast(i))); @@ -746,38 +1203,42 @@ TEST_CASE("nvs api tests, starting with random data in flash", "[nvs][.][long]") char str_buf[128]; snprintf(str_buf, sizeof(str_buf), str, i + count * 1024); TEST_ESP_OK(nvs_set_str(handle_2, "key", str_buf)); - + int32_t v1; TEST_ESP_OK(nvs_get_i32(handle_1, "foo", &v1)); CHECK(0x23456789 % (i + 1) == v1); - + int32_t v2; TEST_ESP_OK(nvs_get_i32(handle_2, "foo", &v2)); CHECK(static_cast(i) == v2); - + char buf[128]; size_t buf_len = sizeof(buf); - + TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len)); - + CHECK(0 == strcmp(buf, str_buf)); nvs_close(handle_2); } nvs_close(handle_1); } -} + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); +} extern "C" void nvs_dump(const char *partName); class RandomTest { - - static const size_t nKeys = 9; + + static const size_t nKeys = 11; int32_t v1 = 0, v2 = 0; uint64_t v3 = 0, v4 = 0; static const size_t strBufLen = 1024; + static const size_t smallBlobLen = Page::CHUNK_MAX_SIZE / 3; + static const size_t largeBlobLen = Page::CHUNK_MAX_SIZE * 3; char v5[strBufLen], v6[strBufLen], v7[strBufLen], v8[strBufLen], v9[strBufLen]; + uint8_t v10[smallBlobLen], v11[largeBlobLen]; bool written[nKeys]; - + public: RandomTest() { @@ -785,17 +1246,17 @@ class RandomTest { } template - esp_err_t doRandomThings(nvs_handle handle, TGen gen, size_t& count) { - - const char* keys[] = {"foo", "bar", "longkey_0123456", "another key", "param1", "param2", "param3", "param4", "param5"}; - const ItemType types[] = {ItemType::I32, ItemType::I32, ItemType::U64, ItemType::U64, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ}; - - void* values[] = {&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8, &v9}; - + esp_err_t doRandomThings(nvs_handle_t handle, TGen gen, size_t& count) { + + const char* keys[] = {"foo", "bar", "longkey_0123456", "another key", "param1", "param2", "param3", "param4", "param5", "singlepage", "multipage"}; + const ItemType types[] = {ItemType::I32, ItemType::I32, ItemType::U64, ItemType::U64, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::BLOB, ItemType::BLOB}; + + void* values[] = {&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8, &v9, &v10, &v11}; + const size_t nKeys = sizeof(keys) / sizeof(keys[0]); static_assert(nKeys == sizeof(types) / sizeof(types[0]), ""); static_assert(nKeys == sizeof(values) / sizeof(values[0]), ""); - + auto randomRead = [&](size_t index) -> esp_err_t { switch (types[index]) { case ItemType::I32: @@ -814,7 +1275,7 @@ class RandomTest { } break; } - + case ItemType::U64: { uint64_t val; @@ -831,7 +1292,7 @@ class RandomTest { } break; } - + case ItemType::SZ: { char buf[strBufLen]; @@ -849,19 +1310,47 @@ class RandomTest { } break; } - + + case ItemType::BLOB: + { + uint32_t blobBufLen = 0; + if(strncmp(keys[index],"singlepage", sizeof("singlepage")) == 0) { + blobBufLen = smallBlobLen ; + } else { + blobBufLen = largeBlobLen ; + + } + uint8_t buf[blobBufLen]; + memset(buf, 0, blobBufLen); + + size_t len = blobBufLen; + auto err = nvs_get_blob(handle, keys[index], buf, &len); + if (err == ESP_ERR_FLASH_OP_FAIL) { + return err; + } + if (!written[index]) { + REQUIRE(err == ESP_ERR_NVS_NOT_FOUND); + } + else { + REQUIRE(err == ESP_OK); + REQUIRE(memcmp(buf, reinterpret_cast(values[index]), blobBufLen) == 0); + } + break; + } + + default: assert(0); } return ESP_OK; }; - + auto randomWrite = [&](size_t index) -> esp_err_t { switch (types[index]) { case ItemType::I32: { int32_t val = static_cast(gen()); - + auto err = nvs_set_i32(handle, keys[index], val); if (err == ESP_ERR_FLASH_OP_FAIL) { return err; @@ -876,11 +1365,11 @@ class RandomTest { *reinterpret_cast(values[index]) = val; break; } - + case ItemType::U64: { uint64_t val = static_cast(gen()); - + auto err = nvs_set_u64(handle, keys[index], val); if (err == ESP_ERR_FLASH_OP_FAIL) { return err; @@ -895,19 +1384,19 @@ class RandomTest { *reinterpret_cast(values[index]) = val; break; } - + case ItemType::SZ: { char buf[strBufLen]; size_t len = strBufLen; - + size_t strLen = gen() % (strBufLen - 1); std::generate_n(buf, strLen, [&]() -> char { const char c = static_cast(gen() % 127); return (c < 32) ? 32 : c; }); buf[strLen] = 0; - + auto err = nvs_set_str(handle, keys[index], buf); if (err == ESP_ERR_FLASH_OP_FAIL) { return err; @@ -922,23 +1411,53 @@ class RandomTest { strncpy(reinterpret_cast(values[index]), buf, strBufLen); break; } - + + case ItemType::BLOB: + { + uint32_t blobBufLen = 0; + if(strncmp(keys[index],"singlepage", sizeof("singlepage")) == 0) { + blobBufLen = smallBlobLen ; + } else { + blobBufLen = largeBlobLen ; + } + uint8_t buf[blobBufLen]; + memset(buf, 0, blobBufLen); + size_t blobLen = gen() % blobBufLen; + std::generate_n(buf, blobLen, [&]() -> uint8_t { + return static_cast(gen() % 256); + }); + + auto err = nvs_set_blob(handle, keys[index], buf, blobLen); + if (err == ESP_ERR_FLASH_OP_FAIL) { + return err; + } + if (err == ESP_ERR_NVS_REMOVE_FAILED) { + written[index] = true; + memcpy(reinterpret_cast(values[index]), buf, blobBufLen); + return ESP_ERR_FLASH_OP_FAIL; + } + REQUIRE(err == ESP_OK); + written[index] = true; + memcpy(reinterpret_cast(values[index]), buf, blobBufLen); + break; + } + default: assert(0); } return ESP_OK; }; - - + + for (; count != 0; --count) { - size_t index = gen() % nKeys; + size_t index = gen() % (nKeys); switch (gen() % 3) { case 0: // read, 1/3 if (randomRead(index) == ESP_ERR_FLASH_OP_FAIL) { return ESP_ERR_FLASH_OP_FAIL; } break; - + default: // write, 2/3 if (randomWrite(index) == ESP_ERR_FLASH_OP_FAIL) { return ESP_ERR_FLASH_OP_FAIL; @@ -948,8 +1467,20 @@ class RandomTest { } return ESP_OK; } -}; + esp_err_t handleExternalWriteAtIndex(uint8_t index, const void* value, const size_t len ) { + if(index == 9) { /* This is only done for small-page blobs for now*/ + if(len > smallBlobLen) { + return ESP_FAIL; + } + memcpy(v10, value, len); + written[index] = true; + return ESP_OK; + } else { + return ESP_FAIL; + } + } +}; TEST_CASE("monkey test", "[nvs][monkey]") { @@ -957,49 +1488,54 @@ TEST_CASE("monkey test", "[nvs][monkey]") std::mt19937 gen(rd()); uint32_t seed = 3; gen.seed(seed); - - SpiFlashEmulator emu(10); - emu.randomize(seed); - emu.clearStats(); - - const uint32_t NVS_FLASH_SECTOR = 6; - const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); - - TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - - nvs_handle handle; + + PartitionEmulationFixture f(0, 10); + f.emu.randomize(seed); + f.emu.clearStats(); + + const uint32_t NVS_FLASH_SECTOR = 2; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 8; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_handle_t handle; TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); RandomTest test; size_t count = 1000; CHECK(test.doRandomThings(handle, gen, count) == ESP_OK); - - s_perf << "Monkey test: nErase=" << emu.getEraseOps() << " nWrite=" << emu.getWriteOps() << std::endl; + + s_perf << "Monkey test: nErase=" << f.emu.getEraseOps() << " nWrite=" << f.emu.getWriteOps() << std::endl; + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } -TEST_CASE("test recovery from sudden poweroff", "[.][long][nvs][recovery][monkey]") +TEST_CASE("test recovery from sudden poweroff", "[long][nvs][recovery][monkey]") { std::random_device rd; std::mt19937 gen(rd()); uint32_t seed = 3; gen.seed(seed); const size_t iter_count = 2000; - - SpiFlashEmulator emu(10); - - const uint32_t NVS_FLASH_SECTOR = 6; - const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); - + + PartitionEmulationFixture f(0, 10); + + const uint32_t NVS_FLASH_SECTOR = 2; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 8; + + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + size_t totalOps = 0; int lastPercent = -1; for (uint32_t errDelay = 0; ; ++errDelay) { INFO(errDelay); - emu.randomize(seed); - emu.clearStats(); - emu.failAfter(errDelay); + f.emu.randomize(seed); + f.emu.clearStats(); + f.emu.failAfter(errDelay); RandomTest test; - + if (totalOps != 0) { int percent = errDelay * 100 / totalOps; if (percent > lastPercent) { @@ -1007,12 +1543,14 @@ TEST_CASE("test recovery from sudden poweroff", "[.][long][nvs][recovery][monkey lastPercent = percent; } } - - nvs_handle handle; + + nvs_handle_t handle; size_t count = iter_count; - if (nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) == ESP_OK) { + if (NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN) == ESP_OK) { if (nvs_open("namespace1", NVS_READWRITE, &handle) == ESP_OK) { if(test.doRandomThings(handle, gen, count) != ESP_ERR_FLASH_OP_FAIL) { nvs_close(handle); @@ -1020,9 +1558,12 @@ TEST_CASE("test recovery from sudden poweroff", "[.][long][nvs][recovery][monkey } nvs_close(handle); } + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } - - TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); auto res = test.doRandomThings(handle, gen, count); if (res != ESP_OK) { @@ -1030,35 +1571,40 @@ TEST_CASE("test recovery from sudden poweroff", "[.][long][nvs][recovery][monkey CHECK(0); } nvs_close(handle); - totalOps = emu.getEraseOps() + emu.getWriteBytes() / 4; + totalOps = f.emu.getEraseOps() + f.emu.getWriteBytes() / 4; + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } } - TEST_CASE("test for memory leaks in open/set", "[leaks]") { - SpiFlashEmulator emu(10); + PartitionEmulationFixture f(0, 10); const uint32_t NVS_FLASH_SECTOR = 6; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; - emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); - TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + for (int i = 0; i < 100000; ++i) { - nvs_handle light_handle = 0; + nvs_handle_t light_handle = 0; char lightbulb[1024] = {12, 13, 14, 15, 16}; TEST_ESP_OK(nvs_open("light", NVS_READWRITE, &light_handle)); TEST_ESP_OK(nvs_set_blob(light_handle, "key", lightbulb, sizeof(lightbulb))); TEST_ESP_OK(nvs_commit(light_handle)); nvs_close(light_handle); } + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } TEST_CASE("duplicate items are removed", "[nvs][dupes]") { - SpiFlashEmulator emu(3); + PartitionEmulationFixture f(0, 3); { // create one item nvs::Page p; - p.load(0); + p.load(&f.part, 0); p.writeItem(1, "opmode", 3); } { @@ -1066,14 +1612,14 @@ TEST_CASE("duplicate items are removed", "[nvs][dupes]") nvs::Item item(1, ItemType::U8, 1, "opmode"); item.data[0] = 2; item.crc32 = item.calculateCrc32(); - emu.write(3 * 32, reinterpret_cast(&item), sizeof(item)); - emu.write(4 * 32, reinterpret_cast(&item), sizeof(item)); + f.emu.write(3 * 32, reinterpret_cast(&item), sizeof(item)); + f.emu.write(4 * 32, reinterpret_cast(&item), sizeof(item)); uint32_t mask = 0xFFFFFFEA; - emu.write(32, &mask, 4); + f.emu.write(32, &mask, 4); } { // load page and check that second item persists - nvs::Storage s; + nvs::Storage s(&f.part); s.init(0, 3); uint8_t val; ESP_ERROR_CHECK(s.readItem(1, "opmode", val)); @@ -1081,7 +1627,7 @@ TEST_CASE("duplicate items are removed", "[nvs][dupes]") } { Page p; - p.load(0); + p.load(&f.part, 0); CHECK(p.getErasedEntryCount() == 2); CHECK(p.getUsedEntryCount() == 1); } @@ -1089,30 +1635,30 @@ TEST_CASE("duplicate items are removed", "[nvs][dupes]") TEST_CASE("recovery after failure to write data", "[nvs]") { - SpiFlashEmulator emu(3); + PartitionEmulationFixture f(0, 3); const char str[] = "value 0123456789abcdef012345678value 0123456789abcdef012345678"; // make flash write fail exactly in Page::writeEntryData - emu.failAfter(17); + f.emu.failAfter(17); { - Storage storage; + Storage storage(&f.part); TEST_ESP_OK(storage.init(0, 3)); - + TEST_ESP_ERR(storage.writeItem(1, ItemType::SZ, "key", str, strlen(str)), ESP_ERR_FLASH_OP_FAIL); - + // check that repeated operations cause an error TEST_ESP_ERR(storage.writeItem(1, ItemType::SZ, "key", str, strlen(str)), ESP_ERR_NVS_INVALID_STATE); - + uint8_t val; TEST_ESP_ERR(storage.readItem(1, ItemType::U8, "key", &val, sizeof(val)), ESP_ERR_NVS_NOT_FOUND); } { // load page and check that data was erased Page p; - p.load(0); + p.load(&f.part, 0); CHECK(p.getErasedEntryCount() == 3); CHECK(p.getUsedEntryCount() == 0); - + // try to write again TEST_ESP_OK(p.writeItem(1, ItemType::SZ, "key", str, strlen(str))); } @@ -1120,25 +1666,25 @@ TEST_CASE("recovery after failure to write data", "[nvs]") TEST_CASE("crc errors in item header are handled", "[nvs]") { - SpiFlashEmulator emu(3); - Storage storage; + PartitionEmulationFixture f(0, 3); + Storage storage(&f.part); // prepare some data TEST_ESP_OK(storage.init(0, 3)); TEST_ESP_OK(storage.writeItem(0, "ns1", static_cast(1))); TEST_ESP_OK(storage.writeItem(1, "value1", static_cast(1))); TEST_ESP_OK(storage.writeItem(1, "value2", static_cast(2))); - + // corrupt item header uint32_t val = 0; - emu.write(32 * 3, &val, 4); - + f.emu.write(32 * 3, &val, 4); + // check that storage can recover TEST_ESP_OK(storage.init(0, 3)); TEST_ESP_OK(storage.readItem(1, "value2", val)); CHECK(val == 2); // check that the corrupted item is no longer present TEST_ESP_ERR(ESP_ERR_NVS_NOT_FOUND, storage.readItem(1, "value1", val)); - + // add more items to make the page full for (size_t i = 0; i < Page::ENTRY_COUNT; ++i) { char item_name[Item::MAX_KEY_LENGTH + 1]; @@ -1148,8 +1694,8 @@ TEST_CASE("crc errors in item header are handled", "[nvs]") // corrupt another item on the full page val = 0; - emu.write(32 * 4, &val, 4); - + f.emu.write(32 * 4, &val, 4); + // check that storage can recover TEST_ESP_OK(storage.init(0, 3)); // check that the corrupted item is no longer present @@ -1158,13 +1704,13 @@ TEST_CASE("crc errors in item header are handled", "[nvs]") TEST_CASE("crc error in variable length item is handled", "[nvs]") { - SpiFlashEmulator emu(3); + PartitionEmulationFixture f(0, 3); const uint64_t before_val = 0xbef04e; const uint64_t after_val = 0xaf7e4; // write some data { Page p; - p.load(0); + p.load(&f.part, 0); TEST_ESP_OK(p.writeItem(0, "before", before_val)); const char* str = "foobar"; TEST_ESP_OK(p.writeItem(0, ItemType::SZ, "key", str, strlen(str))); @@ -1172,13 +1718,13 @@ TEST_CASE("crc error in variable length item is handled", "[nvs]") } // corrupt some data uint32_t w; - CHECK(emu.read(&w, 32 * 3 + 8, sizeof(w))); + CHECK(f.emu.read(&w, 32 * 3 + 8, sizeof(w))); w &= 0xf000000f; - CHECK(emu.write(32 * 3 + 8, &w, sizeof(w))); + CHECK(f.emu.write(32 * 3 + 8, &w, sizeof(w))); // load and check { Page p; - p.load(0); + p.load(&f.part, 0); CHECK(p.getUsedEntryCount() == 2); CHECK(p.getErasedEntryCount() == 2); @@ -1194,16 +1740,16 @@ TEST_CASE("crc error in variable length item is handled", "[nvs]") TEST_CASE("read/write failure (TW8406)", "[nvs]") { - SpiFlashEmulator emu(3); - nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3); + PartitionEmulationFixture f(0, 3); + NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3); for (int attempts = 0; attempts < 3; ++attempts) { int i = 0; - nvs_handle light_handle = 0; + nvs_handle_t light_handle = 0; char key[15] = {0}; char data[76] = {12, 13, 14, 15, 16}; uint8_t number = 20; size_t data_len = sizeof(data); - + ESP_ERROR_CHECK(nvs_open("LIGHT", NVS_READWRITE, &light_handle)); ESP_ERROR_CHECK(nvs_set_u8(light_handle, "RecordNum", number)); for (i = 0; i < number; ++i) { @@ -1211,7 +1757,7 @@ TEST_CASE("read/write failure (TW8406)", "[nvs]") ESP_ERROR_CHECK(nvs_set_blob(light_handle, key, data, sizeof(data))); } nvs_commit(light_handle); - + uint8_t get_number = 0; ESP_ERROR_CHECK(nvs_get_u8(light_handle, "RecordNum", &get_number)); REQUIRE(number == get_number); @@ -1222,38 +1768,44 @@ TEST_CASE("read/write failure (TW8406)", "[nvs]") } nvs_close(light_handle); } + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } TEST_CASE("nvs_flash_init checks for an empty page", "[nvs]") { - const size_t blob_size = Page::BLOB_MAX_SIZE; + const size_t blob_size = Page::CHUNK_MAX_SIZE; uint8_t blob[blob_size] = {0}; - SpiFlashEmulator emu(5); - TEST_ESP_OK( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5) ); - nvs_handle handle; + PartitionEmulationFixture f(0, 8); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5) ); + nvs_handle_t handle; TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); // Fill first page TEST_ESP_OK( nvs_set_blob(handle, "1a", blob, blob_size) ); - TEST_ESP_OK( nvs_set_blob(handle, "1b", blob, blob_size) ); // Fill second page TEST_ESP_OK( nvs_set_blob(handle, "2a", blob, blob_size) ); - TEST_ESP_OK( nvs_set_blob(handle, "2b", blob, blob_size) ); // Fill third page TEST_ESP_OK( nvs_set_blob(handle, "3a", blob, blob_size) ); - TEST_ESP_OK( nvs_set_blob(handle, "3b", blob, blob_size) ); TEST_ESP_OK( nvs_commit(handle) ); nvs_close(handle); + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); // first two pages are now full, third one is writable, last two are empty // init should fail - TEST_ESP_ERR( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3), ESP_ERR_NVS_NO_FREE_PAGES ); + TEST_ESP_ERR( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3), + ESP_ERR_NVS_NO_FREE_PAGES ); + + // in case this test fails, to not affect other tests + nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME); } TEST_CASE("multiple partitions access check", "[nvs]") { SpiFlashEmulator emu(10); - TEST_ESP_OK( nvs_flash_init_custom("nvs1", 0, 5) ); - TEST_ESP_OK( nvs_flash_init_custom("nvs2", 5, 5) ); - nvs_handle handle1, handle2; + PartitionEmulation p0(&emu, 0 * SPI_FLASH_SEC_SIZE, 5 * SPI_FLASH_SEC_SIZE, "nvs1"); + PartitionEmulation p1(&emu, 5 * SPI_FLASH_SEC_SIZE, 5 * SPI_FLASH_SEC_SIZE, "nvs2"); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&p0, 0, 5) ); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&p1, 5, 5) ); + nvs_handle_t handle1, handle2; TEST_ESP_OK( nvs_open_from_partition("nvs1", "test", NVS_READWRITE, &handle1) ); TEST_ESP_OK( nvs_open_from_partition("nvs2", "test", NVS_READWRITE, &handle2) ); TEST_ESP_OK( nvs_set_i32(handle1, "foo", 0xdeadbeef)); @@ -1263,15 +1815,18 @@ TEST_CASE("multiple partitions access check", "[nvs]") TEST_ESP_OK( nvs_get_i32(handle2, "foo", &v2)); CHECK(v1 == 0xdeadbeef); CHECK(v2 == 0xcafebabe); + + TEST_ESP_OK(nvs_flash_deinit_partition(p0.get_partition_name())); + TEST_ESP_OK(nvs_flash_deinit_partition(p1.get_partition_name())); } TEST_CASE("nvs page selection takes into account free entries also not just erased entries", "[nvs]") { - const size_t blob_size = Page::BLOB_MAX_SIZE; + const size_t blob_size = Page::CHUNK_MAX_SIZE/2; uint8_t blob[blob_size] = {0}; - SpiFlashEmulator emu(3); - TEST_ESP_OK( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3) ); - nvs_handle handle; + PartitionEmulationFixture f(0, 3); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3) ); + nvs_handle_t handle; TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); // Fill first page TEST_ESP_OK( nvs_set_blob(handle, "1a", blob, blob_size/3) ); @@ -1284,8 +1839,1689 @@ TEST_CASE("nvs page selection takes into account free entries also not just eras TEST_ESP_OK( nvs_set_blob(handle, "3a", blob, 4) ); TEST_ESP_OK( nvs_commit(handle) ); nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("calculate used and free space", "[nvs]") +{ + PartitionEmulationFixture f(0, 6); + nvs_flash_deinit(); + TEST_ESP_ERR(nvs_get_stats(NULL, NULL), ESP_ERR_INVALID_ARG); + nvs_stats_t stat1; + nvs_stats_t stat2; + TEST_ESP_ERR(nvs_get_stats(NULL, &stat1), ESP_ERR_NVS_NOT_INITIALIZED); + CHECK(stat1.free_entries == 0); + CHECK(stat1.namespace_count == 0); + CHECK(stat1.total_entries == 0); + CHECK(stat1.used_entries == 0); + + nvs_handle_t handle = 0; + size_t h_count_entries; + TEST_ESP_ERR(nvs_get_used_entry_count(handle, &h_count_entries), ESP_ERR_NVS_INVALID_HANDLE); + CHECK(h_count_entries == 0); + + // init nvs + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 6)); + + TEST_ESP_ERR(nvs_get_used_entry_count(handle, &h_count_entries), ESP_ERR_NVS_INVALID_HANDLE); + CHECK(h_count_entries == 0); + + Page p; + // after erase. empty partition + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + CHECK(stat1.free_entries != 0); + CHECK(stat1.namespace_count == 0); + CHECK(stat1.total_entries == 6 * p.ENTRY_COUNT); + CHECK(stat1.used_entries == 0); + + // create namespace test_k1 + nvs_handle_t handle_1; + TEST_ESP_OK(nvs_open("test_k1", NVS_READWRITE, &handle_1)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + CHECK(stat2.free_entries + 1 == stat1.free_entries); + CHECK(stat2.namespace_count == 1); + CHECK(stat2.total_entries == stat1.total_entries); + CHECK(stat2.used_entries == 1); + + // create pair key-value com + TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x12345678)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + CHECK(stat1.free_entries + 1 == stat2.free_entries); + CHECK(stat1.namespace_count == 1); + CHECK(stat1.total_entries == stat2.total_entries); + CHECK(stat1.used_entries == 2); + + // change value in com + TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x01234567)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + CHECK(stat2.free_entries == stat1.free_entries); + CHECK(stat2.namespace_count == 1); + CHECK(stat2.total_entries != 0); + CHECK(stat2.used_entries == 2); + + // create pair key-value ru + TEST_ESP_OK(nvs_set_i32(handle_1, "ru", 0x00FF00FF)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + CHECK(stat1.free_entries + 1 == stat2.free_entries); + CHECK(stat1.namespace_count == 1); + CHECK(stat1.total_entries != 0); + CHECK(stat1.used_entries == 3); + + // amount valid pair in namespace 1 + size_t h1_count_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_1, &h1_count_entries)); + CHECK(h1_count_entries == 2); + + nvs_handle_t handle_2; + // create namespace test_k2 + TEST_ESP_OK(nvs_open("test_k2", NVS_READWRITE, &handle_2)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + CHECK(stat2.free_entries + 1 == stat1.free_entries); + CHECK(stat2.namespace_count == 2); + CHECK(stat2.total_entries == stat1.total_entries); + CHECK(stat2.used_entries == 4); + + // create pair key-value + TEST_ESP_OK(nvs_set_i32(handle_2, "su1", 0x00000001)); + TEST_ESP_OK(nvs_set_i32(handle_2, "su2", 0x00000002)); + TEST_ESP_OK(nvs_set_i32(handle_2, "sus", 0x00000003)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + CHECK(stat1.free_entries + 3 == stat2.free_entries); + CHECK(stat1.namespace_count == 2); + CHECK(stat1.total_entries == stat2.total_entries); + CHECK(stat1.used_entries == 7); + + CHECK(stat1.total_entries == (stat1.used_entries + stat1.free_entries)); + + // amount valid pair in namespace 2 + size_t h2_count_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_2, &h2_count_entries)); + CHECK(h2_count_entries == 3); + + CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + stat1.namespace_count)); + + nvs_close(handle_1); + nvs_close(handle_2); + + size_t temp = h2_count_entries; + TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, &h2_count_entries), ESP_ERR_NVS_INVALID_HANDLE); + CHECK(h2_count_entries == 0); + h2_count_entries = temp; + TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, NULL), ESP_ERR_INVALID_ARG); + + nvs_handle_t handle_3; + // create namespace test_k3 + TEST_ESP_OK(nvs_open("test_k3", NVS_READWRITE, &handle_3)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + CHECK(stat2.free_entries + 1 == stat1.free_entries); + CHECK(stat2.namespace_count == 3); + CHECK(stat2.total_entries == stat1.total_entries); + CHECK(stat2.used_entries == 8); + + // create pair blobs + uint32_t blob[12]; + TEST_ESP_OK(nvs_set_blob(handle_3, "bl1", &blob, sizeof(blob))); + TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); + CHECK(stat1.free_entries + 4 == stat2.free_entries); + CHECK(stat1.namespace_count == 3); + CHECK(stat1.total_entries == stat2.total_entries); + CHECK(stat1.used_entries == 12); + + // amount valid pair in namespace 2 + size_t h3_count_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &h3_count_entries)); + CHECK(h3_count_entries == 4); + + CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + h3_count_entries + stat1.namespace_count)); + + nvs_close(handle_3); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +// TODO: leaks memory +TEST_CASE("Recovery from power-off when the entry being erased is not on active page", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE/2 ; + size_t read_size = blob_size; + uint8_t blob[blob_size] = {0x11}; + PartitionEmulationFixture f(0, 3); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3) ); + nvs_handle_t handle; + TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); + + f.emu.clearStats(); + f.emu.failAfter(Page::CHUNK_MAX_SIZE/4 + 75); + TEST_ESP_OK( nvs_set_blob(handle, "1a", blob, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "1b", blob, blob_size) ); + + TEST_ESP_ERR( nvs_erase_key(handle, "1a"), ESP_ERR_FLASH_OP_FAIL ); + + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3) ); + + /* Check 1a is erased fully*/ + TEST_ESP_ERR( nvs_get_blob(handle, "1a", blob, &read_size), ESP_ERR_NVS_NOT_FOUND); + + /* Check 2b is still accessible*/ + TEST_ESP_OK( nvs_get_blob(handle, "1b", blob, &read_size)); + + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +// TODO: leaks memory +TEST_CASE("Recovery from power-off when page is being freed.", "[nvs]") +{ + const size_t blob_size = (Page::ENTRY_COUNT-3) * Page::ENTRY_SIZE; + size_t read_size = blob_size/2; + uint8_t blob[blob_size] = {0}; + PartitionEmulationFixture f(0, 3); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3)); + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("test", NVS_READWRITE, &handle)); + // Fill first page + TEST_ESP_OK(nvs_set_blob(handle, "1a", blob, blob_size/3)); + TEST_ESP_OK(nvs_set_blob(handle, "1b", blob, blob_size/3)); + TEST_ESP_OK(nvs_set_blob(handle, "1c", blob, blob_size/4)); + // Fill second page + TEST_ESP_OK(nvs_set_blob(handle, "2a", blob, blob_size/2)); + TEST_ESP_OK(nvs_set_blob(handle, "2b", blob, blob_size/2)); + + TEST_ESP_OK(nvs_erase_key(handle, "1c")); + + f.emu.clearStats(); + f.emu.failAfter(6 * Page::ENTRY_COUNT); + TEST_ESP_ERR(nvs_set_blob(handle, "1d", blob, blob_size/4), ESP_ERR_FLASH_OP_FAIL); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3)); + + read_size = blob_size/3; + TEST_ESP_OK( nvs_get_blob(handle, "1a", blob, &read_size)); + TEST_ESP_OK( nvs_get_blob(handle, "1b", blob, &read_size)); + + read_size = blob_size /4; + TEST_ESP_ERR( nvs_get_blob(handle, "1c", blob, &read_size), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR( nvs_get_blob(handle, "1d", blob, &read_size), ESP_ERR_NVS_NOT_FOUND); + + read_size = blob_size /2; + TEST_ESP_OK( nvs_get_blob(handle, "2a", blob, &read_size)); + TEST_ESP_OK( nvs_get_blob(handle, "2b", blob, &read_size)); + + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Multi-page blobs are supported", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *2; + uint8_t blob[blob_size] = {0}; + PartitionEmulationFixture f(0, 5); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5)); + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("test", NVS_READWRITE, &handle)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Failures are handled while storing multi-page blobs", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *7; + uint8_t blob[blob_size] = {0}; + PartitionEmulationFixture f(0, 5); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5)); + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("test", NVS_READWRITE, &handle)); + TEST_ESP_ERR(nvs_set_blob(handle, "abc", blob, blob_size), ESP_ERR_NVS_VALUE_TOO_LONG); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, Page::CHUNK_MAX_SIZE*2)); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Reading multi-page blobs", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size]; + uint8_t blob_read[blob_size]; + size_t read_size = blob_size; + PartitionEmulationFixture f(0, 5); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5)); + nvs_handle_t handle; + memset(blob, 0x11, blob_size); + memset(blob_read, 0xee, blob_size); + TEST_ESP_OK(nvs_open("readTest", NVS_READWRITE, &handle)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob, blob_read, blob_size) == 0); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Modification of values for Multi-page blobs are supported", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *2; + uint8_t blob[blob_size] = {0}; + uint8_t blob_read[blob_size] = {0xfe};; + uint8_t blob2[blob_size] = {0x11}; + uint8_t blob3[blob_size] = {0x22}; + uint8_t blob4[blob_size] ={ 0x33}; + size_t read_size = blob_size; + PartitionEmulationFixture f(0, 6); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 6) ); + nvs_handle_t handle; + memset(blob, 0x11, blob_size); + memset(blob2, 0x22, blob_size); + memset(blob3, 0x33, blob_size); + memset(blob4, 0x44, blob_size); + memset(blob_read, 0xff, blob_size); + TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob2, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob3, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob4, blob_size) ); + TEST_ESP_OK( nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob4, blob_read, blob_size) == 0); + TEST_ESP_OK( nvs_commit(handle) ); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Modification from single page blob to multi-page", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size] = {0}; + uint8_t blob_read[blob_size] = {0xff}; + size_t read_size = blob_size; + PartitionEmulationFixture f(0, 5); + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5) ); + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("Test", NVS_READWRITE, &handle) ); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, Page::CHUNK_MAX_SIZE/2)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob, blob_read, blob_size) == 0); + TEST_ESP_OK(nvs_commit(handle) ); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Modification from multi-page to single page", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size] = {0}; + uint8_t blob_read[blob_size] = {0xff}; + size_t read_size = blob_size; + PartitionEmulationFixture f(0, 5); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5) ); + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("Test", NVS_READWRITE, &handle) ); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, Page::CHUNK_MAX_SIZE/2)); + TEST_ESP_OK(nvs_set_blob(handle, "abc2", blob, blob_size)); + TEST_ESP_OK(nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob, blob_read, Page::CHUNK_MAX_SIZE) == 0); + TEST_ESP_OK(nvs_commit(handle) ); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Multi-page blob erased using nvs_erase_key should not be found when probed for just length", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size] = {0}; + size_t read_size = blob_size; + PartitionEmulationFixture f(0, 5); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 5)); + nvs_handle handle; + TEST_ESP_OK(nvs_open("Test", NVS_READWRITE, &handle)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_erase_key(handle, "abc")); + TEST_ESP_ERR(nvs_get_blob(handle, "abc", NULL, &read_size), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + + +TEST_CASE("Check that orphaned blobs are erased during init", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3 ; + uint8_t blob[blob_size] = {0x11}; + uint8_t blob2[blob_size] = {0x22}; + uint8_t blob3[blob_size] = {0x33}; + PartitionEmulationFixture f(0, 5); + Storage storage(&f.part); + + TEST_ESP_OK(storage.init(0, 5)); + + TEST_ESP_OK(storage.writeItem(1, ItemType::BLOB, "key", blob, sizeof(blob))); + + + TEST_ESP_OK(storage.init(0, 5)); + /* Check that multi-page item is still available.**/ + TEST_ESP_OK(storage.readItem(1, ItemType::BLOB, "key", blob, sizeof(blob))); + + TEST_ESP_ERR(storage.writeItem(1, ItemType::BLOB, "key2", blob, sizeof(blob)), ESP_ERR_NVS_NOT_ENOUGH_SPACE); + + Page p; + p.load(&f.part, 3); // This is where index will be placed. + p.erase(); + + TEST_ESP_OK(storage.init(0, 5)); + + TEST_ESP_ERR(storage.readItem(1, ItemType::BLOB, "key", blob, sizeof(blob)), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(storage.writeItem(1, ItemType::BLOB, "key3", blob, sizeof(blob))); +} + +TEST_CASE("nvs blob fragmentation test", "[nvs]") +{ + PartitionEmulationFixture f(0, 4); + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 4) ); + const size_t BLOB_SIZE = 3500; + uint8_t *blob = (uint8_t*) malloc(BLOB_SIZE); + CHECK(blob != NULL); + memset(blob, 0xEE, BLOB_SIZE); + const uint32_t magic = 0xff33eaeb; + nvs_handle_t h; + TEST_ESP_OK( nvs_open("blob_tests", NVS_READWRITE, &h) ); + for (int i = 0; i < 128; i++) { + INFO("Iteration " << i << "...\n"); + TEST_ESP_OK( nvs_set_u32(h, "magic", magic) ); + TEST_ESP_OK( nvs_set_blob(h, "blob", blob, BLOB_SIZE) ); + char seq_buf[16]; + sprintf(seq_buf, "seq%d", i); + TEST_ESP_OK( nvs_set_u32(h, seq_buf, i) ); + } + free(blob); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("nvs code handles errors properly when partition is near to full", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE * 0.3 ; + uint8_t blob[blob_size] = {0x11}; + PartitionEmulationFixture f(0, 5); + Storage storage(&f.part); + char nvs_key[16] = ""; + + TEST_ESP_OK(storage.init(0, 5)); + + /* Four pages should fit roughly 12 blobs*/ + for(uint8_t count = 1; count <= 12; count++) { + sprintf(nvs_key, "key:%u", count); + TEST_ESP_OK(storage.writeItem(1, ItemType::BLOB, nvs_key, blob, sizeof(blob))); + } + + for(uint8_t count = 13; count <= 20; count++) { + sprintf(nvs_key, "key:%u", count); + TEST_ESP_ERR(storage.writeItem(1, ItemType::BLOB, nvs_key, blob, sizeof(blob)), ESP_ERR_NVS_NOT_ENOUGH_SPACE); + } +} + +TEST_CASE("Check for nvs version incompatibility", "[nvs]") +{ + PartitionEmulationFixture f(0, 3); + + int32_t val1 = 0x12345678; + Page p; + p.load(&f.part, 0); + TEST_ESP_OK(p.setVersion(Page::NVS_VERSION - 1)); + TEST_ESP_OK(p.writeItem(1, ItemType::I32, "foo", &val1, sizeof(val1))); + + TEST_ESP_ERR(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3), + ESP_ERR_NVS_NEW_VERSION_FOUND); + + // if something went wrong, clean up + nvs_flash_deinit_partition(f.part.get_partition_name()); +} + +TEST_CASE("Check that NVS supports old blob format without blob index", "[nvs]") +{ + SpiFlashEmulator emu("../nvs_partition_generator/part_old_blob_format.bin"); + PartitionEmulation part(&emu, 0, 2 * SPI_FLASH_SEC_SIZE, "test"); + nvs_handle_t handle; + + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(&part, 0, 2) ); + TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READWRITE, &handle)); + + char buf[64] = {0}; + size_t buflen = 64; + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + buflen = 64; + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); + + Page p; + p.load(&part, 0); + + /* Check that item is stored in old format without blob index*/ + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "dummyHex2BinKey")); + + /* Modify the blob so that it is stored in the new format*/ + hexdata[0] = hexdata[1] = hexdata[2] = 0x99; + TEST_ESP_OK(nvs_set_blob(handle, "dummyHex2BinKey", hexdata, sizeof(hexdata))); + + Page p2; + p2.load(&part, 0); + + /* Check the type of the blob. Expect type mismatch since the blob is stored in new format*/ + TEST_ESP_ERR(p2.findItem(1, ItemType::BLOB, "dummyHex2BinKey"), ESP_ERR_NVS_TYPE_MISMATCH); + + /* Check that index is present for the modified blob according to new format*/ + TEST_ESP_OK(p2.findItem(1, ItemType::BLOB_IDX, "dummyHex2BinKey")); + + /* Read the blob in new format and check the contents*/ + buflen = 64; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); + + TEST_ESP_OK(nvs_flash_deinit_partition(part.get_partition_name())); +} + +// TODO: leaks memory +TEST_CASE("monkey test with old-format blob present", "[nvs][monkey]") +{ + std::random_device rd; + std::mt19937 gen(rd()); + uint32_t seed = 3; + gen.seed(seed); + + PartitionEmulationFixture f(0, 10); + f.emu.randomize(seed); + f.emu.clearStats(); + + const uint32_t NVS_FLASH_SECTOR = 2; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 8; + static const size_t smallBlobLen = Page::CHUNK_MAX_SIZE / 3; + + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + RandomTest test; + + for ( uint8_t it = 0; it < 10; it++) { + size_t count = 200; + + /* Erase index and chunks for the blob with "singlepage" key */ + for (uint8_t num = NVS_FLASH_SECTOR; num < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; num++) { + Page p; + p.load(&f.part, num); + p.eraseItem(1, ItemType::BLOB, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY); + p.eraseItem(1, ItemType::BLOB_IDX, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY); + p.eraseItem(1, ItemType::BLOB_DATA, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY); + } + + /* Now write "singlepage" blob in old format*/ + for (uint8_t num = NVS_FLASH_SECTOR; num < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; num++) { + Page p; + p.load(&f.part, num); + if (p.state() == Page::PageState::ACTIVE) { + uint8_t buf[smallBlobLen]; + size_t blobLen = gen() % smallBlobLen; + + if(blobLen > p.getVarDataTailroom()) { + blobLen = p.getVarDataTailroom(); + } + + std::generate_n(buf, blobLen, [&]() -> uint8_t { + return static_cast(gen() % 256); + }); + + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB, "singlepage", buf, blobLen, Item::CHUNK_ANY)); + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "singlepage")); + test.handleExternalWriteAtIndex(9, buf, blobLen); // This assumes "singlepage" is always at index 9 + + break; + } + } + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); + /* Initialize again */ + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + /* Perform random things */ + auto res = test.doRandomThings(handle, gen, count); + if (res != ESP_OK) { + nvs_dump(NVS_DEFAULT_PART_NAME); + CHECK(0); + } + + /* Check that only one version is present for "singlepage". Its possible that last iteration did not write + * anything for "singlepage". So either old version or new version should be present.*/ + bool oldVerPresent = false, newVerPresent = false; + + for (uint8_t num = NVS_FLASH_SECTOR; num < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; num++) { + Page p; + p.load(&f.part, num); + if(!oldVerPresent && p.findItem(1, ItemType::BLOB, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY) == ESP_OK) { + oldVerPresent = true; + } + + if(!newVerPresent && p.findItem(1, ItemType::BLOB_IDX, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY) == ESP_OK) { + newVerPresent = true; + } + } + CHECK(oldVerPresent != newVerPresent); + } + + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); + s_perf << "Monkey test: nErase=" << f.emu.getEraseOps() << " nWrite=" << f.emu.getWriteOps() << std::endl; +} + +TEST_CASE("Recovery from power-off during modification of blob present in old-format (same page)", "[nvs]") +{ + std::random_device rd; + std::mt19937 gen(rd()); + uint32_t seed = 3; + gen.seed(seed); + + PartitionEmulationFixture f(0, 3); + f.emu.clearStats(); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3)); + + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + uint8_t hexdata_old[] = {0x11, 0x12, 0x13, 0xbb, 0xcc, 0xee}; + size_t buflen = sizeof(hexdata); + uint8_t buf[Page::CHUNK_MAX_SIZE]; + + /* Power-off when blob was being written on the same page where its old version in old format + * was present*/ + Page p; + p.load(&f.part, 0); + /* Write blob in old-format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB, "singlepage", hexdata_old, sizeof(hexdata_old))); + + /* Write blob in new format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB_DATA, "singlepage", hexdata, sizeof(hexdata), 0)); + /* All pages are stored. Now store the index.*/ + Item item; + item.blobIndex.dataSize = sizeof(hexdata); + item.blobIndex.chunkCount = 1; + item.blobIndex.chunkStart = VerOffset::VER_0_OFFSET; + + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB_IDX, "singlepage", item.data, sizeof(item.data))); + + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "singlepage")); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); + /* Initialize again */ + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3)); + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + TEST_ESP_OK( nvs_get_blob(handle, "singlepage", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + Page p2; + p2.load(&f.part, 0); + TEST_ESP_ERR(p2.findItem(1, ItemType::BLOB, "singlepage"), ESP_ERR_NVS_TYPE_MISMATCH); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +TEST_CASE("Recovery from power-off during modification of blob present in old-format (different page)", "[nvs]") +{ + std::random_device rd; + std::mt19937 gen(rd()); + uint32_t seed = 3; + gen.seed(seed); + + PartitionEmulationFixture f(0, 3); + f.emu.clearStats(); + + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3)); + + nvs_handle_t handle; + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + uint8_t hexdata_old[] = {0x11, 0x12, 0x13, 0xbb, 0xcc, 0xee}; + size_t buflen = sizeof(hexdata); + uint8_t buf[Page::CHUNK_MAX_SIZE]; + + + /* Power-off when blob was being written on the different page where its old version in old format + * was present*/ + Page p; + p.load(&f.part, 0); + /* Write blob in old-format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB, "singlepage", hexdata_old, sizeof(hexdata_old))); + + /* Write blob in new format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB_DATA, "singlepage", hexdata, sizeof(hexdata), 0)); + /* All pages are stored. Now store the index.*/ + Item item; + item.blobIndex.dataSize = sizeof(hexdata); + item.blobIndex.chunkCount = 1; + item.blobIndex.chunkStart = VerOffset::VER_0_OFFSET; + p.markFull(); + Page p2; + p2.load(&f.part, 1); + p2.setSeqNumber(1); + + TEST_ESP_OK(p2.writeItem(1, ItemType::BLOB_IDX, "singlepage", item.data, sizeof(item.data))); + + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "singlepage")); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); + /* Initialize again */ + TEST_ESP_OK(NVSPartitionManager::get_instance()->init_custom(&f.part, 0, 3)); + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + TEST_ESP_OK( nvs_get_blob(handle, "singlepage", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + Page p3; + p3.load(&f.part, 0); + TEST_ESP_ERR(p3.findItem(1, ItemType::BLOB, "singlepage"), ESP_ERR_NVS_NOT_FOUND); + + TEST_ESP_OK(nvs_flash_deinit_partition(f.part.get_partition_name())); +} + +static void check_nvs_part_gen_args(SpiFlashEmulator *spi_flash_emulator, + char const *part_name, + int size, + char const *filename, + bool is_encr, + nvs_sec_cfg_t* xts_cfg) +{ + nvs_handle_t handle; + + esp_partition_t esp_part; + esp_part.encrypted = false; // we're not testing generic flash encryption here, only the legacy NVS encryption + esp_part.address = 0; + esp_part.size = size * SPI_FLASH_SEC_SIZE; + strncpy(esp_part.label, part_name, PART_NAME_MAX_SIZE); + shared_ptr part; + + if (is_encr) { + NVSEncryptedPartition *enc_part = new NVSEncryptedPartition(&esp_part); + TEST_ESP_OK(enc_part->init(xts_cfg)); + part.reset(enc_part); + } else { + part.reset(new PartitionEmulation(spi_flash_emulator, 0, size, part_name)); + } + + TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(part.get(), 0, size) ); + + TEST_ESP_OK( nvs_open_from_partition(part_name, "dummyNamespace", NVS_READONLY, &handle)); + uint8_t u8v; + TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v)); + CHECK(u8v == 127); + int8_t i8v; + TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v)); + CHECK(i8v == -128); + uint16_t u16v; + TEST_ESP_OK( nvs_get_u16(handle, "dummyU16Key", &u16v)); + CHECK(u16v == 32768); + uint32_t u32v; + TEST_ESP_OK( nvs_get_u32(handle, "dummyU32Key", &u32v)); + CHECK(u32v == 4294967295); + int32_t i32v; + TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v)); + CHECK(i32v == -2147483648); + + char buf[64] = {0}; + size_t buflen = 64; + TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen)); + CHECK(strncmp(buf, "0A:0B:0C:0D:0E:0F", buflen) == 0); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + buflen = 64; + int j; + TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); + + buflen = 64; + uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}; + TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen)); + CHECK(memcmp(buf, hexfiledata, buflen) == 0); + + buflen = 64; + uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0"; + TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen)); + CHECK(memcmp(buf, strfiledata, buflen) == 0); + + char bin_data[5200]; + size_t bin_len = sizeof(bin_data); + char binfiledata[5200]; + ifstream file; + file.open(filename); + file.read(binfiledata,5200); + TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); + CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); + + file.close(); + + nvs_close(handle); + + TEST_ESP_OK(nvs_flash_deinit_partition(part_name)); +} + + +TEST_CASE("check and read data from partition generated via partition generation utility with multipage blob support disabled", "[nvs_part_gen]") +{ + int status; + int childpid = fork(); + if (childpid == 0) { + exit(execlp("cp", " cp", + "-rf", + "../nvs_partition_generator/testdata", + ".",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) != -1); + + childpid = fork(); + + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "generate", + "../nvs_partition_generator/sample_singlepage_blob.csv", + "partition_single_page.bin", + "0x3000", + "--version", + "1", + "--outdir", + "../nvs_partition_generator",NULL)); + } else { + CHECK(childpid > 0); + int status; + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + } + } + + SpiFlashEmulator emu("../nvs_partition_generator/partition_single_page.bin"); + + check_nvs_part_gen_args(&emu, "test", 3, "../nvs_partition_generator/testdata/sample_singlepage_blob.bin", false, NULL); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } +} + +TEST_CASE("check and read data from partition generated via partition generation utility with multipage blob support enabled", "[nvs_part_gen]") +{ + int status; + int childpid = fork(); + if (childpid == 0) { + exit(execlp("cp", " cp", + "-rf", + "../nvs_partition_generator/testdata", + ".",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "generate", + "../nvs_partition_generator/sample_multipage_blob.csv", + "partition_multipage_blob.bin", + "0x4000", + "--version", + "2", + "--outdir", + "../nvs_partition_generator",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + } + } + + SpiFlashEmulator emu("../nvs_partition_generator/partition_multipage_blob.bin"); + + check_nvs_part_gen_args(&emu, "test", 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin",false,NULL); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } +} + +TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support disabled", "[mfg_gen]") +{ + int childpid = fork(); + int status; + + if (childpid == 0) { + exit(execlp("bash", "bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test && \ + cp -rf ../../../tools/mass_mfg/testdata mfg_testdata && \ + cp -rf ../nvs_partition_generator/testdata . && \ + mkdir -p ../../../tools/mass_mfg/host_test", NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../../../tools/mass_mfg/mfg_gen.py", + "generate", + "../../../tools/mass_mfg/samples/sample_config.csv", + "../../../tools/mass_mfg/samples/sample_values_singlepage_blob.csv", + "Test", + "0x3000", + "--outdir", + "../../../tools/mass_mfg/host_test", + "--version", + "1",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "generate", + "../../../tools/mass_mfg/host_test/csv/Test-1.csv", + "../nvs_partition_generator/Test-1-partition.bin", + "0x3000", + "--version", + "1",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + + } + + } + + SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin"); + check_nvs_part_gen_args(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL); + + SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin"); + check_nvs_part_gen_args(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL); + + + childpid = fork(); + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + rm -rf mfg_testdata | \ + rm -rf testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + +} + +TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support enabled", "[mfg_gen]") +{ + int childpid = fork(); + int status; + + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \ + cp -rf ../nvs_partition_generator/testdata . | \ + mkdir -p ../../../tools/mass_mfg/host_test",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../../../tools/mass_mfg/mfg_gen.py", + "generate", + "../../../tools/mass_mfg/samples/sample_config.csv", + "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv", + "Test", + "0x4000", + "--outdir", + "../../../tools/mass_mfg/host_test", + "--version", + "2",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "generate", + "../../../tools/mass_mfg/host_test/csv/Test-1.csv", + "../nvs_partition_generator/Test-1-partition.bin", + "0x4000", + "--version", + "2",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + + } + + } + + SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin"); + check_nvs_part_gen_args(&emu1, "test", 4, "mfg_testdata/sample_multipage_blob.bin", false, NULL); + + SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin"); + check_nvs_part_gen_args(&emu2, "test", 4, "testdata/sample_multipage_blob.bin", false, NULL); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + rm -rf mfg_testdata | \ + rm -rf testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + +} + +#if CONFIG_NVS_ENCRYPTION +TEST_CASE("check underlying xts code for 32-byte size sector encryption", "[nvs]") +{ + auto toHex = [](char ch) { + if(ch >= '0' && ch <= '9') + return ch - '0'; + else if(ch >= 'a' && ch <= 'f') + return ch - 'a' + 10; + else if(ch >= 'A' && ch <= 'F') + return ch - 'A' + 10; + else + return 0; + }; + + auto toHexByte = [toHex](char* c) { + return 16 * toHex(c[0]) + toHex(c[1]); + }; + + auto toHexStream = [toHexByte](char* src, uint8_t* dest) { + uint32_t cnt =0; + char* p = src; + while(*p != '\0' && *(p + 1) != '\0') + { + dest[cnt++] = toHexByte(p); p += 2; + } + }; + + uint8_t eky_hex[2 * NVS_KEY_SIZE]; + uint8_t ptxt_hex[Page::ENTRY_SIZE], ctxt_hex[Page::ENTRY_SIZE], ba_hex[16]; + mbedtls_aes_xts_context ectx[1]; + mbedtls_aes_xts_context dctx[1]; + + char eky[][2 * NVS_KEY_SIZE + 1] = { + "0000000000000000000000000000000000000000000000000000000000000000", + "1111111111111111111111111111111111111111111111111111111111111111" + }; + char tky[][2 * NVS_KEY_SIZE + 1] = { + "0000000000000000000000000000000000000000000000000000000000000000", + "2222222222222222222222222222222222222222222222222222222222222222" + }; + char blk_addr[][2*16 + 1] = { + "00000000000000000000000000000000", + "33333333330000000000000000000000" + }; + + char ptxt[][2 * Page::ENTRY_SIZE + 1] = { + "0000000000000000000000000000000000000000000000000000000000000000", + "4444444444444444444444444444444444444444444444444444444444444444" + }; + char ctxt[][2 * Page::ENTRY_SIZE + 1] = { + "d456b4fc2e620bba6ffbed27b956c9543454dd49ebd8d8ee6f94b65cbe158f73", + "e622334f184bbce129a25b2ac76b3d92abf98e22df5bdd15af471f3db8946a85" + }; + + mbedtls_aes_xts_init(ectx); + mbedtls_aes_xts_init(dctx); + + for(uint8_t cnt = 0; cnt < sizeof(eky)/sizeof(eky[0]); cnt++) { + toHexStream(eky[cnt], eky_hex); + toHexStream(tky[cnt], &eky_hex[NVS_KEY_SIZE]); + toHexStream(ptxt[cnt], ptxt_hex); + toHexStream(ctxt[cnt], ctxt_hex); + toHexStream(blk_addr[cnt], ba_hex); + + CHECK(!mbedtls_aes_xts_setkey_enc(ectx, eky_hex, 2 * NVS_KEY_SIZE * 8)); + CHECK(!mbedtls_aes_xts_setkey_enc(dctx, eky_hex, 2 * NVS_KEY_SIZE * 8)); + + CHECK(!mbedtls_aes_crypt_xts(ectx, MBEDTLS_AES_ENCRYPT, Page::ENTRY_SIZE, ba_hex, ptxt_hex, ptxt_hex)); + + CHECK(!memcmp(ptxt_hex, ctxt_hex, Page::ENTRY_SIZE)); + } +} + +TEST_CASE("test nvs apis with encryption enabled", "[nvs]") +{ + nvs_handle_t handle_1; + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + + nvs_sec_cfg_t xts_cfg; + for(int count = 0; count < NVS_KEY_SIZE; count++) { + xts_cfg.eky[count] = 0x11; + xts_cfg.tky[count] = 0x22; + } + EncryptedPartitionFixture fixture(&xts_cfg, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN); + fixture.emu.randomize(100); + fixture.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + for (uint16_t i = NVS_FLASH_SECTOR; i + init_custom(&fixture.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); + + TEST_ESP_ERR(nvs_open("namespace1", NVS_READONLY, &handle_1), ESP_ERR_NVS_NOT_FOUND); + + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle_1)); + TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x12345678)); + TEST_ESP_OK(nvs_set_i32(handle_1, "foo", 0x23456789)); + + nvs_handle_t handle_2; + TEST_ESP_OK(nvs_open("namespace2", NVS_READWRITE, &handle_2)); + TEST_ESP_OK(nvs_set_i32(handle_2, "foo", 0x3456789a)); + const char* str = "value 0123456789abcdef0123456789abcdef"; + TEST_ESP_OK(nvs_set_str(handle_2, "key", str)); + + int32_t v1; + TEST_ESP_OK(nvs_get_i32(handle_1, "foo", &v1)); + CHECK(0x23456789 == v1); + + int32_t v2; + TEST_ESP_OK(nvs_get_i32(handle_2, "foo", &v2)); + CHECK(0x3456789a == v2); + + char buf[strlen(str) + 1]; + size_t buf_len = sizeof(buf); + + size_t buf_len_needed; + TEST_ESP_OK(nvs_get_str(handle_2, "key", NULL, &buf_len_needed)); + CHECK(buf_len_needed == buf_len); + + size_t buf_len_short = buf_len - 1; + TEST_ESP_ERR(ESP_ERR_NVS_INVALID_LENGTH, nvs_get_str(handle_2, "key", buf, &buf_len_short)); + CHECK(buf_len_short == buf_len); + + size_t buf_len_long = buf_len + 1; + TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len_long)); + CHECK(buf_len_long == buf_len); + + TEST_ESP_OK(nvs_get_str(handle_2, "key", buf, &buf_len)); + + CHECK(0 == strcmp(buf, str)); + nvs_close(handle_1); + nvs_close(handle_2); + TEST_ESP_OK(nvs_flash_deinit()); + +} + +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled", "[nvs_part_gen]") +{ + int status; + int childpid = fork(); + if (childpid == 0) { + exit(execlp("cp", " cp", + "-rf", + "../nvs_partition_generator/testdata", + ".",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "encrypt", + "../nvs_partition_generator/sample_multipage_blob.csv", + "partition_encrypted.bin", + "0x4000", + "--inputkey", + "../nvs_partition_generator/testdata/sample_encryption_keys.bin", + "--outdir", + "../nvs_partition_generator",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + } + } + + SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted.bin"); + + nvs_sec_cfg_t cfg; + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = 0x11; + cfg.tky[count] = 0x22; + } + + check_nvs_part_gen_args(&emu, NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + +} + +TEST_CASE("test decrypt functionality for encrypted data", "[nvs_part_gen]") +{ + + //retrieving the temporary test data + int status = system("cp -rf ../nvs_partition_generator/testdata ."); + CHECK(status == 0); + + //encoding data from sample_multipage_blob.csv + status = system("python ../nvs_partition_generator/nvs_partition_gen.py generate ../nvs_partition_generator/sample_multipage_blob.csv partition_encoded.bin 0x5000 --outdir ../nvs_partition_generator"); + CHECK(status == 0); + + //encrypting data from sample_multipage_blob.csv + status = system("python ../nvs_partition_generator/nvs_partition_gen.py encrypt ../nvs_partition_generator/sample_multipage_blob.csv partition_encrypted.bin 0x5000 --inputkey ../nvs_partition_generator/testdata/sample_encryption_keys.bin --outdir ../nvs_partition_generator"); + CHECK(status == 0); + + //decrypting data from partition_encrypted.bin + status = system("python ../nvs_partition_generator/nvs_partition_gen.py decrypt ../nvs_partition_generator/partition_encrypted.bin ../nvs_partition_generator/testdata/sample_encryption_keys.bin ../nvs_partition_generator/partition_decrypted.bin"); + CHECK(status == 0); + + status = system("diff ../nvs_partition_generator/partition_decrypted.bin ../nvs_partition_generator/partition_encoded.bin"); + CHECK(status == 0); + CHECK(WEXITSTATUS(status) == 0); + + + //cleaning up the temporary test data + status = system("rm -rf testdata"); + CHECK(status == 0); + +} + +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using keygen", "[nvs_part_gen]") +{ + int childpid = fork(); + int status; + + if (childpid == 0) { + exit(execlp("cp", " cp", + "-rf", + "../nvs_partition_generator/testdata", + ".",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "../nvs_partition_generator/keys",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "encrypt", + "../nvs_partition_generator/sample_multipage_blob.csv", + "partition_encrypted_using_keygen.bin", + "0x4000", + "--keygen", + "--outdir", + "../nvs_partition_generator",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + } + } + + + DIR *dir; + struct dirent *file; + char *filename; + char *files; + char *file_ext; + + dir = opendir("../nvs_partition_generator/keys"); + while ((file = readdir(dir)) != NULL) + { + filename = file->d_name; + files = strrchr(filename, '.'); + if (files != NULL) + { + file_ext = files+1; + if (strncmp(file_ext,"bin",3) == 0) + { + break; + } + } + } + + std::string encr_file = std::string("../nvs_partition_generator/keys/") + std::string(filename); + SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted_using_keygen.bin"); + + char buffer[64]; + FILE *fp; + + fp = fopen(encr_file.c_str(),"rb"); + fread(buffer,sizeof(buffer),1,fp); + + fclose(fp); + + nvs_sec_cfg_t cfg; + + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = buffer[count] & 255; + cfg.tky[count] = buffer[count+32] & 255; + } + + check_nvs_part_gen_args(&emu, NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); + } +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using inputkey", "[nvs_part_gen]") +{ + int childpid = fork(); + int status; + + DIR *dir; + struct dirent *file; + char *filename; + char *files; + char *file_ext; + + dir = opendir("../nvs_partition_generator/keys"); + while ((file = readdir(dir)) != NULL) + { + filename = file->d_name; + files = strrchr(filename, '.'); + if (files != NULL) + { + file_ext = files+1; + if (strncmp(file_ext,"bin",3) == 0) + { + break; + } + } + } + + std::string encr_file = std::string("../nvs_partition_generator/keys/") + std::string(filename); + + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "encrypt", + "../nvs_partition_generator/sample_multipage_blob.csv", + "partition_encrypted_using_keyfile.bin", + "0x4000", + "--inputkey", + encr_file.c_str(), + "--outdir", + "../nvs_partition_generator",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + } + + SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted_using_keyfile.bin"); + + char buffer[64]; + FILE *fp; + + fp = fopen(encr_file.c_str(),"rb"); + fread(buffer,sizeof(buffer),1,fp); + + fclose(fp); + + nvs_sec_cfg_t cfg; + + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = buffer[count] & 255; + cfg.tky[count] = buffer[count+32] & 255; + } + + check_nvs_part_gen_args(&emu, NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "../nvs_partition_generator/keys",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + } + } + +} + +TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using sample inputkey", "[mfg_gen]") +{ + int childpid = fork(); + int status; + + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \ + cp -rf ../nvs_partition_generator/testdata . | \ + mkdir -p ../../../tools/mass_mfg/host_test",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../../../tools/mass_mfg/mfg_gen.py", + "generate", + "../../../tools/mass_mfg/samples/sample_config.csv", + "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv", + "Test", + "0x4000", + "--outdir", + "../../../tools/mass_mfg/host_test", + "--version", + "2", + "--inputkey", + "mfg_testdata/sample_encryption_keys.bin",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "encrypt", + "../../../tools/mass_mfg/host_test/csv/Test-1.csv", + "../nvs_partition_generator/Test-1-partition-encrypted.bin", + "0x4000", + "--version", + "2", + "--inputkey", + "testdata/sample_encryption_keys.bin",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + + } + + } + + SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin"); + + nvs_sec_cfg_t cfg; + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = 0x11; + cfg.tky[count] = 0x22; + } + + check_nvs_part_gen_args(&emu1, NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg); + + SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition-encrypted.bin"); + + check_nvs_part_gen_args(&emu2, NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg); + + + childpid = fork(); + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + rm -rf mfg_testdata | \ + rm -rf testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + +} + +TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using new generated key", "[mfg_gen]") +{ + int childpid = fork(); + int status; + + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \ + cp -rf ../nvs_partition_generator/testdata . | \ + mkdir -p ../../../tools/mass_mfg/host_test",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../../../tools/mass_mfg/mfg_gen.py", + "generate-key", + "--outdir", + "../../../tools/mass_mfg/host_test", + "--keyfile", + "encr_keys_host_test.bin",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../../../tools/mass_mfg/mfg_gen.py", + "generate", + "../../../tools/mass_mfg/samples/sample_config.csv", + "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv", + "Test", + "0x4000", + "--outdir", + "../../../tools/mass_mfg/host_test", + "--version", + "2", + "--inputkey", + "../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "encrypt", + "../../../tools/mass_mfg/host_test/csv/Test-1.csv", + "../nvs_partition_generator/Test-1-partition-encrypted.bin", + "0x4000", + "--version", + "2", + "--inputkey", + "../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + + } + + } + + } + + + SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin"); + + char buffer[64]; + FILE *fp; + + fp = fopen("../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin","rb"); + fread(buffer,sizeof(buffer),1,fp); + + fclose(fp); + + nvs_sec_cfg_t cfg; + + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = buffer[count] & 255; + cfg.tky[count] = buffer[count+32] & 255; + } + + check_nvs_part_gen_args(&emu1, NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg); + + SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition-encrypted.bin"); + + check_nvs_part_gen_args(&emu2, NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf keys | \ + rm -rf mfg_testdata | \ + rm -rf testdata | \ + rm -rf ../../../tools/mass_mfg/host_test",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + +} +#endif + +/* Add new tests above */ +/* This test has to be the final one */ + TEST_CASE("dump all performance data", "[nvs]") { std::cout << "====================" << std::endl << "Dumping benchmarks" << std::endl; diff --git a/components/nvs_flash/test_nvs_host/test_nvs_cxx_api.cpp b/components/nvs_flash/test_nvs_host/test_nvs_cxx_api.cpp new file mode 100644 index 000000000..da1d006be --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_nvs_cxx_api.cpp @@ -0,0 +1,193 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "catch.hpp" +#include +#include +#include "nvs_test_api.h" +#include "nvs_handle_simple.hpp" +#include "nvs_partition_manager.hpp" +#include "spi_flash_emulation.h" + +#include "test_fixtures.hpp" + +#include + +using namespace std; + +TEST_CASE("NVSHandleSimple CXX api open invalid arguments", "[nvs cxx]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + esp_err_t result; + shared_ptr handle; + + REQUIRE(nvs::NVSPartitionManager::get_instance()-> + init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) == ESP_OK); + + handle = nvs::open_nvs_handle_from_partition(nullptr, "ns_1", NVS_READWRITE, &result); + CHECK(result == ESP_ERR_INVALID_ARG); + CHECK(!handle); + + handle = nvs::open_nvs_handle_from_partition("test", nullptr, NVS_READWRITE, &result); + CHECK(result == ESP_ERR_INVALID_ARG); + CHECK(!handle); + + nvs::NVSPartitionManager::get_instance()->deinit_partition("test"); +} + +TEST_CASE("NVSHandleSimple CXX api open partition uninitialized", "[nvs cxx]") +{ + SpiFlashEmulator emu(10); + esp_err_t result; + shared_ptr handle; + + handle = nvs::open_nvs_handle_from_partition("test", "ns_1", NVS_READWRITE, &result); + bool result_expected = result == ESP_ERR_NVS_NOT_INITIALIZED || result == ESP_ERR_NVS_PART_NOT_FOUND; + CHECK(result_expected); + CHECK(!handle); +} + +TEST_CASE("NVSHandleSimple CXX api open successful", "[nvs cxx]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + esp_err_t result; + shared_ptr handle; + + REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + handle = nvs::open_nvs_handle_from_partition("test", "ns_1", NVS_READWRITE, &result); + CHECK(result == ESP_OK); + CHECK(handle); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 1); + + handle.reset(); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + nvs::NVSPartitionManager::get_instance()->deinit_partition("test"); +} + +TEST_CASE("NVSHandleSimple CXX api open default part successful", "[nvs cxx]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10); + esp_err_t result; + shared_ptr handle; + + REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + handle = nvs::open_nvs_handle("ns_1", NVS_READWRITE, &result); + CHECK(result == ESP_OK); + CHECK(handle); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 1); + + handle.reset(); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + nvs::NVSPartitionManager::get_instance()->deinit_partition("nvs"); +} + +TEST_CASE("NVSHandleSimple CXX api open default part ns NULL", "[nvs cxx]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10); + esp_err_t result; + shared_ptr handle; + + REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + handle = nvs::open_nvs_handle(nullptr, NVS_READWRITE, &result); + CHECK(result == ESP_ERR_INVALID_ARG); + CHECK(!handle); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + nvs::NVSPartitionManager::get_instance()->deinit_partition("nvs"); +} + +TEST_CASE("NVSHandleSimple CXX api read/write string", "[nvs cxx]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10); + char read_buffer [256]; + esp_err_t result; + shared_ptr handle; + + REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + handle = nvs::open_nvs_handle("test_ns", NVS_READWRITE, &result); + CHECK(result == ESP_OK); + REQUIRE(handle); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 1); + + CHECK(handle->set_string("test", "test string") == ESP_OK); + CHECK(handle->commit() == ESP_OK); + CHECK(handle->get_string("test", read_buffer, sizeof(read_buffer)) == ESP_OK); + + CHECK(string(read_buffer) == "test string"); + + nvs::NVSPartitionManager::get_instance()->deinit_partition("nvs"); +} + +TEST_CASE("NVSHandleSimple CXX api read/write blob", "[nvs cxx]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10); + const char blob [6] = {15, 16, 17, 18, 19}; + char read_blob[6] = {0}; + esp_err_t result; + shared_ptr handle; + + REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 0); + + handle = nvs::open_nvs_handle("test_ns", NVS_READWRITE, &result); + CHECK(result == ESP_OK); + REQUIRE(handle); + + CHECK(nvs::NVSPartitionManager::get_instance()->open_handles_size() == 1); + + CHECK(handle->set_blob("test", blob, sizeof(blob)) == ESP_OK); + CHECK(handle->commit() == ESP_OK); + CHECK(handle->get_blob("test", read_blob, sizeof(read_blob)) == ESP_OK); + + CHECK(vector(blob, blob + sizeof(blob)) == vector(read_blob, read_blob + sizeof(read_blob))); + + nvs::NVSPartitionManager::get_instance()->deinit_partition("nvs"); +} diff --git a/components/nvs_flash/test_nvs_host/test_nvs_handle.cpp b/components/nvs_flash/test_nvs_host/test_nvs_handle.cpp new file mode 100644 index 000000000..d9f709e7a --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_nvs_handle.cpp @@ -0,0 +1,281 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "catch.hpp" +#include +#include +#include "nvs_test_api.h" +#include "nvs_handle_simple.hpp" +#include "nvs_partition_manager.hpp" +#include "spi_flash_emulation.h" + +#include "test_fixtures.hpp" + +#include +#include + +using namespace std; +using namespace nvs; + +TEST_CASE("NVSHandleSimple closes its reference in PartitionManager", "[partition_mgr]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle("test", "ns_1", NVS_READWRITE, &handle) == ESP_OK); + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 1); + + delete handle; + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition("test") == ESP_OK); + +} + +TEST_CASE("NVSHandleSimple multiple open and closes with PartitionManager", "[partition_mgr]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + + NVSHandleSimple *handle1; + NVSHandleSimple *handle2; + + REQUIRE(NVSPartitionManager::get_instance()->open_handle("test", "ns_1", NVS_READWRITE, &handle1) == ESP_OK); + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 1); + + REQUIRE(NVSPartitionManager::get_instance()->open_handle("test", "ns_1", NVS_READWRITE, &handle2) == ESP_OK); + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 2); + + delete handle1; + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 1); + + delete handle2; + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition("test") == ESP_OK); + +} + +TEST_CASE("NVSHandleSimple readonly fails", "[partition_mgr]") +{ + PartitionEmulationFixture f(0, 10); + + NVSPartitionManager::get_instance()->deinit_partition(NVS_DEFAULT_PART_NAME); + NVSHandleSimple *handle_1; + NVSHandleSimple *handle_2; + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + CHECK(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) == ESP_OK); + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + + // first, creating namespace... + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READWRITE, &handle_1) == ESP_OK); + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 1); + + delete handle_1; + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READONLY, &handle_2) == ESP_OK); + CHECK(handle_2->set_item("key", 47) == ESP_ERR_NVS_READ_ONLY); + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 1); + + delete handle_2; + + CHECK(NVSPartitionManager::get_instance()->open_handles_size() == 0); + // without deinit it affects "nvs api tests" + CHECK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME) == ESP_OK); +} + +TEST_CASE("NVSHandleSimple set/get char", "[partition_mgr]") +{ + enum class TestEnum : char { + FOO = -1, + BEER, + BAR + }; + + PartitionEmulationFixture f(0, 10); + + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READWRITE, &handle) == ESP_OK); + + char test_e = 'a'; + char test_e_read = 'z'; + + CHECK(handle->set_item("key", test_e) == ESP_OK); + + CHECK(handle->get_item("key", test_e_read) == ESP_OK); + CHECK(test_e == test_e_read); + + delete handle; + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(NVS_DEFAULT_PART_NAME) == ESP_OK); +} + +TEST_CASE("NVSHandleSimple correctly sets/gets int enum", "[partition_mgr]") +{ + enum class TestEnum : int { + FOO, + BAR + }; + + PartitionEmulationFixture f(0, 10); + + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READWRITE, &handle) == ESP_OK); + + TestEnum test_e = TestEnum::BAR; + TestEnum test_e_read = TestEnum::FOO; + + CHECK(handle->set_item("key", test_e) == ESP_OK); + + CHECK(handle->get_item("key", test_e_read) == ESP_OK); + CHECK(test_e == test_e_read); + + delete handle; + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(NVS_DEFAULT_PART_NAME) == ESP_OK); +} + +TEST_CASE("NVSHandleSimple correctly sets/gets int enum with negative values", "[partition_mgr]") +{ + enum class TestEnum : int { + FOO = -1, + BEER, + BAR + }; + + PartitionEmulationFixture f(0, 10); + + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READWRITE, &handle) == ESP_OK); + + TestEnum test_e = TestEnum::FOO; + TestEnum test_e_read = TestEnum::BEER; + + CHECK(handle->set_item("key", test_e) == ESP_OK); + + CHECK(handle->get_item("key", test_e_read) == ESP_OK); + CHECK(test_e == test_e_read); + + delete handle; + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(NVS_DEFAULT_PART_NAME) == ESP_OK); +} + +TEST_CASE("NVSHandleSimple correctly sets/gets uint8_t enum", "[partition_mgr]") +{ + enum class TestEnum : uint8_t { + FOO, + BAR + }; + + PartitionEmulationFixture f(0, 10); + + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READWRITE, &handle) == ESP_OK); + + TestEnum test_e = TestEnum::BAR; + TestEnum test_e_read = TestEnum::FOO; + + CHECK(handle->set_item("key", test_e) == ESP_OK); + + CHECK(handle->get_item("key", test_e_read) == ESP_OK); + CHECK(test_e == test_e_read); + + delete handle; + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(NVS_DEFAULT_PART_NAME) == ESP_OK); +} + +TEST_CASE("NVSHandleSimple correctly sets/gets char enum", "[partition_mgr]") +{ + enum class TestEnum : char { + FOO = -1, + BEER, + BAR + }; + + PartitionEmulationFixture f(0, 10); + + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle(NVS_DEFAULT_PART_NAME, "ns_1", NVS_READWRITE, &handle) == ESP_OK); + + TestEnum test_e = TestEnum::BAR; + TestEnum test_e_read = TestEnum::FOO; + + CHECK(handle->set_item("key", test_e) == ESP_OK); + + CHECK(handle->get_item("key", test_e_read) == ESP_OK); + CHECK(test_e == test_e_read); + + delete handle; + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(NVS_DEFAULT_PART_NAME) == ESP_OK); +} diff --git a/components/nvs_flash/test_nvs_host/test_nvs_initialization.cpp b/components/nvs_flash/test_nvs_host/test_nvs_initialization.cpp new file mode 100644 index 000000000..7731d1da5 --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_nvs_initialization.cpp @@ -0,0 +1,46 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "catch.hpp" +#include "nvs_partition_manager.hpp" +#include "spi_flash_emulation.h" +#include "esp_partition.h" +#include "nvs.h" +#include + +using namespace nvs; + +TEST_CASE("nvs_flash_init_partition_ptr fails due to nullptr arg", "[nvs_custom_part]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + SpiFlashEmulator emu(10); + + CHECK(nvs_flash_init_partition_ptr(nullptr) == ESP_ERR_INVALID_ARG); +} + +TEST_CASE("nvs_flash_init_partition_ptr inits one partition", "[nvs_custom_part]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + SpiFlashEmulator emu(10); + + esp_partition_t partition = {}; + strcpy(partition.label, "test"); + partition.address = NVS_FLASH_SECTOR * SPI_FLASH_SEC_SIZE; + partition.size = NVS_FLASH_SECTOR_COUNT_MIN * SPI_FLASH_SEC_SIZE; + + CHECK(nvs_flash_init_partition_ptr(&partition) == ESP_OK); + CHECK(NVSPartitionManager::get_instance()->lookup_storage_from_name("test") != nullptr); + CHECK(NVSPartitionManager::get_instance()->deinit_partition("test") == ESP_OK); +} diff --git a/components/nvs_flash/test_nvs_host/test_nvs_partition.cpp b/components/nvs_flash/test_nvs_host/test_nvs_partition.cpp new file mode 100644 index 000000000..4385b9a6d --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_nvs_partition.cpp @@ -0,0 +1,55 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "catch.hpp" +#include +#include +#include "nvs_test_api.h" +#include "nvs_handle_simple.hpp" +#include "nvs_partition.hpp" +#include "spi_flash_emulation.h" + +#include "test_fixtures.hpp" + +#include + +using namespace std; +using namespace nvs; + +TEST_CASE("encrypted partition read size must be item size", "[nvs]") +{ + char foo [32] = { }; + nvs_sec_cfg_t xts_cfg; + for(int count = 0; count < NVS_KEY_SIZE; count++) { + xts_cfg.eky[count] = 0x11; + xts_cfg.tky[count] = 0x22; + } + EncryptedPartitionFixture fix(&xts_cfg); + + CHECK(fix.part.read(0, foo, sizeof (foo) -1) == ESP_ERR_INVALID_SIZE); +} + +TEST_CASE("encrypted partition write size must be mod item size", "[nvs]") +{ + char foo [64] = { }; + nvs_sec_cfg_t xts_cfg; + for(int count = 0; count < NVS_KEY_SIZE; count++) { + xts_cfg.eky[count] = 0x11; + xts_cfg.tky[count] = 0x22; + } + EncryptedPartitionFixture fix(&xts_cfg); + + CHECK(fix.part.write(0, foo, sizeof (foo) -1) == ESP_ERR_INVALID_SIZE); + CHECK(fix.part.write(0, foo, sizeof (foo)) == ESP_OK); + CHECK(fix.part.write(0, foo, sizeof (foo) * 2) == ESP_OK); +} diff --git a/components/nvs_flash/test_nvs_host/test_nvs_storage.cpp b/components/nvs_flash/test_nvs_host/test_nvs_storage.cpp new file mode 100644 index 000000000..a0ace089c --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_nvs_storage.cpp @@ -0,0 +1,60 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "catch.hpp" +#include +#include "nvs_test_api.h" +#include "nvs_storage.hpp" +#include "nvs_partition_manager.hpp" +#include "spi_flash_emulation.h" + +#include "test_fixtures.hpp" + +#include + +using namespace std; +using namespace nvs; + +TEST_CASE("Storage iterator recognizes blob with VerOffset::VER_1_OFFSET", "[nvs_storage]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + uint8_t blob [] = {0x0, 0x1, 0x2, 0x3}; + uint8_t blob_new [] = {0x3, 0x2, 0x1, 0x0}; + Storage *storage = NVSPartitionManager::get_instance()->lookup_storage_from_name("test"); + uint8_t ns_index; + storage->createOrOpenNamespace("test_ns", true, ns_index); + + CHECK(storage->writeItem(ns_index, ItemType::BLOB, "test_blob", blob, sizeof(blob)) == ESP_OK); + + // changing provokes a blob with version offset 1 (VerOffset::VER_1_OFFSET) + CHECK(storage->writeItem(ns_index, ItemType::BLOB, "test_blob", blob_new, sizeof(blob_new)) == ESP_OK); + + nvs_opaque_iterator_t it; + it.storage = storage; + it.type = NVS_TYPE_ANY; + + // Central check: does the iterator recognize the blob with version 1? + REQUIRE(storage->findEntry(&it, "test_ns")); + + CHECK(string(it.entry_info.namespace_name) == string("test_ns")); + CHECK(string(it.entry_info.key) == string("test_blob")); + CHECK(it.entry_info.type == NVS_TYPE_BLOB); + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition("test") == ESP_OK); +} diff --git a/components/nvs_flash/test_nvs_host/test_partition_manager.cpp b/components/nvs_flash/test_nvs_host/test_partition_manager.cpp new file mode 100644 index 000000000..b2db3a9cf --- /dev/null +++ b/components/nvs_flash/test_nvs_host/test_partition_manager.cpp @@ -0,0 +1,91 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "catch.hpp" +#include +#include +#include "nvs_test_api.h" +#include "nvs_handle_simple.hpp" +#include "nvs_partition_manager.hpp" +#include "spi_flash_emulation.h" +#include "nvs_test_api.h" + +#include "test_fixtures.hpp" + +using namespace nvs; + +TEST_CASE("Partition manager initializes storage", "[partition_mgr]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) == ESP_OK); + CHECK(NVSPartitionManager::get_instance()->lookup_storage_from_name("test") != nullptr); + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(f.part.get_partition_name()) == ESP_OK); +} + +TEST_CASE("Partition manager de-initializes storage", "[partition_mgr]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) == ESP_OK); + CHECK(NVSPartitionManager::get_instance()->lookup_storage_from_name("test") != nullptr); + CHECK(NVSPartitionManager::get_instance()->deinit_partition("test") == ESP_OK); + CHECK(NVSPartitionManager::get_instance()->lookup_storage_from_name("test") == nullptr); +} + +TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + SpiFlashEmulator emu(10); + PartitionEmulation part_0(&emu, NVS_FLASH_SECTOR * SPI_FLASH_SEC_SIZE, NVS_FLASH_SECTOR_COUNT_MIN * SPI_FLASH_SEC_SIZE, "test1"); + PartitionEmulation part_1(&emu, NVS_FLASH_SECTOR * SPI_FLASH_SEC_SIZE, NVS_FLASH_SECTOR_COUNT_MIN * SPI_FLASH_SEC_SIZE, "test2"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&part_0, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + // TODO: why does this work, actually? same sectors used as above + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&part_1, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + Storage *storage1 = NVSPartitionManager::get_instance()->lookup_storage_from_name("test1"); + REQUIRE(storage1 != nullptr); + Storage *storage2 = NVSPartitionManager::get_instance()->lookup_storage_from_name("test2"); + REQUIRE(storage2 != nullptr); + + CHECK(storage1 != storage2); + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(part_0.get_partition_name()) == ESP_OK); + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition(part_1.get_partition_name()) == ESP_OK); +} + +TEST_CASE("Partition manager invalidates handle on partition de-init", "[partition_mgr]") +{ + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + PartitionEmulationFixture f(0, 10, "test"); + + REQUIRE(NVSPartitionManager::get_instance()->init_custom(&f.part, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN) + == ESP_OK); + + NVSHandleSimple *handle; + REQUIRE(NVSPartitionManager::get_instance()->open_handle("test", "ns_1", NVS_READWRITE, &handle) == ESP_OK); + CHECK(handle->erase_all() == ESP_OK); + + REQUIRE(NVSPartitionManager::get_instance()->deinit_partition("test") == ESP_OK); + + CHECK(handle->erase_all() == ESP_ERR_NVS_INVALID_HANDLE); + + delete handle; +} diff --git a/components/nvs_flash/test_nvs_host/test_spi_flash_emulation.cpp b/components/nvs_flash/test_nvs_host/test_spi_flash_emulation.cpp index 0c77aa966..353e1212a 100644 --- a/components/nvs_flash/test_nvs_host/test_spi_flash_emulation.cpp +++ b/components/nvs_flash/test_nvs_host/test_spi_flash_emulation.cpp @@ -13,7 +13,9 @@ // limitations under the License. #include "catch.hpp" #include "esp_spi_flash.h" +#include "esp_partition.h" #include "spi_flash_emulation.h" +#include using namespace std; @@ -23,14 +25,21 @@ bool range_empty_n(Tit it_begin, size_t n) return all_of(it_begin, it_begin + n, bind(equal_to(), placeholders::_1, 0xffffffff)); } +struct FlashEmuFixture { + FlashEmuFixture(size_t sectors) : esp_part(), emu(sectors) { } + + esp_partition_t esp_part; + SpiFlashEmulator emu; +}; + TEST_CASE("flash starts with all bytes == 0xff", "[spi_flash_emu]") { - SpiFlashEmulator emu(4); + FlashEmuFixture f(4); uint8_t sector[SPI_FLASH_SEC_SIZE]; for (int i = 0; i < 4; ++i) { - CHECK(spi_flash_read(0, sector, sizeof(sector)) == ESP_OK); + CHECK(esp_partition_read(&f.esp_part, 0, sector, sizeof(sector)) == ESP_OK); for (auto v: sector) { CHECK(v == 0xff); } @@ -39,116 +48,138 @@ TEST_CASE("flash starts with all bytes == 0xff", "[spi_flash_emu]") TEST_CASE("invalid writes are checked", "[spi_flash_emu]") { - SpiFlashEmulator emu(1); + FlashEmuFixture f(1); uint32_t val = 0; - CHECK(spi_flash_write(0, &val, 4) == ESP_OK); + CHECK(esp_partition_write(&f.esp_part, 0, &val, 4) == ESP_OK); val = 1; - CHECK(spi_flash_write(0, &val, 4) == ESP_ERR_FLASH_OP_FAIL); + CHECK(esp_partition_write(&f.esp_part, 0, &val, 4) == ESP_ERR_FLASH_OP_FAIL); } TEST_CASE("out of bounds writes fail", "[spi_flash_emu]") { - SpiFlashEmulator emu(4); + FlashEmuFixture f(4); uint32_t vals[8]; std::fill_n(vals, 8, 0); - CHECK(spi_flash_write(0, vals, sizeof(vals)) == ESP_OK); + CHECK(esp_partition_write(&f.esp_part, 0, &vals, sizeof(vals)) == ESP_OK); - CHECK(spi_flash_write(4*4096 - sizeof(vals), vals, sizeof(vals)) == ESP_OK); + CHECK(esp_partition_write(&f.esp_part, 4*4096 - sizeof(vals), &vals, sizeof(vals)) == ESP_OK); - CHECK(spi_flash_write(4*4096 - sizeof(vals) + 4, vals, sizeof(vals)) == ESP_ERR_FLASH_OP_FAIL); + CHECK(esp_partition_write(&f.esp_part, 4*4096 - sizeof(vals) + 4, &vals, sizeof(vals)) == ESP_ERR_FLASH_OP_FAIL); } - TEST_CASE("after erase the sector is set to 0xff", "[spi_flash_emu]") { - SpiFlashEmulator emu(4); + FlashEmuFixture f(4); uint32_t val1 = 0xab00cd12; - CHECK(spi_flash_write(0, &val1, sizeof(val1)) == ESP_OK); + CHECK(esp_partition_write(&f.esp_part, 0, &val1, sizeof(val1)) == ESP_OK); uint32_t val2 = 0x5678efab; - CHECK(spi_flash_write(4096 - 4, &val2, sizeof(val2)) == ESP_OK); + CHECK(esp_partition_write(&f.esp_part, 4096 - 4, &val2, sizeof(val2)) == ESP_OK); + + CHECK(f.emu.words()[0] == val1); + CHECK(range_empty_n(f.emu.words() + 1, 4096 / 4 - 2)); + CHECK(f.emu.words()[4096 / 4 - 1] == val2); + + CHECK(esp_partition_erase_range(&f.esp_part, 0, SPI_FLASH_SEC_SIZE) == ESP_OK); + + CHECK(f.emu.words()[0] == 0xffffffff); + CHECK(range_empty_n(f.emu.words() + 1, 4096 / 4 - 2)); + CHECK(f.emu.words()[4096 / 4 - 1] == 0xffffffff); +} + +TEST_CASE("EMU raw read function works", "[spi_flash_emu]") +{ + FlashEmuFixture f(4); + uint32_t value = 0xdeadbeef; + uint32_t read_value = 0; + CHECK(esp_partition_write(&f.esp_part, 0, &value, sizeof(value)) == ESP_OK); + + CHECK(esp_partition_read(&f.esp_part, 0, &read_value, sizeof(&read_value)) == ESP_OK); + + CHECK(read_value == 0xdeadbeef); +} - CHECK(emu.words()[0] == val1); - CHECK(range_empty_n(emu.words() + 1, 4096 / 4 - 2)); - CHECK(emu.words()[4096 / 4 - 1] == val2); +TEST_CASE("EMU raw write function works", "[spi_flash_emu]") +{ + FlashEmuFixture f(4); + uint32_t value = 0xdeadbeef; + uint32_t read_value = 0; + CHECK(esp_partition_write(&f.esp_part, 0, &value, sizeof(value)) == ESP_OK); - CHECK(spi_flash_erase_sector(0) == ESP_OK); + CHECK(esp_partition_read(&f.esp_part, 0, &read_value, sizeof(&read_value)) == ESP_OK); - CHECK(emu.words()[0] == 0xffffffff); - CHECK(range_empty_n(emu.words() + 1, 4096 / 4 - 2)); - CHECK(emu.words()[4096 / 4 - 1] == 0xffffffff); + CHECK(read_value == 0xdeadbeef); } TEST_CASE("read/write/erase operation times are calculated correctly", "[spi_flash_emu]") { - SpiFlashEmulator emu(1); + FlashEmuFixture f(1); uint8_t data[512]; - spi_flash_read(0, data, 4); - CHECK(emu.getTotalTime() == 7); - CHECK(emu.getReadOps() == 1); - CHECK(emu.getReadBytes() == 4); - emu.clearStats(); - spi_flash_read(0, data, 8); - CHECK(emu.getTotalTime() == 5); - CHECK(emu.getReadOps() == 1); - CHECK(emu.getReadBytes() == 8); - emu.clearStats(); - spi_flash_read(0, data, 16); - CHECK(emu.getTotalTime() == 6); - CHECK(emu.getReadOps() == 1); - CHECK(emu.getReadBytes() == 16); - emu.clearStats(); - spi_flash_read(0, data, 128); - CHECK(emu.getTotalTime() == 18); - CHECK(emu.getReadOps() == 1); - CHECK(emu.getReadBytes() == 128); - emu.clearStats(); - spi_flash_read(0, data, 256); - CHECK(emu.getTotalTime() == 32); - emu.clearStats(); - spi_flash_read(0, data, (128+256)/2); - CHECK(emu.getTotalTime() == (18+32)/2); - emu.clearStats(); - - spi_flash_write(0, data, 4); - CHECK(emu.getTotalTime() == 19); - CHECK(emu.getWriteOps() == 1); - CHECK(emu.getWriteBytes() == 4); - emu.clearStats(); - CHECK(emu.getWriteOps() == 0); - CHECK(emu.getWriteBytes() == 0); - spi_flash_write(0, data, 8); - CHECK(emu.getTotalTime() == 23); - emu.clearStats(); - spi_flash_write(0, data, 16); - CHECK(emu.getTotalTime() == 35); - CHECK(emu.getWriteOps() == 1); - CHECK(emu.getWriteBytes() == 16); - emu.clearStats(); - spi_flash_write(0, data, 128); - CHECK(emu.getTotalTime() == 205); - emu.clearStats(); - spi_flash_write(0, data, 256); - CHECK(emu.getTotalTime() == 417); - emu.clearStats(); - spi_flash_write(0, data, (128+256)/2); - CHECK(emu.getTotalTime() == (205+417)/2); - emu.clearStats(); - - spi_flash_erase_sector(0); - CHECK(emu.getEraseOps() == 1); - CHECK(emu.getTotalTime() == 37142); + esp_partition_read(&f.esp_part, 0, data, 4); + CHECK(f.emu.getTotalTime() == 7); + CHECK(f.emu.getReadOps() == 1); + CHECK(f.emu.getReadBytes() == 4); + f.emu.clearStats(); + esp_partition_read(&f.esp_part, 0, data, 8); + CHECK(f.emu.getTotalTime() == 5); + CHECK(f.emu.getReadOps() == 1); + CHECK(f.emu.getReadBytes() == 8); + f.emu.clearStats(); + esp_partition_read(&f.esp_part, 0, data, 16); + CHECK(f.emu.getTotalTime() == 6); + CHECK(f.emu.getReadOps() == 1); + CHECK(f.emu.getReadBytes() == 16); + f.emu.clearStats(); + esp_partition_read(&f.esp_part, 0, data, 128); + CHECK(f.emu.getTotalTime() == 18); + CHECK(f.emu.getReadOps() == 1); + CHECK(f.emu.getReadBytes() == 128); + f.emu.clearStats(); + esp_partition_read(&f.esp_part, 0, data, 256); + CHECK(f.emu.getTotalTime() == 32); + f.emu.clearStats(); + esp_partition_read(&f.esp_part, 0, data, (128+256)/2); + CHECK(f.emu.getTotalTime() == (18+32)/2); + f.emu.clearStats(); + + esp_partition_write(&f.esp_part, 0, data, 4); + CHECK(f.emu.getTotalTime() == 19); + CHECK(f.emu.getWriteOps() == 1); + CHECK(f.emu.getWriteBytes() == 4); + f.emu.clearStats(); + CHECK(f.emu.getWriteOps() == 0); + CHECK(f.emu.getWriteBytes() == 0); + esp_partition_write(&f.esp_part, 0, data, 8); + CHECK(f.emu.getTotalTime() == 23); + f.emu.clearStats(); + esp_partition_write(&f.esp_part, 0, data, 16); + CHECK(f.emu.getTotalTime() == 35); + CHECK(f.emu.getWriteOps() == 1); + CHECK(f.emu.getWriteBytes() == 16); + f.emu.clearStats(); + esp_partition_write(&f.esp_part, 0, data, 128); + CHECK(f.emu.getTotalTime() == 205); + f.emu.clearStats(); + esp_partition_write(&f.esp_part, 0, data, 256); + CHECK(f.emu.getTotalTime() == 417); + f.emu.clearStats(); + esp_partition_write(&f.esp_part, 0, data, (128+256)/2); + CHECK(f.emu.getTotalTime() == (205+417)/2); + f.emu.clearStats(); + + esp_partition_erase_range(&f.esp_part, 0, SPI_FLASH_SEC_SIZE); + CHECK(f.emu.getEraseOps() == 1); + CHECK(f.emu.getTotalTime() == 37142); } TEST_CASE("data is randomized predictably", "[spi_flash_emu]") { SpiFlashEmulator emu1(3); emu1.randomize(0x12345678); - + SpiFlashEmulator emu2(3); emu2.randomize(0x12345678); - + CHECK(std::equal(emu1.bytes(), emu1.bytes() + emu1.size(), emu2.bytes())); } - diff --git a/components/spi_flash/CMakeLists.txt b/components/spi_flash/CMakeLists.txt index b2a0f715c..255eff8e7 100644 --- a/components/spi_flash/CMakeLists.txt +++ b/components/spi_flash/CMakeLists.txt @@ -5,6 +5,10 @@ if(BOOTLOADER_BUILD) set(srcs "${srcs}" "port/port.c") set(priv_requires "bootloader_support") else() + if(CONFIG_ENABLE_TH25Q16HB_PATCH_0) + list(APPEND srcs "src/patch/th25q16hb.c") + endif() + set(priv_requires "esp8266" "freertos" "bootloader_support") endif() diff --git a/components/spi_flash/Kconfig b/components/spi_flash/Kconfig new file mode 100644 index 000000000..3d97ba109 --- /dev/null +++ b/components/spi_flash/Kconfig @@ -0,0 +1,12 @@ +menu "SPI Flash" + + menu "Patch" + config ENABLE_TH25Q16HB_PATCH_0 + bool "Enable TH25Q16HB Patch 0" + default n + help + WARNING: If you don't use TH25Q16HB, you must not enable this option. + Although you use TH25Q16HB, you should ask your flash manufacturer + if your flash need use this patch. + endmenu +endmenu diff --git a/components/spi_flash/component.mk b/components/spi_flash/component.mk index f5eb97ca6..c4dfc940a 100644 --- a/components/spi_flash/component.mk +++ b/components/spi_flash/component.mk @@ -15,4 +15,8 @@ CFLAGS += -DPARTITION_QUEUE_HEADER=\"sys/queue.h\" ifdef IS_BOOTLOADER_BUILD COMPONENT_SRCDIRS += port COMPONENT_OBJS += port/port.o +else +ifdef CONFIG_ENABLE_TH25Q16HB_PATCH_0 +COMPONENT_SRCDIRS += src/patch +endif endif diff --git a/components/spi_flash/include/spi_flash.h b/components/spi_flash/include/spi_flash.h index d51756d72..7dfeb6a24 100644 --- a/components/spi_flash/include/spi_flash.h +++ b/components/spi_flash/include/spi_flash.h @@ -204,6 +204,15 @@ int esp_patition_table_init_data(void *partition_info); int esp_patition_copy_ota1_to_ota0(const void *partition_info); #endif +#ifdef CONFIG_ENABLE_TH25Q16HB_PATCH_0 +/** + * @brief Apply TH25Q16HB patch 0 to avoid some hardware issues. + * + * @return 0 if success or others if failed + */ +int th25q16hb_apply_patch_0(void); +#endif + #ifdef __cplusplus } #endif diff --git a/components/spi_flash/linker.lf b/components/spi_flash/linker.lf index 9ac365ea5..27695eb70 100644 --- a/components/spi_flash/linker.lf +++ b/components/spi_flash/linker.lf @@ -2,3 +2,4 @@ archive: libspi_flash.a entries: spi_flash_raw (noflash) + th25q16hb (noflash) diff --git a/components/spi_flash/src/patch/th25q16hb.c b/components/spi_flash/src/patch/th25q16hb.c new file mode 100644 index 000000000..425096f9b --- /dev/null +++ b/components/spi_flash/src/patch/th25q16hb.c @@ -0,0 +1,709 @@ +// Copyright 2023 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "esp_log.h" +#include "esp_attr.h" +#include "spi_flash.h" +#include "priv/esp_spi_flash_raw.h" +#include "FreeRTOS.h" + +#include "esp8266/eagle_soc.h" +#include "esp8266/pin_mux_register.h" +#include "esp8266/spi_register.h" +#include "esp8266/spi_struct.h" + +#define SPI_FLASH SPI0 +#define SPI_BLOCK_SIZE 32 +#define ADDR_SHIFT_BITS 8 + +#if 0 +typedef int (*__ets_printf_t)(const char *fmt, ...); +#define ROM_PRINTF(_fmt, ...) ((__ets_printf_t)(0x400024cc))(_fmt, ##__VA_ARGS__) +#else +#define ROM_PRINTF(_fmt, ...) +#endif + +#define TOCHAR(_v) #_v +#define PRINT_STEP(_s) ROM_PRINTF("Step %d\n", (_s)); +#define JUMP_TO_STEP(_s) { ROM_PRINTF("Jump to " TOCHAR(_s) "\n"); goto _s; } +#define GOTO_FAILED(_s) { ROM_PRINTF("ERROR: " TOCHAR(_s) " failed\n"); ret = -EIO; JUMP_TO_STEP(step17); } + +#define write_u8_dummy(_c, _a, _d8,_d) {uint32_t __data = _d8; spi_trans(1, (_c), 8, (_a), 24, (uint8_t *)&__data, 1, (_d));} +#define write_u8(_c, _a, _d8) write_u8_dummy((_c), (_a), (_d8), 0) + +extern void Cache_Read_Disable_2(void); +extern void Cache_Read_Enable_2(); +extern void vPortEnterCritical(void); +extern void vPortExitCritical(void); +extern uint32_t spi_flash_get_id(void); + +static void delay(int ms) +{ + for (volatile int i = 0; i < ms; i++) { + for (volatile int j = 0; j < 7800; j++) { + } + } +} + +#if 0 +static void dump_hex(const uint8_t *ptr, int n) +{ + const uint8_t *s1 = ptr; + const int line_bytes = 16; + + ROM_PRINTF("\nHex:\n"); + for (int i = 0; i < n ; i += line_bytes) + { + int m = MIN(n - i, line_bytes); + + ROM_PRINTF("\t"); + for (int j = 0; j < m; j++) + { + ROM_PRINTF("%02x ", s1[i + j]); + } + + ROM_PRINTF("\n"); + } + + ROM_PRINTF("\n"); +} + +static void dump_hex_compare(const uint8_t *s1, const uint8_t *s2, int n) +{ + const int line_bytes = 16; + + ROM_PRINTF("\nHex:\n"); + for (int i = 0; i < n ; i += line_bytes) + { + int m = MIN(n - i, line_bytes); + + ROM_PRINTF("\t"); + for (int j = 0; j < m; j++) + { + ROM_PRINTF("%02x:%02x ", s1[i + j], s2[i + j]); + } + + ROM_PRINTF("\n"); + } + + ROM_PRINTF("\n"); +} +#endif + +typedef struct spi_state { + uint32_t io_mux_reg; + uint32_t spi_clk_reg; + uint32_t spi_ctrl_reg; + uint32_t spi_user_reg; +} spi_state_t; + +static void spi_enter(spi_state_t *state) +{ + vPortEnterCritical(); + Cache_Read_Disable_2(); + + Wait_SPI_Idle(&g_rom_flashchip); + + state->io_mux_reg = READ_PERI_REG(PERIPHS_IO_MUX_CONF_U); + state->spi_clk_reg = SPI_FLASH.clock.val; + state->spi_ctrl_reg = SPI_FLASH.ctrl.val; + state->spi_user_reg = SPI_FLASH.user.val; + + SPI_FLASH.user.usr_command = 1; + SPI_FLASH.user.flash_mode = 0; + SPI_FLASH.user.usr_miso_highpart = 0; + + CLEAR_PERI_REG_MASK(PERIPHS_IO_MUX_CONF_U, SPI0_CLK_EQU_SYS_CLK); + + SPI_FLASH.user.cs_setup = 1; + SPI_FLASH.user.cs_hold = 1; + SPI_FLASH.user.usr_mosi = 1; + + SPI_FLASH.user.usr_command = 1; + SPI_FLASH.user.flash_mode = 0; + + SPI_FLASH.ctrl.fread_qio = 0; + SPI_FLASH.ctrl.fread_dio = 0; + SPI_FLASH.ctrl.fread_quad = 0; + SPI_FLASH.ctrl.fread_dual = 0; + + SPI_FLASH.clock.val = 0; + SPI_FLASH.clock.clkcnt_l = 3; + SPI_FLASH.clock.clkcnt_h = 1; + SPI_FLASH.clock.clkcnt_n = 3; + + SPI_FLASH.ctrl.fastrd_mode = 1; + + while (SPI_FLASH.cmd.usr) { + ; + } +} + +static void spi_exit(spi_state_t *state) +{ + while (SPI_FLASH.cmd.usr) { + ; + } + + WRITE_PERI_REG(PERIPHS_IO_MUX_CONF_U, state->io_mux_reg); + + SPI_FLASH.ctrl.val = state->spi_ctrl_reg; + SPI_FLASH.clock.val = state->spi_clk_reg; + SPI_FLASH.user.val = state->spi_user_reg; + + Cache_Read_Enable_2(); + vPortExitCritical(); +} + +static void spi_trans_block(bool write_mode, + uint32_t cmd, + uint32_t cmd_bits, + uint32_t addr, + uint32_t addr_bits, + uint8_t *data, + uint32_t data_bytes, + uint32_t dummy_bits) +{ + if ((uint32_t)data & 0x3) { + ROM_PRINTF("ERROR: data=%p\n", data); + return; + } + + if (cmd_bits) { + SPI_FLASH.user.usr_command = 1; + SPI_FLASH.user2.usr_command_value = cmd; + SPI_FLASH.user2.usr_command_bitlen = cmd_bits - 1; + } else { + SPI_FLASH.user.usr_command = 0; + } + + if (addr_bits) { + SPI_FLASH.user.usr_addr = 1; + SPI_FLASH.addr = addr << ADDR_SHIFT_BITS; + SPI_FLASH.user1.usr_addr_bitlen = addr_bits - 1; + } else { + SPI_FLASH.user.usr_addr = 0; + } + + if (dummy_bits) { + SPI_FLASH.user.usr_dummy = 1; + SPI_FLASH.user1.usr_dummy_cyclelen = dummy_bits - 1; + } else { + SPI_FLASH.user.usr_dummy = 0; + } + + if (write_mode && data_bytes) { + int words = (data_bytes + 3) / 4; + uint32_t *p = (uint32_t *)data; + + SPI_FLASH.user.usr_mosi = 1; + SPI_FLASH.user.usr_miso = 0; + SPI_FLASH.user1.usr_mosi_bitlen = data_bytes * 8 - 1; + + for (int i = 0; i < words; i++) { + SPI_FLASH.data_buf[i] = p[i]; + } + } else if (!write_mode && data_bytes) { + int words = (data_bytes + 3) / 4; + + SPI_FLASH.user.usr_mosi = 0; + SPI_FLASH.user.usr_miso = 1; + SPI_FLASH.user1.usr_miso_bitlen = data_bytes * 8 - 1; + + for (int i = 0; i < words; i++) { + SPI_FLASH.data_buf[i] = 0; + } + } else { + SPI_FLASH.user.usr_mosi = 0; + SPI_FLASH.user.usr_miso = 0; + } + + SPI_FLASH.cmd.usr = 1; + while (SPI_FLASH.cmd.usr) { + ; + } + + if (!write_mode && data_bytes) { + int words = (data_bytes + 3) / 4; + uint32_t *p = (uint32_t *)data; + + for (int i = 0; i < words; i++) { + p[i] = SPI_FLASH.data_buf[i]; + } + } +} + +static void spi_trans(bool write_mode, + uint32_t cmd, + uint32_t cmd_bits, + uint32_t addr, + uint32_t addr_bits, + uint8_t *data, + uint32_t data_bytes, + uint32_t dummy_bits) + +{ + if (!data_bytes || data_bytes <= SPI_BLOCK_SIZE) { + return spi_trans_block(write_mode, cmd, cmd_bits, addr, + addr_bits, data, data_bytes, dummy_bits); + } + + for (int i = 0; i < data_bytes; i += SPI_BLOCK_SIZE) { + uint32_t n = MIN(SPI_BLOCK_SIZE, data_bytes - i); + + spi_trans_block(write_mode, cmd, cmd_bits, addr + i, + addr_bits, data + i, n, dummy_bits); + } +} + +static void write_cmd(uint32_t cmd) +{ + spi_trans(1, cmd, 8, 0, 0, NULL, 0, 0); +} + +static void write_buffer(uint32_t addr, uint8_t *buffer, int size) +{ + for (int i = 0; i < size; i += SPI_BLOCK_SIZE) { + int n = MIN(size - i, SPI_BLOCK_SIZE); + + write_cmd(0x6); + spi_trans(1, 0x42, 8, addr + i, 24, buffer + i, n, 0); + delay(3); + } +} + +static void read_buffer(uint32_t addr, uint8_t *buffer, int n) +{ + spi_trans(0, 0x48, 8, addr, 24, buffer, n, 8); +} + +static void erase_sector(uint32_t addr) +{ + write_cmd(0x6); + spi_trans(1, 0x44, 8, addr, 24, NULL, 0, 0); + delay(8); +} + +int th25q16hb_apply_patch_0(void) +{ + int ret = 0; + uint32_t flash_id; + int count; + spi_state_t state; + uint8_t *buffer256_0; + uint8_t *buffer256_1; + uint8_t *buffer256_2; + uint8_t *buffer1024; + + flash_id = spi_flash_get_id(); + if (flash_id != 0x1560eb) { + ROM_PRINTF("WARN: id=0x%x, is not TH25Q16HB\n", flash_id); + return 0; + } + + buffer1024 = heap_caps_malloc(1024, MALLOC_CAP_8BIT); + if (!buffer1024) { + return -ENOMEM; + } + + buffer256_0 = buffer1024; + buffer256_1 = buffer1024 + 256; + buffer256_2 = buffer1024 + 512; + + spi_enter(&state); + + // Step 1 + PRINT_STEP(1); + write_cmd(0x33); + + // Step 2 + PRINT_STEP(2); + write_cmd(0xcc); + + // Step 3 + PRINT_STEP(3); + write_cmd(0xaa); + + // Step 4 + PRINT_STEP(4); + write_u8(0xfa, 0x1200d, 0x3); + write_u8_dummy(0xfa, 0x2200d, 0x3, 1); + + // Step 5-1 + PRINT_STEP(5); + read_buffer(0xbed, buffer256_0, 3); + if (buffer256_0[0] == 0xff && + buffer256_0[1] == 0xff && + buffer256_0[2] == 0xff) { + ROM_PRINTF("INFO: check done 0\n"); + } else if (buffer256_0[0] == 0x55 && + buffer256_0[1] == 0xff && + buffer256_0[2] == 0xff) { + JUMP_TO_STEP(step10); + } else if (buffer256_0[0] == 0x55 && + buffer256_0[1] == 0x55 && + buffer256_0[2] == 0xff) { + JUMP_TO_STEP(step14); + } else if (buffer256_0[0] == 0x55 && + buffer256_0[1] == 0x55 && + buffer256_0[2] == 0x55) { + JUMP_TO_STEP(step17); + } else { + ROM_PRINTF("ERROR: 0xbed=0x%x 0xbee=0x%x 0xbef=0x%x\n", + buffer256_0[0], buffer256_0[1], buffer256_0[2]); + GOTO_FAILED(5-1); + } + + // Step 5-2 + read_buffer(0x50d, buffer256_0, 1); + buffer256_0[0] &= 0x7f; + if (buffer256_0[0] == 0x7c) { + JUMP_TO_STEP(step17); + } else if (buffer256_0[0] == 0x3c) { + ROM_PRINTF("INFO: check done 1\n"); + } else { + ROM_PRINTF("ERROR: 0x50d=0x%x\n", buffer256_0[0]); + GOTO_FAILED(5-2); + } + + // Step 6 + PRINT_STEP(6); + for (count = 0; count < 3; count++) { + erase_sector(0); + + bool check_done = true; + read_buffer(0x0, buffer1024, 1024); + for (int i = 0; i < 1024; i++) { + if (buffer1024[i] != 0xff) { + check_done = false; + ROM_PRINTF("ERROR: buffer1024[%d]=0x%x\n", i, buffer1024[i]); + break; + } + } + + if (check_done) { + break; + } + } + if (count >= 3) { + GOTO_FAILED(6) + } + + // Step 7-1.1 + PRINT_STEP(7); + read_buffer(0x400, buffer256_0, 256); + read_buffer(0x400, buffer256_1, 256); + read_buffer(0x400, buffer256_2, 256); + if (memcmp(buffer256_0, buffer256_1, 256) || + memcmp(buffer256_0, buffer256_2, 256)) { + GOTO_FAILED(7-1.1); + } + + write_buffer(0, buffer256_0, 256); + write_buffer(0x200, buffer256_0, 256); + + // Step 7-1.2 + for (count = 0; count < 3; count++) { + read_buffer(0, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + + read_buffer(0x200, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + } + if (count < 3) { + GOTO_FAILED(7-1.2); + } + + // Step 7-2.1 + read_buffer(0x500, buffer256_0, 256); + read_buffer(0x500, buffer256_1, 256); + read_buffer(0x500, buffer256_2, 256); + if (memcmp(buffer256_0, buffer256_1, 256) || + memcmp(buffer256_0, buffer256_2, 256)) { + GOTO_FAILED(7-2.1); + } + write_buffer(0x100, buffer256_0, 256); + write_buffer(0x300, buffer256_0, 256); + + // Step 7-2.2 + for (count = 0; count < 3; count++) { + read_buffer(0x100, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + read_buffer(0x300, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + } + if (count < 3) { + GOTO_FAILED(7-2.2); + } + + // Step 8 + PRINT_STEP(8); + read_buffer(0x0, buffer256_0, 1); + read_buffer(0x23, buffer256_1, 1); + if (buffer256_0[0] != 0x13 || buffer256_1[0] != 0x14) { + ROM_PRINTF("ERROR: 0x0=0x%x 0x23=0x%x\n", buffer256_0[0], buffer256_1[0]); + GOTO_FAILED(8); + } + + // Step 9-1 + PRINT_STEP(9); + read_buffer(0x140, buffer256_0, 2); + if (buffer256_0[0] != 0 || buffer256_0[1] != 0xff) { + ROM_PRINTF("ERROR: 0x140=0x%x 0x141=0x%x\n", buffer256_0[0], buffer256_0[1]); + GOTO_FAILED(9-1); + } + + // Step 9-2 + buffer256_0[0] = 0x55; + write_buffer(0xaed, buffer256_0, 1); + write_buffer(0xbed, buffer256_0, 1); + for (count = 0; count < 3; count++) { + read_buffer(0xaed, buffer256_0, 1); + if (buffer256_0[0] != 0x55) { + break; + } + + read_buffer(0xbed, buffer256_0, 1); + if (buffer256_0[0] != 0x55) { + break; + } + } + if (count < 3) { + GOTO_FAILED(9-2); + } + +step10: + // Step 10-1 + PRINT_STEP(10); + for (count = 0; count < 3; count++) { + erase_sector(0x400); + + bool check_done = true; + read_buffer(0x400, buffer1024, 1024); + for (int i = 0; i < 1024; i++) { + if (buffer1024[i] != 0xff) { + check_done = false; + ROM_PRINTF("ERROR: buffer1024[%d]=0x%x\n", i, buffer1024[i]); + break; + } + } + + if (check_done) { + break; + } + } + if (count >= 3) { + GOTO_FAILED(10-1); + } + + // Step 10-3.1 + read_buffer(0, buffer256_0, 256); + read_buffer(0, buffer256_1, 256); + read_buffer(0, buffer256_2, 256); + if (memcmp(buffer256_0, buffer256_1, 256) || + memcmp(buffer256_0, buffer256_2, 256)) { + GOTO_FAILED(10-3.1); + } + write_buffer(0x400, buffer256_0, 256); + write_buffer(0x600, buffer256_0, 256); + + // Step 10-3.2 + for (count = 0; count < 3; count++) { + read_buffer(0x400, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + + read_buffer(0x600, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + } + if (count < 3) { + GOTO_FAILED(10-3.2); + } + + // Step 10-3.3 + read_buffer(0x100, buffer256_0, 256); + read_buffer(0x100, buffer256_1, 256); + read_buffer(0x100, buffer256_2, 256); + if (memcmp(buffer256_0, buffer256_1, 256) || + memcmp(buffer256_0, buffer256_2, 256)) { + GOTO_FAILED(10-3.3); + } + buffer256_0[9] = 0x79; + buffer256_0[13] = (buffer256_0[13] & 0x3f) | + ((~buffer256_0[13]) & 0x80) | + 0x40; + write_buffer(0x500, buffer256_0, 256); + write_buffer(0x700, buffer256_0, 256); + + // Step 10-3.4 + for (count = 0; count < 3; count++) { + read_buffer(0x500, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + + read_buffer(0x700, buffer256_1, 256); + if (memcmp(buffer256_0, buffer256_1, 256)) { + break; + } + } + if (count < 3) { + GOTO_FAILED(10-3.4); + } + + // Step11-1 + PRINT_STEP(11); + + for (count = 0; count < 3; count++) { + read_buffer(0x400, buffer256_0, 1); + read_buffer(0x423, buffer256_1, 1); + if (buffer256_0[0] != 0x13 || buffer256_1[0] != 0x14) { + ROM_PRINTF("ERROR: 0x400=0x%x 0x423=0x%x\n", buffer256_0[0], buffer256_1[0]); + break; + } + } + if (count < 3) { + GOTO_FAILED(11-1); + } + + // Step11-2.1 + for (count = 0; count < 3; count++) { + read_buffer(0x540, buffer256_0, 2); + if (buffer256_0[0] != 0 || buffer256_0[1] != 0xff) { + ROM_PRINTF("ERROR: 0x540=0x%x 0x541=0x%x\n", buffer256_0[0], buffer256_0[1]); + break; + } + } + if (count < 3) { + GOTO_FAILED(11-2); + } + + // Step11-2.2 + for (count = 0; count < 3; count++) { + read_buffer(0x50d, buffer256_0, 1); + buffer256_0[0] &= 0x7f; + if (buffer256_0[0] != 0x7c) { + ROM_PRINTF("ERROR: 0x50d=0x%x\n", buffer256_0[0]); + break; + } + } + if (count < 3) { + GOTO_FAILED(11-2); + } + + // Step 12 + PRINT_STEP(12); + buffer256_0[0] = 0x55; + write_buffer(0xaee, buffer256_0, 1); + write_buffer(0xbee, buffer256_0, 1); + + // Step 13 + PRINT_STEP(13); + for (count = 0; count < 3; count++) { + read_buffer(0xaee, buffer256_0, 1); + if (buffer256_0[0] != 0x55) { + break; + } + + read_buffer(0xbee, buffer256_0, 1); + if (buffer256_0[0] != 0x55) { + break; + } + } + if (count < 3) { + GOTO_FAILED(13); + } + +step14: + // Step 14 + PRINT_STEP(14); + for (count = 0; count < 3; count++) { + erase_sector(0); + + bool check_done = true; + read_buffer(0x0, buffer1024, 1024); + for (int i = 0; i < 1024; i++) { + if (buffer1024[i] != 0xff) { + check_done = false; + ROM_PRINTF("ERROR: buffer1024[%d]=0x%x\n", i, buffer1024[i]); + break; + } + } + + if (check_done) { + break; + } + } + if (count >= 3) { + GOTO_FAILED(14); + } + + // Step 15 + PRINT_STEP(15); + buffer256_0[0] = 0x55; + write_buffer(0xaef, buffer256_0, 1); + write_buffer(0xbef, buffer256_0, 1); + + // Step 16 + PRINT_STEP(16); + for (count = 0; count < 3; count++) { + read_buffer(0xaef, buffer256_0, 1); + if (buffer256_0[0] != 0x55) { + break; + } + + read_buffer(0xbef, buffer256_0, 1); + if (buffer256_0[0] != 0x55) { + break; + } + } + if (count < 3) { + GOTO_FAILED(16); + } + +step17: + // Step 17 + PRINT_STEP(17); + write_cmd(0x55); + + // Step 18 + PRINT_STEP(18); + write_cmd(0x88); + + spi_exit(&state); + + heap_caps_free(buffer1024); + + if (!ret) { + ROM_PRINTF("INFO: Patch for TH25Q16HB is done\n"); + } + + return ret; +} diff --git a/components/spi_flash/src/spi_flash.c b/components/spi_flash/src/spi_flash.c index 93240e9d2..221b25951 100644 --- a/components/spi_flash/src/spi_flash.c +++ b/components/spi_flash/src/spi_flash.c @@ -259,6 +259,12 @@ bool spi_flash_check_wr_protect(void) special_flash_write_status(0x01, status&(~(SPI_GD25Q32_FLASH_WRITE_PROTECT_STATUS)), 1, true); } } + } else if (flash_id == 0x164068) { + if(spi_flash_read_status(&status)==0) { //Read status Ok + if(status&SPI_GD25Q32_FLASH_WRITE_PROTECT_STATUS) { + special_flash_write_status(0x01, status&(~(SPI_GD25Q32_FLASH_WRITE_PROTECT_STATUS)), 1, true); + } + } } //Others else { @@ -435,6 +441,10 @@ void user_spi_flash_dio_to_qio_pre_init(void) to_qio = true; } //ENBALE FLASH QIO 0X01H+0X00+0X02 + } else if (flash_id == 0x164068) { + if (flash_gd25q32c_enable_QIO_mode() == true) { + to_qio = true; + } } else { if (spi_flash_enable_qmode_raw(&g_rom_flashchip) == ESP_OK) { to_qio = true; diff --git a/components/spi_flash/src/spi_flash_raw.c b/components/spi_flash/src/spi_flash_raw.c index dee89962b..10abf29e0 100644 --- a/components/spi_flash/src/spi_flash_raw.c +++ b/components/spi_flash/src/spi_flash_raw.c @@ -39,7 +39,9 @@ uint32_t spi_flash_get_id_raw(esp_rom_spiflash_chip_t *chip) { uint32_t rdid = 0; +#ifndef BOOTLOADER_BUILD Cache_Read_Disable(); +#endif Wait_SPI_Idle(chip); @@ -49,7 +51,9 @@ uint32_t spi_flash_get_id_raw(esp_rom_spiflash_chip_t *chip) rdid = READ_PERI_REG(PERIPHS_SPI_FLASH_C0)&0xffffff; +#ifndef BOOTLOADER_BUILD Cache_Read_Enable_New(); +#endif return rdid; } @@ -150,8 +154,11 @@ bool spi_user_cmd_raw(esp_rom_spiflash_chip_t *chip, spi_cmd_dir_t mode, spi_cmd { int idx = 0; +#ifndef BOOTLOADER_BUILD // Cache Disable Cache_Read_Disable_2(); +#endif + //wait spi idle if((mode & SPI_RAW) == 0) { Wait_SPI_Idle(chip); @@ -272,8 +279,11 @@ bool spi_user_cmd_raw(esp_rom_spiflash_chip_t *chip, spi_cmd_dir_t mode, spi_cmd if((mode & SPI_RAW) == 0) { Wait_SPI_Idle(chip); } + +#ifndef BOOTLOADER_BUILD //enable icache Cache_Read_Enable_2(); +#endif return true; } diff --git a/components/tcpip_adapter/tcpip_adapter_lwip.c b/components/tcpip_adapter/tcpip_adapter_lwip.c index d3a31d635..3faabd07e 100644 --- a/components/tcpip_adapter/tcpip_adapter_lwip.c +++ b/components/tcpip_adapter/tcpip_adapter_lwip.c @@ -74,7 +74,6 @@ static esp_err_t tcpip_adapter_reset_ip_info(tcpip_adapter_if_t tcpip_if); static esp_err_t tcpip_adapter_start_ip_lost_timer(tcpip_adapter_if_t tcpip_if); static void tcpip_adapter_dhcpc_cb(struct netif *netif); static void tcpip_adapter_ip_lost_timer(void *arg); -static void tcpip_adapter_dhcpc_done(TimerHandle_t arg); static bool tcpip_inited = false; static const char* TAG = "tcpip_adapter"; @@ -85,9 +84,25 @@ ESP_EVENT_DEFINE_BASE(IP_EVENT); err_t ethernetif_init(struct netif* netif); void system_station_got_ip_set(); -static int dhcp_fail_time = 0; static tcpip_adapter_ip_info_t esp_ip[TCPIP_ADAPTER_IF_MAX]; -static TimerHandle_t *dhcp_check_timer; + +static void tcpip_adapter_set_wifi_ps(bool flag) +{ + static bool is_ps_set = false; + if (flag) { + if (!is_ps_set) { + ESP_LOGD(TAG, "enable wifi ps"); + esp_wifi_ps_lock(); + is_ps_set = true; + } + } else { + if (is_ps_set) { + ESP_LOGD(TAG, "disable wifi ps"); + esp_wifi_ps_unlock(); + is_ps_set = false; + } + } +} static void tcpip_adapter_dhcps_cb(u8_t client_ip[4]) { @@ -192,11 +207,6 @@ void tcpip_adapter_init(void) IP4_ADDR(&esp_ip[TCPIP_ADAPTER_IF_AP].ip, 192, 168 , 4, 1); IP4_ADDR(&esp_ip[TCPIP_ADAPTER_IF_AP].gw, 192, 168 , 4, 1); IP4_ADDR(&esp_ip[TCPIP_ADAPTER_IF_AP].netmask, 255, 255 , 255, 0); - - dhcp_check_timer = xTimerCreate("check_dhcp", 500 / portTICK_RATE_MS, true, NULL, tcpip_adapter_dhcpc_done); - if (!dhcp_check_timer) { - ESP_LOGI(TAG, "TCPIP adapter timer create error"); - } } } @@ -287,56 +297,6 @@ static int tcpip_adapter_sta_recv_cb(void *buffer, uint16_t len, void *eb) return tcpip_adapter_recv_cb(esp_netif[ESP_IF_WIFI_STA], buffer, len, eb); } -static void tcpip_adapter_dhcpc_done(TimerHandle_t xTimer) -{ - bool unlock_ps = true; - struct dhcp *clientdhcp = netif_dhcp_data(esp_netif[TCPIP_ADAPTER_IF_STA]) ; - struct netif *netif = esp_netif[TCPIP_ADAPTER_IF_STA]; - - if (!netif) { - ESP_LOGD(TAG, "null netif=%p", netif); - return; - } - -#if LWIP_IPV4 && LWIP_AUTOIP - struct autoip *autoip = netif_autoip_data(netif); -#endif - - xTimerStop(dhcp_check_timer, 0); - if (netif_is_up(esp_netif[TCPIP_ADAPTER_IF_STA])) { - if ((clientdhcp && clientdhcp->state == DHCP_STATE_BOUND) -#if LWIP_IPV4 && LWIP_AUTOIP - || (autoip && autoip->state == AUTOIP_STATE_ANNOUNCING) -#endif - ) { - /*send event here*/ - tcpip_adapter_dhcpc_cb(esp_netif[TCPIP_ADAPTER_IF_STA]); - ESP_LOGD(TAG,"ip:" IPSTR ",mask:" IPSTR ",gw:" IPSTR "\n", IP2STR(ip_2_ip4(&(esp_netif[0]->ip_addr))), - IP2STR(ip_2_ip4(&(esp_netif[0]->netmask))), IP2STR(ip_2_ip4(&(esp_netif[0]->gw)))); - dhcp_fail_time = 0; - } else if (dhcp_fail_time < (CONFIG_IP_LOST_TIMER_INTERVAL * 1000 / 500)) { - ESP_LOGD(TAG,"dhcpc time(ms): %d\n", dhcp_fail_time * 500); - dhcp_fail_time ++; - xTimerReset(dhcp_check_timer, 0); - unlock_ps = false; - } else { - dhcp_fail_time = 0; - ESP_LOGD(TAG,"ERROR dhcp get ip error\n"); - } - } else { - dhcp_fail_time = 0; - tcpip_adapter_release_dhcp(esp_netif[TCPIP_ADAPTER_IF_STA]); - - dhcpc_status[TCPIP_ADAPTER_IF_STA] = TCPIP_ADAPTER_DHCP_INIT; - - tcpip_adapter_reset_ip_info(TCPIP_ADAPTER_IF_STA); - } - - if (unlock_ps) { - esp_wifi_ps_unlock(); - } -} - static esp_err_t tcpip_adapter_update_default_netif(void) { if ((esp_netif[TCPIP_ADAPTER_IF_STA] != NULL) && netif_is_up(esp_netif[TCPIP_ADAPTER_IF_STA])) { @@ -865,8 +825,14 @@ esp_err_t tcpip_adapter_set_dns_info(tcpip_adapter_if_t tcpip_if, tcpip_adapter_ return ESP_ERR_TCPIP_ADAPTER_INVALID_PARAMS; } +#if CONFIG_LWIP_IPV6 + if (!IP_IS_V4(&dns->ip) && !IP_IS_V6(&dns->ip)) { + IP_SET_TYPE_VAL(dns->ip, IPADDR_TYPE_V4); + } +#else ESP_LOGD(TAG, "set dns if=%d type=%d dns=%x", tcpip_if, type, ip_2_ip4(&(dns->ip))->addr); IP_SET_TYPE_VAL(dns->ip, IPADDR_TYPE_V4); +#endif if (tcpip_if == TCPIP_ADAPTER_IF_STA || tcpip_if == TCPIP_ADAPTER_IF_ETH) { dns_setserver(type, &(dns->ip)); @@ -1073,6 +1039,7 @@ static void tcpip_adapter_dhcpc_cb(struct netif *netif) } } + tcpip_adapter_set_wifi_ps(false); return; } @@ -1171,16 +1138,13 @@ esp_err_t tcpip_adapter_dhcpc_start(tcpip_adapter_if_t tcpip_if) return ESP_OK; } - esp_wifi_ps_lock(); + tcpip_adapter_set_wifi_ps(true); if (tcpip_adapter_start_dhcp(p_netif) != ERR_OK) { - esp_wifi_ps_unlock(); + tcpip_adapter_set_wifi_ps(false); ESP_LOGD(TAG, "dhcp client start failed"); return ESP_ERR_TCPIP_ADAPTER_DHCPC_START_FAILED; } - - dhcp_fail_time = 0; - xTimerReset(dhcp_check_timer, portMAX_DELAY); ESP_LOGD(TAG, "dhcp client start successfully"); dhcpc_status[tcpip_if] = TCPIP_ADAPTER_DHCP_STARTED; return ESP_OK; diff --git a/components/wpa_supplicant/src/ap/ap_config.c b/components/wpa_supplicant/src/ap/ap_config.c index 14f73547e..5fc87cd5d 100644 --- a/components/wpa_supplicant/src/ap/ap_config.c +++ b/components/wpa_supplicant/src/ap/ap_config.c @@ -130,14 +130,12 @@ static int hostapd_derive_psk(struct hostapd_ssid *ssid) wpa_hexdump_ascii_key(MSG_DEBUG, "PSK (ASCII passphrase)", (u8 *) ssid->wpa_passphrase, strlen(ssid->wpa_passphrase)); -#ifdef ESP_SUPPLICANT - memcpy(ssid->wpa_psk->psk, esp_wifi_ap_get_prof_pmk_internal(), PMK_LEN); -#else + /* It's too SLOW */ pbkdf2_sha1(ssid->wpa_passphrase, - ssid->ssid, ssid->ssid_len, + (const char *)ssid->ssid, ssid->ssid_len, 4096, ssid->wpa_psk->psk, PMK_LEN); -#endif + wpa_hexdump_key(MSG_DEBUG, "PSK (from passphrase)", ssid->wpa_psk->psk, PMK_LEN); return 0; diff --git a/components/wpa_supplicant/src/common/wpa_common.h b/components/wpa_supplicant/src/common/wpa_common.h index f88e8a6fa..fd1d9995a 100644 --- a/components/wpa_supplicant/src/common/wpa_common.h +++ b/components/wpa_supplicant/src/common/wpa_common.h @@ -110,6 +110,8 @@ #define WPA_CAPABILITY_MFPR BIT(6) #define WPA_CAPABILITY_MFPC BIT(7) #define WPA_CAPABILITY_PEERKEY_ENABLED BIT(9) +#define WPA_CAPABILITY_SPP_CAPABLE BIT(10) +#define WPA_CAPABILITY_SPP_REQUIRED BIT(11) /* IEEE 802.11r */ diff --git a/components/wpa_supplicant/src/eap_peer/eap_peap.c b/components/wpa_supplicant/src/eap_peer/eap_peap.c index 54e0ac0c6..38890f4bb 100644 --- a/components/wpa_supplicant/src/eap_peer/eap_peap.c +++ b/components/wpa_supplicant/src/eap_peer/eap_peap.c @@ -66,6 +66,7 @@ struct eap_peap_data { u8 cmk[20]; int soh; /* Whether IF-TNCCS-SOH (Statement of Health; Microsoft NAP) * is enabled. */ + enum { NO_AUTH, FOR_INITIAL, ALWAYS } phase2_auth; }; @@ -114,6 +115,19 @@ eap_peap_parse_phase1(struct eap_peap_data *data, wpa_printf(MSG_DEBUG, "EAP-PEAP: Require cryptobinding"); } + if (os_strstr(phase1, "phase2_auth=0")) { + data->phase2_auth = NO_AUTH; + wpa_printf(MSG_DEBUG, + "EAP-PEAP: Do not require Phase 2 authentication"); + } else if (os_strstr(phase1, "phase2_auth=1")) { + data->phase2_auth = FOR_INITIAL; + wpa_printf(MSG_DEBUG, + "EAP-PEAP: Require Phase 2 authentication for initial connection"); + } else if (os_strstr(phase1, "phase2_auth=2")) { + data->phase2_auth = ALWAYS; + wpa_printf(MSG_DEBUG, + "EAP-PEAP: Require Phase 2 authentication for all cases"); + } #ifdef EAP_TNC if (os_strstr(phase1, "tnc=soh2")) { data->soh = 2; @@ -145,6 +159,7 @@ eap_peap_init(struct eap_sm *sm) data->force_peap_version = -1; data->peap_outer_success = 2; data->crypto_binding = OPTIONAL_BINDING; + data->phase2_auth = FOR_INITIAL; if (config && config->phase1 && eap_peap_parse_phase1(data, config->phase1) < 0) { @@ -449,6 +464,18 @@ eap_tlv_validate_cryptobinding(struct eap_sm *sm, return 0; } +static bool peap_phase2_sufficient(struct eap_sm *sm, + struct eap_peap_data *data) +{ + if ((data->phase2_auth == ALWAYS || + (data->phase2_auth == FOR_INITIAL && + !tls_connection_resumed(sm->ssl_ctx, data->ssl.conn) && + !data->ssl.client_cert_conf) || + data->phase2_eap_started) && + !data->phase2_eap_success) + return false; + return true; +} /** * eap_tlv_process - Process a received EAP-TLV message and generate a response @@ -565,6 +592,11 @@ eap_tlv_process(struct eap_sm *sm, struct eap_peap_data *data, " - force failed Phase 2"); resp_status = EAP_TLV_RESULT_FAILURE; ret->decision = DECISION_FAIL; + } else if (!peap_phase2_sufficient(sm, data)) { + wpa_printf(MSG_INFO, + "EAP-PEAP: Server indicated Phase 2 success, but sufficient Phase 2 authentication has not been completed"); + resp_status = EAP_TLV_RESULT_FAILURE; + ret->decision = DECISION_FAIL; } else { resp_status = EAP_TLV_RESULT_SUCCESS; ret->decision = DECISION_UNCOND_SUCC; @@ -939,8 +971,7 @@ eap_peap_decrypt(struct eap_sm *sm, struct eap_peap_data *data, /* EAP-Success within TLS tunnel is used to indicate * shutdown of the TLS channel. The authentication has * been completed. */ - if (data->phase2_eap_started && - !data->phase2_eap_success) { + if (!peap_phase2_sufficient(sm, data)) { wpa_printf(MSG_DEBUG, "EAP-PEAP: Phase 2 " "Success used to indicate success, " "but Phase 2 EAP was not yet " @@ -1194,7 +1225,7 @@ eap_peap_has_reauth_data(struct eap_sm *sm, void *priv) { struct eap_peap_data *data = priv; return tls_connection_established(sm->ssl_ctx, data->ssl.conn) && - data->phase2_success; + data->phase2_success && data->phase2_auth != ALWAYS; } diff --git a/components/wpa_supplicant/src/eap_peer/eap_tls_common.c b/components/wpa_supplicant/src/eap_peer/eap_tls_common.c index 9c93f6d0c..31299b7f6 100755 --- a/components/wpa_supplicant/src/eap_peer/eap_tls_common.c +++ b/components/wpa_supplicant/src/eap_peer/eap_tls_common.c @@ -84,7 +84,7 @@ static void eap_tls_params_from_conf1(struct tls_connection_params *params, static int eap_tls_params_from_conf(struct eap_sm *sm, struct eap_ssl_data *data, struct tls_connection_params *params, - struct eap_peer_config *config) + struct eap_peer_config *config, int phase2) { os_memset(params, 0, sizeof(*params)); if (sm->workaround && data->eap_type != EAP_TYPE_FAST) { @@ -119,6 +119,12 @@ static int eap_tls_params_from_conf(struct eap_sm *sm, return -1; } + if (!phase2) + data->client_cert_conf = params->client_cert || + params->client_cert_blob || + params->private_key || + params->private_key_blob; + return 0; } @@ -196,7 +202,7 @@ int eap_peer_tls_ssl_init(struct eap_sm *sm, struct eap_ssl_data *data, data->eap = sm; data->eap_type = eap_type; data->ssl_ctx = sm->ssl_ctx; - if (eap_tls_params_from_conf(sm, data, ¶ms, config) < 0) /* no phase2 */ + if (eap_tls_params_from_conf(sm, data, ¶ms, config, data->phase2) < 0) /* no phase2 */ return -1; if (eap_tls_init_connection(sm, data, config, ¶ms) < 0) diff --git a/components/wpa_supplicant/src/eap_peer/eap_tls_common.h b/components/wpa_supplicant/src/eap_peer/eap_tls_common.h index 1a5e0f89e..50390c4ce 100644 --- a/components/wpa_supplicant/src/eap_peer/eap_tls_common.h +++ b/components/wpa_supplicant/src/eap_peer/eap_tls_common.h @@ -73,6 +73,11 @@ struct eap_ssl_data { * eap_type - EAP method used in Phase 1 (EAP_TYPE_TLS/PEAP/TTLS/FAST) */ u8 eap_type; + + /** + * client_cert_conf: Whether client certificate has been configured + */ + bool client_cert_conf; }; diff --git a/components/wpa_supplicant/src/rsn_supp/wpa_ie.c b/components/wpa_supplicant/src/rsn_supp/wpa_ie.c index d648a4542..5ad24bb99 100644 --- a/components/wpa_supplicant/src/rsn_supp/wpa_ie.c +++ b/components/wpa_supplicant/src/rsn_supp/wpa_ie.c @@ -225,6 +225,9 @@ static int wpa_gen_wpa_ie_rsn(u8 *rsn_ie, size_t rsn_ie_len, } } #endif /* CONFIG_IEEE80211W */ + + capab |= WPA_CAPABILITY_SPP_CAPABLE; + WPA_PUT_LE16(pos, capab); pos += 2; diff --git a/docs/en/api-reference/wifi/esp_wifi.rst b/docs/en/api-reference/wifi/esp_wifi.rst index d7c24b176..2368cbd26 100644 --- a/docs/en/api-reference/wifi/esp_wifi.rst +++ b/docs/en/api-reference/wifi/esp_wifi.rst @@ -10,7 +10,7 @@ The WiFi libraries provide support for configuring and monitoring the ESP8266 Wi - AP mode (aka Soft-AP mode or Access Point mode). Stations connect to the ESP8266. - Combined AP-STA mode (ESP8266 is concurrently an access point and a station connected to another access point). -- Various security modes for the above (WPA, WPA2, WEP, etc.) +- Various security modes for the above (WPA, WPA2, WPA3, WEP, etc.) - Scanning for access points (active & passive scanning). - Promiscuous mode monitoring of IEEE802.11 WiFi packets. diff --git a/docs/requirements.txt b/docs/requirements.txt index f3884f2ee..a5347f528 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,17 +1,17 @@ # This is a list of python packages used to generate documentation. This file is used with pip: # pip install --user -r requirements.txt # -sphinx>=1.8.4 -breathe==4.11.1 -sphinx-rtd-theme -sphinx-notfound-page -sphinxcontrib-blockdiag>=1.5.5, <2.0.0 -sphinxcontrib-seqdiag>=0.8.5, <2.0.0 -sphinxcontrib-actdiag>=0.8.5, <2.0.0 -sphinxcontrib-nwdiag>=0.9.5, <2.0.0 -blockdiag>=1.5.4, <2.0.0 -seqdiag>=0.9.6, <2.0.0 -actdiag>=0.5.4, <2.0.0 -nwdiag>=1.0.4, <2.0.0 -recommonmark -future>=0.16.0 # for ../tools/gen_esp_err_to_name.py +setuptools<57.5.0 +sphinx==2.3.1 +breathe==4.14.1 +sphinx-rtd-theme==1.0.0 +sphinx-notfound-page==0.7.1 +sphinxcontrib-blockdiag==2.0.0 +sphinxcontrib-seqdiag==2.0.0 +sphinxcontrib-actdiag==2.0.0 +sphinxcontrib-nwdiag==2.0.0 +sphinxcontrib-wavedrom==2.0.0 +funcparserlib==0.3.6 +nwdiag==2.0.0 +recommonmark==0.7.0 +future==0.16.0 # for ../tools/gen_esp_err_to_name.py \ No newline at end of file diff --git a/docs/setuptools.requirements.txt b/docs/setuptools.requirements.txt new file mode 100644 index 000000000..1f5449f58 --- /dev/null +++ b/docs/setuptools.requirements.txt @@ -0,0 +1 @@ +setuptools<57.5.0 \ No newline at end of file diff --git a/docs/sphinx-known-warnings.txt b/docs/sphinx-known-warnings.txt index 69b3877a4..8334d71be 100644 --- a/docs/sphinx-known-warnings.txt +++ b/docs/sphinx-known-warnings.txt @@ -86,3 +86,7 @@ spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::cmd spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::addr spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::mosi spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::miso +spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::cmd : 5 +spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::addr : 7 +spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::mosi : 10 +spi.inc:line: WARNING: Duplicate declaration, uint32_t spi_trans_t::miso : 10 diff --git a/examples/get-started/hello_world/sdkconfig.ci.2MB b/examples/get-started/hello_world/sdkconfig.ci.2MB new file mode 100644 index 000000000..b6c645057 --- /dev/null +++ b/examples/get-started/hello_world/sdkconfig.ci.2MB @@ -0,0 +1 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_2MB=y diff --git a/examples/get-started/hello_world/sdkconfig.ci.4MB b/examples/get-started/hello_world/sdkconfig.ci.4MB new file mode 100644 index 000000000..df4c6ef13 --- /dev/null +++ b/examples/get-started/hello_world/sdkconfig.ci.4MB @@ -0,0 +1 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y diff --git a/examples/protocols/sockets/README.md b/examples/protocols/sockets/README.md index 9fbc850db..1679cb584 100644 --- a/examples/protocols/sockets/README.md +++ b/examples/protocols/sockets/README.md @@ -1,7 +1,7 @@ # BSD Socket API Examples -This directory contains simple examples demonstrating BSD Socket API. +This directory contains simple examples demonstrating BSD Socket API. Each example, contains README.md file with mode detailed informations about that particular example. For more general informations about all examples, see the README.md file in the upper level 'examples' directory. Examples: @@ -26,8 +26,8 @@ http://wiki.treck.com/Introduction_to_BSD_Sockets ## Host tools -There are many host-side tools which can be used to interact with the UDP/TCP server/client example. -One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. +There are many host-side tools which can be used to interact with the UDP/TCP server/client example. +One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. Note: please replace `192.168.0.167 3333` with desired IPV4/IPV6 address (displayed in monitor console) and port number in the following commands. In addition to those tools, simple Python scripts can be found under sockets/scripts directory. Every script is designed to interact with one of the examples. @@ -74,7 +74,7 @@ IPV6 = 'FE80::32AE:A4FF:FE80:5288' ## Hardware Required -This example can be run on any commonly available ESP32 development board. +This example can be run on any commonly available ESP8266 development board. ## Configure the project @@ -97,6 +97,3 @@ make -j4 flash monitor (To exit the serial monitor, type ``Ctrl-]``.) See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects. - - - diff --git a/examples/protocols/sockets/tcp_client/README.md b/examples/protocols/sockets/tcp_client/README.md index be9e9f2d4..a927f7a78 100644 --- a/examples/protocols/sockets/tcp_client/README.md +++ b/examples/protocols/sockets/tcp_client/README.md @@ -9,8 +9,8 @@ The application creates a TCP socket and tries to connect to the server with pre In order to create TCP server that communicates with TCP Client example, choose one of the following options. -There are many host-side tools which can be used to interact with the UDP/TCP server/client. -One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. +There are many host-side tools which can be used to interact with the UDP/TCP server/client. +One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. Note: please replace `192.168.0.167 3333` with desired IPV4/IPV6 address (displayed in monitor console) and port number in the following command. In addition to those tools, simple Python scripts can be found under sockets/scripts directory. Every script is designed to interact with one of the examples. @@ -30,7 +30,7 @@ PORT = 3333; ## Hardware Required -This example can be run on any commonly available ESP32 development board. +This example can be run on any commonly available ESP8266 development board. ## Configure the project diff --git a/examples/protocols/sockets/tcp_server/README.md b/examples/protocols/sockets/tcp_server/README.md index f43dc0791..828bf8e56 100644 --- a/examples/protocols/sockets/tcp_server/README.md +++ b/examples/protocols/sockets/tcp_server/README.md @@ -9,8 +9,8 @@ The application creates a TCP socket with the specified port number and waits fo In order to create TCP client that communicates with TCP server example, choose one of the following options. -There are many host-side tools which can be used to interact with the UDP/TCP server/client. -One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. +There are many host-side tools which can be used to interact with the UDP/TCP server/client. +One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. Note: please replace `192.168.0.167 3333` with desired IPV4/IPV6 address (displayed in monitor console) and port number in the following command. In addition to those tools, simple Python scripts can be found under sockets/scripts directory. Every script is designed to interact with one of the examples. @@ -32,7 +32,7 @@ IPV6 = 'FE80::32AE:A4FF:FE80:5288' ## Hardware Required -This example can be run on any commonly available ESP32 development board. +This example can be run on any commonly available ESP8266 development board. ## Configure the project diff --git a/examples/protocols/sockets/udp_client/README.md b/examples/protocols/sockets/udp_client/README.md index 80994e2c0..01b10b9de 100644 --- a/examples/protocols/sockets/udp_client/README.md +++ b/examples/protocols/sockets/udp_client/README.md @@ -9,8 +9,8 @@ The application creates UDP socket and sends message to the predefined port and In order to create UDP server that communicates with UDP Client example, choose one of the following options. -There are many host-side tools which can be used to interact with the UDP/TCP server/client. -One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. +There are many host-side tools which can be used to interact with the UDP/TCP server/client. +One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. Note: please replace `192.168.0.167 3333` with desired IPV4/IPV6 address (displayed in monitor console) and port number in the following commands. In addition to those tools, simple Python scripts can be found under sockets/scripts directory. Every script is designed to interact with one of the examples. @@ -40,7 +40,7 @@ PORT = 3333; ## Hardware Required -This example can be run on any commonly available ESP32 development board. +This example can be run on any commonly available ESP8266 development board. ## Configure the project diff --git a/examples/protocols/sockets/udp_server/README.md b/examples/protocols/sockets/udp_server/README.md index 8fe64a73e..a3e17de49 100644 --- a/examples/protocols/sockets/udp_server/README.md +++ b/examples/protocols/sockets/udp_server/README.md @@ -9,8 +9,8 @@ The application creates UDP socket with the specified port number and waits for In order to create UDP client that communicates with UDP server example, choose one of the following options. -There are many host-side tools which can be used to interact with the UDP/TCP server/client. -One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. +There are many host-side tools which can be used to interact with the UDP/TCP server/client. +One command line tool is [netcat](http://netcat.sourceforge.net) which can send and receive many kinds of packets. Note: please replace `192.168.0.167 3333` with desired IPV4/IPV6 address (displayed in monitor console) and port number in the following commands. In addition to those tools, simple Python scripts can be found under sockets/scripts directory. Every script is designed to interact with one of the examples. @@ -42,7 +42,7 @@ IPV6 = 'FE80::32AE:A4FF:FE80:5288' ## Hardware Required -This example can be run on any commonly available ESP32 development board. +This example can be run on any commonly available ESP8266 development board. ## Configure the project diff --git a/examples/provisioning/legacy/custom_config/README.md b/examples/provisioning/legacy/custom_config/README.md index 0af5006c8..80e0bbf97 100644 --- a/examples/provisioning/legacy/custom_config/README.md +++ b/examples/provisioning/legacy/custom_config/README.md @@ -22,7 +22,7 @@ Also, it uses a component provided with this example `custom_provisioning` which ### Hardware Required -Example should be able to run on any commonly available ESP32 development board. +Example should be able to run on any commonly available ESP8266 development board. ### Application Required diff --git a/examples/provisioning/legacy/softap_prov/README.md b/examples/provisioning/legacy/softap_prov/README.md index c59bc0bd5..c916bfdf2 100644 --- a/examples/provisioning/legacy/softap_prov/README.md +++ b/examples/provisioning/legacy/softap_prov/README.md @@ -23,7 +23,7 @@ This example can be used, as it is, for adding a provisioning service to any app ### Hardware Required -Example should be able to run on any commonly available ESP32 development board. +Example should be able to run on any commonly available ESP8266 development board. ### Application Required diff --git a/examples/system/factory-test/components/rf_test/lib/librftest.a b/examples/system/factory-test/components/rf_test/lib/librftest.a old mode 100755 new mode 100644 index d76a39ea6..fdf36b009 Binary files a/examples/system/factory-test/components/rf_test/lib/librftest.a and b/examples/system/factory-test/components/rf_test/lib/librftest.a differ diff --git a/examples/wifi/smart_config/main/Kconfig.projbuild b/examples/wifi/smart_config/main/Kconfig.projbuild index 533a7140c..8d27ee8e0 100644 --- a/examples/wifi/smart_config/main/Kconfig.projbuild +++ b/examples/wifi/smart_config/main/Kconfig.projbuild @@ -12,7 +12,7 @@ config ESP_TOUCH bool "ESPTouch" config AIRKISS bool "AirKiss" -config ESP_TOUCH-AIRKISS +config ESP_TOUCH_AIRKISS bool "ESPTouch and AirKiss" config ESP_TOUCH_V2 bool "ESPTouch-V2" @@ -22,7 +22,7 @@ config ESP_SMARTCONFIG_TYPE int default 0 if ESP_TOUCH default 1 if AIRKISS - default 2 if ESP_TOUCH-AIRKISS + default 2 if ESP_TOUCH_AIRKISS default 3 if ESP_TOUCH_V2 endmenu diff --git a/requirements.txt b/requirements.txt index a43c6d4ae..6a9793304 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ setuptools click>=5.0 pyserial>=3.0 future>=0.15.2 -cryptography>=2.1.4 +cryptography>=2.1.4,<35 pyparsing>=2.0.3,<2.4.0 pyelftools>=0.22 diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index b30f5c51f..fbeaf65ae 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -118,53 +118,75 @@ build_example () { local EXAMPLE_DIR=$(dirname "${MAKE_FILE}") local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") - local EXAMPLE_BUILD_DIR="${ID}_${EXAMPLE_NAME}" + local EXAMPLE_BUILD_DIRS=() - if [[ -f "example_builds/${EXAMPLE_BUILD_DIR}/build/ci_build_success" ]]; then - echo "Project ${EXAMPLE_BUILD_DIR} has been built and skip building ..." + # count number of CI sdkconfig files + SDKCONFIG_CI_FILES=$( find ${EXAMPLE_DIR}/ -type f -name sdkconfig.ci.* | sort ) + if [[ -z ${SDKCONFIG_CI_FILES} ]]; then + EXAMPLE_BUILD_DIRS[0]="${ID}_${EXAMPLE_NAME}" else - echo "Building ${EXAMPLE_BUILD_DIR}..." - mkdir -p "example_builds/${EXAMPLE_BUILD_DIR}" - cp -r "${EXAMPLE_DIR}/"* "example_builds/${EXAMPLE_BUILD_DIR}/" - pushd "example_builds/${EXAMPLE_BUILD_DIR}" - # be stricter in the CI build than the default IDF settings - export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} - - # sdkconfig files are normally not checked into git, but may be present when - # a developer runs this script locally - rm -f sdkconfig - - # If sdkconfig.ci file is present, append it to sdkconfig.defaults, - # replacing environment variables - if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then - cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults - fi - - # build non-verbose first - local BUILDLOG=${LOG_PATH}/ex_${EXAMPLE_BUILD_DIR}_log.txt - touch ${BUILDLOG} - - local FLASH_ARGS=build/download.config - - make clean >>${BUILDLOG} 2>&1 && - make defconfig >>${BUILDLOG} 2>&1 && - make all -j4 >>${BUILDLOG} 2>&1 && - make ota >>${BUILDLOG} 2>&1 && - make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} && - touch build/ci_build_success || - { - RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; - } - - tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || : - test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty" - - cat ${BUILDLOG} - popd - - grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || : + COUNT=0 + for CI_FILE in ${SDKCONFIG_CI_FILES} + do + EXAMPLE_BUILD_DIRS[COUNT]="${ID}_${EXAMPLE_NAME}_${CI_FILE##*.}" + COUNT=$(( $COUNT + 1 )) + done fi + + for EXAMPLE_BUILD_DIR in ${EXAMPLE_BUILD_DIRS[*]} + do + if [[ -f "example_builds/${EXAMPLE_BUILD_DIR}/build/ci_build_success" ]]; then + echo "Project ${EXAMPLE_BUILD_DIR} has been built and skip building ..." + else + echo "Building ${EXAMPLE_BUILD_DIR}..." + mkdir -p "example_builds/${EXAMPLE_BUILD_DIR}" + cp -r "${EXAMPLE_DIR}/"* "example_builds/${EXAMPLE_BUILD_DIR}/" + + if [[ -n ${SDKCONFIG_CI_FILES} ]]; then + cp "example_builds/${EXAMPLE_BUILD_DIR}/sdkconfig.ci.${EXAMPLE_BUILD_DIR##*_}" "example_builds/${EXAMPLE_BUILD_DIR}/sdkconfig.ci" + rm example_builds/${EXAMPLE_BUILD_DIR}/sdkconfig.ci.* + fi + + pushd "example_builds/${EXAMPLE_BUILD_DIR}" + # be stricter in the CI build than the default IDF settings + export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" + export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} + + # sdkconfig files are normally not checked into git, but may be present when + # a developer runs this script locally + rm -f sdkconfig + + # If sdkconfig.ci file is present, append it to sdkconfig.defaults, + # replacing environment variables + if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then + cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults + fi + + # build non-verbose first + local BUILDLOG=${LOG_PATH}/ex_${EXAMPLE_BUILD_DIR}_log.txt + touch ${BUILDLOG} + + local FLASH_ARGS=build/download.config + + make clean >>${BUILDLOG} 2>&1 && + make defconfig >>${BUILDLOG} 2>&1 && + make all -j4 >>${BUILDLOG} 2>&1 && + make ota >>${BUILDLOG} 2>&1 && + make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} && + touch build/ci_build_success || + { + RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; + } + + tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || : + test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty" + + cat ${BUILDLOG} + popd + + grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || : + fi + done } EXAMPLE_NUM=0 diff --git a/tools/ci/build_examples_cmake.sh b/tools/ci/build_examples_cmake.sh index 914dbcc2e..24063bad9 100755 --- a/tools/ci/build_examples_cmake.sh +++ b/tools/ci/build_examples_cmake.sh @@ -132,47 +132,70 @@ build_example () { local EXAMPLE_DIR=$(dirname "${CMAKELISTS}") local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") - local EXAMPLE_BUILD_DIR="${ID}_${EXAMPLE_NAME}" + local EXAMPLE_BUILD_DIRS=() - if [[ -f "example_builds/${EXAMPLE_BUILD_DIR}/build/ci_build_success" ]]; then - echo "Project ${EXAMPLE_NAME} has been built and skip building ..." + # count number of CI sdkconfig files + SDKCONFIG_CI_FILES=$( find ${EXAMPLE_DIR}/ -type f -name sdkconfig.ci.* | sort ) + if [[ -z ${SDKCONFIG_CI_FILES} ]]; then + EXAMPLE_BUILD_DIRS[0]="${ID}_${EXAMPLE_NAME}" else - echo "Building ${EXAMPLE_BUILD_DIR}..." - mkdir -p "example_builds/${EXAMPLE_BUILD_DIR}" - cp -r "${EXAMPLE_DIR}/"* "example_builds/${EXAMPLE_BUILD_DIR}" - pushd "example_builds/${EXAMPLE_BUILD_DIR}" - # be stricter in the CI build than the default IDF settings - export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} - - prepare_build ${EXAMPLE_NAME} - - # sdkconfig files are normally not checked into git, but may be present when - # a developer runs this script locally - rm -f sdkconfig - - # If sdkconfig.ci file is present, append it to sdkconfig.defaults, - # replacing environment variables - if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then - cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults - fi - - # build non-verbose first - local BUILDLOG=${LOG_PATH}/ex_${EXAMPLE_BUILD_DIR}_log.txt - touch ${BUILDLOG} - - idf.py build >>${BUILDLOG} 2>&1 && - cp build/flash_project_args build/download.config && # backwards compatible download.config filename - touch build/ci_build_success || - { - RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; - } - - cat ${BUILDLOG} - popd - - grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || : + COUNT=0 + for CI_FILE in ${SDKCONFIG_CI_FILES} + do + echo "${COUNT} ${CI_FILE}" + EXAMPLE_BUILD_DIRS[COUNT]="${ID}_${EXAMPLE_NAME}_${CI_FILE##*.}" + COUNT=$(( $COUNT + 1 )) + done fi + + for EXAMPLE_BUILD_DIR in ${EXAMPLE_BUILD_DIRS[*]} + do + if [[ -f "example_builds/${EXAMPLE_BUILD_DIR}/build/ci_build_success" ]]; then + echo "Project ${EXAMPLE_BUILD_DIR} has been built and skip building ..." + else + echo "Building ${EXAMPLE_BUILD_DIR}..." + mkdir -p "example_builds/${EXAMPLE_BUILD_DIR}" + cp -r "${EXAMPLE_DIR}/"* "example_builds/${EXAMPLE_BUILD_DIR}/" + + if [[ -n ${SDKCONFIG_CI_FILES} ]]; then + cp "example_builds/${EXAMPLE_BUILD_DIR}/sdkconfig.ci.${EXAMPLE_BUILD_DIR##*_}" "example_builds/${EXAMPLE_BUILD_DIR}/sdkconfig.ci" + rm example_builds/${EXAMPLE_BUILD_DIR}/sdkconfig.ci.* + fi + + pushd "example_builds/${EXAMPLE_BUILD_DIR}" + # be stricter in the CI build than the default IDF settings + export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" + export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} + + prepare_build ${EXAMPLE_NAME} + + # sdkconfig files are normally not checked into git, but may be present when + # a developer runs this script locally + rm -f sdkconfig + + # If sdkconfig.ci file is present, append it to sdkconfig.defaults, + # replacing environment variables + if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then + cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults + fi + + # build non-verbose first + local BUILDLOG=${LOG_PATH}/ex_${EXAMPLE_BUILD_DIR}_log.txt + touch ${BUILDLOG} + + idf.py build >>${BUILDLOG} 2>&1 && + cp build/flash_project_args build/download.config && # backwards compatible download.config filename + touch build/ci_build_success || + { + RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; + } + + cat ${BUILDLOG} + popd + + grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || : + fi + done } EXAMPLE_NUM=0