/* * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include #include "sdkconfig.h" #include "esp_attr.h" #include "esp_log.h" #include "esp_check.h" #include "esp_heap_caps.h" #include "soc/soc_caps.h" #include "hal/cache_types.h" #include "hal/cache_hal.h" #include "hal/cache_ll.h" #include "hal/mmu_types.h" #include "hal/mmu_hal.h" #include "hal/mmu_ll.h" #include "esp_private/cache_utils.h" #include "esp_private/esp_cache_esp32_private.h" #include "esp_private/esp_mmu_map_private.h" #include "ext_mem_layout.h" #include "esp_mmu_map.h" //This is for size align #define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1)) //This is for vaddr align #define ALIGN_DOWN_BY(num, align) ((num) & (~((align) - 1))) //This flag indicates the memory region is merged, we don't care about it anymore #define MEM_REGION_MERGED -1 /** * We have some hw related tests for vaddr region capabilities * Use this macro to disable paddr check as we need to reuse certain paddr blocks */ #define ENABLE_PADDR_CHECK !ESP_MMAP_TEST_ALLOW_MAP_TO_MAPPED_PADDR static DRAM_ATTR const char *TAG = "mmap"; /** * @brief MMU Memory Mapping Driver * * Driver Backgrounds: * * -------------------------------------------------------------------------------------------------------- * Memory Pool | * -------------------------------------------------------------------------------------------------------- * | Memory Region 0 | Memory Region 1 | ... | * -------------------------------------------------------------------------------------------------------- * | Block 0 | Slot 0 | Block 1 | Block 2 | ... | Slot 1 (final slot) | ... | * -------------------------------------------------------------------------------------------------------- * * - A block is a piece of vaddr range that is dynamically mapped. Blocks are doubly linked: * Block 0 <-> Block 1 <-> Block 2 * - A Slot is the vaddr range between 2 blocks. */ /** * Struct for a block */ typedef struct mem_block_ { uint32_t laddr_start; //linear address start of this block uint32_t laddr_end; //linear address end of this block intptr_t vaddr_start; //virtual address start of this block intptr_t vaddr_end; //virtual address end of this block size_t size; //size of this block, should be aligned to MMU page size int caps; //caps of this block, `mmu_mem_caps_t` uint32_t paddr_start; //physical address start of this block uint32_t paddr_end; //physical address end of this block mmu_target_t target; //physical target that this block is mapped to TAILQ_ENTRY(mem_block_) entries; //link entry } mem_block_t; /** * Struct for a memory region */ typedef struct mem_region_ { cache_bus_mask_t bus_id; //cache bus mask of this region uint32_t start; //linear address start of this region uint32_t end; //linear address end of this region size_t region_size; //region size, in bytes uint32_t free_head; //linear address free head of this region size_t max_slot_size; //max slot size within this region int caps; //caps of this region, `mmu_mem_caps_t` mmu_target_t targets; //physical targets that this region is supported TAILQ_HEAD(mem_block_head_, mem_block_) mem_block_head; //link head of allocated blocks within this region } mem_region_t; typedef struct { /** * number of memory regions that are available, after coalescing, this number should be smaller than or equal to `SOC_MMU_LINEAR_ADDRESS_REGION_NUM` */ uint32_t num_regions; /** * This saves the available MMU linear address regions, * after reserving flash .rodata and .text, and after coalescing. * Only the first `num_regions` items are valid */ mem_region_t mem_regions[SOC_MMU_LINEAR_ADDRESS_REGION_NUM]; } mmu_ctx_t; static mmu_ctx_t s_mmu_ctx; #if ENABLE_PADDR_CHECK static bool s_is_enclosed(uint32_t block_start, uint32_t block_end, uint32_t new_block_start, uint32_t new_block_size); static bool s_is_overlapped(uint32_t block_start, uint32_t block_end, uint32_t new_block_start, uint32_t new_block_size); #endif //#if ENABLE_PADDR_CHECK #if CONFIG_APP_BUILD_USE_FLASH_SECTIONS static cache_bus_mask_t s_get_bus_mask(uint32_t vaddr_start, uint32_t len) { #if CACHE_LL_EXT_MEM_VIA_L2CACHE return cache_ll_l2_get_bus(0, vaddr_start, len); #else return cache_ll_l1_get_bus(0, vaddr_start, len); #endif } static void s_reserve_irom_region(mem_region_t *hw_mem_regions, int region_nums) { /** * We follow the way how 1st bootloader load flash .text: * * - Now IBUS addresses (between `_instruction_reserved_start` and `_instruction_reserved_end`) are consecutive on all chips, * we strongly rely on this to calculate the .text length */ extern int _instruction_reserved_start; extern int _instruction_reserved_end; size_t irom_len_to_reserve = (uint32_t)&_instruction_reserved_end - (uint32_t)&_instruction_reserved_start; assert((mmu_ll_vaddr_to_laddr((uint32_t)&_instruction_reserved_end) - mmu_ll_vaddr_to_laddr((uint32_t)&_instruction_reserved_start)) == irom_len_to_reserve); irom_len_to_reserve += (uint32_t)&_instruction_reserved_start - ALIGN_DOWN_BY((uint32_t)&_instruction_reserved_start, CONFIG_MMU_PAGE_SIZE); irom_len_to_reserve = ALIGN_UP_BY(irom_len_to_reserve, CONFIG_MMU_PAGE_SIZE); cache_bus_mask_t bus_mask = s_get_bus_mask((uint32_t)&_instruction_reserved_start, irom_len_to_reserve); for (int i = 0; i < SOC_MMU_LINEAR_ADDRESS_REGION_NUM; i++) { if (bus_mask & hw_mem_regions[i].bus_id) { if (hw_mem_regions[i].region_size <= irom_len_to_reserve) { hw_mem_regions[i].free_head = hw_mem_regions[i].end; hw_mem_regions[i].max_slot_size = 0; irom_len_to_reserve -= hw_mem_regions[i].region_size; } else { hw_mem_regions[i].free_head = hw_mem_regions[i].free_head + irom_len_to_reserve; hw_mem_regions[i].max_slot_size -= irom_len_to_reserve; } } } } static void s_reserve_drom_region(mem_region_t *hw_mem_regions, int region_nums) { /** * Similarly, we follow the way how 1st bootloader load flash .rodata: */ extern int _rodata_reserved_start; extern int _rodata_reserved_end; size_t drom_len_to_reserve = (uint32_t)&_rodata_reserved_end - (uint32_t)&_rodata_reserved_start; assert((mmu_ll_vaddr_to_laddr((uint32_t)&_rodata_reserved_end) - mmu_ll_vaddr_to_laddr((uint32_t)&_rodata_reserved_start)) == drom_len_to_reserve); drom_len_to_reserve += (uint32_t)&_rodata_reserved_start - ALIGN_DOWN_BY((uint32_t)&_rodata_reserved_start, CONFIG_MMU_PAGE_SIZE); drom_len_to_reserve = ALIGN_UP_BY(drom_len_to_reserve, CONFIG_MMU_PAGE_SIZE); cache_bus_mask_t bus_mask = s_get_bus_mask((uint32_t)&_rodata_reserved_start, drom_len_to_reserve); for (int i = 0; i < SOC_MMU_LINEAR_ADDRESS_REGION_NUM; i++) { if (bus_mask & hw_mem_regions[i].bus_id) { if (hw_mem_regions[i].region_size <= drom_len_to_reserve) { hw_mem_regions[i].free_head = hw_mem_regions[i].end; hw_mem_regions[i].max_slot_size = 0; drom_len_to_reserve -= hw_mem_regions[i].region_size; } else { hw_mem_regions[i].free_head = hw_mem_regions[i].free_head + drom_len_to_reserve; hw_mem_regions[i].max_slot_size -= drom_len_to_reserve; } } } } #endif //#if CONFIG_APP_BUILD_USE_FLASH_SECTIONS #if SOC_MMU_PER_EXT_MEM_TARGET static inline uint32_t s_get_mmu_id_from_target(mmu_target_t target) { return (target == MMU_TARGET_FLASH0) ? MMU_LL_FLASH_MMU_ID : MMU_LL_PSRAM_MMU_ID; } #endif void esp_mmu_map_init(void) { mem_region_t hw_mem_regions[SOC_MMU_LINEAR_ADDRESS_REGION_NUM] = {}; for (int i = 0; i < SOC_MMU_LINEAR_ADDRESS_REGION_NUM; i++) { hw_mem_regions[i].start = g_mmu_mem_regions[i].start; hw_mem_regions[i].end = g_mmu_mem_regions[i].end; hw_mem_regions[i].region_size = g_mmu_mem_regions[i].size; hw_mem_regions[i].max_slot_size = g_mmu_mem_regions[i].size; hw_mem_regions[i].free_head = g_mmu_mem_regions[i].start; hw_mem_regions[i].bus_id = g_mmu_mem_regions[i].bus_id; hw_mem_regions[i].caps = g_mmu_mem_regions[i].caps; hw_mem_regions[i].targets = g_mmu_mem_regions[i].targets; #if CONFIG_IDF_TARGET_ESP32 || CONFIG_IDF_TARGET_ESP32S2 assert(__builtin_popcount(hw_mem_regions[i].bus_id) == 1); #endif assert(hw_mem_regions[i].region_size % CONFIG_MMU_PAGE_SIZE == 0); } #if CONFIG_APP_BUILD_USE_FLASH_SECTIONS //First reserve memory regions used for irom and drom, as we must follow the way how 1st bootloader load them s_reserve_irom_region(hw_mem_regions, SOC_MMU_LINEAR_ADDRESS_REGION_NUM); s_reserve_drom_region(hw_mem_regions, SOC_MMU_LINEAR_ADDRESS_REGION_NUM); #endif //#if CONFIG_APP_BUILD_USE_FLASH_SECTIONS if (SOC_MMU_LINEAR_ADDRESS_REGION_NUM > 1) { //Now we can coalesce adjacent regions for (int i = 1; i < SOC_MMU_LINEAR_ADDRESS_REGION_NUM; i++) { mem_region_t *a = &hw_mem_regions[i - 1]; mem_region_t *b = &hw_mem_regions[i]; if ((b->free_head == a->end) && (b->caps == a->caps) && (b->targets == a->targets)) { a->caps = MEM_REGION_MERGED; b->bus_id |= a->bus_id; b->start = a->start; b->region_size += a->region_size; b->free_head = a->free_head; b->max_slot_size += a->max_slot_size; } } } //Count the mem regions left after coalescing uint32_t region_num = 0; for (int i = 0; i < SOC_MMU_LINEAR_ADDRESS_REGION_NUM; i++) { if (hw_mem_regions[i].caps != MEM_REGION_MERGED) { region_num++; } } ESP_EARLY_LOGV(TAG, "after coalescing, %" PRIu32 " regions are left", region_num); //Initialise `s_mmu_ctx.mem_regions[]`, as we've done all static allocation, to prepare available virtual memory regions uint32_t available_region_idx = 0; s_mmu_ctx.num_regions = region_num; for (int i = 0; i < SOC_MMU_LINEAR_ADDRESS_REGION_NUM; i++) { if (hw_mem_regions[i].caps == MEM_REGION_MERGED) { continue; } memcpy(&s_mmu_ctx.mem_regions[available_region_idx], &hw_mem_regions[i], sizeof(mem_region_t)); available_region_idx++; } for (int i = 0; i < available_region_idx; i++) { TAILQ_INIT(&s_mmu_ctx.mem_regions[i].mem_block_head); } assert(available_region_idx == region_num); } static esp_err_t s_mem_caps_check(mmu_mem_caps_t caps) { if (caps & MMU_MEM_CAP_EXEC) { if ((caps & MMU_MEM_CAP_8BIT) || (caps & MMU_MEM_CAP_WRITE)) { //None of the executable memory are expected to be 8-bit accessible or writable. return ESP_ERR_INVALID_ARG; } caps |= MMU_MEM_CAP_32BIT; } return ESP_OK; } esp_err_t esp_mmu_map_get_max_consecutive_free_block_size(mmu_mem_caps_t caps, mmu_target_t target, size_t *out_len) { ESP_RETURN_ON_FALSE(out_len, ESP_ERR_INVALID_ARG, TAG, "null pointer"); ESP_RETURN_ON_ERROR(s_mem_caps_check(caps), TAG, "invalid caps"); *out_len = 0; size_t max = 0; for (int i = 0; i < s_mmu_ctx.num_regions; i++) { if (((s_mmu_ctx.mem_regions[i].caps & caps) == caps) && ((s_mmu_ctx.mem_regions[i].targets & target) == target)) { if (s_mmu_ctx.mem_regions[i].max_slot_size > max) { max = s_mmu_ctx.mem_regions[i].max_slot_size; } } } *out_len = max; return ESP_OK; } static int32_t s_find_available_region(mem_region_t *mem_regions, uint32_t region_nums, size_t size, mmu_mem_caps_t caps, mmu_target_t target) { int32_t found_region_id = -1; for (int i = 0; i < region_nums; i++) { if (((mem_regions[i].caps & caps) == caps) && ((mem_regions[i].targets & target) == target)) { if (mem_regions[i].max_slot_size >= size) { found_region_id = i; break; } } } return found_region_id; } esp_err_t esp_mmu_map_reserve_block_with_caps(size_t size, mmu_mem_caps_t caps, mmu_target_t target, const void **out_ptr) { ESP_RETURN_ON_FALSE(out_ptr, ESP_ERR_INVALID_ARG, TAG, "null pointer"); ESP_RETURN_ON_ERROR(s_mem_caps_check(caps), TAG, "invalid caps"); size_t aligned_size = ALIGN_UP_BY(size, CONFIG_MMU_PAGE_SIZE); uint32_t laddr = 0; int32_t found_region_id = s_find_available_region(s_mmu_ctx.mem_regions, s_mmu_ctx.num_regions, aligned_size, caps, target); if (found_region_id == -1) { ESP_EARLY_LOGE(TAG, "no such vaddr range"); return ESP_ERR_NOT_FOUND; } laddr = (uint32_t)s_mmu_ctx.mem_regions[found_region_id].free_head; s_mmu_ctx.mem_regions[found_region_id].free_head += aligned_size; s_mmu_ctx.mem_regions[found_region_id].max_slot_size -= aligned_size; ESP_EARLY_LOGV(TAG, "found laddr is 0x%" PRIx32, laddr); uint32_t vaddr = 0; if (caps & MMU_MEM_CAP_EXEC) { vaddr = mmu_ll_laddr_to_vaddr(laddr, MMU_VADDR_INSTRUCTION, target); } else { vaddr = mmu_ll_laddr_to_vaddr(laddr, MMU_VADDR_DATA, target); } *out_ptr = (void *)vaddr; return ESP_OK; } IRAM_ATTR esp_err_t esp_mmu_paddr_find_caps(const esp_paddr_t paddr, mmu_mem_caps_t *out_caps) { mem_region_t *region = NULL; mem_block_t *mem_block = NULL; bool found = false; mem_block_t *found_block = NULL; if (out_caps == NULL) { return ESP_ERR_INVALID_ARG; } for (int i = 0; i < s_mmu_ctx.num_regions; i++) { region = &s_mmu_ctx.mem_regions[i]; TAILQ_FOREACH(mem_block, ®ion->mem_block_head, entries) { if (mem_block == TAILQ_FIRST(®ion->mem_block_head) || mem_block == TAILQ_LAST(®ion->mem_block_head, mem_block_head_)) { //we don't care the dummy_head and the dummy_tail continue; } //now we are only traversing the actual dynamically allocated blocks, dummy_head and dummy_tail are excluded already if (mem_block->paddr_start == paddr) { found = true; found_block = mem_block; break; } } } if (!found) { return ESP_ERR_NOT_FOUND; } *out_caps = found_block->caps; return ESP_OK; } static void IRAM_ATTR NOINLINE_ATTR s_do_cache_invalidate(uint32_t vaddr_start, uint32_t size) { #if CONFIG_IDF_TARGET_ESP32 /** * On ESP32, due to hardware limitation, we don't have an * easy way to sync between cache and external memory wrt * certain range. So we do a full sync here */ cache_sync(); #else //Other chips cache_hal_invalidate_addr(vaddr_start, size); #endif // CONFIG_IDF_TARGET_ESP32 } #if SOC_MMU_PER_EXT_MEM_TARGET FORCE_INLINE_ATTR uint32_t s_mapping_operation(mmu_target_t target, uint32_t vaddr_start, esp_paddr_t paddr_start, uint32_t size) { uint32_t actual_mapped_len = 0; uint32_t mmu_id = s_get_mmu_id_from_target(target); mmu_hal_map_region(mmu_id, target, vaddr_start, paddr_start, size, &actual_mapped_len); return actual_mapped_len; } #else FORCE_INLINE_ATTR uint32_t s_mapping_operation(mmu_target_t target, uint32_t vaddr_start, esp_paddr_t paddr_start, uint32_t size) { uint32_t actual_mapped_len = 0; mmu_hal_map_region(0, target, vaddr_start, paddr_start, size, &actual_mapped_len); #if (SOC_MMU_PERIPH_NUM == 2) #if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE mmu_hal_map_region(1, target, vaddr_start, paddr_start, size, &actual_mapped_len); #endif // #if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE #endif // #if (SOC_MMU_PERIPH_NUM == 2) return actual_mapped_len; } #endif static void IRAM_ATTR NOINLINE_ATTR s_do_mapping(mmu_target_t target, uint32_t vaddr_start, esp_paddr_t paddr_start, uint32_t size) { /** * Disable Cache, after this function, involved code and data should be placed in internal RAM. * * @note we call this for now, but this will be refactored to move out of `spi_flash` */ spi_flash_disable_interrupts_caches_and_other_cpu(); uint32_t actual_mapped_len = s_mapping_operation(target, vaddr_start, paddr_start, size); cache_bus_mask_t bus_mask = cache_ll_l1_get_bus(0, vaddr_start, size); cache_ll_l1_enable_bus(0, bus_mask); #if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE bus_mask = cache_ll_l1_get_bus(0, vaddr_start, size); cache_ll_l1_enable_bus(1, bus_mask); #endif s_do_cache_invalidate(vaddr_start, size); //enable Cache, after this function, internal RAM access is no longer mandatory spi_flash_enable_interrupts_caches_and_other_cpu(); ESP_EARLY_LOGV(TAG, "actual_mapped_len is 0x%"PRIx32, actual_mapped_len); } esp_err_t esp_mmu_map(esp_paddr_t paddr_start, size_t size, mmu_target_t target, mmu_mem_caps_t caps, int flags, void **out_ptr) { esp_err_t ret = ESP_FAIL; ESP_RETURN_ON_FALSE(out_ptr, ESP_ERR_INVALID_ARG, TAG, "null pointer"); #if !SOC_SPIRAM_SUPPORTED || CONFIG_IDF_TARGET_ESP32 ESP_RETURN_ON_FALSE(!(target & MMU_TARGET_PSRAM0), ESP_ERR_NOT_SUPPORTED, TAG, "PSRAM is not supported"); #endif ESP_RETURN_ON_FALSE((paddr_start % CONFIG_MMU_PAGE_SIZE == 0), ESP_ERR_INVALID_ARG, TAG, "paddr must be rounded up to the nearest multiple of CONFIG_MMU_PAGE_SIZE"); ESP_RETURN_ON_ERROR(s_mem_caps_check(caps), TAG, "invalid caps"); size_t aligned_size = ALIGN_UP_BY(size, CONFIG_MMU_PAGE_SIZE); int32_t found_region_id = s_find_available_region(s_mmu_ctx.mem_regions, s_mmu_ctx.num_regions, aligned_size, caps, target); if (found_region_id == -1) { ESP_EARLY_LOGE(TAG, "no such vaddr range"); return ESP_ERR_NOT_FOUND; } //Now we're sure we can find an available block inside a certain region mem_region_t *found_region = &s_mmu_ctx.mem_regions[found_region_id]; mem_block_t *dummy_head = NULL; mem_block_t *dummy_tail = NULL; mem_block_t *new_block = NULL; if (TAILQ_EMPTY(&found_region->mem_block_head)) { dummy_head = (mem_block_t *)heap_caps_calloc(1, sizeof(mem_block_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); ESP_GOTO_ON_FALSE(dummy_head, ESP_ERR_NO_MEM, err, TAG, "no mem"); dummy_head->laddr_start = found_region->free_head; dummy_head->laddr_end = found_region->free_head; //We don't care vaddr or paddr address for dummy head dummy_head->size = 0; dummy_head->caps = caps; TAILQ_INSERT_HEAD(&found_region->mem_block_head, dummy_head, entries); dummy_tail = (mem_block_t *)heap_caps_calloc(1, sizeof(mem_block_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); ESP_GOTO_ON_FALSE(dummy_tail, ESP_ERR_NO_MEM, err, TAG, "no mem"); dummy_tail->laddr_start = found_region->end; dummy_tail->laddr_end = found_region->end; //We don't care vaddr or paddr address for dummy tail dummy_tail->size = 0; dummy_tail->caps = caps; TAILQ_INSERT_TAIL(&found_region->mem_block_head, dummy_tail, entries); } //Check if paddr is overlapped mem_block_t *mem_block = NULL; #if ENABLE_PADDR_CHECK bool is_enclosed = false; bool is_overlapped = false; bool allow_overlap = flags & ESP_MMU_MMAP_FLAG_PADDR_SHARED; TAILQ_FOREACH(mem_block, &found_region->mem_block_head, entries) { if (target == mem_block->target) { if ((s_is_enclosed(mem_block->paddr_start, mem_block->paddr_end, paddr_start, aligned_size))) { //the to-be-mapped paddr block is mapped already is_enclosed = true; break; } if (!allow_overlap && (s_is_overlapped(mem_block->paddr_start, mem_block->paddr_end, paddr_start, aligned_size))) { is_overlapped = true; break; } } } if (is_enclosed) { ESP_LOGW(TAG, "paddr block is mapped already, vaddr_start: %p, size: 0x%x", (void *)mem_block->vaddr_start, mem_block->size); *out_ptr = (void *)mem_block->vaddr_start; return ESP_ERR_INVALID_STATE; } if (!allow_overlap && is_overlapped) { ESP_LOGE(TAG, "paddr block is overlapped with an already mapped paddr block"); return ESP_ERR_INVALID_ARG; } #endif //#if ENABLE_PADDR_CHECK new_block = (mem_block_t *)heap_caps_calloc(1, sizeof(mem_block_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); ESP_GOTO_ON_FALSE(new_block, ESP_ERR_NO_MEM, err, TAG, "no mem"); //Reserve this block as it'll be mapped bool found = false; // Get the end address of the dummy_head block, which is always first block on the list uint32_t last_end = TAILQ_FIRST(&found_region->mem_block_head)->laddr_end; size_t slot_len = 0; size_t max_slot_len = 0; mem_block_t *found_block = NULL; //This stands for the block we found, whose slot between its prior block is where we will insert the new block to TAILQ_FOREACH(mem_block, &found_region->mem_block_head, entries) { slot_len = mem_block->laddr_start - last_end; if (!found) { if (slot_len >= aligned_size) { //Found it found = true; found_block = mem_block; slot_len -= aligned_size; new_block->laddr_start = last_end; } } max_slot_len = (slot_len > max_slot_len) ? slot_len : max_slot_len; last_end = mem_block->laddr_end; } assert(found); //insert the to-be-mapped new block to the list TAILQ_INSERT_BEFORE(found_block, new_block, entries); //Finally, we update the max_slot_size found_region->max_slot_size = max_slot_len; //Now we fill others according to the found `new_block->laddr_start` new_block->laddr_end = new_block->laddr_start + aligned_size; new_block->size = aligned_size; new_block->caps = caps; new_block->paddr_start = paddr_start; new_block->paddr_end = paddr_start + aligned_size; new_block->target = target; if (caps & MMU_MEM_CAP_EXEC) { new_block->vaddr_start = mmu_ll_laddr_to_vaddr(new_block->laddr_start, MMU_VADDR_INSTRUCTION, target); new_block->vaddr_end = mmu_ll_laddr_to_vaddr(new_block->laddr_end, MMU_VADDR_INSTRUCTION, target); } else { new_block->vaddr_start = mmu_ll_laddr_to_vaddr(new_block->laddr_start, MMU_VADDR_DATA, target); new_block->vaddr_end = mmu_ll_laddr_to_vaddr(new_block->laddr_end, MMU_VADDR_DATA, target); } //do mapping s_do_mapping(target, new_block->vaddr_start, paddr_start, aligned_size); *out_ptr = (void *)new_block->vaddr_start; return ESP_OK; err: if (dummy_tail) { free(dummy_tail); } if (dummy_head) { free(dummy_head); } return ret; } #if SOC_MMU_PER_EXT_MEM_TARGET FORCE_INLINE_ATTR void s_unmapping_operation(uint32_t vaddr_start, uint32_t size) { mmu_target_t target = mmu_ll_vaddr_to_target(vaddr_start); uint32_t mmu_id = s_get_mmu_id_from_target(target); mmu_hal_unmap_region(mmu_id, vaddr_start, size); } #else FORCE_INLINE_ATTR void s_unmapping_operation(uint32_t vaddr_start, uint32_t size) { mmu_hal_unmap_region(0, vaddr_start, size); #if (SOC_MMU_PERIPH_NUM == 2) #if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE mmu_hal_unmap_region(1, vaddr_start, size); #endif // #if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE #endif // #if (SOC_MMU_PERIPH_NUM == 2) } #endif static void IRAM_ATTR NOINLINE_ATTR s_do_unmapping(uint32_t vaddr_start, uint32_t size) { /** * Disable Cache, after this function, involved code and data should be placed in internal RAM. * * @note we call this for now, but this will be refactored to move out of `spi_flash` */ spi_flash_disable_interrupts_caches_and_other_cpu(); s_unmapping_operation(vaddr_start, size); //enable Cache, after this function, internal RAM access is no longer mandatory spi_flash_enable_interrupts_caches_and_other_cpu(); } esp_err_t esp_mmu_unmap(void *ptr) { ESP_RETURN_ON_FALSE(ptr, ESP_ERR_INVALID_ARG, TAG, "null pointer"); mem_region_t *region = NULL; mem_block_t *mem_block = NULL; uint32_t ptr_laddr = mmu_ll_vaddr_to_laddr((uint32_t)ptr); size_t slot_len = 0; for (int i = 0; i < s_mmu_ctx.num_regions; i++) { if (ptr_laddr >= s_mmu_ctx.mem_regions[i].free_head && ptr_laddr < s_mmu_ctx.mem_regions[i].end) { region = &s_mmu_ctx.mem_regions[i]; } } ESP_RETURN_ON_FALSE(region, ESP_ERR_NOT_FOUND, TAG, "munmap target pointer is outside external memory regions"); bool found = false; mem_block_t *found_block = NULL; TAILQ_FOREACH(mem_block, ®ion->mem_block_head, entries) { if (mem_block == TAILQ_FIRST(®ion->mem_block_head) || mem_block == TAILQ_LAST(®ion->mem_block_head, mem_block_head_)) { //we don't care the dummy_head and the dummy_tail continue; } //now we are only traversing the actual dynamically allocated blocks, dummy_head and dummy_tail are excluded already if (mem_block->laddr_start == ptr_laddr) { slot_len = TAILQ_NEXT(mem_block, entries)->laddr_start - TAILQ_PREV(mem_block, mem_block_head_, entries)->laddr_end; region->max_slot_size = (slot_len > region->max_slot_size) ? slot_len : region->max_slot_size; found = true; found_block = mem_block; break; } } ESP_RETURN_ON_FALSE(found, ESP_ERR_NOT_FOUND, TAG, "munmap target pointer isn't mapped yet"); //do unmap s_do_unmapping(mem_block->vaddr_start, mem_block->size); //remove the already unmapped block from the list TAILQ_REMOVE(®ion->mem_block_head, found_block, entries); free(found_block); return ESP_OK; } esp_err_t esp_mmu_map_dump_mapped_blocks(FILE* stream) { char line[100]; for (int i = 0; i < s_mmu_ctx.num_regions; i++) { fprintf(stream, "region %d:\n", i); fprintf(stream, "%-15s %-14s %-14s %-12s %-12s %-12s\n", "Bus ID", "Start", "Free Head", "End", "Caps", "Max Slot Size"); char *buf = line; size_t len = sizeof(line); memset(line, 0x0, len); snprintf(buf, len, "0x%-13x 0x%-12"PRIx32" 0x%-11"PRIx32" 0x%-10"PRIx32" 0x%-10x 0x%-8x\n", s_mmu_ctx.mem_regions[i].bus_id, s_mmu_ctx.mem_regions[i].start, s_mmu_ctx.mem_regions[i].free_head, s_mmu_ctx.mem_regions[i].end, s_mmu_ctx.mem_regions[i].caps, s_mmu_ctx.mem_regions[i].max_slot_size); fputs(line, stream); fprintf(stream, "mapped blocks:\n"); fprintf(stream, "%-4s %-13s %-12s %-12s %-6s %-13s %-11s\n", "ID", "Vaddr Start", "Vaddr End", "Block Size", "Caps", "Paddr Start", "Paddr End"); mem_region_t *region = &s_mmu_ctx.mem_regions[i]; mem_block_t *mem_block = NULL; int id = 0; TAILQ_FOREACH(mem_block, ®ion->mem_block_head, entries) { if (mem_block != TAILQ_FIRST(®ion->mem_block_head) && mem_block != TAILQ_LAST(®ion->mem_block_head, mem_block_head_)) { snprintf(buf, len, "%-4d 0x%-11x 0x%-10x 0x%-10x 0x%-4x 0x%-11"PRIx32" 0x%-8"PRIx32"\n", id, mem_block->vaddr_start, mem_block->vaddr_end, mem_block->size, mem_block->caps, mem_block->paddr_start, mem_block->paddr_end); fputs(line, stream); id++; } } fprintf(stream, "\n"); } return ESP_OK; } /*--------------------------------------------------------------- Private dump functions, IRAM Safe ---------------------------------------------------------------*/ esp_err_t IRAM_ATTR esp_mmu_map_dump_mapped_blocks_private(void) { for (int i = 0; i < s_mmu_ctx.num_regions; i++) { mem_region_t *region = &s_mmu_ctx.mem_regions[i]; mem_block_t *mem_block = NULL; TAILQ_FOREACH(mem_block, ®ion->mem_block_head, entries) { if (mem_block != TAILQ_FIRST(®ion->mem_block_head) && mem_block != TAILQ_LAST(®ion->mem_block_head, mem_block_head_)) { ESP_DRAM_LOGI(TAG, "block vaddr_start: 0x%x", mem_block->vaddr_start); ESP_DRAM_LOGI(TAG, "block vaddr_end: 0x%x", mem_block->vaddr_end); ESP_DRAM_LOGI(TAG, "block size: 0x%x", mem_block->size); ESP_DRAM_LOGI(TAG, "block caps: 0x%x", mem_block->caps); ESP_DRAM_LOGI(TAG, "block paddr_start: 0x%x", mem_block->paddr_start); ESP_DRAM_LOGI(TAG, "block paddr_end: 0x%x", mem_block->paddr_end); } } ESP_DRAM_LOGI(TAG, "region bus_id: 0x%x", s_mmu_ctx.mem_regions[i].bus_id); ESP_DRAM_LOGI(TAG, "region start: 0x%x", s_mmu_ctx.mem_regions[i].start); ESP_DRAM_LOGI(TAG, "region end: 0x%x", s_mmu_ctx.mem_regions[i].end); ESP_DRAM_LOGI(TAG, "region caps: 0x%x", s_mmu_ctx.mem_regions[i].caps); } return ESP_OK; } /*--------------------------------------------------------------- Helper APIs for conversion between vaddr and paddr ---------------------------------------------------------------*/ static bool NOINLINE_ATTR IRAM_ATTR s_vaddr_to_paddr(uint32_t vaddr, esp_paddr_t *out_paddr, mmu_target_t *out_target) { //we call this for now, but this will be refactored to move out of `spi_flash` spi_flash_disable_interrupts_caches_and_other_cpu(); bool is_mapped = mmu_hal_vaddr_to_paddr(0, vaddr, out_paddr, out_target); #if SOC_MMU_PER_EXT_MEM_TARGET if (!is_mapped) { is_mapped = mmu_hal_vaddr_to_paddr(1, vaddr, out_paddr, out_target); } #endif spi_flash_enable_interrupts_caches_and_other_cpu(); return is_mapped; } esp_err_t esp_mmu_vaddr_to_paddr(void *vaddr, esp_paddr_t *out_paddr, mmu_target_t *out_target) { ESP_RETURN_ON_FALSE(vaddr && out_paddr, ESP_ERR_INVALID_ARG, TAG, "null pointer"); ESP_RETURN_ON_FALSE(mmu_hal_check_valid_ext_vaddr_region(0, (uint32_t)vaddr, 1, MMU_VADDR_DATA | MMU_VADDR_INSTRUCTION), ESP_ERR_INVALID_ARG, TAG, "not a valid external virtual address"); esp_paddr_t paddr = 0; mmu_target_t target = 0; bool is_mapped = s_vaddr_to_paddr((uint32_t)vaddr, &paddr, &target); ESP_RETURN_ON_FALSE(is_mapped, ESP_ERR_NOT_FOUND, TAG, "vaddr isn't mapped"); *out_paddr = paddr; *out_target = target; return ESP_OK; } static bool NOINLINE_ATTR IRAM_ATTR s_paddr_to_vaddr(esp_paddr_t paddr, mmu_target_t target, mmu_vaddr_t type, uint32_t *out_vaddr) { //we call this for now, but this will be refactored to move out of `spi_flash` spi_flash_disable_interrupts_caches_and_other_cpu(); uint32_t mmu_id = 0; #if SOC_MMU_PER_EXT_MEM_TARGET mmu_id = s_get_mmu_id_from_target(target); #endif bool found = mmu_hal_paddr_to_vaddr(mmu_id, paddr, target, type, out_vaddr); spi_flash_enable_interrupts_caches_and_other_cpu(); return found; } esp_err_t esp_mmu_paddr_to_vaddr(esp_paddr_t paddr, mmu_target_t target, mmu_vaddr_t type, void **out_vaddr) { ESP_RETURN_ON_FALSE(out_vaddr, ESP_ERR_INVALID_ARG, TAG, "null pointer"); uint32_t vaddr = 0; bool found = false; found = s_paddr_to_vaddr(paddr, target, type, &vaddr); ESP_RETURN_ON_FALSE(found, ESP_ERR_NOT_FOUND, TAG, "paddr isn't mapped"); *out_vaddr = (void *)vaddr; return ESP_OK; } #if ENABLE_PADDR_CHECK /*--------------------------------------------------------------- Helper functions to check block ---------------------------------------------------------------*/ /** * Check if a new block is enclosed by another, e.g. * * This is enclosed: * * new_block_start new_block_end * |-------- New Block --------| * |--------------- Block ---------------| * block_start block_end * * @note Note the difference between `s_is_overlapped()` below * * @param block_start An original block start * @param block_end An original block end * @param new_block_start New block start * @param new_block_size New block size * * @return True: new block is enclosed; False: new block is not enclosed */ static bool s_is_enclosed(uint32_t block_start, uint32_t block_end, uint32_t new_block_start, uint32_t new_block_size) { bool is_enclosed = false; uint32_t new_block_end = new_block_start + new_block_size; if ((new_block_start >= block_start) && (new_block_end <= block_end)) { is_enclosed = true; } else { is_enclosed = false; } return is_enclosed; } /** * Check if a new block is overlapped by another, e.g. * * This is overlapped: * * new_block_start new_block_end * |---------- New Block ----------| * |--------------- Block ---------------| * block_start block_end * * @note Note the difference between `s_is_enclosed()` above * * @param block_start An original block start * @param block_end An original block end * @param new_block_start New block start * @param new_block_size New block size * * @return True: new block is overlapped; False: new block is not overlapped */ static bool s_is_overlapped(uint32_t block_start, uint32_t block_end, uint32_t new_block_start, uint32_t new_block_size) { bool is_overlapped = false; uint32_t new_block_end = new_block_start + new_block_size; if (((new_block_start < block_start) && (new_block_end > block_start)) || ((new_block_start < block_end) && (new_block_end > block_end))) { is_overlapped = true; } else { is_overlapped = false; } return is_overlapped; } #endif //#if ENABLE_PADDR_CHECK