Merge 6634fea239 into 49ce7a6075

To get more insight to firmware size changes when code changes.

Signed-off-by: Damien George <damien@micropython.org>

.github/workflows/code_size.yml

@@ -8,9 +8,15 @@ on:
       - 'tools/**'
       - 'py/**'
       - 'extmod/**'
+      - 'shared/**'
       - 'lib/**'
       - 'ports/bare-arm/**'
+      - 'ports/mimxrt/**'
       - 'ports/minimal/**'
+      - 'ports/rp2/**'
+      - 'ports/samd/**'
+      - 'ports/stm32/**'
+      - 'ports/unix/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/ports_unix.yml

@@ -88,10 +88,11 @@ jobs:
         (cd ports/unix && gcov -o build-coverage/py ../../py/*.c || true)
         (cd ports/unix && gcov -o build-coverage/extmod ../../extmod/*.c || true)
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true
         verbose: true
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Print failures
       if: failure()
       run: tests/run-tests.py --print-failures

extmod/btstack/btstack.cmake

@@ -16,45 +16,45 @@ target_include_directories(micropy_extmod_btstack INTERFACE
 
 target_sources(micropy_extmod_btstack INTERFACE
     ${BTSTACK_LIB_DIR}/platform/embedded/hci_dump_embedded_stdout.c
-    ${BTSTACK_LIB_DIR}/src/ad_parser.c
+    #${BTSTACK_LIB_DIR}/src/ad_parser.c
     ${BTSTACK_LIB_DIR}/src/ble/gatt-service/ancs_client.c
     ${BTSTACK_LIB_DIR}/src/ble/att_db.c
     ${BTSTACK_LIB_DIR}/src/ble/att_db_util.c
     ${BTSTACK_LIB_DIR}/src/ble/att_dispatch.c
     ${BTSTACK_LIB_DIR}/src/ble/att_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/battery_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/cycling_power_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/cycling_speed_and_cadence_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/device_information_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/heart_rate_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/hids_device.c
-    ${BTSTACK_LIB_DIR}/src/mesh/gatt-service/mesh_provisioning_service_server.c
-    ${BTSTACK_LIB_DIR}/src/mesh/gatt-service/mesh_proxy_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/nordic_spp_service_server.c
-    ${BTSTACK_LIB_DIR}/src/ble/gatt-service/ublox_spp_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/battery_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/cycling_power_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/cycling_speed_and_cadence_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/device_information_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/heart_rate_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/hids_device.c
+    #${BTSTACK_LIB_DIR}/src/mesh/gatt-service/mesh_provisioning_service_server.c
+    #${BTSTACK_LIB_DIR}/src/mesh/gatt-service/mesh_proxy_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/nordic_spp_service_server.c
+    #${BTSTACK_LIB_DIR}/src/ble/gatt-service/ublox_spp_service_server.c
     ${BTSTACK_LIB_DIR}/src/ble/gatt_client.c
-    ${BTSTACK_LIB_DIR}/src/ble/le_device_db_memory.c
+    ${BTSTACK_LIB_DIR}/src/ble/le_device_db_tlv.c
     ${BTSTACK_LIB_DIR}/src/ble/sm.c
-    ${BTSTACK_LIB_DIR}/src/btstack_audio.c
-    ${BTSTACK_LIB_DIR}/src/btstack_base64_decoder.c
+    #${BTSTACK_LIB_DIR}/src/btstack_audio.c
+    #${BTSTACK_LIB_DIR}/src/btstack_base64_decoder.c
     ${BTSTACK_LIB_DIR}/src/btstack_crypto.c
-    ${BTSTACK_LIB_DIR}/src/btstack_hid_parser.c
+    #${BTSTACK_LIB_DIR}/src/btstack_hid_parser.c
     ${BTSTACK_LIB_DIR}/src/btstack_linked_list.c
     ${BTSTACK_LIB_DIR}/src/btstack_memory.c
     ${BTSTACK_LIB_DIR}/src/btstack_memory_pool.c
-    ${BTSTACK_LIB_DIR}/src/btstack_resample.c
-    ${BTSTACK_LIB_DIR}/src/btstack_ring_buffer.c
+    #${BTSTACK_LIB_DIR}/src/btstack_resample.c
+    #${BTSTACK_LIB_DIR}/src/btstack_ring_buffer.c
     ${BTSTACK_LIB_DIR}/src/btstack_run_loop.c
     ${BTSTACK_LIB_DIR}/src/btstack_run_loop_base.c
-    ${BTSTACK_LIB_DIR}/src/btstack_slip.c
+    #${BTSTACK_LIB_DIR}/src/btstack_slip.c
     ${BTSTACK_LIB_DIR}/src/btstack_tlv.c
-    ${BTSTACK_LIB_DIR}/src/btstack_tlv_none.c
+    #${BTSTACK_LIB_DIR}/src/btstack_tlv_none.c
     ${BTSTACK_LIB_DIR}/src/btstack_util.c
     ${BTSTACK_LIB_DIR}/src/hci.c
     ${BTSTACK_LIB_DIR}/src/hci_cmd.c
     ${BTSTACK_LIB_DIR}/src/hci_dump.c
-    ${BTSTACK_LIB_DIR}/src/hci_transport_em9304_spi.c
-    ${BTSTACK_LIB_DIR}/src/hci_transport_h4.c
+    #${BTSTACK_LIB_DIR}/src/hci_transport_em9304_spi.c
+    #${BTSTACK_LIB_DIR}/src/hci_transport_h4.c
     ${BTSTACK_LIB_DIR}/src/l2cap.c
     ${BTSTACK_LIB_DIR}/src/l2cap_signaling.c
 )

extmod/btstack/btstack.mk

@@ -34,7 +34,7 @@ INC += -I$(BTSTACK_DIR)/3rd-party/yxml
 
 SRC_BTSTACK_C = \
 	$(addprefix lib/btstack/src/, $(SRC_FILES)) \
-	$(addprefix lib/btstack/src/ble/, $(filter-out %_tlv.c, $(SRC_BLE_FILES))) \
+	$(addprefix lib/btstack/src/ble/, $(filter-out le_device_db_memory.c, $(SRC_BLE_FILES))) \
 	lib/btstack/platform/embedded/hci_dump_embedded_stdout.c \
 
 ifeq ($(MICROPY_BLUETOOTH_BTSTACK_USB),1)

extmod/btstack/btstack_config_common.h

@@ -35,7 +35,7 @@
 #define MAX_NR_LE_DEVICE_DB_ENTRIES 4
 
 // Link Key DB and LE Device DB using TLV on top of Flash Sector interface
-// #define NVM_NUM_DEVICE_DB_ENTRIES 16
+#define NVM_NUM_DEVICE_DB_ENTRIES 16
 
 // We don't give btstack a malloc, so use a fixed-size ATT DB.
 #define MAX_ATT_DB_SIZE 512

extmod/btstack/modbluetooth_btstack.c

@@ -34,6 +34,8 @@
 #include "extmod/modbluetooth.h"
 
 #include "lib/btstack/src/btstack.h"
+#include "lib/btstack/src/ble/le_device_db_tlv.h"
+#include "lib/btstack/src/btstack_tlv.h"
 
 #define DEBUG_printf(...) // printf("btstack: " __VA_ARGS__)
 
@@ -300,6 +302,12 @@ static void btstack_packet_handler_generic(uint8_t packet_type, uint16_t channel
             desc->sm_connection_authenticated,
             desc->sm_le_db_index != -1,
             desc->sm_actual_encryption_key_size);
+    } else if (event_type == SM_EVENT_PASSKEY_DISPLAY_NUMBER) {
+        mp_bluetooth_gap_on_passkey_action(sm_event_passkey_display_number_get_handle(packet), MP_BLUETOOTH_PASSKEY_ACTION_DISPLAY, sm_event_passkey_display_number_get_passkey(packet));
+    } else if (event_type == SM_EVENT_PASSKEY_INPUT_NUMBER) {
+        mp_bluetooth_gap_on_passkey_action(sm_event_passkey_input_number_get_handle(packet), MP_BLUETOOTH_PASSKEY_ACTION_INPUT, 0);
+    } else if (event_type == SM_EVENT_NUMERIC_COMPARISON_REQUEST) {
+        mp_bluetooth_gap_on_passkey_action(sm_event_numeric_comparison_request_get_handle(packet), MP_BLUETOOTH_PASSKEY_ACTION_NUMERIC_COMPARISON, sm_event_numeric_comparison_request_get_passkey(packet));
         #endif // MICROPY_PY_BLUETOOTH_ENABLE_PAIRING_BONDING
     } else if (event_type == HCI_EVENT_DISCONNECTION_COMPLETE) {
         DEBUG_printf("  --> hci disconnect complete\n");
@@ -372,7 +380,10 @@ static void btstack_packet_handler_generic(uint8_t packet_type, uint16_t channel
     }
 }
 
-static btstack_packet_callback_registration_t hci_event_callback_registration = {
+static btstack_packet_callback_registration_t mp_hci_event_callback_registration = {
+    .callback = &btstack_packet_handler_generic
+};
+static btstack_packet_callback_registration_t mp_sm_event_callback_registration = {
     .callback = &btstack_packet_handler_generic
 };
 
@@ -590,10 +601,13 @@ static void deinit_stack(void) {
     hci_deinit();
     btstack_memory_deinit();
     btstack_run_loop_deinit();
+    btstack_crypto_deinit();
 
     MP_STATE_PORT(bluetooth_btstack_root_pointers) = NULL;
 }
 
+static const btstack_tlv_t btstack_tlv_mp;
+
 int mp_bluetooth_init(void) {
     DEBUG_printf("mp_bluetooth_init\n");
 
@@ -623,6 +637,8 @@ int mp_bluetooth_init(void) {
     mp_bluetooth_btstack_state = MP_BLUETOOTH_BTSTACK_STATE_STARTING;
 
     l2cap_init();
+    btstack_tlv_set_instance(&btstack_tlv_mp, NULL);
+    le_device_db_tlv_configure(&btstack_tlv_mp, NULL);
     le_device_db_init();
     sm_init();
 
@@ -641,7 +657,9 @@ int mp_bluetooth_init(void) {
     #endif // MICROPY_PY_BLUETOOTH_ENABLE_GATT_CLIENT
 
     // Register for HCI events.
-    hci_add_event_handler(&hci_event_callback_registration);
+    hci_add_event_handler(&mp_hci_event_callback_registration);
+
+    sm_add_event_handler(&mp_sm_event_callback_registration);
 
     // Register for ATT server events.
     att_server_register_packet_handler(&btstack_packet_handler_att_server);
@@ -771,10 +789,14 @@ void mp_bluetooth_set_address_mode(uint8_t addr_mode) {
             set_random_address();
             break;
         }
-        case MP_BLUETOOTH_ADDRESS_MODE_RPA:
-        case MP_BLUETOOTH_ADDRESS_MODE_NRPA:
-            // Not yet supported.
-            mp_raise_OSError(MP_EINVAL);
+        case MP_BLUETOOTH_ADDRESS_MODE_RPA: {
+            gap_random_address_set_mode(GAP_RANDOM_ADDRESS_RESOLVABLE);
+            break;
+        }
+        case MP_BLUETOOTH_ADDRESS_MODE_NRPA: {
+            gap_random_address_set_mode(GAP_RANDOM_ADDRESS_NON_RESOLVABLE);
+            break;
+        }
     }
 }
 
@@ -1264,7 +1286,26 @@ int mp_bluetooth_gap_pair(uint16_t conn_handle) {
 
 int mp_bluetooth_gap_passkey(uint16_t conn_handle, uint8_t action, mp_int_t passkey) {
     DEBUG_printf("mp_bluetooth_gap_passkey: conn_handle=%d action=%d passkey=%d\n", conn_handle, action, (int)passkey);
-    return MP_EOPNOTSUPP;
+    switch (action) {
+        case MP_BLUETOOTH_PASSKEY_ACTION_INPUT: {
+            sm_passkey_input(conn_handle, passkey);
+            break;
+        }
+        case MP_BLUETOOTH_PASSKEY_ACTION_DISPLAY: {
+            sm_use_fixed_passkey_in_display_role(passkey);
+            break;
+        }
+        case MP_BLUETOOTH_PASSKEY_ACTION_NUMERIC_COMPARISON: {
+            if (passkey != 0) {
+                sm_numeric_comparison_confirm(conn_handle);
+            }
+            break;
+        }
+        default: {
+            return MP_EINVAL;
+        }
+    }
+    return 0;
 }
 
 #endif // MICROPY_PY_BLUETOOTH_ENABLE_PAIRING_BONDING
@@ -1545,4 +1586,41 @@ int mp_bluetooth_l2cap_recvinto(uint16_t conn_handle, uint16_t cid, uint8_t *buf
 
 MP_REGISTER_ROOT_POINTER(struct _mp_bluetooth_btstack_root_pointers_t *bluetooth_btstack_root_pointers);
 
+#if MICROPY_PY_BLUETOOTH_ENABLE_PAIRING_BONDING
+
+static int btstack_tlv_mp_get_tag(void *context, uint32_t tag, uint8_t *buffer, uint32_t buffer_size) {
+    UNUSED(context);
+    const uint8_t *data;
+    size_t data_len;
+    if (!mp_bluetooth_gap_on_get_secret(0, 0, (uint8_t *)&tag, sizeof(tag), &data, &data_len)) {
+        return -1;
+    }
+    if (data_len > buffer_size) {
+        return -1;
+    }
+    memcpy(buffer, data, data_len);
+    return data_len;
+}
+
+static int btstack_tlv_mp_store_tag(void *context, uint32_t tag, const uint8_t *data, uint32_t data_size) {
+    UNUSED(context);
+    if (mp_bluetooth_gap_on_set_secret(0, (uint8_t *)&tag, sizeof(tag), (uint8_t *)data, data_size)) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+static void btstack_tlv_mp_delete_tag(void *context, uint32_t tag) {
+    mp_bluetooth_gap_on_set_secret(0, (uint8_t *)&tag, sizeof(tag), NULL, 0);
+}
+
+static const btstack_tlv_t btstack_tlv_mp = {
+    &btstack_tlv_mp_get_tag,
+    &btstack_tlv_mp_store_tag,
+    &btstack_tlv_mp_delete_tag,
+};
+
+#endif // MICROPY_PY_BLUETOOTH_ENABLE_PAIRING_BONDING
+
 #endif // MICROPY_PY_BLUETOOTH && MICROPY_BLUETOOTH_BTSTACK

ports/rp2/CMakeLists.txt

@@ -260,6 +260,7 @@ if(MICROPY_PY_BLUETOOTH)
         MICROPY_PY_BLUETOOTH=1
         MICROPY_PY_BLUETOOTH_USE_SYNC_EVENTS=1
         MICROPY_PY_BLUETOOTH_ENABLE_CENTRAL_MODE=1
+        MICROPY_PY_BLUETOOTH_ENABLE_PAIRING_BONDING=1
     )
 endif()
 

ports/rp2/README.md

@@ -37,7 +37,7 @@ You can also build the standard CMake way.  The final firmware is found in
 the top-level of the CMake build directory (`build` by default) and is
 called `firmware.uf2`.
 
-If you are using a different board other than a Rasoberry Pi Pico, then you should
+If you are using a board other than a Raspberry Pi Pico, you should
 pass the board name to the build; e.g. for Raspberry Pi Pico W:
 
     $ make BOARD=RPI_PICO_W submodules

ports/rp2/modmachine.c

@@ -127,10 +127,10 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     const uint32_t xosc_hz = XOSC_MHZ * 1000000;
 
-    uint32_t my_interrupts = save_and_disable_interrupts();
+    uint32_t my_interrupts = mp_thread_begin_atomic_section();
     #if MICROPY_PY_NETWORK_CYW43
     if (cyw43_has_pending && cyw43_poll != NULL) {
-        restore_interrupts(my_interrupts);
+        mp_thread_end_atomic_section(my_interrupts);
         return;
     }
     #endif
@@ -165,8 +165,15 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
     } else {
         uint32_t sleep_en0 = clocks_hw->sleep_en0;
         uint32_t sleep_en1 = clocks_hw->sleep_en1;
+        bool timer3_enabled = irq_is_enabled(3);
+
         clocks_hw->sleep_en0 = CLOCKS_SLEEP_EN0_CLK_RTC_RTC_BITS;
         if (use_timer_alarm) {
+            // Make sure ALARM3/IRQ3 is enabled on _this_ core
+            timer_hw->inte |= 1 << 3;
+            if (!timer3_enabled) {
+                irq_set_enabled(3, true);
+            }
             // Use timer alarm to wake.
             clocks_hw->sleep_en1 = CLOCKS_SLEEP_EN1_CLK_SYS_TIMER_BITS;
             timer_hw->alarm[3] = timer_hw->timerawl + delay_ms * 1000;
@@ -177,6 +184,9 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
         scb_hw->scr |= M0PLUS_SCR_SLEEPDEEP_BITS;
         __wfi();
         scb_hw->scr &= ~M0PLUS_SCR_SLEEPDEEP_BITS;
+        if (!timer3_enabled) {
+            irq_set_enabled(3, false);
+        }
         clocks_hw->sleep_en0 = sleep_en0;
         clocks_hw->sleep_en1 = sleep_en1;
     }
@@ -186,7 +196,7 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     // Bring back all clocks.
     clocks_init();
-    restore_interrupts(my_interrupts);
+    mp_thread_end_atomic_section(my_interrupts);
 }
 
 NORETURN static void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args) {

py/obj.h

@@ -753,20 +753,20 @@ typedef struct _mp_obj_full_type_t {
 // Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE.
 // Generated with:
 // for i in range(13):
-//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .name = _name, .flags = _flags{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slots = { v1, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
+//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
 
 // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take
 // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking

py/objarray.c

@@ -424,6 +424,13 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) {
     if (self->free < len) {
         self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz);
         self->free = 0;
+
+        if (self_in == arg_in) {
+            // Get arg_bufinfo again in case self->items has moved
+            //
+            // (Note not possible to handle case that arg_in is a memoryview into self)
+            mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ);
+        }
     } else {
         self->free -= len;
     }
@@ -456,7 +463,8 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                 #if MICROPY_PY_ARRAY_SLICE_ASSIGN
                 // Assign
                 size_t src_len;
-                void *src_items;
+                uint8_t *src_items;
+                size_t src_offs = 0;
                 size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL);
                 if (mp_obj_is_obj(value) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(value))->type, subscr) == array_subscr) {
                     // value is array, bytearray or memoryview
@@ -469,7 +477,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                     src_items = src_slice->items;
                     #if MICROPY_PY_BUILTINS_MEMORYVIEW
                     if (mp_obj_is_type(value, &mp_type_memoryview)) {
-                        src_items = (uint8_t *)src_items + (src_slice->memview_offset * item_sz);
+                        src_offs = src_slice->memview_offset * item_sz;
                     }
                     #endif
                 } else if (mp_obj_is_type(value, &mp_type_bytes)) {
@@ -504,13 +512,17 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                         // TODO: alloc policy; at the moment we go conservative
                         o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz);
                         o->free = len_adj;
+                        // m_renew may have moved o->items
+                        if (src_items == dest_items) {
+                            src_items = o->items;
+                        }
                         dest_items = o->items;
                     }
                     mp_seq_replace_slice_grow_inplace(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, len_adj, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, len_adj, item_sz);
                 } else {
                     mp_seq_replace_slice_no_grow(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, item_sz);
                     // Clear "freed" elements at the end of list
                     // TODO: This is actually only needed for typecode=='O'
                     mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz);

py/objfun.h

@@ -56,14 +56,14 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
 #if MICROPY_EMIT_NATIVE
 
 static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
-    mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
+    mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
     o->base.type = &mp_type_fun_native;
     return MP_OBJ_FROM_PTR(o);
 }
 
 static inline mp_obj_t mp_obj_new_fun_viper(const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
     mp_obj_fun_bc_t *o = mp_obj_malloc(mp_obj_fun_bc_t, &mp_type_fun_viper);
-    o->bytecode = fun_data;
+    o->bytecode = (const byte *)fun_data;
     o->context = mc;
     o->child_table = child_table;
     return MP_OBJ_FROM_PTR(o);
@@ -101,9 +101,9 @@ static inline void *mp_obj_fun_native_get_generator_resume(const mp_obj_fun_bc_t
 
 #if MICROPY_EMIT_INLINE_ASM
 static inline mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) {
-    mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
+    mp_obj_fun_asm_t *o = (mp_obj_fun_asm_t *)mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
     o->n_args = n_args;
-    o->fun_data = fun_data;
+    o->fun_data = (const byte *)fun_data;
     o->type_sig = type_sig;
     return MP_OBJ_FROM_PTR(o);
 }

tests/basics/bytearray_add.py

@@ -15,4 +15,11 @@ print(b)
 
 # this inplace add tests the code when the buffer doesn't need to be increased
 b = bytearray()
-b += b''
+b += b""
+
+# extend a bytearray from itself
+b = bytearray(b"abcdefgh")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation, as above
+    b.extend(b)
+print(b)

tests/basics/bytearray_add_self.py

@@ -0,0 +1,8 @@
+# add a bytearray to itself
+# This is not supported by CPython as of 3.11.18.
+
+b = bytearray(b"123456789")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation increases chance 'b' has to relocate
+    b += b
+print(b)

tests/basics/bytearray_add_self.py.exp

@@ -0,0 +1 @@
+bytearray(b'123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789')

tests/basics/bytearray_slice_assign.py

@@ -18,7 +18,7 @@ l = bytearray(x)
 l[1:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[1:3]
+# del l[1:3]
 print(l)
 
 l = bytearray(x)
@@ -28,7 +28,7 @@ l = bytearray(x)
 l[:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:3]
+# del l[:3]
 print(l)
 
 l = bytearray(x)
@@ -38,7 +38,7 @@ l = bytearray(x)
 l[:-3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:-3]
+# del l[:-3]
 print(l)
 
 # slice assignment that extends the array
@@ -61,8 +61,14 @@ b[1:1] = b"12345"
 print(b)
 
 # Growth of bytearray via slice extension
-b = bytearray(b'12345678')
-b.append(57) # expand and add a bit of unused space at end of the bytearray
+b = bytearray(b"12345678")
+b.append(57)  # expand and add a bit of unused space at end of the bytearray
 for i in range(400):
-    b[-1:] = b'ab' # grow slowly into the unused space
+    b[-1:] = b"ab"  # grow slowly into the unused space
+print(len(b), b)
+
+# Growth of bytearray via slice extension from itself
+b = bytearray(b"1234567")
+for i in range(3):
+    b[-1:] = b
 print(len(b), b)

tests/cpydiff/types_memoryview_invalid.py

@@ -0,0 +1,12 @@
+"""
+categories: Types,memoryview
+description: memoryview can become invalid if its target is resized
+cause: CPython prevents a ``bytearray`` or ``io.bytesIO`` object from changing size while there is a ``memoryview`` object that references it. MicroPython requires the programmer to manually ensure that an object is not resized while any ``memoryview`` references it.
+
+In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime.
+workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it.
+"""
+b = bytearray(b"abcdefg")
+m = memoryview(b)
+b.extend(b"hijklmnop")
+print(b, bytes(m))