Merge e2d9925b9a into 49ce7a6075

To get more insight to firmware size changes when code changes.

Signed-off-by: Damien George <damien@micropython.org>

.github/workflows/code_size.yml

@@ -8,9 +8,15 @@ on:
       - 'tools/**'
       - 'py/**'
       - 'extmod/**'
+      - 'shared/**'
       - 'lib/**'
       - 'ports/bare-arm/**'
+      - 'ports/mimxrt/**'
       - 'ports/minimal/**'
+      - 'ports/rp2/**'
+      - 'ports/samd/**'
+      - 'ports/stm32/**'
+      - 'ports/unix/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/ports_unix.yml

@@ -88,10 +88,11 @@ jobs:
         (cd ports/unix && gcov -o build-coverage/py ../../py/*.c || true)
         (cd ports/unix && gcov -o build-coverage/extmod ../../extmod/*.c || true)
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true
         verbose: true
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Print failures
       if: failure()
       run: tests/run-tests.py --print-failures

docs/library/machine.I2S.rst

@@ -82,7 +82,7 @@ other things.  For these drivers see:
 Constructor
 -----------
 
-.. class:: I2S(id, *, sck, ws, sd, mck=None, mode, bits, format, rate, ibuf)
+.. class:: I2S(id, *, sck, ws=None, sd, mck=None, mode, bits, format, rate, ibuf)
 
    Construct an I2S object of the given id:
 
@@ -91,7 +91,8 @@ Constructor
    Keyword-only parameters that are supported on all ports:
 
      - ``sck`` is a pin object for the serial clock line
-     - ``ws`` is a pin object for the word select line
+     - ``ws`` is a pin object for the word select line;
+       can be left unset if using ``I2S.PDM_RX`` mode
      - ``sd`` is a pin object for the serial data line
      - ``mck`` is a pin object for the master clock line;
        master clock frequency is sampling rate * 256
@@ -155,6 +156,10 @@ Constants
 
    for initialising the I2S bus ``mode`` to transmit
 
+.. data:: I2S.PDM_RX
+
+   for initialising the I2S bus ``mode`` to receive in PDM mode
+
 .. data:: I2S.STEREO
 
    for initialising the I2S bus ``format`` to stereo

extmod/machine_i2s.c

@@ -344,7 +344,11 @@ static void copy_appbuf_to_ringbuf_non_blocking(machine_i2s_obj_t *self) {
 MP_NOINLINE static void machine_i2s_init_helper(machine_i2s_obj_t *self, size_t n_pos_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     static const mp_arg_t allowed_args[] = {
         { MP_QSTR_sck,      MP_ARG_KW_ONLY | MP_ARG_REQUIRED | MP_ARG_OBJ,   {.u_obj = MP_OBJ_NULL} },
+        #if MICROPY_PY_MACHINE_PDM
+        { MP_QSTR_ws,       MP_ARG_KW_ONLY | MP_ARG_OBJ,   {.u_obj = mp_const_none} },
+        #else
         { MP_QSTR_ws,       MP_ARG_KW_ONLY | MP_ARG_REQUIRED | MP_ARG_OBJ,   {.u_obj = MP_OBJ_NULL} },
+        #endif
         { MP_QSTR_sd,       MP_ARG_KW_ONLY | MP_ARG_REQUIRED | MP_ARG_OBJ,   {.u_obj = MP_OBJ_NULL} },
         #if MICROPY_PY_MACHINE_I2S_MCK
         { MP_QSTR_mck,      MP_ARG_KW_ONLY | MP_ARG_OBJ,   {.u_obj = mp_const_none} },
@@ -517,6 +521,9 @@ static const mp_rom_map_elem_t machine_i2s_locals_dict_table[] = {
     // Constants
     { MP_ROM_QSTR(MP_QSTR_RX),              MP_ROM_INT(MICROPY_PY_MACHINE_I2S_CONSTANT_RX) },
     { MP_ROM_QSTR(MP_QSTR_TX),              MP_ROM_INT(MICROPY_PY_MACHINE_I2S_CONSTANT_TX) },
+    #if MICROPY_PY_MACHINE_PDM
+    { MP_ROM_QSTR(MP_QSTR_PDM_RX),          MP_ROM_INT(MICROPY_PY_MACHINE_I2S_PDM_RX) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_STEREO),          MP_ROM_INT(STEREO) },
     { MP_ROM_QSTR(MP_QSTR_MONO),            MP_ROM_INT(MONO) },
 };
@@ -525,10 +532,18 @@ MP_DEFINE_CONST_DICT(machine_i2s_locals_dict, machine_i2s_locals_dict_table);
 static mp_uint_t machine_i2s_stream_read(mp_obj_t self_in, void *buf_in, mp_uint_t size, int *errcode) {
     machine_i2s_obj_t *self = MP_OBJ_TO_PTR(self_in);
 
+    #if MICROPY_PY_MACHINE_PDM
+    if ((self->mode != MICROPY_PY_MACHINE_I2S_CONSTANT_RX) &&
+        (self->mode != MICROPY_PY_MACHINE_I2S_PDM_RX)) {
+        *errcode = MP_EPERM;
+        return MP_STREAM_ERROR;
+    }
+    #else
     if (self->mode != MICROPY_PY_MACHINE_I2S_CONSTANT_RX) {
         *errcode = MP_EPERM;
         return MP_STREAM_ERROR;
     }
+    #endif
 
     uint8_t appbuf_sample_size_in_bytes = (self->bits / 8) * (self->format == STEREO ? 2: 1);
     if (size % appbuf_sample_size_in_bytes != 0) {
@@ -623,10 +638,18 @@ static mp_uint_t machine_i2s_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_
         ret = 0;
 
         if (flags & MP_STREAM_POLL_RD) {
+            #if MICROPY_PY_MACHINE_PDM
+            if ((self->mode != MICROPY_PY_MACHINE_I2S_CONSTANT_RX) &&
+                (self->mode != MICROPY_PY_MACHINE_I2S_PDM_RX)) {
+                *errcode = MP_EPERM;
+                return MP_STREAM_ERROR;
+            }
+            #else
             if (self->mode != MICROPY_PY_MACHINE_I2S_CONSTANT_RX) {
                 *errcode = MP_EPERM;
                 return MP_STREAM_ERROR;
             }
+            #endif
 
             #if MICROPY_PY_MACHINE_I2S_RING_BUF
             if (!ringbuf_is_empty(&self->ring_buffer)) {

ports/esp32/machine_i2s.c

@@ -29,6 +29,9 @@
 
 #include "py/mphal.h"
 #include "driver/i2s_std.h"
+#if MICROPY_PY_MACHINE_PDM
+#include "driver/i2s_pdm.h"
+#endif
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 #include "freertos/queue.h"
@@ -125,6 +128,9 @@ static int8_t get_frame_mapping_index(i2s_data_bit_width_t bits, format_t format
 static i2s_data_bit_width_t get_dma_bits(uint8_t mode, i2s_data_bit_width_t bits) {
     if (mode == MICROPY_PY_MACHINE_I2S_CONSTANT_TX) {
         return bits;
+    } else if (mode == MICROPY_PY_MACHINE_I2S_PDM_RX) { // PDM Rx
+        // fixed to 16-bit per ESP-IDF documentation
+        return I2S_DATA_BIT_WIDTH_16BIT;
     } else { // Master Rx
         // read 32 bit samples for I2S hardware.  e.g. MEMS microphones
         return I2S_DATA_BIT_WIDTH_32BIT;
@@ -132,7 +138,8 @@ static i2s_data_bit_width_t get_dma_bits(uint8_t mode, i2s_data_bit_width_t bits
 }
 
 static i2s_slot_mode_t get_dma_format(uint8_t mode, format_t format) {
-    if (mode == MICROPY_PY_MACHINE_I2S_CONSTANT_TX) {
+    if ((mode == MICROPY_PY_MACHINE_I2S_CONSTANT_TX) ||
+        (mode == MICROPY_PY_MACHINE_I2S_PDM_RX)) {
         if (format == MONO) {
             return I2S_SLOT_MODE_MONO;
         } else {  // STEREO
@@ -305,15 +312,20 @@ i2s_event_callbacks_t i2s_callbacks_null = {
 };
 
 static void mp_machine_i2s_init_helper(machine_i2s_obj_t *self, mp_arg_val_t *args) {
+    int8_t mode = args[ARG_mode].u_int;
+    int8_t ws = -1;
+
     // are Pins valid?
     int8_t sck = args[ARG_sck].u_obj == MP_OBJ_NULL ? -1 : machine_pin_get_id(args[ARG_sck].u_obj);
-    int8_t ws = args[ARG_ws].u_obj == MP_OBJ_NULL ? -1 : machine_pin_get_id(args[ARG_ws].u_obj);
+    if (mode != (MICROPY_PY_MACHINE_I2S_PDM_RX)) {
+        ws = args[ARG_ws].u_obj == MP_OBJ_NULL ? -1 : machine_pin_get_id(args[ARG_ws].u_obj);
+    }
     int8_t sd = args[ARG_sd].u_obj == MP_OBJ_NULL ? -1 : machine_pin_get_id(args[ARG_sd].u_obj);
 
     // is Mode valid?
-    int8_t mode = args[ARG_mode].u_int;
     if ((mode != (MICROPY_PY_MACHINE_I2S_CONSTANT_RX)) &&
-        (mode != (MICROPY_PY_MACHINE_I2S_CONSTANT_TX))) {
+        (mode != (MICROPY_PY_MACHINE_I2S_CONSTANT_TX)) &&
+        (mode != (MICROPY_PY_MACHINE_I2S_PDM_RX))) {
         mp_raise_ValueError(MP_ERROR_TEXT("invalid mode"));
     }
 
@@ -368,33 +380,64 @@ static void mp_machine_i2s_init_helper(machine_i2s_obj_t *self, mp_arg_val_t *ar
         ESP_ERROR_CHECK(i2s_new_channel(&chan_config, NULL, &self->i2s_chan_handle));
     }
 
-    i2s_std_slot_config_t slot_cfg = I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG(get_dma_bits(mode, bits), get_dma_format(mode, format));
-    slot_cfg.slot_mask = I2S_STD_SLOT_BOTH;
+    if (mode != MICROPY_PY_MACHINE_I2S_PDM_RX) {
+        i2s_std_slot_config_t slot_cfg = I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG(get_dma_bits(mode, bits), get_dma_format(mode, format));
+        slot_cfg.slot_mask = I2S_STD_SLOT_BOTH;
 
-    i2s_std_config_t std_cfg = {
-        .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(self->rate),
-        .slot_cfg = slot_cfg,
-        .gpio_cfg = {
-            .mclk = I2S_GPIO_UNUSED,
-            .bclk = self->sck,
-            .ws = self->ws,
-            .invert_flags = {
-                .mclk_inv = false,
-                .bclk_inv = false,
-                .ws_inv = false,
+        i2s_std_config_t std_cfg = {
+            .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(self->rate),
+            .slot_cfg = slot_cfg,
+            .gpio_cfg = {
+                .mclk = I2S_GPIO_UNUSED,
+                .bclk = self->sck,
+                .ws = self->ws,
+                .invert_flags = {
+                    .mclk_inv = false,
+                    .bclk_inv = false,
+                    .ws_inv = false,
+                },
             },
-        },
-    };
+        };
 
-    if (mode == MICROPY_PY_MACHINE_I2S_CONSTANT_TX) {
-        std_cfg.gpio_cfg.dout = self->sd;
-        std_cfg.gpio_cfg.din = I2S_GPIO_UNUSED;
-    } else { // rx
-        std_cfg.gpio_cfg.dout = I2S_GPIO_UNUSED;
-        std_cfg.gpio_cfg.din = self->sd;
+        if (mode == MICROPY_PY_MACHINE_I2S_CONSTANT_TX) {
+            std_cfg.gpio_cfg.dout = self->sd;
+            std_cfg.gpio_cfg.din = I2S_GPIO_UNUSED;
+        } else { // rx
+            std_cfg.gpio_cfg.dout = I2S_GPIO_UNUSED;
+            std_cfg.gpio_cfg.din = self->sd;
+        }
+
+        ESP_ERROR_CHECK(i2s_channel_init_std_mode(self->i2s_chan_handle, &std_cfg));
+    } else { // PDM mode
+        #if MICROPY_PY_MACHINE_PDM
+        // PDM can only be in id 0
+        if (self->i2s_id != 0) {
+            mp_raise_ValueError(MP_ERROR_TEXT("invalid i2s id for PDM mode"));
+        }
+
+        i2s_pdm_rx_slot_config_t slot_cfg = I2S_PDM_RX_SLOT_DEFAULT_CONFIG(get_dma_bits(mode, bits), get_dma_format(mode, format));
+
+        slot_cfg.slot_mask = I2S_PDM_SLOT_BOTH;
+
+        i2s_pdm_rx_config_t pdm_cfg = {
+            .clk_cfg = I2S_PDM_RX_CLK_DEFAULT_CONFIG(self->rate),
+            .slot_cfg = slot_cfg,
+            .gpio_cfg = {
+                .clk = self->sck,
+                .din = self->sd,
+                .invert_flags = {
+                    .clk_inv = false,
+                },
+            },
+        };
+        pdm_cfg.clk_cfg.dn_sample_mode = I2S_PDM_DSR_MAX;
+
+        ESP_ERROR_CHECK(i2s_channel_init_pdm_rx_mode(self->i2s_chan_handle, &pdm_cfg));
+        #else
+        mp_raise_ValueError(MP_ERROR_TEXT("invalid mode"));
+        #endif
     }
 
-    ESP_ERROR_CHECK(i2s_channel_init_std_mode(self->i2s_chan_handle, &std_cfg));
     ESP_ERROR_CHECK(i2s_channel_register_event_callback(self->i2s_chan_handle, &i2s_callbacks, self));
     ESP_ERROR_CHECK(i2s_channel_enable(self->i2s_chan_handle));
 }

ports/esp32/mpconfigport.h

@@ -149,6 +149,10 @@
 #define MICROPY_PY_MACHINE_I2S_FINALISER    (1)
 #define MICROPY_PY_MACHINE_I2S_CONSTANT_RX  (I2S_DIR_RX)
 #define MICROPY_PY_MACHINE_I2S_CONSTANT_TX  (I2S_DIR_TX)
+#ifndef MICROPY_PY_MACHINE_PDM
+#define MICROPY_PY_MACHINE_PDM              (SOC_I2S_SUPPORTS_PDM_RX)
+#endif
+#define MICROPY_PY_MACHINE_I2S_PDM_RX       ((I2S_DIR_RX) | (BIT(2)))
 #define MICROPY_PY_MACHINE_UART             (1)
 #define MICROPY_PY_MACHINE_UART_INCLUDEFILE "ports/esp32/machine_uart.c"
 #define MICROPY_PY_MACHINE_UART_SENDBREAK   (1)

ports/rp2/README.md

@@ -37,7 +37,7 @@ You can also build the standard CMake way.  The final firmware is found in
 the top-level of the CMake build directory (`build` by default) and is
 called `firmware.uf2`.
 
-If you are using a different board other than a Rasoberry Pi Pico, then you should
+If you are using a board other than a Raspberry Pi Pico, you should
 pass the board name to the build; e.g. for Raspberry Pi Pico W:
 
     $ make BOARD=RPI_PICO_W submodules

ports/rp2/modmachine.c

@@ -127,10 +127,10 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     const uint32_t xosc_hz = XOSC_MHZ * 1000000;
 
-    uint32_t my_interrupts = save_and_disable_interrupts();
+    uint32_t my_interrupts = mp_thread_begin_atomic_section();
     #if MICROPY_PY_NETWORK_CYW43
     if (cyw43_has_pending && cyw43_poll != NULL) {
-        restore_interrupts(my_interrupts);
+        mp_thread_end_atomic_section(my_interrupts);
         return;
     }
     #endif
@@ -165,8 +165,15 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
     } else {
         uint32_t sleep_en0 = clocks_hw->sleep_en0;
         uint32_t sleep_en1 = clocks_hw->sleep_en1;
+        bool timer3_enabled = irq_is_enabled(3);
+
         clocks_hw->sleep_en0 = CLOCKS_SLEEP_EN0_CLK_RTC_RTC_BITS;
         if (use_timer_alarm) {
+            // Make sure ALARM3/IRQ3 is enabled on _this_ core
+            timer_hw->inte |= 1 << 3;
+            if (!timer3_enabled) {
+                irq_set_enabled(3, true);
+            }
             // Use timer alarm to wake.
             clocks_hw->sleep_en1 = CLOCKS_SLEEP_EN1_CLK_SYS_TIMER_BITS;
             timer_hw->alarm[3] = timer_hw->timerawl + delay_ms * 1000;
@@ -177,6 +184,9 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
         scb_hw->scr |= M0PLUS_SCR_SLEEPDEEP_BITS;
         __wfi();
         scb_hw->scr &= ~M0PLUS_SCR_SLEEPDEEP_BITS;
+        if (!timer3_enabled) {
+            irq_set_enabled(3, false);
+        }
         clocks_hw->sleep_en0 = sleep_en0;
         clocks_hw->sleep_en1 = sleep_en1;
     }
@@ -186,7 +196,7 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     // Bring back all clocks.
     clocks_init();
-    restore_interrupts(my_interrupts);
+    mp_thread_end_atomic_section(my_interrupts);
 }
 
 NORETURN static void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args) {

py/obj.h

@@ -753,20 +753,20 @@ typedef struct _mp_obj_full_type_t {
 // Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE.
 // Generated with:
 // for i in range(13):
-//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .name = _name, .flags = _flags{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slots = { v1, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
+//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
 
 // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take
 // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking

py/objarray.c

@@ -424,6 +424,13 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) {
     if (self->free < len) {
         self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz);
         self->free = 0;
+
+        if (self_in == arg_in) {
+            // Get arg_bufinfo again in case self->items has moved
+            //
+            // (Note not possible to handle case that arg_in is a memoryview into self)
+            mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ);
+        }
     } else {
         self->free -= len;
     }
@@ -456,7 +463,8 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                 #if MICROPY_PY_ARRAY_SLICE_ASSIGN
                 // Assign
                 size_t src_len;
-                void *src_items;
+                uint8_t *src_items;
+                size_t src_offs = 0;
                 size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL);
                 if (mp_obj_is_obj(value) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(value))->type, subscr) == array_subscr) {
                     // value is array, bytearray or memoryview
@@ -469,7 +477,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                     src_items = src_slice->items;
                     #if MICROPY_PY_BUILTINS_MEMORYVIEW
                     if (mp_obj_is_type(value, &mp_type_memoryview)) {
-                        src_items = (uint8_t *)src_items + (src_slice->memview_offset * item_sz);
+                        src_offs = src_slice->memview_offset * item_sz;
                     }
                     #endif
                 } else if (mp_obj_is_type(value, &mp_type_bytes)) {
@@ -504,13 +512,17 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                         // TODO: alloc policy; at the moment we go conservative
                         o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz);
                         o->free = len_adj;
+                        // m_renew may have moved o->items
+                        if (src_items == dest_items) {
+                            src_items = o->items;
+                        }
                         dest_items = o->items;
                     }
                     mp_seq_replace_slice_grow_inplace(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, len_adj, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, len_adj, item_sz);
                 } else {
                     mp_seq_replace_slice_no_grow(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, item_sz);
                     // Clear "freed" elements at the end of list
                     // TODO: This is actually only needed for typecode=='O'
                     mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz);

py/objfun.h

@@ -56,14 +56,14 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
 #if MICROPY_EMIT_NATIVE
 
 static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
-    mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
+    mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
     o->base.type = &mp_type_fun_native;
     return MP_OBJ_FROM_PTR(o);
 }
 
 static inline mp_obj_t mp_obj_new_fun_viper(const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
     mp_obj_fun_bc_t *o = mp_obj_malloc(mp_obj_fun_bc_t, &mp_type_fun_viper);
-    o->bytecode = fun_data;
+    o->bytecode = (const byte *)fun_data;
     o->context = mc;
     o->child_table = child_table;
     return MP_OBJ_FROM_PTR(o);
@@ -101,9 +101,9 @@ static inline void *mp_obj_fun_native_get_generator_resume(const mp_obj_fun_bc_t
 
 #if MICROPY_EMIT_INLINE_ASM
 static inline mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) {
-    mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
+    mp_obj_fun_asm_t *o = (mp_obj_fun_asm_t *)mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
     o->n_args = n_args;
-    o->fun_data = fun_data;
+    o->fun_data = (const byte *)fun_data;
     o->type_sig = type_sig;
     return MP_OBJ_FROM_PTR(o);
 }

tests/basics/bytearray_add.py

@@ -15,4 +15,11 @@ print(b)
 
 # this inplace add tests the code when the buffer doesn't need to be increased
 b = bytearray()
-b += b''
+b += b""
+
+# extend a bytearray from itself
+b = bytearray(b"abcdefgh")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation, as above
+    b.extend(b)
+print(b)

tests/basics/bytearray_add_self.py

@@ -0,0 +1,8 @@
+# add a bytearray to itself
+# This is not supported by CPython as of 3.11.18.
+
+b = bytearray(b"123456789")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation increases chance 'b' has to relocate
+    b += b
+print(b)

tests/basics/bytearray_add_self.py.exp

@@ -0,0 +1 @@
+bytearray(b'123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789')

tests/basics/bytearray_slice_assign.py

@@ -18,7 +18,7 @@ l = bytearray(x)
 l[1:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[1:3]
+# del l[1:3]
 print(l)
 
 l = bytearray(x)
@@ -28,7 +28,7 @@ l = bytearray(x)
 l[:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:3]
+# del l[:3]
 print(l)
 
 l = bytearray(x)
@@ -38,7 +38,7 @@ l = bytearray(x)
 l[:-3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:-3]
+# del l[:-3]
 print(l)
 
 # slice assignment that extends the array
@@ -61,8 +61,14 @@ b[1:1] = b"12345"
 print(b)
 
 # Growth of bytearray via slice extension
-b = bytearray(b'12345678')
-b.append(57) # expand and add a bit of unused space at end of the bytearray
+b = bytearray(b"12345678")
+b.append(57)  # expand and add a bit of unused space at end of the bytearray
 for i in range(400):
-    b[-1:] = b'ab' # grow slowly into the unused space
+    b[-1:] = b"ab"  # grow slowly into the unused space
+print(len(b), b)
+
+# Growth of bytearray via slice extension from itself
+b = bytearray(b"1234567")
+for i in range(3):
+    b[-1:] = b
 print(len(b), b)

tests/cpydiff/types_memoryview_invalid.py

@@ -0,0 +1,12 @@
+"""
+categories: Types,memoryview
+description: memoryview can become invalid if its target is resized
+cause: CPython prevents a ``bytearray`` or ``io.bytesIO`` object from changing size while there is a ``memoryview`` object that references it. MicroPython requires the programmer to manually ensure that an object is not resized while any ``memoryview`` references it.
+
+In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime.
+workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it.
+"""
+b = bytearray(b"abcdefg")
+m = memoryview(b)
+b.extend(b"hijklmnop")
+print(b, bytes(m))