Merge fd7bc4f12b into 49ce7a6075

To get more insight to firmware size changes when code changes.

Signed-off-by: Damien George <damien@micropython.org>

.github/workflows/code_size.yml

@@ -8,9 +8,15 @@ on:
       - 'tools/**'
       - 'py/**'
       - 'extmod/**'
+      - 'shared/**'
       - 'lib/**'
       - 'ports/bare-arm/**'
+      - 'ports/mimxrt/**'
       - 'ports/minimal/**'
+      - 'ports/rp2/**'
+      - 'ports/samd/**'
+      - 'ports/stm32/**'
+      - 'ports/unix/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/ports_unix.yml

@@ -88,10 +88,11 @@ jobs:
         (cd ports/unix && gcov -o build-coverage/py ../../py/*.c || true)
         (cd ports/unix && gcov -o build-coverage/extmod ../../extmod/*.c || true)
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true
         verbose: true
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Print failures
       if: failure()
       run: tests/run-tests.py --print-failures

docs/library/builtins.rst

@@ -77,7 +77,7 @@ Functions and types
       In MicroPython, `byteorder` parameter must be positional (this is
       compatible with CPython).
 
-   .. method:: to_bytes(size, byteorder)
+   .. method:: to_bytes(size, byteorder, / signed=False)
 
       In MicroPython, `byteorder` parameter must be positional (this is
       compatible with CPython).

ports/rp2/README.md

@@ -37,7 +37,7 @@ You can also build the standard CMake way.  The final firmware is found in
 the top-level of the CMake build directory (`build` by default) and is
 called `firmware.uf2`.
 
-If you are using a different board other than a Rasoberry Pi Pico, then you should
+If you are using a board other than a Raspberry Pi Pico, you should
 pass the board name to the build; e.g. for Raspberry Pi Pico W:
 
     $ make BOARD=RPI_PICO_W submodules

ports/rp2/modmachine.c

@@ -127,10 +127,10 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     const uint32_t xosc_hz = XOSC_MHZ * 1000000;
 
-    uint32_t my_interrupts = save_and_disable_interrupts();
+    uint32_t my_interrupts = mp_thread_begin_atomic_section();
     #if MICROPY_PY_NETWORK_CYW43
     if (cyw43_has_pending && cyw43_poll != NULL) {
-        restore_interrupts(my_interrupts);
+        mp_thread_end_atomic_section(my_interrupts);
         return;
     }
     #endif
@@ -165,8 +165,15 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
     } else {
         uint32_t sleep_en0 = clocks_hw->sleep_en0;
         uint32_t sleep_en1 = clocks_hw->sleep_en1;
+        bool timer3_enabled = irq_is_enabled(3);
+
         clocks_hw->sleep_en0 = CLOCKS_SLEEP_EN0_CLK_RTC_RTC_BITS;
         if (use_timer_alarm) {
+            // Make sure ALARM3/IRQ3 is enabled on _this_ core
+            timer_hw->inte |= 1 << 3;
+            if (!timer3_enabled) {
+                irq_set_enabled(3, true);
+            }
             // Use timer alarm to wake.
             clocks_hw->sleep_en1 = CLOCKS_SLEEP_EN1_CLK_SYS_TIMER_BITS;
             timer_hw->alarm[3] = timer_hw->timerawl + delay_ms * 1000;
@@ -177,6 +184,9 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
         scb_hw->scr |= M0PLUS_SCR_SLEEPDEEP_BITS;
         __wfi();
         scb_hw->scr &= ~M0PLUS_SCR_SLEEPDEEP_BITS;
+        if (!timer3_enabled) {
+            irq_set_enabled(3, false);
+        }
         clocks_hw->sleep_en0 = sleep_en0;
         clocks_hw->sleep_en1 = sleep_en1;
     }
@@ -186,7 +196,7 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     // Bring back all clocks.
     clocks_init();
-    restore_interrupts(my_interrupts);
+    mp_thread_end_atomic_section(my_interrupts);
 }
 
 NORETURN static void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args) {

ports/unix/modffi.c

@@ -442,7 +442,7 @@ static unsigned long long ffi_get_int_value(mp_obj_t o) {
         return MP_OBJ_SMALL_INT_VALUE(o);
     } else {
         unsigned long long res;
-        mp_obj_int_to_bytes_impl(o, MP_ENDIANNESS_BIG, sizeof(res), (byte *)&res);
+        mp_obj_int_to_bytes_impl(o, MP_ENDIANNESS_BIG, true, sizeof(res), (byte *)&res);
         return res;
     }
 }

py/binary.c

@@ -444,7 +444,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p
         default:
             #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
             if (mp_obj_is_exact_type(val_in, &mp_type_int)) {
-                mp_obj_int_to_bytes_impl(val_in, struct_type == '>', size, p);
+                mp_obj_int_to_bytes_impl(val_in, struct_type == '>', true, size, p);
                 return;
             }
             #endif
@@ -482,7 +482,7 @@ void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_
             #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
             if (mp_obj_is_exact_type(val_in, &mp_type_int)) {
                 size_t size = mp_binary_get_size('@', typecode, NULL);
-                mp_obj_int_to_bytes_impl(val_in, MP_ENDIANNESS_BIG,
+                mp_obj_int_to_bytes_impl(val_in, MP_ENDIANNESS_BIG, true,
                     size, (uint8_t *)p + index * size);
                 return;
             }

py/misc.h

@@ -334,4 +334,45 @@ typedef const char *mp_rom_error_text_t;
 // For now, forward directly to MP_COMPRESSED_ROM_TEXT.
 #define MP_ERROR_TEXT(x) (mp_rom_error_text_t)MP_COMPRESSED_ROM_TEXT(x)
 
+// Macro and inline function to measure the length (in bytes) needed to hold an
+// unambiguous representation of a small (C representable) integer.
+//
+// Implemented inline as there's a lot the compiler can optimise based on the size of INT.
+#define MP_INT_REPR_LEN(INT, IS_SIGNED) mp_int_repr_len_helper(&(INT), sizeof(INT), IS_SIGNED)
+
+static inline int mp_int_repr_len_helper(void *pint, int size, bool is_signed) {
+    int i;
+    byte *b = (byte *)pint;
+    byte ext_byte = 0x00;
+    int result = size;
+    int msb_idx, step;
+
+    #if MP_ENDIANNESS_LITTLE
+    msb_idx = size - 1;
+    step = -1;
+    #else
+    msb_idx = 0;
+    step = 1;
+    #endif
+
+    if (is_signed && (b[msb_idx] & 0x80)) {
+        ext_byte = 0xFF; // Negative number
+    }
+
+    // Count down the number of most significant bytes that don't contain
+    // any significant values (i.e. equal to the extension byte).
+    for (i = msb_idx; i >= 0 && i <= size - 1; i += step) {
+        if (b[i] != ext_byte) {
+            if (is_signed && (b[i] & 0x80) != (ext_byte & 0x80)) {
+                // Add one additional byte to hold the sign bit
+                result++;
+            }
+            break;
+        }
+        result--;
+    }
+
+    return result;
+}
+
 #endif // MICROPY_INCLUDED_PY_MISC_H

py/mpz.c

@@ -1589,7 +1589,7 @@ bool mpz_as_uint_checked(const mpz_t *i, mp_uint_t *value) {
     return true;
 }
 
-void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf) {
+bool mpz_as_bytes(const mpz_t *z, bool big_endian, bool as_signed, size_t len, byte *buf) {
     byte *b = buf;
     if (big_endian) {
         b += len;
@@ -1598,6 +1598,8 @@ void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf) {
     int bits = 0;
     mpz_dbl_dig_t d = 0;
     mpz_dbl_dig_t carry = 1;
+    size_t olen = len; // bytes in output buffer
+    bool ok = true;
     for (size_t zlen = z->len; zlen > 0; --zlen) {
         bits += DIG_SIZE;
         d = (d << DIG_SIZE) | *zdig++;
@@ -1607,28 +1609,32 @@ void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf) {
                 val = (~val & 0xff) + carry;
                 carry = val >> 8;
             }
+
+            if (!olen) {
+                // Buffer is full, only OK if all remaining bytes are zeroes
+                ok = ok && ((byte)val == 0);
+                continue;
+            }
+
             if (big_endian) {
                 *--b = val;
-                if (b == buf) {
-                    return;
-                }
             } else {
                 *b++ = val;
-                if (b == buf + len) {
-                    return;
-                }
             }
+            olen--;
         }
     }
 
-    // fill remainder of buf with zero/sign extension of the integer
-    if (big_endian) {
-        len = b - buf;
+    if (as_signed && olen == 0 && len > 0) {
+        // If output exhausted then ensure there was enough space for the sign bit
+        byte most_sig = big_endian ? buf[0] : buf[len - 1];
+        ok = ok && (bool)(most_sig & 0x80) == (bool)z->neg;
     } else {
-        len = buf + len - b;
-        buf = b;
+        // fill remainder of buf with zero/sign extension of the integer
+        memset(big_endian ? buf : b, z->neg ? 0xff : 0x00, olen);
     }
-    memset(buf, z->neg ? 0xff : 0x00, len);
+
+    return ok;
 }
 
 #if MICROPY_PY_BUILTINS_FLOAT

py/mpz.h

@@ -93,9 +93,9 @@ typedef int8_t mpz_dbl_dig_signed_t;
 typedef struct _mpz_t {
     // Zero has neg=0, len=0.  Negative zero is not allowed.
     size_t neg : 1;
-    size_t fixed_dig : 1;
-    size_t alloc : (8 * sizeof(size_t) - 2);
-    size_t len;
+    size_t fixed_dig : 1; // flag, 'dig' buffer cannot be reallocated
+    size_t alloc : (8 * sizeof(size_t) - 2); // number of entries allocated in 'dig'
+    size_t len; // number of entries used in 'dig'
     mpz_dig_t *dig;
 } mpz_t;
 
@@ -145,7 +145,8 @@ static inline size_t mpz_max_num_bits(const mpz_t *z) {
 mp_int_t mpz_hash(const mpz_t *z);
 bool mpz_as_int_checked(const mpz_t *z, mp_int_t *value);
 bool mpz_as_uint_checked(const mpz_t *z, mp_uint_t *value);
-void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf);
+// Returns true if 'z' fit into 'len' bytes of 'buf' without overflowing, 'buf' is truncated otherwise.
+bool mpz_as_bytes(const mpz_t *z, bool big_endian, bool as_signed, size_t len, byte *buf);
 #if MICROPY_PY_BUILTINS_FLOAT
 mp_float_t mpz_as_float(const mpz_t *z);
 #endif

py/obj.h

@@ -753,20 +753,20 @@ typedef struct _mp_obj_full_type_t {
 // Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE.
 // Generated with:
 // for i in range(13):
-//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .name = _name, .flags = _flags{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slots = { v1, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
+//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
 
 // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take
 // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking

py/objarray.c

@@ -424,6 +424,13 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) {
     if (self->free < len) {
         self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz);
         self->free = 0;
+
+        if (self_in == arg_in) {
+            // Get arg_bufinfo again in case self->items has moved
+            //
+            // (Note not possible to handle case that arg_in is a memoryview into self)
+            mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ);
+        }
     } else {
         self->free -= len;
     }
@@ -456,7 +463,8 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                 #if MICROPY_PY_ARRAY_SLICE_ASSIGN
                 // Assign
                 size_t src_len;
-                void *src_items;
+                uint8_t *src_items;
+                size_t src_offs = 0;
                 size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL);
                 if (mp_obj_is_obj(value) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(value))->type, subscr) == array_subscr) {
                     // value is array, bytearray or memoryview
@@ -469,7 +477,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                     src_items = src_slice->items;
                     #if MICROPY_PY_BUILTINS_MEMORYVIEW
                     if (mp_obj_is_type(value, &mp_type_memoryview)) {
-                        src_items = (uint8_t *)src_items + (src_slice->memview_offset * item_sz);
+                        src_offs = src_slice->memview_offset * item_sz;
                     }
                     #endif
                 } else if (mp_obj_is_type(value, &mp_type_bytes)) {
@@ -504,13 +512,17 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                         // TODO: alloc policy; at the moment we go conservative
                         o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz);
                         o->free = len_adj;
+                        // m_renew may have moved o->items
+                        if (src_items == dest_items) {
+                            src_items = o->items;
+                        }
                         dest_items = o->items;
                     }
                     mp_seq_replace_slice_grow_inplace(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, len_adj, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, len_adj, item_sz);
                 } else {
                     mp_seq_replace_slice_no_grow(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, item_sz);
                     // Clear "freed" elements at the end of list
                     // TODO: This is actually only needed for typecode=='O'
                     mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz);

py/objfun.h

@@ -56,14 +56,14 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
 #if MICROPY_EMIT_NATIVE
 
 static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
-    mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
+    mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
     o->base.type = &mp_type_fun_native;
     return MP_OBJ_FROM_PTR(o);
 }
 
 static inline mp_obj_t mp_obj_new_fun_viper(const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
     mp_obj_fun_bc_t *o = mp_obj_malloc(mp_obj_fun_bc_t, &mp_type_fun_viper);
-    o->bytecode = fun_data;
+    o->bytecode = (const byte *)fun_data;
     o->context = mc;
     o->child_table = child_table;
     return MP_OBJ_FROM_PTR(o);
@@ -101,9 +101,9 @@ static inline void *mp_obj_fun_native_get_generator_resume(const mp_obj_fun_bc_t
 
 #if MICROPY_EMIT_INLINE_ASM
 static inline mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) {
-    mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
+    mp_obj_fun_asm_t *o = (mp_obj_fun_asm_t *)mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
     o->n_args = n_args;
-    o->fun_data = fun_data;
+    o->fun_data = (const byte *)fun_data;
     o->type_sig = type_sig;
     return MP_OBJ_FROM_PTR(o);
 }

py/objint.c

@@ -420,35 +420,59 @@ static mp_obj_t int_from_bytes(size_t n_args, const mp_obj_t *args) {
 static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(int_from_bytes_fun_obj, 3, 4, int_from_bytes);
 static MP_DEFINE_CONST_CLASSMETHOD_OBJ(int_from_bytes_obj, MP_ROM_PTR(&int_from_bytes_fun_obj));
 
-static mp_obj_t int_to_bytes(size_t n_args, const mp_obj_t *args) {
-    // TODO: Support signed param (assumes signed=False)
-    (void)n_args;
+static mp_obj_t int_to_bytes(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    // Only supported kwarg is 'signed'
+    enum { ARG_signed };
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_signed, MP_ARG_BOOL, {.u_bool = false} },
+    };
 
-    mp_int_t len = mp_obj_get_int(args[1]);
-    if (len < 0) {
+    // Parse positional args
+    mp_obj_t self = pos_args[0];
+    mp_int_t dlen = mp_obj_get_int(pos_args[1]);
+    if (dlen < 0) {
         mp_raise_ValueError(NULL);
     }
-    bool big_endian = args[2] != MP_OBJ_NEW_QSTR(MP_QSTR_little);
+    bool big_endian = pos_args[2] != MP_OBJ_NEW_QSTR(MP_QSTR_little);
+
+    // parse kwargs
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args - 3, pos_args + 3, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+    bool as_signed = args[ARG_signed].u_bool;
+
+    bool overflow;
 
     vstr_t vstr;
-    vstr_init_len(&vstr, len);
+    vstr_init_len(&vstr, dlen);
     byte *data = (byte *)vstr.buf;
-    memset(data, 0, len);
 
     #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
-    if (!mp_obj_is_small_int(args[0])) {
-        mp_obj_int_to_bytes_impl(args[0], big_endian, len, data);
+    if (!mp_obj_is_small_int(self)) {
+        overflow = !mp_obj_int_to_bytes_impl(self, big_endian, as_signed, dlen, data);
     } else
     #endif
     {
-        mp_int_t val = MP_OBJ_SMALL_INT_VALUE(args[0]);
-        size_t l = MIN((size_t)len, sizeof(val));
-        mp_binary_set_int(l, big_endian, data + (big_endian ? (len - l) : 0), val);
+        mp_int_t val = MP_OBJ_SMALL_INT_VALUE(self);
+        if (val < 0 && !as_signed) {
+            mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("can't convert negative int to unsigned"));
+        }
+        int slen = MP_INT_REPR_LEN(val, as_signed);
+        memset(data, val < 0 ? 0xFF : 0x00, dlen);
+        if (slen <= dlen) {
+            mp_binary_set_int(slen, big_endian, data + (big_endian ? (dlen - slen) : 0), val);
+            overflow = false;
+        } else {
+            overflow = true;
+        }
+    }
+
+    if (overflow) {
+        mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("int too big to convert"));
     }
 
     return mp_obj_new_bytes_from_vstr(&vstr);
 }
-static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(int_to_bytes_obj, 3, 4, int_to_bytes);
+static MP_DEFINE_CONST_FUN_OBJ_KW(int_to_bytes_obj, 3, int_to_bytes);
 
 static const mp_rom_map_elem_t int_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_from_bytes), MP_ROM_PTR(&int_from_bytes_obj) },

py/objint.h

@@ -55,7 +55,8 @@ char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size,
     int base, const char *prefix, char base_char, char comma);
 mp_int_t mp_obj_int_hash(mp_obj_t self_in);
 mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf);
-void mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf);
+// Returns true if 'self_in' fit into 'len' bytes of 'buf' without overflowing, 'buf' is truncated otherwise.
+bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, bool as_signed, size_t len, byte *buf);
 int mp_obj_int_sign(mp_obj_t self_in);
 mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in);
 mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in);

py/objint_longlong.c

@@ -57,10 +57,12 @@ mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf
     return mp_obj_new_int_from_ll(value);
 }
 
-void mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf) {
+bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, bool as_signed, size_t len, byte *buf) {
     assert(mp_obj_is_exact_type(self_in, &mp_type_int));
     mp_obj_int_t *self = self_in;
     long long val = self->val;
+    size_t slen = MP_INT_REPR_LEN(val, as_signed);
+    bool ok = slen <= len;
     if (big_endian) {
         byte *b = buf + len;
         while (b > buf) {
@@ -73,6 +75,7 @@ void mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byt
             val >>= 8;
         }
     }
+    return ok;
 }
 
 int mp_obj_int_sign(mp_obj_t self_in) {

py/objint_mpz.c

@@ -112,10 +112,13 @@ mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf
     return MP_OBJ_FROM_PTR(o);
 }
 
-void mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf) {
+bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, bool as_signed, size_t len, byte *buf) {
     assert(mp_obj_is_exact_type(self_in, &mp_type_int));
     mp_obj_int_t *self = MP_OBJ_TO_PTR(self_in);
-    mpz_as_bytes(&self->mpz, big_endian, len, buf);
+    if (self->mpz.neg && !as_signed) {
+        mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("can't convert negative int to unsigned"));
+    }
+    return mpz_as_bytes(&self->mpz, big_endian, as_signed, len, buf);
 }
 
 int mp_obj_int_sign(mp_obj_t self_in) {

tests/basics/bytearray_add.py

@@ -15,4 +15,11 @@ print(b)
 
 # this inplace add tests the code when the buffer doesn't need to be increased
 b = bytearray()
-b += b''
+b += b""
+
+# extend a bytearray from itself
+b = bytearray(b"abcdefgh")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation, as above
+    b.extend(b)
+print(b)

tests/basics/bytearray_add_self.py

@@ -0,0 +1,8 @@
+# add a bytearray to itself
+# This is not supported by CPython as of 3.11.18.
+
+b = bytearray(b"123456789")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation increases chance 'b' has to relocate
+    b += b
+print(b)

tests/basics/bytearray_add_self.py.exp

@@ -0,0 +1 @@
+bytearray(b'123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789')

tests/basics/bytearray_slice_assign.py

@@ -18,7 +18,7 @@ l = bytearray(x)
 l[1:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[1:3]
+# del l[1:3]
 print(l)
 
 l = bytearray(x)
@@ -28,7 +28,7 @@ l = bytearray(x)
 l[:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:3]
+# del l[:3]
 print(l)
 
 l = bytearray(x)
@@ -38,7 +38,7 @@ l = bytearray(x)
 l[:-3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:-3]
+# del l[:-3]
 print(l)
 
 # slice assignment that extends the array
@@ -61,8 +61,14 @@ b[1:1] = b"12345"
 print(b)
 
 # Growth of bytearray via slice extension
-b = bytearray(b'12345678')
-b.append(57) # expand and add a bit of unused space at end of the bytearray
+b = bytearray(b"12345678")
+b.append(57)  # expand and add a bit of unused space at end of the bytearray
 for i in range(400):
-    b[-1:] = b'ab' # grow slowly into the unused space
+    b[-1:] = b"ab"  # grow slowly into the unused space
+print(len(b), b)
+
+# Growth of bytearray via slice extension from itself
+b = bytearray(b"1234567")
+for i in range(3):
+    b[-1:] = b
 print(len(b), b)

tests/basics/int_bytes.py

@@ -1,3 +1,5 @@
+import sys
+
 print((10).to_bytes(1, "little"))
 print((111111).to_bytes(4, "little"))
 print((100).to_bytes(10, "little"))
@@ -20,3 +22,63 @@ try:
     (1).to_bytes(-1, "little")
 except ValueError:
     print("ValueError")
+
+# zero byte destination should also raise an error
+try:
+    (1).to_bytes(0, "little")
+except OverflowError:
+    print("OverflowError")
+
+# except for converting 0 to a zero-length byte array
+print((0).to_bytes(0, "big"))
+
+# byte length can fit the integer directly
+print((0xFF).to_bytes(1, "little"))
+print((0xFF).to_bytes(1, "big"))
+print((0xEFF).to_bytes(2, "little"))
+print((0xEFF).to_bytes(2, "big"))
+print((0xCDEFF).to_bytes(3, "little"))
+print((0xCDEFF).to_bytes(3, "big"))
+
+# OverFlowError if not big enough
+
+try:
+    (0x123).to_bytes(1, "big")
+except OverflowError:
+    print("OverflowError")
+
+try:
+    (0x12345).to_bytes(2, "big")
+except OverflowError:
+    print("OverflowError")
+
+try:
+    (0x1234567).to_bytes(3, "big")
+except OverflowError:
+    print("OverflowError")
+
+
+# negative representations
+
+print((-1).to_bytes(1, "little", signed=True))
+print((-1).to_bytes(3, "little", signed=True))
+print((-1).to_bytes(1, "big", signed=True))
+print((-1).to_bytes(3, "big", signed=True))
+print((-128).to_bytes(1, "big", signed=True))
+print((-32768).to_bytes(2, "big", signed=True))
+print((-(1 << 23)).to_bytes(3, "big", signed=True))
+
+try:
+    print((-129).to_bytes(1, "big", signed=True))
+except OverflowError:
+    print("OverflowError")
+
+try:
+    print((-32769).to_bytes(2, "big", signed=True))
+except OverflowError:
+    print("OverflowError")
+
+try:
+    print(((-1 << 23) - 1).to_bytes(2, "big", signed=True))
+except OverflowError:
+    print("OverflowError")

tests/basics/int_bytes_int64.py

@@ -0,0 +1,41 @@
+import sys
+
+# Depending on the port, the numbers in this test may be implemented as "small"
+# native 64 bit ints, arbitrary precision large ints, or large integers using 64-bit
+# long longs.
+
+try:
+    x = int.from_bytes(b"\x6F\xAB\xCD\x12\x34\x56\x78\xFB", "big")
+except OverflowError:
+    print("SKIP")  # Port can't represent this size of integer at all
+    raise SystemExit
+
+print(hex(x))
+b = x.to_bytes(8, "little")
+print(b)
+print(x.to_bytes(8, "big"))
+
+# padding in output
+print(x.to_bytes(20, "little"))
+print(x.to_bytes(20, "big"))
+
+# check that extra zero bytes don't change the internal int value
+print(int.from_bytes(b + bytes(10), "little") == x)
+
+# can't write to a zero-length bytes object
+try:
+    x.to_bytes(0, "little")
+except OverflowError:
+    print("OverflowError")
+
+# or one that it too short
+try:
+    x.to_bytes(7, "big")
+except OverflowError:
+    print("OverflowError")
+
+# negative representations
+
+print((-x).to_bytes(8, "little", signed=True))
+print((-x).to_bytes(20, "big", signed=True))
+print((-x).to_bytes(20, "little", signed=True))

tests/basics/int_bytes_intbig.py

@@ -1,3 +1,5 @@
+import sys
+
 print((2**64).to_bytes(9, "little"))
 print((2**64).to_bytes(9, "big"))
 
@@ -10,5 +12,40 @@ print(ib)
 print(il.to_bytes(20, "little"))
 print(ib.to_bytes(20, "big"))
 
+# check padding comes out correctly
+print(il.to_bytes(40, "little"))
+print(ib.to_bytes(40, "big"))
+
 # check that extra zero bytes don't change the internal int value
 print(int.from_bytes(b + bytes(10), "little") == int.from_bytes(b, "little"))
+
+# can't write to a zero-length bytes object
+try:
+    ib.to_bytes(0, "little")
+except OverflowError:
+    print("OverflowError")
+
+# or one that it too short
+try:
+    ib.to_bytes(18, "big")
+except OverflowError:
+    print("OverflowError")
+
+# negative representations
+
+print((-ib).to_bytes(20, "big", signed=True))
+print((ib * -ib).to_bytes(40, "big", signed=True))
+
+# case where an additional byte is needed for sign bit
+ib = (2**64) - 1
+print(ib.to_bytes(8, "little"))
+
+ib *= -1
+
+try:
+    print((ib).to_bytes(8, "little", signed=True))
+except OverflowError:
+    print("OverflowError")
+
+print((ib).to_bytes(9, "little", signed=True))
+print((ib).to_bytes(9, "big", signed=True))

tests/cpydiff/types_memoryview_invalid.py

@@ -0,0 +1,12 @@
+"""
+categories: Types,memoryview
+description: memoryview can become invalid if its target is resized
+cause: CPython prevents a ``bytearray`` or ``io.bytesIO`` object from changing size while there is a ``memoryview`` object that references it. MicroPython requires the programmer to manually ensure that an object is not resized while any ``memoryview`` references it.
+
+In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime.
+workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it.
+"""
+b = bytearray(b"abcdefg")
+m = memoryview(b)
+b.extend(b"hijklmnop")
+print(b, bytes(m))