Merge 3035029363 into e60e8079a7

Signed-off-by: Damien George <damien@micropython.org>

.github/workflows/code_size.yml

@@ -8,9 +8,15 @@ on:
       - 'tools/**'
       - 'py/**'
       - 'extmod/**'
+      - 'shared/**'
       - 'lib/**'
       - 'ports/bare-arm/**'
+      - 'ports/mimxrt/**'
       - 'ports/minimal/**'
+      - 'ports/rp2/**'
+      - 'ports/samd/**'
+      - 'ports/stm32/**'
+      - 'ports/unix/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/ports_unix.yml

@@ -88,10 +88,11 @@ jobs:
         (cd ports/unix && gcov -o build-coverage/py ../../py/*.c || true)
         (cd ports/unix && gcov -o build-coverage/extmod ../../extmod/*.c || true)
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true
         verbose: true
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Print failures
       if: failure()
       run: tests/run-tests.py --print-failures

extmod/extmod.cmake

@@ -57,6 +57,8 @@ set(MICROPY_SOURCE_EXTMOD
     ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c
     ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_lfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_map.c
+    ${MICROPY_EXTMOD_DIR}/vfs_map_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_posix.c
     ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_reader.c

extmod/extmod.mk

@@ -59,6 +59,8 @@ SRC_EXTMOD_C += \
 	extmod/vfs_fat_diskio.c \
 	extmod/vfs_fat_file.c \
 	extmod/vfs_lfs.c \
+	extmod/vfs_map.c \
+	extmod/vfs_map_file.c \
 	extmod/vfs_posix.c \
 	extmod/vfs_posix_file.c \
 	extmod/vfs_reader.c \

extmod/modasyncio.c

@@ -156,7 +156,7 @@ static MP_DEFINE_CONST_OBJ_TYPE(
 // Task class
 
 // This is the core asyncio context with cur_task, _task_queue and CancelledError.
-static mp_obj_t asyncio_context = MP_OBJ_NULL;
+mp_obj_t mp_asyncio_context = MP_OBJ_NULL;
 
 static mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 1, 2, false);
@@ -168,7 +168,7 @@ static mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n
     self->state = TASK_STATE_RUNNING_NOT_WAITED_ON;
     self->ph_key = MP_OBJ_NEW_SMALL_INT(0);
     if (n_args == 2) {
-        asyncio_context = args[1];
+        mp_asyncio_context = args[1];
     }
     return MP_OBJ_FROM_PTR(self);
 }
@@ -186,7 +186,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) {
         return mp_const_false;
     }
     // Can't cancel self (not supported yet).
-    mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task));
+    mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task));
     if (self_in == cur_task) {
         mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("can't cancel self"));
     }
@@ -195,7 +195,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) {
         self = MP_OBJ_TO_PTR(self->data);
     }
 
-    mp_obj_t _task_queue = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue));
+    mp_obj_t _task_queue = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue));
 
     // Reschedule Task as a cancelled task.
     mp_obj_t dest[3];
@@ -218,7 +218,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) {
         task_queue_push(2, dest);
     }
 
-    self->data = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError));
+    self->data = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError));
 
     return mp_const_true;
 }
@@ -278,7 +278,7 @@ static mp_obj_t task_iternext(mp_obj_t self_in) {
         nlr_raise(self->data);
     } else {
         // Put calling task on waiting queue.
-        mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task));
+        mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task));
         mp_obj_t args[2] = { self->state, cur_task };
         task_queue_push(2, args);
         // Set calling task's data to this task that it waits on, to double-link it.

extmod/modvfs.c

@@ -31,6 +31,7 @@
 #include "extmod/vfs.h"
 #include "extmod/vfs_fat.h"
 #include "extmod/vfs_lfs.h"
+#include "extmod/vfs_map.h"
 #include "extmod/vfs_posix.h"
 
 #if !MICROPY_VFS
@@ -51,6 +52,9 @@ static const mp_rom_map_elem_t vfs_module_globals_table[] = {
     #if MICROPY_VFS_LFS2
     { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) },
     #endif
+    #if MICROPY_VFS_MAP
+    { MP_ROM_QSTR(MP_QSTR_VfsMap), MP_ROM_PTR(&mp_type_vfs_map) },
+    #endif
     #if MICROPY_VFS_POSIX
     { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) },
     #endif

extmod/vfs.c

@@ -208,6 +208,17 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args
         { MP_QSTR_mkfs, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_FALSE} },
     };
 
+    if (n_args == 1) {
+        // query mount table
+        const char *path = mp_obj_str_get_str(pos_args[0]);
+        for (mp_vfs_mount_t *vfs = MP_STATE_VM(vfs_mount_table); vfs != NULL; vfs = vfs->next) {
+            if (strncmp(path, vfs->str, vfs->len) == 0 && path[vfs->len] == '\0') {
+                return vfs->obj;
+            }
+        }
+        return mp_const_none;
+    }
+
     // parse args
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all(n_args - 2, pos_args + 2, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
@@ -262,7 +273,7 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args
 
     return mp_const_none;
 }
-MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 2, mp_vfs_mount);
+MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 1, mp_vfs_mount);
 
 mp_obj_t mp_vfs_umount(mp_obj_t mnt_in) {
     // remove vfs from the mount table

extmod/vfs.h

@@ -55,6 +55,7 @@
 // At the moment the VFS protocol just has import_stat, but could be extended to other methods
 typedef struct _mp_vfs_proto_t {
     mp_import_stat_t (*import_stat)(void *self, const char *path);
+    size_t (*memmap)(void *self, const char *path, const void **ptr_out);
 } mp_vfs_proto_t;
 
 typedef struct _mp_vfs_blockdev_t {

extmod/vfs_map.c

@@ -0,0 +1,325 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "py/runtime.h"
+#include "py/mperrno.h"
+#include "extmod/vfs.h"
+#include "extmod/vfs_map.h"
+
+#if MICROPY_VFS_MAP
+
+#define MAGIC_LEN (2)
+#define MAGIC_BYTE0 ('M')
+#define MAGIC_BYTE1 ('F')
+
+#define GET_LE16(p) ((p)[0] | (p)[1] << 8)
+#define GET_LE32(p) ((p)[0] | (p)[1] << 8 | (p)[2] << 16 | (p)[3] << 24)
+
+struct _mp_obj_vfs_map_t {
+    mp_obj_base_t base;
+    mp_obj_t memory;
+    mp_obj_t device;
+    const uint8_t *filesystem;
+};
+
+mp_import_stat_t mp_vfs_map_search_filesystem(mp_obj_vfs_map_t *self, const char *path, size_t *size_out, const uint8_t **data_out) {
+    if (!(self->filesystem[0] == MAGIC_BYTE0 && self->filesystem[1] == MAGIC_BYTE1)) {
+        return MP_IMPORT_STAT_NO_EXIST;
+    }
+    if (path[0] == '/') {
+        ++path;
+    }
+    size_t path_len = strlen(path);
+    const uint8_t *fs = self->filesystem + MAGIC_LEN;
+    for (;;) {
+        uint16_t nlen = GET_LE16(fs);
+        fs += 2;
+        if (nlen == 0) {
+            return MP_IMPORT_STAT_NO_EXIST;
+        }
+        uint32_t flen;
+        mp_import_stat_t type;
+        if (nlen & 0x8000) {
+            // A directory.
+            nlen &= 0x7fff;
+            flen = 0;
+            type = MP_IMPORT_STAT_DIR;
+        } else {
+            // A file.
+            flen = GET_LE32(fs);
+            fs += 4;
+            type = MP_IMPORT_STAT_FILE;
+        }
+        if (path_len == nlen && memcmp(path, fs, path_len) == 0) {
+            if (size_out != NULL) {
+                *size_out = flen;
+                *data_out = fs + nlen;
+            }
+            return type;
+        }
+        fs += nlen + flen;
+    }
+}
+
+static inline const char *vfs_map_get_path_str(mp_obj_vfs_map_t *self, mp_obj_t path) {
+    return mp_obj_str_get_str(path);
+}
+
+static mp_obj_t vfs_map_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
+    mp_arg_check_num(n_args, n_kw, 1, 2, false);
+
+    mp_obj_vfs_map_t *self = m_new_obj(mp_obj_vfs_map_t);
+    self->base.type = type;
+    self->memory = args[0];
+    self->device = n_args == 1 ? mp_const_none : args[1];
+
+    mp_buffer_info_t bufinfo;
+    if (mp_get_buffer(self->memory, &bufinfo, MP_BUFFER_READ)) {
+        self->filesystem = bufinfo.buf;
+    } else {
+        self->filesystem = (const uint8_t *)(uintptr_t)mp_obj_get_int(self->memory);
+    }
+
+    return MP_OBJ_FROM_PTR(self);
+}
+
+static mp_obj_t vfs_map_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) {
+    (void)self_in;
+    (void)readonly;
+    if (mp_obj_is_true(mkfs)) {
+        mp_raise_OSError(MP_EPERM);
+    }
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_3(vfs_map_mount_obj, vfs_map_mount);
+
+static mp_obj_t vfs_map_umount(mp_obj_t self_in) {
+    (void)self_in;
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(vfs_map_umount_obj, vfs_map_umount);
+
+// mp_vfs_map_file_open is implemented in vfs_map_file.c.
+static MP_DEFINE_CONST_FUN_OBJ_3(vfs_map_open_obj, mp_vfs_map_file_open);
+
+static mp_obj_t vfs_map_chdir(mp_obj_t self_in, mp_obj_t path_in) {
+    (void)self_in;
+    (void)path_in;
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_chdir_obj, vfs_map_chdir);
+
+typedef struct _vfs_map_ilistdir_it_t {
+    mp_obj_base_t base;
+    mp_fun_1_t iternext;
+    bool is_str;
+    const char *path;
+    const uint8_t *index;
+} vfs_map_ilistdir_it_t;
+
+static mp_obj_t vfs_map_ilistdir_it_iternext(mp_obj_t self_in) {
+    vfs_map_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in);
+
+    if (self->index == NULL) {
+        return MP_OBJ_STOP_ITERATION;
+    }
+
+    const char *path = self->path;
+    if (path[0] == '/' || (path[0] == '.' && path[1] == '\0')) {
+        ++path;
+    }
+    size_t path_len = strlen(path);
+
+    for (;;) {
+        uint16_t nlen = GET_LE16(self->index);
+        self->index += 2;
+        if (nlen == 0) {
+            self->index = NULL;
+            return MP_OBJ_STOP_ITERATION;
+        }
+
+        uint32_t flen;
+        uint32_t type;
+        if (nlen & 0x8000) {
+            // dir
+            nlen &= 0x7fff;
+            flen = 0;
+            type = MP_S_IFDIR;
+        } else {
+            // file
+            flen = GET_LE32(self->index);
+            self->index += 4;
+            type = MP_S_IFREG;
+        }
+
+        const uint8_t *nstr = self->index;
+        self->index += nlen + flen;
+
+        if ((path_len == 0 && memchr(nstr, '/', nlen) == NULL)
+            || (path_len < nlen && nstr[path_len] == '/' && memcmp(path, nstr, path_len) == 0 && memchr(nstr + path_len + 1, '/', nlen - path_len - 1) == NULL)) {
+            // Make 4-tuple with info about this entry: (name, attr, inode, size)
+            mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(4, NULL));
+
+            if (path_len > 0) {
+                nstr += path_len + 1;
+                nlen -= path_len + 1;
+            }
+
+            if (self->is_str) {
+                t->items[0] = mp_obj_new_str((const char *)nstr, nlen);
+            } else {
+                t->items[0] = mp_obj_new_bytes(nstr, nlen);
+            }
+
+            t->items[1] = MP_OBJ_NEW_SMALL_INT(type);
+            t->items[2] = MP_OBJ_NEW_SMALL_INT(0);
+            t->items[3] = mp_obj_new_int(flen);
+
+            return MP_OBJ_FROM_PTR(t);
+        }
+    }
+}
+
+static mp_obj_t vfs_map_ilistdir(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+    vfs_map_ilistdir_it_t *iter = m_new_obj(vfs_map_ilistdir_it_t);
+    iter->base.type = &mp_type_polymorph_iter;
+    iter->iternext = vfs_map_ilistdir_it_iternext;
+    iter->is_str = mp_obj_get_type(path_in) == &mp_type_str;
+    iter->path = mp_obj_str_get_str(path_in);
+    if (!(self->filesystem[0] == MAGIC_BYTE0 && self->filesystem[1] == MAGIC_BYTE1)) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    if (iter->path[0] == '\0'
+        || (iter->path[0] == '.' && iter->path[1] == '\0')
+        || (iter->path[0] == '/' && iter->path[1] == '\0')) {
+        // pass
+    } else if (mp_vfs_map_search_filesystem(self, iter->path, NULL, NULL) != MP_IMPORT_STAT_DIR) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    iter->index = self->filesystem + MAGIC_LEN;
+    return MP_OBJ_FROM_PTR(iter);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_ilistdir_obj, vfs_map_ilistdir);
+
+static mp_obj_t vfs_map_stat(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+    const char *path = vfs_map_get_path_str(self, path_in);
+    size_t file_size;
+    const uint8_t *file_data;
+    mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &file_size, &file_data);
+    if (stat == MP_IMPORT_STAT_NO_EXIST) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
+    t->items[0] = MP_OBJ_NEW_SMALL_INT(stat == MP_IMPORT_STAT_FILE ? MP_S_IFREG : MP_S_IFDIR); // st_mode
+    t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino
+    t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // st_dev
+    t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // st_nlink
+    t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid
+    t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid
+    t->items[6] = MP_OBJ_NEW_SMALL_INT(file_size); // st_size
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime
+    return MP_OBJ_FROM_PTR(t);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_stat_obj, vfs_map_stat);
+
+static mp_obj_t vfs_map_statvfs(mp_obj_t self_in, mp_obj_t path_in) {
+    (void)self_in;
+    (void)path_in;
+    mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
+    t->items[0] = MP_OBJ_NEW_SMALL_INT(0); // f_bsize
+    t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // f_frsize
+    t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // f_blocks
+    t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // f_bfree
+    t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // f_bavail
+    t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // f_files
+    t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(65535); // f_namemax
+    return MP_OBJ_FROM_PTR(t);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_statvfs_obj, vfs_map_statvfs);
+
+static mp_obj_t vfs_map_device(mp_obj_t self_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+    return self->device;
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(vfs_map_device_obj, vfs_map_device);
+
+static const mp_rom_map_elem_t vfs_map_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_map_mount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&vfs_map_umount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_map_open_obj) },
+
+    { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_map_chdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_map_ilistdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_map_stat_obj) },
+    { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_map_statvfs_obj) },
+
+    { MP_ROM_QSTR(MP_QSTR_device), MP_ROM_PTR(&vfs_map_device_obj) },
+};
+static MP_DEFINE_CONST_DICT(vfs_map_locals_dict, vfs_map_locals_dict_table);
+
+static mp_import_stat_t mp_vfs_map_import_stat(void *self_in, const char *path) {
+    mp_obj_vfs_map_t *self = self_in;
+    return mp_vfs_map_search_filesystem(self, path, NULL, NULL);
+}
+
+static size_t mp_vfs_map_memmap(void *self_in, const char *path, const void **ptr_out) {
+    mp_obj_vfs_map_t *self = self_in;
+    size_t size;
+    const uint8_t *data;
+    mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &size, &data);
+    if (stat == MP_IMPORT_STAT_FILE) {
+        *ptr_out = data;
+        return size;
+    } else {
+        *ptr_out = NULL;
+        return 0;
+    }
+}
+
+static const mp_vfs_proto_t vfs_map_proto = {
+    .import_stat = mp_vfs_map_import_stat,
+    .memmap = mp_vfs_map_memmap,
+};
+
+MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_map,
+    MP_QSTR_VfsMap,
+    MP_TYPE_FLAG_NONE,
+    make_new, vfs_map_make_new,
+    protocol, &vfs_map_proto,
+    locals_dict, &vfs_map_locals_dict
+    );
+
+#endif // MICROPY_VFS_MAP

extmod/vfs_map.h

@@ -0,0 +1,39 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_EXTMOD_VFS_MAP_H
+#define MICROPY_INCLUDED_EXTMOD_VFS_MAP_H
+
+#include "py/builtin.h"
+#include "py/obj.h"
+
+typedef struct _mp_obj_vfs_map_t mp_obj_vfs_map_t;
+
+extern const mp_obj_type_t mp_type_vfs_map;
+
+mp_import_stat_t mp_vfs_map_search_filesystem(mp_obj_vfs_map_t *self, const char *path, size_t *size_out, const uint8_t **data_out);
+mp_obj_t mp_vfs_map_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in);
+
+#endif // MICROPY_INCLUDED_EXTMOD_VFS_MAP_H

extmod/vfs_map_file.c

@@ -0,0 +1,202 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "py/reader.h"
+#include "py/runtime.h"
+#include "py/stream.h"
+#include "extmod/vfs_map.h"
+
+#if MICROPY_VFS_MAP
+
+typedef struct _mp_obj_vfs_map_file_t {
+    mp_obj_base_t base;
+    size_t file_size;
+    size_t file_offset;
+    const uint8_t *file_data;
+} mp_obj_vfs_map_file_t;
+
+static const mp_obj_type_t mp_type_vfs_map_fileio;
+static const mp_obj_type_t mp_type_vfs_map_textio;
+
+static uintptr_t mp_vfs_map_reader_readbyte(void *data) {
+    mp_obj_vfs_map_file_t *self = data;
+    if (self->file_offset >= self->file_size) {
+        return MP_READER_EOF;
+    }
+    return self->file_data[self->file_offset++];
+}
+
+static intptr_t mp_vfs_map_reader_ioctl(void *data, uintptr_t request, uintptr_t arg) {
+    mp_obj_vfs_map_file_t *self = data;
+
+    if (request == MP_READER_CLOSE) {
+        return 0;
+    } else if (request == MP_READER_MEMMAP) {
+        mp_reader_ioctl_memmap_t *memmap = (mp_reader_ioctl_memmap_t *)arg;
+        memmap->ptr = self->file_data + self->file_offset;
+        self->file_offset += memmap->len;
+        return 0;
+    }
+
+    return -MP_EINVAL;
+}
+
+mp_obj_t mp_vfs_map_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+
+    const char *mode_s = mp_obj_str_get_str(mode_in);
+    const mp_obj_type_t *type = &mp_type_vfs_map_textio;
+    while (*mode_s) {
+        switch (*mode_s++) {
+            case 'r':
+                break;
+            case 'w':
+            case 'a':
+            case '+':
+                mp_raise_OSError(MP_EROFS);
+            case 'b':
+                type = &mp_type_vfs_map_fileio;
+                break;
+            case 't':
+                type = &mp_type_vfs_map_textio;
+                break;
+        }
+    }
+
+    mp_obj_vfs_map_file_t *o = m_new_obj(mp_obj_vfs_map_file_t);
+    o->base.type = type;
+    o->file_offset = 0;
+
+    const char *path = mp_obj_str_get_str(path_in);
+    mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &o->file_size, &o->file_data);
+    if (stat == MP_IMPORT_STAT_NO_EXIST) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+static mp_obj_t vfs_map_file___exit__(size_t n_args, const mp_obj_t *args) {
+    (void)n_args;
+    return mp_stream_close(args[0]);
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(vfs_map_file___exit___obj, 4, 4, vfs_map_file___exit__);
+
+static mp_uint_t vfs_map_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
+    mp_obj_vfs_map_file_t *self = MP_OBJ_TO_PTR(o_in);
+    size_t remain = self->file_size - self->file_offset;
+    if (size > remain) {
+        size = remain;
+    }
+    memcpy(buf, self->file_data + self->file_offset, size);
+    self->file_offset += size;
+    return size;
+}
+
+static mp_uint_t vfs_map_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) {
+    mp_obj_vfs_map_file_t *self = MP_OBJ_TO_PTR(o_in);
+
+    switch (request) {
+        case MP_STREAM_FLUSH:
+            return 0;
+        case MP_STREAM_SEEK: {
+            struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg;
+            if (s->whence == 0) { // SEEK_SET
+                self->file_offset = MIN(self->file_size, (size_t)s->offset);
+            } else if (s->whence == 1) { // SEEK_CUR
+                self->file_offset = MIN(self->file_size, self->file_offset + s->offset);
+            } else { // SEEK_END
+                self->file_offset = self->file_size;
+            }
+            s->offset = self->file_offset;
+            return 0;
+        }
+        case MP_STREAM_CLOSE:
+            return 0;
+        case MP_STREAM_POLL: {
+            mp_uint_t ret = 0;
+            if (arg & MP_STREAM_POLL_RD) {
+                ret |= MP_STREAM_POLL_RD;
+            }
+            return ret;
+        }
+        case MP_STREAM_INIT_READER: {
+            mp_reader_t *reader = (mp_reader_t *)arg;
+            reader->data = self;
+            reader->readbyte = mp_vfs_map_reader_readbyte;
+            reader->ioctl = mp_vfs_map_reader_ioctl;
+            return 0;
+        }
+        default:
+            *errcode = MP_EINVAL;
+            return MP_STREAM_ERROR;
+    }
+}
+
+static const mp_rom_map_elem_t vfs_map_rawfile_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) },
+    { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) },
+    { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) },
+    { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) },
+    { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
+    { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) },
+    { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_map_file___exit___obj) },
+};
+static MP_DEFINE_CONST_DICT(vfs_map_rawfile_locals_dict, vfs_map_rawfile_locals_dict_table);
+
+static const mp_stream_p_t vfs_map_fileio_stream_p = {
+    .read = vfs_map_file_read,
+    .ioctl = vfs_map_file_ioctl,
+};
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_map_fileio,
+    MP_QSTR_FileIO,
+    MP_TYPE_FLAG_ITER_IS_STREAM,
+    protocol, &vfs_map_fileio_stream_p,
+    locals_dict, &vfs_map_rawfile_locals_dict
+    );
+
+static const mp_stream_p_t vfs_map_textio_stream_p = {
+    .read = vfs_map_file_read,
+    .ioctl = vfs_map_file_ioctl,
+    .is_text = true,
+};
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_map_textio,
+    MP_QSTR_TextIOWrapper,
+    MP_TYPE_FLAG_ITER_IS_STREAM,
+    protocol, &vfs_map_textio_stream_p,
+    locals_dict, &vfs_map_rawfile_locals_dict
+    );
+
+#endif // MICROPY_VFS_MAP

extmod/vfs_reader.c

@@ -49,7 +49,7 @@ typedef struct _mp_reader_vfs_t {
     byte buf[];
 } mp_reader_vfs_t;
 
-static mp_uint_t mp_reader_vfs_readbyte(void *data) {
+static uintptr_t mp_reader_vfs_readbyte(void *data) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
     if (reader->bufpos >= reader->buflen) {
         if (reader->buflen < reader->bufsize) {
@@ -70,13 +70,39 @@ static mp_uint_t mp_reader_vfs_readbyte(void *data) {
     return reader->buf[reader->bufpos++];
 }
 
-static void mp_reader_vfs_close(void *data) {
+static intptr_t mp_reader_vfs_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
-    mp_stream_close(reader->file);
-    m_del_obj(mp_reader_vfs_t, reader);
+
+    if (request == MP_READER_CLOSE) {
+        mp_stream_close(reader->file);
+        m_del_obj(mp_reader_vfs_t, reader);
+        return 0;
+    }
+
+    return -MP_EINVAL;
 }
 
 void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
+    #if MICROPY_VFS_MAP
+    const char *path_out;
+    mp_vfs_mount_t *vfs = mp_vfs_lookup_path(qstr_str(filename), &path_out);
+    if (!(vfs == MP_VFS_NONE || vfs == MP_VFS_ROOT)) {
+        // If the mounted object has the VFS protocol, call its memmap helper.
+        const mp_obj_type_t *type = mp_obj_get_type(vfs->obj);
+        if (MP_OBJ_TYPE_HAS_SLOT(type, protocol)) {
+            const mp_vfs_proto_t *proto = MP_OBJ_TYPE_GET_SLOT(type, protocol);
+            if (proto->memmap != NULL) {
+                const void *data;
+                size_t size = proto->memmap(MP_OBJ_TO_PTR(vfs->obj), path_out, &data);
+                if (data != NULL) {
+                    mp_reader_new_mem(reader, data, size, (size_t)-1);
+                    return;
+                }
+            }
+        }
+    }
+    #endif
+
     mp_obj_t args[2] = {
         MP_OBJ_NEW_QSTR(filename),
         MP_OBJ_NEW_QSTR(MP_QSTR_rb),
@@ -85,6 +111,15 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
 
     const mp_stream_p_t *stream_p = mp_get_stream(file);
     int errcode = 0;
+
+    #if 0 && MICROPY_VFS_MAP
+    // Check if the stream can initialising a reader itself.
+    mp_uint_t reader_ret = stream_p->ioctl(file, MP_STREAM_INIT_READER, (uintptr_t)reader, &errcode);
+    if (reader_ret == 0) {
+        return;
+    }
+    #endif
+
     mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode);
     if (bufsize == MP_STREAM_ERROR || bufsize == 0) {
         // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl
@@ -98,13 +133,14 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
     rf->file = file;
     rf->bufsize = bufsize;
     rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+
     if (errcode != 0) {
         mp_raise_OSError(errcode);
     }
     rf->bufpos = 0;
     reader->data = rf;
     reader->readbyte = mp_reader_vfs_readbyte;
-    reader->close = mp_reader_vfs_close;
+    reader->ioctl = mp_reader_vfs_ioctl;
 }
 
 #endif // MICROPY_READER_VFS

ports/esp32/esp32_partition.c

@@ -265,6 +265,16 @@ static MP_DEFINE_CONST_FUN_OBJ_1(esp32_partition_mark_app_valid_cancel_rollback_
 static MP_DEFINE_CONST_CLASSMETHOD_OBJ(esp32_partition_mark_app_valid_cancel_rollback_obj,
     MP_ROM_PTR(&esp32_partition_mark_app_valid_cancel_rollback_fun_obj));
 
+static mp_obj_t esp32_partition_mmap(mp_obj_t self_in) {
+    esp32_partition_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    const void *ptr;
+    esp_partition_mmap_handle_t handle;
+    // TODO need to esp_partition_munmap(handle) on soft reset
+    check_esp_err(esp_partition_mmap(self->part, 0, self->part->size, ESP_PARTITION_MMAP_DATA, &ptr, &handle));
+    return mp_obj_new_memoryview('B', self->part->size, (void *)ptr);
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(esp32_partition_mmap_obj, esp32_partition_mmap);
+
 static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_find), MP_ROM_PTR(&esp32_partition_find_obj) },
 
@@ -276,6 +286,7 @@ static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_set_boot), MP_ROM_PTR(&esp32_partition_set_boot_obj) },
     { MP_ROM_QSTR(MP_QSTR_mark_app_valid_cancel_rollback), MP_ROM_PTR(&esp32_partition_mark_app_valid_cancel_rollback_obj) },
     { MP_ROM_QSTR(MP_QSTR_get_next_update), MP_ROM_PTR(&esp32_partition_get_next_update_obj) },
+    { MP_ROM_QSTR(MP_QSTR_mmap), MP_ROM_PTR(&esp32_partition_mmap_obj) },
 
     { MP_ROM_QSTR(MP_QSTR_BOOT), MP_ROM_INT(ESP32_PARTITION_BOOT) },
     { MP_ROM_QSTR(MP_QSTR_RUNNING), MP_ROM_INT(ESP32_PARTITION_RUNNING) },

ports/esp32/modules/_boot.py

@@ -1,7 +1,14 @@
 import gc
 import vfs
+from esp32 import Partition
 from flashbdev import bdev
 
+if mapfs := Partition.find(Partition.TYPE_DATA, label="mapfs"):
+    import sys
+
+    sys.path.insert(0, "/mapfs")
+del mapfs
+
 try:
     if bdev:
         vfs.mount(bdev, "/")

ports/esp32/mpconfigport.h

@@ -71,6 +71,7 @@
 #define MICROPY_USE_INTERNAL_PRINTF         (0) // ESP32 SDK requires its own printf
 #define MICROPY_SCHEDULER_DEPTH             (8)
 #define MICROPY_VFS                         (1)
+#define MICROPY_VFS_MAP                     (1)
 
 // control over Python builtins
 #define MICROPY_PY_STR_BYTES_CMP_WARN       (1)

ports/esp32/partitions-4MiB.csv

@@ -3,5 +3,6 @@
 # Name,   Type, SubType, Offset,  Size, Flags
 nvs,      data, nvs,     0x9000,  0x6000,
 phy_init, data, phy,     0xf000,  0x1000,
-factory,  app,  factory, 0x10000, 0x1F0000,
+factory,  app,  factory, 0x10000, 0x1D0000,
+mapfs,    data, 0x8f,    0x1E0000, 0x20000,
 vfs,      data, fat,     0x200000, 0x200000,

ports/esp8266/lexerstr32.c

@@ -26,6 +26,7 @@
  */
 
 #include "py/lexer.h"
+#include "py/mperrno.h"
 
 #if MICROPY_ENABLE_COMPILER
 
@@ -35,7 +36,7 @@ typedef struct _mp_lexer_str32_buf_t {
     uint8_t byte_off;
 } mp_lexer_str32_buf_t;
 
-static mp_uint_t str32_buf_next_byte(void *sb_in) {
+static uintptr_t str32_buf_next_byte(void *sb_in) {
     mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t *)sb_in;
     byte c = sb->val & 0xff;
     if (c == 0) {
@@ -52,9 +53,13 @@ static mp_uint_t str32_buf_next_byte(void *sb_in) {
     return c;
 }
 
-static void str32_buf_free(void *sb_in) {
+static intptr_t str32_buf_free(void *sb_in, uintptr_t request, uintptr_t arg) {
     mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t *)sb_in;
-    m_del_obj(mp_lexer_str32_buf_t, sb);
+    if (request == MP_READER_CLOSE) {
+        m_del_obj(mp_lexer_str32_buf_t, sb);
+        return 0;
+    }
+    return -MP_EINVAL;
 }
 
 mp_lexer_t *mp_lexer_new_from_str32(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len) {

ports/nrf/modules/os/microbitfs.c

@@ -458,11 +458,18 @@ static mp_uint_t microbit_file_write(mp_obj_t obj, const void *buf, mp_uint_t si
     return size;
 }
 
-static void microbit_file_close(file_descriptor_obj *fd) {
-    if (fd->writable) {
-        flash_write_byte((uint32_t)&(file_system_chunks[fd->start_chunk].header.end_offset), fd->seek_offset);
+static intptr_t microbit_file_ioctl(void *fd_in, uintptr_t request, uintptr_t arg) {
+    file_descriptor_obj *fd = fd_in;
+
+    if (request == MP_READER_CLOSE) {
+        if (fd->writable) {
+            flash_write_byte((uint32_t)&(file_system_chunks[fd->start_chunk].header.end_offset), fd->seek_offset);
+        }
+        fd->open = false;
+        return 0;
     }
-    fd->open = false;
+
+    return -MP_EINVAL;
 }
 
 static mp_obj_t microbit_file_list(void) {
@@ -495,7 +502,7 @@ static mp_obj_t microbit_file_size(mp_obj_t filename) {
     return mp_obj_new_int(len);
 }
 
-static mp_uint_t file_read_byte(file_descriptor_obj *fd) {
+static uintptr_t file_read_byte(file_descriptor_obj *fd) {
     if (file_system_chunks[fd->seek_chunk].next_chunk == UNUSED_CHUNK) {
         uint8_t end_offset = file_system_chunks[fd->start_chunk].header.end_offset;
         if (end_offset == UNUSED_CHUNK || fd->seek_offset == end_offset) {
@@ -516,8 +523,8 @@ mp_lexer_t *os_mbfs_new_reader(const char *filename) {
     }
     mp_reader_t reader;
     reader.data = fd;
-    reader.readbyte = (mp_uint_t(*)(void*))file_read_byte;
-    reader.close = (void(*)(void*))microbit_file_close; // no-op
+    reader.readbyte = (uintptr_t(*)(void*))file_read_byte;
+    reader.close = microbit_file_ioctl;
     return mp_lexer_new(qstr_from_str(filename), reader);
 }
 
@@ -538,7 +545,7 @@ static MP_DEFINE_CONST_FUN_OBJ_1(os_mbfs_file_name_obj, os_mbfs_file_name);
 
 static mp_obj_t os_mbfs_file_close(mp_obj_t self) {
     file_descriptor_obj *fd = (file_descriptor_obj*)self;
-    microbit_file_close(fd);
+    microbit_file_ioctl(fd, MP_READER_CLOSE, 0);
     return mp_const_none;
 }
 static MP_DEFINE_CONST_FUN_OBJ_1(os_mbfs_file_close_obj, os_mbfs_file_close);

ports/nrf/mpconfigport.h

@@ -59,6 +59,12 @@
 
 // options to control how MicroPython is built
 
+// Due to the use of LTO and the unknown distance between nlr.o and nlrthumb.o code,
+// MCUs using the Thumb 1 instruction set must enable this NLR long jump feature.
+#if defined(NRF51822)
+#define MICROPY_NLR_THUMB_USE_LONG_JUMP    (1)
+#endif
+
 #ifndef MICROPY_VFS
 #define MICROPY_VFS                        (CORE_FEAT)
 #endif

ports/rp2/README.md

@@ -37,7 +37,7 @@ You can also build the standard CMake way.  The final firmware is found in
 the top-level of the CMake build directory (`build` by default) and is
 called `firmware.uf2`.
 
-If you are using a different board other than a Rasoberry Pi Pico, then you should
+If you are using a board other than a Raspberry Pi Pico, you should
 pass the board name to the build; e.g. for Raspberry Pi Pico W:
 
     $ make BOARD=RPI_PICO_W submodules

ports/rp2/modmachine.c

@@ -127,10 +127,10 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     const uint32_t xosc_hz = XOSC_MHZ * 1000000;
 
-    uint32_t my_interrupts = save_and_disable_interrupts();
+    uint32_t my_interrupts = mp_thread_begin_atomic_section();
     #if MICROPY_PY_NETWORK_CYW43
     if (cyw43_has_pending && cyw43_poll != NULL) {
-        restore_interrupts(my_interrupts);
+        mp_thread_end_atomic_section(my_interrupts);
         return;
     }
     #endif
@@ -165,8 +165,15 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
     } else {
         uint32_t sleep_en0 = clocks_hw->sleep_en0;
         uint32_t sleep_en1 = clocks_hw->sleep_en1;
+        bool timer3_enabled = irq_is_enabled(3);
+
         clocks_hw->sleep_en0 = CLOCKS_SLEEP_EN0_CLK_RTC_RTC_BITS;
         if (use_timer_alarm) {
+            // Make sure ALARM3/IRQ3 is enabled on _this_ core
+            timer_hw->inte |= 1 << 3;
+            if (!timer3_enabled) {
+                irq_set_enabled(3, true);
+            }
             // Use timer alarm to wake.
             clocks_hw->sleep_en1 = CLOCKS_SLEEP_EN1_CLK_SYS_TIMER_BITS;
             timer_hw->alarm[3] = timer_hw->timerawl + delay_ms * 1000;
@@ -177,6 +184,9 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
         scb_hw->scr |= M0PLUS_SCR_SLEEPDEEP_BITS;
         __wfi();
         scb_hw->scr &= ~M0PLUS_SCR_SLEEPDEEP_BITS;
+        if (!timer3_enabled) {
+            irq_set_enabled(3, false);
+        }
         clocks_hw->sleep_en0 = sleep_en0;
         clocks_hw->sleep_en1 = sleep_en1;
     }
@@ -186,7 +196,7 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) {
 
     // Bring back all clocks.
     clocks_init();
-    restore_interrupts(my_interrupts);
+    mp_thread_end_atomic_section(my_interrupts);
 }
 
 NORETURN static void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args) {

ports/rp2/modules/_boot.py

@@ -1,7 +1,13 @@
 import vfs
+import sys
 import machine, rp2
 
 
+mapfs = rp2.Flash("mapfs")
+vfs.mount(vfs.VfsMap(mapfs.ioctl(0x100, 0), mapfs), "/mapfs")
+sys.path.insert(0, "/mapfs")
+
+
 # Try to mount the filesystem, and format the flash if it doesn't exist.
 # Note: the flash requires the programming size to be aligned to 256 bytes.
 bdev = rp2.Flash()
@@ -12,4 +18,4 @@ except:
     fs = vfs.VfsLfs2(bdev, progsize=256)
 vfs.mount(fs, "/")
 
-del vfs, bdev, fs
+del vfs, sys, bdev, fs, mapfs

ports/rp2/mpconfigport.h

@@ -151,6 +151,7 @@
 #define MICROPY_VFS                             (1)
 #define MICROPY_VFS_LFS2                        (1)
 #define MICROPY_VFS_FAT                         (1)
+#define MICROPY_VFS_MAP                         (1)
 #define MICROPY_SSL_MBEDTLS                     (1)
 #define MICROPY_PY_LWIP_SOCK_RAW                (MICROPY_PY_LWIP)
 

ports/rp2/rp2_flash.c

@@ -28,6 +28,7 @@
 
 #include "py/mphal.h"
 #include "py/runtime.h"
+#include "py/mperrno.h"
 #include "extmod/vfs.h"
 #include "modrp2.h"
 #include "hardware/flash.h"
@@ -44,6 +45,9 @@ static_assert(MICROPY_HW_FLASH_STORAGE_BYTES % 4096 == 0, "Flash storage size mu
 #define MICROPY_HW_FLASH_STORAGE_BASE (PICO_FLASH_SIZE_BYTES - MICROPY_HW_FLASH_STORAGE_BYTES)
 #endif
 
+#define MICROPY_HW_MAPFS_BASE (512 * 1024) // leave 512k for firmware...
+#define MICROPY_HW_MAPFS_BYTES (MICROPY_HW_FLASH_STORAGE_BASE - MICROPY_HW_MAPFS_BASE)
+
 static_assert(MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big");
 static_assert(MICROPY_HW_FLASH_STORAGE_BASE + MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big");
 
@@ -53,6 +57,12 @@ typedef struct _rp2_flash_obj_t {
     uint32_t flash_size;
 } rp2_flash_obj_t;
 
+static rp2_flash_obj_t rp2_flash_mapfs_obj = {
+    .base = { &rp2_flash_type },
+    .flash_base = MICROPY_HW_MAPFS_BASE,
+    .flash_size = MICROPY_HW_MAPFS_BYTES,
+};
+
 static rp2_flash_obj_t rp2_flash_obj = {
     .base = { &rp2_flash_type },
     .flash_base = MICROPY_HW_FLASH_STORAGE_BASE,
@@ -87,6 +97,15 @@ static void end_critical_flash_section(uint32_t state) {
 }
 
 static mp_obj_t rp2_flash_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) {
+    if (n_args == 1 && n_kw == 0 && mp_obj_is_str(all_args[0])) {
+        if (mp_obj_str_get_qstr(all_args[0]) == MP_QSTR_mapfs) {
+            if (MICROPY_HW_MAPFS_BYTES <= 0) {
+                mp_raise_OSError(MP_ENODEV);
+            }
+            return MP_OBJ_FROM_PTR(&rp2_flash_mapfs_obj);
+        }
+    }
+
     // Parse arguments
     enum { ARG_start, ARG_len };
     static const mp_arg_t allowed_args[] = {
@@ -190,6 +209,8 @@ static mp_obj_t rp2_flash_ioctl(mp_obj_t self_in, mp_obj_t cmd_in, mp_obj_t arg_
             // TODO check return value
             return MP_OBJ_NEW_SMALL_INT(0);
         }
+        case 0x100: // mmap
+            return mp_obj_new_int(XIP_BASE + self->flash_base);
         default:
             return mp_const_none;
     }

ports/stm32/boardctrl.h

@@ -61,6 +61,10 @@
 #define MICROPY_BOARD_TOP_SOFT_RESET_LOOP boardctrl_top_soft_reset_loop
 #endif
 
+#ifndef MICROPY_BOARD_FROZEN_BOOT_FILE
+#define MICROPY_BOARD_FROZEN_BOOT_FILE "_boot.py"
+#endif
+
 #ifndef MICROPY_BOARD_RUN_BOOT_PY
 #define MICROPY_BOARD_RUN_BOOT_PY boardctrl_run_boot_py
 #endif

ports/stm32/boards/NUCLEO_F091RC/mpconfigboard.mk

@@ -8,7 +8,7 @@ MICROPY_VFS_FAT = 0
 MICROPY_VFS_LFS1 ?= 1
 
 # Don't include default frozen modules because MCU is tight on flash space
-FROZEN_MANIFEST ?=
+FROZEN_MANIFEST ?= boards/manifest_minimal.py
 
 # LTO reduces final binary size, may be slower to build depending on gcc version and hardware
 LTO ?= 1

ports/stm32/boards/PYBD_SF2/f722_qspi.ld

@@ -20,7 +20,8 @@ MEMORY
 {
     FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 512K
     FLASH_ISR (rx)  : ORIGIN = 0x08000000, LENGTH = 32K     /* sectors 0,1 */
-    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 480K    /* sectors 2-7 */
+    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 352K    /* sectors 2-6 */
+    FLASH_MAPFS (rx): ORIGIN = 0x08060000, LENGTH = 128K    /* sector 7 */
     FLASH_EXT (rx)  : ORIGIN = 0x90000000, LENGTH = 2048K   /* external QSPI */
     RAM (rwx)       : ORIGIN = 0x20000000, LENGTH = 256K    /* DTCM+SRAM1+SRAM2 */
 }
@@ -40,6 +41,9 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM);
 _heap_start = _ebss; /* heap starts just after statically allocated memory */
 _heap_end = _sstack;
 
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);
+
 /* Define output sections */
 SECTIONS
 {
@@ -51,6 +55,10 @@ SECTIONS
         *lib/mynewt-nimble/*(.text* .rodata*)
         *lib/cyw43-driver/*(.rodata.w4343*_combined)
         *drivers/cyw43/*(.rodata.cyw43_btfw_*)
+        *lib/oofatfs/*(.text* .rodata*)
+        *lib/littlefs/*(.text* .rodata*)
+        *lib/lwip/*(.text* .rodata*)
+        *extmod/*(.text* .rodata*)
         . = ALIGN(4);
     } >FLASH_EXT
 }

ports/stm32/boards/PYBD_SF6/f767.ld

@@ -18,7 +18,8 @@ MEMORY
 {
     FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 2048K
     FLASH_ISR (rx)  : ORIGIN = 0x08000000, LENGTH = 32K     /* sector 0, 32K */
-    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 2016K   /* sectors 1-11 3x32K 1*128K 7*256K */
+    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 1504K   /* sectors 1-11 3x32K 1*128K 7*256K */
+    FLASH_MAPFS (rx): ORIGIN = 0x08180000, LENGTH = 512K
     FLASH_EXT (rx)  : ORIGIN = 0x90000000, LENGTH = 2048K   /* external QSPI */
     RAM (rwx)       : ORIGIN = 0x20000000, LENGTH = 512K    /* DTCM=128k, SRAM1=368K, SRAM2=16K */
 }
@@ -38,4 +39,7 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM);
 _heap_start = _ebss; /* heap starts just after statically allocated memory */
 _heap_end = _sstack;
 
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);
+
 INCLUDE common_bl.ld

ports/stm32/boards/_boot.py

@@ -0,0 +1,8 @@
+import vfs, sys, pyb
+
+
+mapfs = pyb.Flash("mapfs")
+vfs.mount(vfs.VfsMap(mapfs.ioctl(0x100, 0), mapfs), "/mapfs")
+sys.path.insert(0, "/mapfs")
+
+del vfs, sys, pyb, mapfs

ports/stm32/boards/manifest.py

@@ -1,3 +1,4 @@
+freeze(".", "_boot.py", opt=3)
 include("$(MPY_DIR)/extmod/asyncio")
 
 require("dht")

ports/stm32/boards/manifest_minimal.py

@@ -0,0 +1 @@
+freeze(".", "_boot.py", opt=3)

ports/stm32/boards/stm32f091xc.ld

@@ -5,7 +5,8 @@
 /* Specify the memory areas */
 MEMORY
 {
-    FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 236K /* sectors 0-117 */
+    FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 226K /* sectors 0-112 */
+    FLASH_MAPFS (rx): ORIGIN = 0x08036000, LENGTH = 10K /* sectors 113-117 */
     FLASH_FS (r)    : ORIGIN = 0x0803B000, LENGTH = 20K /* sectors 118-127 */
     RAM (xrw)       : ORIGIN = 0x20000000, LENGTH = 32K
 }
@@ -32,3 +33,6 @@ _heap_end = _sstack;
 
 _micropy_hw_internal_flash_storage_start = ORIGIN(FLASH_FS);
 _micropy_hw_internal_flash_storage_end   = ORIGIN(FLASH_FS) + LENGTH(FLASH_FS);
+
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);

ports/stm32/boards/stm32f405.ld

@@ -8,7 +8,8 @@ MEMORY
     FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 1024K /* entire flash */
     FLASH_ISR (rx)  : ORIGIN = 0x08000000, LENGTH = 16K /* sector 0 */
     FLASH_FS (rx)   : ORIGIN = 0x08004000, LENGTH = 112K /* sectors 1,2,3,4 are for filesystem */
-    FLASH_TEXT (rx) : ORIGIN = 0x08020000, LENGTH = 896K /* sectors 5,6,7,8,9,10,11 */
+    FLASH_TEXT (rx) : ORIGIN = 0x08020000, LENGTH = 640K /* sectors 5,6,7,8,9 */
+    FLASH_MAPFS (rx): ORIGIN = 0x080c0000, LENGTH = 256K /* sectors 10,11 */
     CCMRAM (xrw)    : ORIGIN = 0x10000000, LENGTH = 64K
     RAM (xrw)       : ORIGIN = 0x20000000, LENGTH = 128K
 }
@@ -33,3 +34,6 @@ _micropy_hw_internal_flash_storage_ram_cache_start = ORIGIN(CCMRAM);
 _micropy_hw_internal_flash_storage_ram_cache_end = ORIGIN(CCMRAM) + LENGTH(CCMRAM);
 _micropy_hw_internal_flash_storage_start = ORIGIN(FLASH_FS);
 _micropy_hw_internal_flash_storage_end = ORIGIN(FLASH_FS) + LENGTH(FLASH_FS);
+
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);

ports/stm32/mpconfigport.h

@@ -155,6 +155,8 @@
 #define MICROPY_PY_ONEWIRE          (1)
 #endif
 
+#define MICROPY_VFS_MAP             (1)
+
 // fatfs configuration used in ffconf.h
 #define MICROPY_FATFS_ENABLE_LFN       (1)
 #define MICROPY_FATFS_LFN_CODE_PAGE    437 /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */

ports/stm32/qstrdefsport.h

@@ -32,6 +32,7 @@ Q(/flash)
 Q(/flash/lib)
 Q(/sd)
 Q(/sd/lib)
+Q(/mapfs)
 
 // For os.sep
 Q(/)

ports/stm32/storage.c

@@ -259,6 +259,8 @@ int storage_readblocks_ext(uint8_t *dest, uint32_t block, uint32_t offset, uint3
 }
 #endif
 
+static struct _pyb_mapfs_obj_t pyb_mapfs_obj;
+
 typedef struct _pyb_flash_obj_t {
     mp_obj_base_t base;
     uint32_t start; // in bytes
@@ -289,6 +291,11 @@ static mp_obj_t pyb_flash_make_new(const mp_obj_type_t *type, size_t n_args, siz
         { MP_QSTR_start, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = -1} },
         { MP_QSTR_len,   MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = -1} },
     };
+
+    if (n_args == 1 && mp_obj_is_str(all_args[0]) && mp_obj_str_get_qstr(all_args[0]) == MP_QSTR_mapfs) {
+        return MP_OBJ_FROM_PTR(&pyb_mapfs_obj);
+    }
+
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
@@ -475,3 +482,103 @@ void pyb_flash_init_vfs(fs_user_mount_t *vfs) {
 }
 
 #endif
+
+/******************************************************************************/
+// mapfs partition
+//
+// TODO: this doesn't properly implement the block device protocol because it
+// has potentially large sectors but pretends they are small.
+// writing to the start of a sector will erase that whole sector first.
+
+#include "flash.h"
+
+#define MICROPY_HW_MAPFS_BASE (uintptr_t)(&_micropy_hw_mapfs_start)
+#define MICROPY_HW_MAPFS_BYTES (uintptr_t)(&_micropy_hw_mapfs_end - &_micropy_hw_mapfs_start)
+#define MAPFS_BLOCK_SIZE_BYTES (256) // can be anything bigger than alignment requirements of hardware writes
+
+typedef struct _pyb_mapfs_obj_t {
+    mp_obj_base_t base;
+    // uint32_t flash_base;
+    // uint32_t flash_size;
+} pyb_mapfs_obj_t;
+
+extern uint8_t _micropy_hw_mapfs_start;
+extern uint8_t _micropy_hw_mapfs_end;
+
+static const mp_obj_type_t pyb_mapfs_type;
+
+static pyb_mapfs_obj_t pyb_mapfs_obj = {
+    .base = { &pyb_mapfs_type },
+    // .flash_base = MICROPY_HW_MAPFS_BASE,
+    // .flash_size = MICROPY_HW_MAPFS_BYTES,
+};
+
+static mp_obj_t pyb_mapfs_readblocks(size_t n_args, const mp_obj_t *args) {
+    // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(args[0]);
+    uint32_t offset = mp_obj_get_int(args[1]) * MAPFS_BLOCK_SIZE_BYTES;
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_WRITE);
+    memcpy(bufinfo.buf, (void *)(MICROPY_HW_MAPFS_BASE + offset), bufinfo.len);
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_mapfs_readblocks_obj, 3, 3, pyb_mapfs_readblocks);
+
+static mp_obj_t pyb_mapfs_writeblocks(size_t n_args, const mp_obj_t *args) {
+    // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(args[0]);
+    uint32_t offset = mp_obj_get_int(args[1]) * MAPFS_BLOCK_SIZE_BYTES;
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_READ);
+    uint32_t dest = MICROPY_HW_MAPFS_BASE + offset;
+
+    // Compute start and end address of the sector being written.
+    uint32_t sector_size = 0;
+    uint32_t sector_start = 0;
+    int ret = flash_get_sector_info(dest, &sector_start, &sector_size);
+    if (ret < 0) {
+        return MP_OBJ_NEW_SMALL_INT(ret);
+    }
+
+    // Erase sector if writing to the start of it.
+    if (dest == sector_start) {
+        ret = flash_erase(sector_start);
+        if (ret != 0) {
+            return MP_OBJ_NEW_SMALL_INT(ret);
+        }
+    }
+
+    flash_write(dest, bufinfo.buf, bufinfo.len / 4);
+    // TODO check return value
+
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_mapfs_writeblocks_obj, 3, 4, pyb_mapfs_writeblocks);
+
+static mp_obj_t pyb_mapfs_ioctl(mp_obj_t self_in, mp_obj_t cmd_in, mp_obj_t arg_in) {
+    // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    mp_int_t cmd = mp_obj_get_int(cmd_in);
+    switch (cmd) {
+        case MP_BLOCKDEV_IOCTL_BLOCK_COUNT:
+            return MP_OBJ_NEW_SMALL_INT(MICROPY_HW_MAPFS_BYTES / MAPFS_BLOCK_SIZE_BYTES);
+        case MP_BLOCKDEV_IOCTL_BLOCK_SIZE:
+            return MP_OBJ_NEW_SMALL_INT(MAPFS_BLOCK_SIZE_BYTES);
+        case 0x100: // mmap
+            return mp_obj_new_int(MICROPY_HW_MAPFS_BASE);
+        default:
+            return mp_const_none;
+    }
+}
+static MP_DEFINE_CONST_FUN_OBJ_3(pyb_mapfs_ioctl_obj, pyb_mapfs_ioctl);
+
+static const mp_rom_map_elem_t pyb_mapfs_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_readblocks), MP_ROM_PTR(&pyb_mapfs_readblocks_obj) },
+    { MP_ROM_QSTR(MP_QSTR_writeblocks), MP_ROM_PTR(&pyb_mapfs_writeblocks_obj) },
+    { MP_ROM_QSTR(MP_QSTR_ioctl), MP_ROM_PTR(&pyb_mapfs_ioctl_obj) },
+};
+static MP_DEFINE_CONST_DICT(pyb_mapfs_locals_dict, pyb_mapfs_locals_dict_table);
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    pyb_mapfs_type,
+    MP_QSTR_Flash,
+    MP_TYPE_FLAG_NONE,
+    locals_dict, &pyb_mapfs_locals_dict
+    );

ports/unix/variants/mpconfigvariant_common.h

@@ -122,3 +122,5 @@
 #define MICROPY_PY_MACHINE             (1)
 #define MICROPY_PY_MACHINE_PULSE       (1)
 #define MICROPY_PY_MACHINE_PIN_BASE    (1)
+
+#define MICROPY_VFS_MAP                (1)

ports/webassembly/Makefile

@@ -22,6 +22,9 @@ BUILD ?= build-$(VARIANT)
 include ../../py/mkenv.mk
 include $(VARIANT_DIR)/mpconfigvariant.mk
 
+# Use the default frozen manifest, variants may override this.
+FROZEN_MANIFEST ?= variants/manifest.py
+
 # Qstr definitions (must come before including py.mk).
 QSTR_DEFS = qstrdefsport.h
 

ports/webassembly/api.js

@@ -38,7 +38,7 @@ export async function loadMicroPython(options) {
         { heapsize: 1024 * 1024, linebuffer: true },
         options,
     );
-    const Module = {};
+    let Module = {};
     Module.locateFile = (path, scriptDirectory) =>
         url || scriptDirectory + path;
     Module._textDecoder = new TextDecoder();
@@ -83,11 +83,7 @@ export async function loadMicroPython(options) {
             Module.stderr = (c) => stderr(new Uint8Array([c]));
         }
     }
-    const moduleLoaded = new Promise((r) => {
-        Module.postRun = r;
-    });
-    _createMicroPythonModule(Module);
-    await moduleLoaded;
+    Module = await _createMicroPythonModule(Module);
     globalThis.Module = Module;
     proxy_js_init();
     const pyimport = (name) => {
@@ -131,23 +127,31 @@ export async function loadMicroPython(options) {
         },
         pyimport: pyimport,
         runPython(code) {
+            const len = Module.lengthBytesUTF8(code);
+            const buf = Module._malloc(len + 1);
+            Module.stringToUTF8(code, buf, len + 1);
             const value = Module._malloc(3 * 4);
             Module.ccall(
                 "mp_js_do_exec",
                 "number",
-                ["string", "pointer"],
-                [code, value],
+                ["pointer", "number", "pointer"],
+                [buf, len, value],
             );
+            Module._free(buf);
             return proxy_convert_mp_to_js_obj_jsside_with_free(value);
         },
         runPythonAsync(code) {
+            const len = Module.lengthBytesUTF8(code);
+            const buf = Module._malloc(len + 1);
+            Module.stringToUTF8(code, buf, len + 1);
             const value = Module._malloc(3 * 4);
             Module.ccall(
                 "mp_js_do_exec_async",
                 "number",
-                ["string", "pointer"],
-                [code, value],
+                ["pointer", "number", "pointer"],
+                [buf, len, value],
             );
+            Module._free(buf);
             return proxy_convert_mp_to_js_obj_jsside_with_free(value);
         },
         replInit() {
@@ -224,6 +228,16 @@ async function runCLI() {
             }
         });
     } else {
+        // If the script to run ends with a running of the asyncio main loop, then inject
+        // a simple `asyncio.run` hook that starts the main task.  This is primarily to
+        // support running the standard asyncio tests.
+        if (contents.endsWith("asyncio.run(main())\n")) {
+            const asyncio = mp.pyimport("asyncio");
+            asyncio.run = async (task) => {
+                await asyncio.create_task(task);
+            };
+        }
+
         try {
             mp.runPython(contents);
         } catch (error) {

ports/webassembly/asyncio/__init__.py

@@ -0,0 +1,9 @@
+# MicroPython asyncio module, for use with webassembly port
+# MIT license; Copyright (c) 2024 Damien P. George
+
+from .core import *
+from .funcs import wait_for, wait_for_ms, gather
+from .event import Event
+from .lock import Lock
+
+__version__ = (3, 0, 0)

ports/webassembly/asyncio/core.py

@@ -0,0 +1,249 @@
+# MicroPython asyncio module, for use with webassembly port
+# MIT license; Copyright (c) 2019-2024 Damien P. George
+
+from time import ticks_ms as ticks, ticks_diff, ticks_add
+import sys, js, jsffi
+
+# Import TaskQueue and Task from built-in C code.
+from _asyncio import TaskQueue, Task
+
+
+################################################################################
+# Exceptions
+
+
+class CancelledError(BaseException):
+    pass
+
+
+class TimeoutError(Exception):
+    pass
+
+
+# Used when calling Loop.call_exception_handler.
+_exc_context = {"message": "Task exception wasn't retrieved", "exception": None, "future": None}
+
+
+################################################################################
+# Sleep functions
+
+
+# "Yield" once, then raise StopIteration
+class SingletonGenerator:
+    def __init__(self):
+        self.state = None
+        self.exc = StopIteration()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.state is not None:
+            _task_queue.push(cur_task, self.state)
+            self.state = None
+            return None
+        else:
+            self.exc.__traceback__ = None
+            raise self.exc
+
+
+# Pause task execution for the given time (integer in milliseconds, uPy extension)
+# Use a SingletonGenerator to do it without allocating on the heap
+def sleep_ms(t, sgen=SingletonGenerator()):
+    if cur_task is None:
+        # Support top-level asyncio.sleep, via a JavaScript Promise.
+        return jsffi.async_timeout_ms(t)
+    assert sgen.state is None
+    sgen.state = ticks_add(ticks(), max(0, t))
+    return sgen
+
+
+# Pause task execution for the given time (in seconds)
+def sleep(t):
+    return sleep_ms(int(t * 1000))
+
+
+################################################################################
+# Main run loop
+
+asyncio_timer = None
+
+
+class ThenableEvent:
+    def __init__(self, thenable):
+        self.result = None  # Result of the thenable
+        self.waiting = None  # Task waiting on completion of this thenable
+        thenable.then(self.set)
+
+    def set(self, value):
+        # Thenable/Promise is fulfilled, set result and schedule any waiting task.
+        self.result = value
+        if self.waiting:
+            _task_queue.push(self.waiting)
+            self.waiting = None
+            _schedule_run_iter(0)
+
+    def remove(self, task):
+        self.waiting = None
+
+    # async
+    def wait(self):
+        # Set the calling task as the task waiting on this thenable.
+        self.waiting = cur_task
+        # Set calling task's data to this object so it can be removed if needed.
+        cur_task.data = self
+        # Wait for the thenable to fulfill.
+        yield
+        # Return the result of the thenable.
+        return self.result
+
+
+# Ensure the awaitable is a task
+def _promote_to_task(aw):
+    return aw if isinstance(aw, Task) else create_task(aw)
+
+
+def _schedule_run_iter(dt):
+    global asyncio_timer
+    if asyncio_timer is not None:
+        js.clearTimeout(asyncio_timer)
+    asyncio_timer = js.setTimeout(_run_iter, dt)
+
+
+def _run_iter():
+    global cur_task
+    excs_all = (CancelledError, Exception)  # To prevent heap allocation in loop
+    excs_stop = (CancelledError, StopIteration)  # To prevent heap allocation in loop
+    while True:
+        # Wait until the head of _task_queue is ready to run
+        t = _task_queue.peek()
+        if t:
+            # A task waiting on _task_queue; "ph_key" is time to schedule task at
+            dt = max(0, ticks_diff(t.ph_key, ticks()))
+        else:
+            # No tasks can be woken so finished running
+            cur_task = None
+            return
+
+        if dt > 0:
+            # schedule to call again later
+            cur_task = None
+            _schedule_run_iter(dt)
+            return
+
+        # Get next task to run and continue it
+        t = _task_queue.pop()
+        cur_task = t
+        try:
+            # Continue running the coroutine, it's responsible for rescheduling itself
+            exc = t.data
+            if not exc:
+                t.coro.send(None)
+            else:
+                # If the task is finished and on the run queue and gets here, then it
+                # had an exception and was not await'ed on.  Throwing into it now will
+                # raise StopIteration and the code below will catch this and run the
+                # call_exception_handler function.
+                t.data = None
+                t.coro.throw(exc)
+        except excs_all as er:
+            # Check the task is not on any event queue
+            assert t.data is None
+            # This task is done.
+            if t.state:
+                # Task was running but is now finished.
+                waiting = False
+                if t.state is True:
+                    # "None" indicates that the task is complete and not await'ed on (yet).
+                    t.state = None
+                elif callable(t.state):
+                    # The task has a callback registered to be called on completion.
+                    t.state(t, er)
+                    t.state = False
+                    waiting = True
+                else:
+                    # Schedule any other tasks waiting on the completion of this task.
+                    while t.state.peek():
+                        _task_queue.push(t.state.pop())
+                        waiting = True
+                    # "False" indicates that the task is complete and has been await'ed on.
+                    t.state = False
+                if not waiting and not isinstance(er, excs_stop):
+                    # An exception ended this detached task, so queue it for later
+                    # execution to handle the uncaught exception if no other task retrieves
+                    # the exception in the meantime (this is handled by Task.throw).
+                    _task_queue.push(t)
+                # Save return value of coro to pass up to caller.
+                t.data = er
+            elif t.state is None:
+                # Task is already finished and nothing await'ed on the task,
+                # so call the exception handler.
+
+                # Save exception raised by the coro for later use.
+                t.data = exc
+
+                # Create exception context and call the exception handler.
+                _exc_context["exception"] = exc
+                _exc_context["future"] = t
+                Loop.call_exception_handler(_exc_context)
+
+
+# Create and schedule a new task from a coroutine.
+def create_task(coro):
+    if not hasattr(coro, "send"):
+        raise TypeError("coroutine expected")
+    t = Task(coro, globals())
+    _task_queue.push(t)
+    _schedule_run_iter(0)
+    return t
+
+
+################################################################################
+# Event loop wrapper
+
+
+cur_task = None
+
+
+class Loop:
+    _exc_handler = None
+
+    def create_task(coro):
+        return create_task(coro)
+
+    def close():
+        pass
+
+    def set_exception_handler(handler):
+        Loop._exc_handler = handler
+
+    def get_exception_handler():
+        return Loop._exc_handler
+
+    def default_exception_handler(loop, context):
+        print(context["message"], file=sys.stderr)
+        print("future:", context["future"], "coro=", context["future"].coro, file=sys.stderr)
+        sys.print_exception(context["exception"], sys.stderr)
+
+    def call_exception_handler(context):
+        (Loop._exc_handler or Loop.default_exception_handler)(Loop, context)
+
+
+def get_event_loop():
+    return Loop
+
+
+def current_task():
+    if cur_task is None:
+        raise RuntimeError("no running event loop")
+    return cur_task
+
+
+def new_event_loop():
+    global _task_queue
+    _task_queue = TaskQueue()  # TaskQueue of Task instances.
+    return Loop
+
+
+# Initialise default event loop.
+new_event_loop()

ports/webassembly/main.c

@@ -104,7 +104,7 @@ void mp_js_do_import(const char *name, uint32_t *out) {
     }
 }
 
-void mp_js_do_exec(const char *src, uint32_t *out) {
+void mp_js_do_exec(const char *src, size_t len, uint32_t *out) {
     // Collect at the top-level, where there are no root pointers from stack/registers.
     gc_collect_start();
     gc_collect_end();
@@ -112,7 +112,7 @@ void mp_js_do_exec(const char *src, uint32_t *out) {
     mp_parse_input_kind_t input_kind = MP_PARSE_FILE_INPUT;
     nlr_buf_t nlr;
     if (nlr_push(&nlr) == 0) {
-        mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0);
+        mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, len, 0);
         qstr source_name = lex->source_name;
         mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
         mp_obj_t module_fun = mp_compile(&parse_tree, source_name, false);
@@ -125,9 +125,9 @@ void mp_js_do_exec(const char *src, uint32_t *out) {
     }
 }
 
-void mp_js_do_exec_async(const char *src, uint32_t *out) {
+void mp_js_do_exec_async(const char *src, size_t len, uint32_t *out) {
     mp_compile_allow_top_level_await = true;
-    mp_js_do_exec(src, out);
+    mp_js_do_exec(src, len, out);
     mp_compile_allow_top_level_await = false;
 }
 

ports/webassembly/modjsffi.c

@@ -61,12 +61,27 @@ static mp_obj_t mp_jsffi_to_js(mp_obj_t arg) {
 }
 static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_to_js_obj, mp_jsffi_to_js);
 
+// *FORMAT-OFF*
+EM_JS(void, promise_with_timeout_ms, (double ms, uint32_t * out), {
+    const ret = new Promise((resolve) => setTimeout(resolve, ms));
+    proxy_convert_js_to_mp_obj_jsside(ret, out);
+});
+// *FORMAT-ON*
+
+static mp_obj_t mp_jsffi_async_timeout_ms(mp_obj_t arg) {
+    uint32_t out[PVN];
+    promise_with_timeout_ms(mp_obj_get_float_to_d(arg), out);
+    return proxy_convert_js_to_mp_obj_cside(out);
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_async_timeout_ms_obj, mp_jsffi_async_timeout_ms);
+
 static const mp_rom_map_elem_t mp_module_jsffi_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_jsffi) },
 
     { MP_ROM_QSTR(MP_QSTR_JsProxy), MP_ROM_PTR(&mp_type_jsproxy) },
     { MP_ROM_QSTR(MP_QSTR_create_proxy), MP_ROM_PTR(&mp_jsffi_create_proxy_obj) },
     { MP_ROM_QSTR(MP_QSTR_to_js), MP_ROM_PTR(&mp_jsffi_to_js_obj) },
+    { MP_ROM_QSTR(MP_QSTR_async_timeout_ms), MP_ROM_PTR(&mp_jsffi_async_timeout_ms_obj) },
 };
 static MP_DEFINE_CONST_DICT(mp_module_jsffi_globals, mp_module_jsffi_globals_table);
 

ports/webassembly/objjsproxy.c

@@ -346,6 +346,12 @@ typedef struct _jsproxy_gen_t {
 
 mp_vm_return_kind_t jsproxy_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) {
     jsproxy_gen_t *self = MP_OBJ_TO_PTR(self_in);
+
+    if (throw_value) {
+        *ret_val = throw_value;
+        return MP_VM_RETURN_EXCEPTION;
+    }
+
     switch (self->state) {
         case JSOBJ_GEN_STATE_WAITING:
             self->state = JSOBJ_GEN_STATE_COMPLETED;
@@ -468,9 +474,29 @@ static mp_obj_t jsproxy_new_gen(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
 
 /******************************************************************************/
 
+#if MICROPY_PY_ASYNCIO
+extern mp_obj_t mp_asyncio_context;
+#endif
+
 static mp_obj_t jsproxy_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
     mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in);
     if (has_attr(self->ref, "then")) {
+        #if MICROPY_PY_ASYNCIO
+        // When asyncio is running and the caller here is a task, wrap the JavaScript
+        // thenable in a ThenableEvent, and get the task to wait on that event.  This
+        // decouples the task from the thenable and allows cancelling the task.
+        if (mp_asyncio_context != MP_OBJ_NULL) {
+            mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task));
+            if (cur_task != mp_const_none) {
+                mp_obj_t thenable_event_class = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_ThenableEvent));
+                mp_obj_t thenable_event = mp_call_function_1(thenable_event_class, self_in);
+                mp_obj_t dest[2];
+                mp_load_method(thenable_event, MP_QSTR_wait, dest);
+                mp_obj_t wait_gen = mp_call_method_n_kw(0, 0, dest);
+                return mp_getiter(wait_gen, iter_buf);
+            }
+        }
+        #endif
         return jsproxy_new_gen(self_in, iter_buf);
     } else {
         return jsproxy_new_it(self_in, iter_buf);

ports/webassembly/proxy_c.c

@@ -296,10 +296,11 @@ EM_JS(void, js_then_resolve, (uint32_t * ret_value, uint32_t * resolve, uint32_t
     resolve_js(ret_value_js);
 });
 
-EM_JS(void, js_then_reject, (uint32_t * resolve, uint32_t * reject), {
+EM_JS(void, js_then_reject, (uint32_t * ret_value, uint32_t * resolve, uint32_t * reject), {
+    const ret_value_js = proxy_convert_mp_to_js_obj_jsside(ret_value);
     const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve);
     const reject_js = proxy_convert_mp_to_js_obj_jsside(reject);
-    reject_js(null);
+    reject_js(ret_value_js);
 });
 
 // *FORMAT-OFF*
@@ -307,14 +308,34 @@ EM_JS(void, js_then_continue, (int jsref, uint32_t * py_resume, uint32_t * resol
     const py_resume_js = proxy_convert_mp_to_js_obj_jsside(py_resume);
     const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve);
     const reject_js = proxy_convert_mp_to_js_obj_jsside(reject);
-    const ret = proxy_js_ref[jsref].then((x) => {py_resume_js(x, resolve_js, reject_js);}, reject_js);
+    const ret = proxy_js_ref[jsref].then(
+        (result) => {
+            // The Promise is fulfilled on the JavaScript side.  Take the result and
+            // send it to the encapsulating generator on the Python side, so it
+            // becomes the result of the "yield from" that deferred to this Promise.
+            py_resume_js(result, null, resolve_js, reject_js);
+        },
+        (reason) => {
+            // The Promise is rejected on the JavaScript side.  Take the reason and
+            // throw it into the encapsulating generator on the Python side.
+            py_resume_js(null, reason, resolve_js, reject_js);
+        },
+    );
     proxy_convert_js_to_mp_obj_jsside(ret, out);
 });
 // *FORMAT-ON*
 
-static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t resolve, mp_obj_t reject) {
+static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t resolve, mp_obj_t reject) {
+    if (throw_value != MP_OBJ_NULL && throw_value != mp_const_none) {
+        if (send_value == mp_const_none) {
+            send_value = MP_OBJ_NULL;
+        }
+    } else {
+        throw_value = MP_OBJ_NULL;
+    }
+
     mp_obj_t ret_value;
-    mp_vm_return_kind_t ret_kind = mp_resume(self_in, value, MP_OBJ_NULL, &ret_value);
+    mp_vm_return_kind_t ret_kind = mp_resume(self_in, send_value, throw_value, &ret_value);
 
     uint32_t out_resolve[PVN];
     uint32_t out_reject[PVN];
@@ -335,17 +356,19 @@ static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t
         uint32_t out[PVN];
         js_then_continue(ref, out_py_resume, out_resolve, out_reject, out);
         return proxy_convert_js_to_mp_obj_cside(out);
-    } else {
-        // MP_VM_RETURN_EXCEPTION;
-        js_then_reject(out_resolve, out_reject);
-        nlr_raise(ret_value);
+    } else { // ret_kind == MP_VM_RETURN_EXCEPTION;
+        // Pass the exception through as an object to reject the promise (don't raise/throw it).
+        uint32_t out_ret_value[PVN];
+        proxy_convert_mp_to_js_obj_cside(ret_value, out_ret_value);
+        js_then_reject(out_ret_value, out_resolve, out_reject);
+        return mp_const_none;
     }
 }
 
 static mp_obj_t resume_fun(size_t n_args, const mp_obj_t *args) {
-    return proxy_resume_execute(args[0], args[1], args[2], args[3]);
+    return proxy_resume_execute(args[0], args[1], args[2], args[3], args[4]);
 }
-static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 4, 4, resume_fun);
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 5, 5, resume_fun);
 
 void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) {
     nlr_buf_t nlr;
@@ -353,7 +376,7 @@ void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) {
         mp_obj_t obj = proxy_c_get_obj(c_ref);
         mp_obj_t resolve = proxy_convert_js_to_mp_obj_cside(args + 1 * 3);
         mp_obj_t reject = proxy_convert_js_to_mp_obj_cside(args + 2 * 3);
-        mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, resolve, reject);
+        mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, mp_const_none, resolve, reject);
         nlr_pop();
         return proxy_convert_mp_to_js_obj_cside(ret, args);
     } else {

ports/webassembly/variants/manifest.py

@@ -0,0 +1,24 @@
+# The asyncio package is built from the standard implementation but with the
+# core scheduler replaced with a custom scheduler that uses the JavaScript
+# runtime (with setTimeout an Promise's) to contrtol the scheduling.
+
+package(
+    "asyncio",
+    (
+        "event.py",
+        "funcs.py",
+        "lock.py",
+    ),
+    base_path="$(MPY_DIR)/extmod",
+    opt=3,
+)
+
+package(
+    "asyncio",
+    (
+        "__init__.py",
+        "core.py",
+    ),
+    base_path="$(PORT_DIR)",
+    opt=3,
+)

ports/webassembly/variants/pyscript/manifest.py

@@ -1,3 +1,5 @@
+include("$(PORT_DIR)/variants/manifest.py")
+
 require("abc")
 require("base64")
 require("collections")

py/lexer.c

@@ -908,7 +908,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
 
 void mp_lexer_free(mp_lexer_t *lex) {
     if (lex) {
-        lex->reader.close(lex->reader.data);
+        lex->reader.ioctl(lex->reader.data, MP_READER_CLOSE, 0);
         vstr_clear(&lex->vstr);
         #if MICROPY_PY_FSTRINGS
         vstr_clear(&lex->fstring_args);

py/mpconfig.h

@@ -587,6 +587,12 @@
 /*****************************************************************************/
 /* Python internal features                                                  */
 
+// Use a special long jump in nlrthumb.c, which may be necessary if nlr.o and
+// nlrthumb.o are linked far apart from each other.
+#ifndef MICROPY_NLR_THUMB_USE_LONG_JUMP
+#define MICROPY_NLR_THUMB_USE_LONG_JUMP (0)
+#endif
+
 // Whether to enable import of external modules
 // When disabled, only importing of built-in modules is supported
 // When enabled, a port must implement mp_import_stat (among other things)

py/nlrthumb.c

@@ -38,6 +38,14 @@
 
 __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) {
 
+    // If you get a linker error here, indicating that a relocation doesn't
+    // fit, try the following (in that order):
+    //
+    // 1. Ensure that nlr.o nlrthumb.o are linked closely together, i.e.
+    //    there aren't too many other files between them in the linker list
+    //    (PY_CORE_O_BASENAME in py/py.mk)
+    // 2. Set -DMICROPY_NLR_THUMB_USE_LONG_JUMP=1 during the build
+    //
     __asm volatile (
         "str    r4, [r0, #12]       \n" // store r4 into nlr_buf
         "str    r5, [r0, #16]       \n" // store r5 into nlr_buf
@@ -71,7 +79,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) {
         "str    lr, [r0, #8]        \n" // store lr into nlr_buf
         #endif
 
-        #if !defined(__thumb2__)
+        #if MICROPY_NLR_THUMB_USE_LONG_JUMP
         "ldr    r1, nlr_push_tail_var \n"
         "bx     r1                  \n" // do the rest in C
         ".align 2                   \n"

py/obj.h

@@ -753,20 +753,20 @@ typedef struct _mp_obj_full_type_t {
 // Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE.
 // Generated with:
 // for i in range(13):
-//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .name = _name, .flags = _flags{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slots = { v1, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
-#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
+//     print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}")
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } }
+#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } }
 
 // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take
 // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking

py/objarray.c

@@ -424,6 +424,13 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) {
     if (self->free < len) {
         self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz);
         self->free = 0;
+
+        if (self_in == arg_in) {
+            // Get arg_bufinfo again in case self->items has moved
+            //
+            // (Note not possible to handle case that arg_in is a memoryview into self)
+            mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ);
+        }
     } else {
         self->free -= len;
     }
@@ -456,7 +463,8 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                 #if MICROPY_PY_ARRAY_SLICE_ASSIGN
                 // Assign
                 size_t src_len;
-                void *src_items;
+                uint8_t *src_items;
+                size_t src_offs = 0;
                 size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL);
                 if (mp_obj_is_obj(value) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(value))->type, subscr) == array_subscr) {
                     // value is array, bytearray or memoryview
@@ -469,7 +477,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                     src_items = src_slice->items;
                     #if MICROPY_PY_BUILTINS_MEMORYVIEW
                     if (mp_obj_is_type(value, &mp_type_memoryview)) {
-                        src_items = (uint8_t *)src_items + (src_slice->memview_offset * item_sz);
+                        src_offs = src_slice->memview_offset * item_sz;
                     }
                     #endif
                 } else if (mp_obj_is_type(value, &mp_type_bytes)) {
@@ -504,13 +512,17 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                         // TODO: alloc policy; at the moment we go conservative
                         o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz);
                         o->free = len_adj;
+                        // m_renew may have moved o->items
+                        if (src_items == dest_items) {
+                            src_items = o->items;
+                        }
                         dest_items = o->items;
                     }
                     mp_seq_replace_slice_grow_inplace(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, len_adj, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, len_adj, item_sz);
                 } else {
                     mp_seq_replace_slice_no_grow(dest_items, o->len,
-                        slice.start, slice.stop, src_items, src_len, item_sz);
+                        slice.start, slice.stop, src_items + src_offs, src_len, item_sz);
                     // Clear "freed" elements at the end of list
                     // TODO: This is actually only needed for typecode=='O'
                     mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz);

py/objfun.h

@@ -56,14 +56,14 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
 #if MICROPY_EMIT_NATIVE
 
 static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
-    mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
+    mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));
     o->base.type = &mp_type_fun_native;
     return MP_OBJ_FROM_PTR(o);
 }
 
 static inline mp_obj_t mp_obj_new_fun_viper(const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
     mp_obj_fun_bc_t *o = mp_obj_malloc(mp_obj_fun_bc_t, &mp_type_fun_viper);
-    o->bytecode = fun_data;
+    o->bytecode = (const byte *)fun_data;
     o->context = mc;
     o->child_table = child_table;
     return MP_OBJ_FROM_PTR(o);
@@ -101,9 +101,9 @@ static inline void *mp_obj_fun_native_get_generator_resume(const mp_obj_fun_bc_t
 
 #if MICROPY_EMIT_INLINE_ASM
 static inline mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) {
-    mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
+    mp_obj_fun_asm_t *o = (mp_obj_fun_asm_t *)mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm);
     o->n_args = n_args;
-    o->fun_data = fun_data;
+    o->fun_data = (const byte *)fun_data;
     o->type_sig = type_sig;
     return MP_OBJ_FROM_PTR(o);
 }

py/persistentcode.c

@@ -160,6 +160,18 @@ static size_t read_uint(mp_reader_t *reader) {
     return unum;
 }
 
+#if MICROPY_VFS_MAP
+static inline const uint8_t *map_try_read_bytes(mp_reader_t *reader, size_t len) {
+    mp_reader_ioctl_memmap_t memmap;
+    memmap.len = len;
+    intptr_t ret = reader->ioctl(reader->data, MP_READER_MEMMAP, (uintptr_t)&memmap);
+    if (ret < 0) {
+        return NULL;
+    }
+    return memmap.ptr;
+}
+#endif
+
 static qstr load_qstr(mp_reader_t *reader) {
     size_t len = read_uint(reader);
     if (len & 1) {
@@ -167,6 +179,12 @@ static qstr load_qstr(mp_reader_t *reader) {
         return len >> 1;
     }
     len >>= 1;
+    #if MICROPY_VFS_MAP
+    const uint8_t *memmap = map_try_read_bytes(reader, len + 1);
+    if (memmap != NULL) {
+        return qstr_from_strn_static((const char *)memmap, len);
+    }
+    #endif
     char *str = m_new(char, len);
     read_bytes(reader, (byte *)str, len);
     read_byte(reader); // read and discard null terminator
@@ -175,6 +193,22 @@ static qstr load_qstr(mp_reader_t *reader) {
     return qst;
 }
 
+#if MICROPY_VFS_MAP
+static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *data, size_t len) {
+    if (type == &mp_type_str) {
+        qstr q = qstr_find_strn((const char *)data, len);
+        if (q != MP_QSTRnull) {
+            return MP_OBJ_NEW_QSTR(q);
+        }
+    }
+    mp_obj_str_t *o = m_new_obj(mp_obj_str_t);
+    o->base.type = type;
+    o->len = len;
+    o->hash = qstr_compute_hash(data, len);
+    o->data = data;
+    return MP_OBJ_FROM_PTR(o);
+}
+#endif
 static mp_obj_t load_obj(mp_reader_t *reader) {
     byte obj_type = read_byte(reader);
     #if MICROPY_EMIT_MACHINE_CODE
@@ -192,6 +226,7 @@ static mp_obj_t load_obj(mp_reader_t *reader) {
         return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj);
     } else {
         size_t len = read_uint(reader);
+
         if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) {
             read_byte(reader); // skip null terminator
             return mp_const_empty_bytes;
@@ -202,11 +237,25 @@ static mp_obj_t load_obj(mp_reader_t *reader) {
             }
             return MP_OBJ_FROM_PTR(tuple);
         }
+
+
+        const uint8_t *memmap = NULL;
         vstr_t vstr;
-        vstr_init_len(&vstr, len);
-        read_bytes(reader, (byte *)vstr.buf, len);
+        #if MICROPY_VFS_MAP
+        memmap = map_try_read_bytes(reader, len);
+        #endif
+        if (memmap == NULL) {
+            vstr_init_len(&vstr, len);
+            read_bytes(reader, (byte *)vstr.buf, len);
+        }
         if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) {
             read_byte(reader); // skip null terminator
+            #if MICROPY_VFS_MAP
+            if (memmap != NULL) {
+                const mp_obj_type_t *t = obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes;
+                return mp_obj_new_str_static(t, memmap, len);
+            }
+            #endif
             if (obj_type == MP_PERSISTENT_OBJ_STR) {
                 return mp_obj_new_str_from_utf8_vstr(&vstr);
             } else {
@@ -243,10 +292,17 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
     #endif
 
     if (kind == MP_CODE_BYTECODE) {
-        // Allocate memory for the bytecode
-        fun_data = m_new(uint8_t, fun_data_len);
-        // Load bytecode
-        read_bytes(reader, fun_data, fun_data_len);
+        #if MICROPY_VFS_MAP
+        // Try to reference memory-mapped data for the bytecode.
+        fun_data = (uint8_t *)map_try_read_bytes(reader, fun_data_len);
+        #endif
+
+        if (fun_data == NULL) {
+            // Allocate memory for the bytecode.
+            fun_data = m_new(uint8_t, fun_data_len);
+            // Load bytecode.
+            read_bytes(reader, fun_data, fun_data_len);
+        }
 
     #if MICROPY_EMIT_MACHINE_CODE
     } else {
@@ -393,9 +449,14 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
     return rc;
 }
 
+static void reader_close(void *reader_in) {
+    mp_reader_t *reader = reader_in;
+    reader->ioctl(reader->data, MP_READER_CLOSE, 0);
+}
+
 void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) {
     // Set exception handler to close the reader if an exception is raised.
-    MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader->close, reader->data);
+    MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader_close, reader);
     nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback);
 
     byte header[4];

py/qstr.c

@@ -309,7 +309,7 @@ qstr qstr_from_str(const char *str) {
     return qstr_from_strn(str, strlen(str));
 }
 
-qstr qstr_from_strn(const char *str, size_t len) {
+static qstr qstr_from_strn_helper(const char *str, size_t len, bool is_static) {
     QSTR_ENTER();
     qstr q = qstr_find_strn(str, len);
     if (q == 0) {
@@ -321,6 +321,11 @@ qstr qstr_from_strn(const char *str, size_t len) {
             mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("name too long"));
         }
 
+        if (is_static) {
+            assert(str[len] == '\0');
+            goto add;
+        }
+
         // compute number of bytes needed to intern this string
         size_t n_bytes = len + 1;
 
@@ -364,12 +369,23 @@ qstr qstr_from_strn(const char *str, size_t len) {
         // store the interned strings' data
         memcpy(q_ptr, str, len);
         q_ptr[len] = '\0';
-        q = qstr_add(len, q_ptr);
+        str = q_ptr;
+
+    add:
+        q = qstr_add(len, str);
     }
     QSTR_EXIT();
     return q;
 }
 
+qstr qstr_from_strn(const char *str, size_t len) {
+    return qstr_from_strn_helper(str, len, false);
+}
+
+qstr qstr_from_strn_static(const char *str, size_t len) {
+    return qstr_from_strn_helper(str, len, true);
+}
+
 mp_uint_t qstr_hash(qstr q) {
     const qstr_pool_t *pool = find_qstr(&q);
     #if MICROPY_QSTR_BYTES_IN_HASH

py/qstr.h

@@ -101,6 +101,7 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if
 
 qstr qstr_from_str(const char *str);
 qstr qstr_from_strn(const char *str, size_t len);
+qstr qstr_from_strn_static(const char *str, size_t len);
 
 mp_uint_t qstr_hash(qstr q);
 const char *qstr_str(qstr q);

py/reader.c

@@ -39,7 +39,7 @@ typedef struct _mp_reader_mem_t {
     const byte *end;
 } mp_reader_mem_t;
 
-static mp_uint_t mp_reader_mem_readbyte(void *data) {
+static uintptr_t mp_reader_mem_readbyte(void *data) {
     mp_reader_mem_t *reader = (mp_reader_mem_t *)data;
     if (reader->cur < reader->end) {
         return *reader->cur++;
@@ -48,12 +48,23 @@ static mp_uint_t mp_reader_mem_readbyte(void *data) {
     }
 }
 
-static void mp_reader_mem_close(void *data) {
+static intptr_t mp_reader_mem_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_mem_t *reader = (mp_reader_mem_t *)data;
-    if (reader->free_len > 0) {
-        m_del(char, (char *)reader->beg, reader->free_len);
+
+    if (request == MP_READER_CLOSE) {
+        if (reader->free_len > 0 && reader->free_len != (size_t)-1) {
+            m_del(char, (char *)reader->beg, reader->free_len);
+        }
+        m_del_obj(mp_reader_mem_t, reader);
+        return 0;
+    } else if (request == MP_READER_MEMMAP && reader->free_len == (size_t)-1) {
+        mp_reader_ioctl_memmap_t *memmap = (mp_reader_ioctl_memmap_t *)arg;
+        memmap->ptr = reader->cur;
+        reader->cur += memmap->len;
+        return 0;
     }
-    m_del_obj(mp_reader_mem_t, reader);
+
+    return -MP_EINVAL;
 }
 
 void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) {
@@ -64,7 +75,7 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t
     rm->end = buf + len;
     reader->data = rm;
     reader->readbyte = mp_reader_mem_readbyte;
-    reader->close = mp_reader_mem_close;
+    reader->ioctl = mp_reader_mem_ioctl;
 }
 
 #if MICROPY_READER_POSIX
@@ -81,7 +92,7 @@ typedef struct _mp_reader_posix_t {
     byte buf[20];
 } mp_reader_posix_t;
 
-static mp_uint_t mp_reader_posix_readbyte(void *data) {
+static uintptr_t mp_reader_posix_readbyte(void *data) {
     mp_reader_posix_t *reader = (mp_reader_posix_t *)data;
     if (reader->pos >= reader->len) {
         if (reader->len == 0) {
@@ -101,14 +112,20 @@ static mp_uint_t mp_reader_posix_readbyte(void *data) {
     return reader->buf[reader->pos++];
 }
 
-static void mp_reader_posix_close(void *data) {
+static intptr_t mp_reader_posix_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_posix_t *reader = (mp_reader_posix_t *)data;
-    if (reader->close_fd) {
-        MP_THREAD_GIL_EXIT();
-        close(reader->fd);
-        MP_THREAD_GIL_ENTER();
+
+    if (request == MP_READER_CLOSE) {
+        if (reader->close_fd) {
+            MP_THREAD_GIL_EXIT();
+            close(reader->fd);
+            MP_THREAD_GIL_ENTER();
+        }
+        m_del_obj(mp_reader_posix_t, reader);
+        return 0;
     }
-    m_del_obj(mp_reader_posix_t, reader);
+
+    return -MP_EINVAL;
 }
 
 void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) {
@@ -129,7 +146,7 @@ void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) {
     rp->pos = 0;
     reader->data = rp;
     reader->readbyte = mp_reader_posix_readbyte;
-    reader->close = mp_reader_posix_close;
+    reader->ioctl = mp_reader_posix_ioctl;
 }
 
 #if !MICROPY_VFS_POSIX

py/reader.h

@@ -28,15 +28,30 @@
 
 #include "py/obj.h"
 
-// the readbyte function must return the next byte in the input stream
-// it must return MP_READER_EOF if end of stream
-// it can be called again after returning MP_READER_EOF, and in that case must return MP_READER_EOF
-#define MP_READER_EOF ((mp_uint_t)(-1))
+#define MP_READER_EOF ((uintptr_t)(-1))
+
+// Reader ioctl request codes.
+#define MP_READER_CLOSE (1)
+#define MP_READER_MEMMAP (2)
+
+// Used as arg for MP_READER_MEMMAP ioctl request.
+typedef struct _mp_reader_ioctl_memmap_t {
+    size_t len;
+    const uint8_t *ptr;
+} mp_reader_ioctl_memmap_t;
 
 typedef struct _mp_reader_t {
+    // Pointer to the context of this reader, passed as the first argument to the methods below.
     void *data;
-    mp_uint_t (*readbyte)(void *data);
-    void (*close)(void *data);
+
+    // The readbyte function must return the next byte in the input stream.
+    // It must return MP_READER_EOF if end of stream.  It can be called again after returning
+    // MP_READER_EOF, and in that case must return MP_READER_EOF.
+    uintptr_t (*readbyte)(void *data);
+
+    // Ioctl method for performing various control actions.
+    // On error it must return a negative errno value.
+    intptr_t (*ioctl)(void *data, uintptr_t request, uintptr_t arg);
 } mp_reader_t;
 
 void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len);

py/stream.h

@@ -44,6 +44,7 @@
 #define MP_STREAM_SET_DATA_OPTS (9)  // Set data/message options
 #define MP_STREAM_GET_FILENO    (10) // Get fileno of underlying file
 #define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file
+#define MP_STREAM_INIT_READER   (12) // Initialise a mp_reader_t for this stream
 
 // These poll ioctl values are compatible with Linux
 #define MP_STREAM_POLL_RD       (0x0001)

shared/runtime/pyexec.c

@@ -34,6 +34,7 @@
 #include "py/repl.h"
 #include "py/gc.h"
 #include "py/frozenmod.h"
+#include "py/mperrno.h"
 #include "py/mphal.h"
 #if MICROPY_HW_ENABLE_USB
 #include "irq.h"
@@ -135,7 +136,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input
 
         if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) {
             const mp_reader_t *reader = source;
-            reader->close(reader->data);
+            reader->ioctl(reader->data, MP_READER_CLOSE, 0);
         }
 
         // print EOF after normal output
@@ -201,7 +202,7 @@ typedef struct _mp_reader_stdin_t {
     uint16_t window_remain;
 } mp_reader_stdin_t;
 
-static mp_uint_t mp_reader_stdin_readbyte(void *data) {
+static uintptr_t mp_reader_stdin_readbyte(void *data) {
     mp_reader_stdin_t *reader = (mp_reader_stdin_t *)data;
 
     if (reader->eof) {
@@ -233,18 +234,24 @@ static mp_uint_t mp_reader_stdin_readbyte(void *data) {
     return c;
 }
 
-static void mp_reader_stdin_close(void *data) {
+static intptr_t mp_reader_stdin_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_stdin_t *reader = (mp_reader_stdin_t *)data;
-    if (!reader->eof) {
-        reader->eof = true;
-        mp_hal_stdout_tx_strn("\x04", 1); // indicate end to host
-        for (;;) {
-            int c = mp_hal_stdin_rx_chr();
-            if (c == CHAR_CTRL_C || c == CHAR_CTRL_D) {
-                break;
+
+    if (request == MP_READER_CLOSE) {
+        if (!reader->eof) {
+            reader->eof = true;
+            mp_hal_stdout_tx_strn("\x04", 1); // indicate end to host
+            for (;;) {
+                int c = mp_hal_stdin_rx_chr();
+                if (c == CHAR_CTRL_C || c == CHAR_CTRL_D) {
+                    break;
+                }
             }
         }
+        return 0;
     }
+
+    return -MP_EINVAL;
 }
 
 static void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_stdin, uint16_t buf_max) {
@@ -260,7 +267,7 @@ static void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s
     reader_stdin->window_remain = window;
     reader->data = reader_stdin;
     reader->readbyte = mp_reader_stdin_readbyte;
-    reader->close = mp_reader_stdin_close;
+    reader->ioctl = mp_reader_stdin_ioctl;
 }
 
 static int do_reader_stdin(int c) {

tests/basics/bytearray_add.py

@@ -15,4 +15,11 @@ print(b)
 
 # this inplace add tests the code when the buffer doesn't need to be increased
 b = bytearray()
-b += b''
+b += b""
+
+# extend a bytearray from itself
+b = bytearray(b"abcdefgh")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation, as above
+    b.extend(b)
+print(b)

tests/basics/bytearray_add_self.py

@@ -0,0 +1,8 @@
+# add a bytearray to itself
+# This is not supported by CPython as of 3.11.18.
+
+b = bytearray(b"123456789")
+for _ in range(4):
+    c = bytearray(b)  # extra allocation increases chance 'b' has to relocate
+    b += b
+print(b)

tests/basics/bytearray_add_self.py.exp

@@ -0,0 +1 @@
+bytearray(b'123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789')

tests/basics/bytearray_slice_assign.py

@@ -18,7 +18,7 @@ l = bytearray(x)
 l[1:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[1:3]
+# del l[1:3]
 print(l)
 
 l = bytearray(x)
@@ -28,7 +28,7 @@ l = bytearray(x)
 l[:3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:3]
+# del l[:3]
 print(l)
 
 l = bytearray(x)
@@ -38,7 +38,7 @@ l = bytearray(x)
 l[:-3] = bytearray()
 print(l)
 l = bytearray(x)
-#del l[:-3]
+# del l[:-3]
 print(l)
 
 # slice assignment that extends the array
@@ -61,8 +61,14 @@ b[1:1] = b"12345"
 print(b)
 
 # Growth of bytearray via slice extension
-b = bytearray(b'12345678')
-b.append(57) # expand and add a bit of unused space at end of the bytearray
+b = bytearray(b"12345678")
+b.append(57)  # expand and add a bit of unused space at end of the bytearray
 for i in range(400):
-    b[-1:] = b'ab' # grow slowly into the unused space
+    b[-1:] = b"ab"  # grow slowly into the unused space
+print(len(b), b)
+
+# Growth of bytearray via slice extension from itself
+b = bytearray(b"1234567")
+for i in range(3):
+    b[-1:] = b
 print(len(b), b)

tests/cpydiff/types_memoryview_invalid.py

@@ -0,0 +1,12 @@
+"""
+categories: Types,memoryview
+description: memoryview can become invalid if its target is resized
+cause: CPython prevents a ``bytearray`` or ``io.bytesIO`` object from changing size while there is a ``memoryview`` object that references it. MicroPython requires the programmer to manually ensure that an object is not resized while any ``memoryview`` references it.
+
+In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime.
+workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it.
+"""
+b = bytearray(b"abcdefg")
+m = memoryview(b)
+b.extend(b"hijklmnop")
+print(b, bytes(m))

tests/ports/webassembly/asyncio_create_task.mjs

@@ -0,0 +1,44 @@
+// Test asyncio.create_task(), and tasks waiting on a Promise.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+globalThis.p0 = new Promise((resolve, reject) => {
+    resolve(123);
+});
+
+globalThis.p1 = new Promise((resolve, reject) => {
+    setTimeout(() => {
+        console.log("setTimeout resolved");
+        resolve(456);
+    }, 200);
+});
+
+mp.runPython(`
+import js
+import asyncio
+
+async def task(id, promise):
+    print("task start", id)
+    print("task await", id, await promise)
+    print("task await", id, await promise)
+    print("task end", id)
+
+print("start")
+t1 = asyncio.create_task(task(1, js.p0))
+t2 = asyncio.create_task(task(2, js.p1))
+print("t1", t1.done(), t2.done())
+print("end")
+`);
+
+// Wait for p1 to fulfill so t2 can continue.
+await globalThis.p1;
+
+// Wait a little longer so t2 can complete.
+await new Promise((resolve, reject) => {
+    setTimeout(resolve, 10);
+});
+
+mp.runPython(`
+print("restart")
+print("t1", t1.done(), t2.done())
+`);

tests/ports/webassembly/asyncio_create_task.mjs.exp

@@ -0,0 +1,14 @@
+start
+t1 False False
+end
+task start 1
+task start 2
+task await 1 123
+task await 1 123
+task end 1
+setTimeout resolved
+task await 2 456
+task await 2 456
+task end 2
+restart
+t1 True True

tests/ports/webassembly/asyncio_sleep.mjs

@@ -0,0 +1,25 @@
+// Test asyncio.sleep(), both at the top level and within a task.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+await mp.runPythonAsync(`
+import time
+import asyncio
+
+print("main start")
+t0 = time.time()
+await asyncio.sleep(0.25)
+dt = time.time() - t0
+print(0.2 <= dt <= 0.3)
+
+async def task():
+    print("task start")
+    t0 = time.time()
+    await asyncio.sleep(0.25)
+    dt = time.time() - t0
+    print(0.2 <= dt <= 0.3)
+    print("task end")
+
+asyncio.create_task(task())
+print("main end")
+`);

tests/ports/webassembly/asyncio_sleep.mjs.exp

@@ -0,0 +1,6 @@
+main start
+True
+main end
+task start
+True
+task end

tests/run-tests.py

@@ -681,6 +681,17 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         elif args.target == "webassembly":
             skip_tests.add("basics/string_format_modulo.py")  # can't print nulls to stdout
             skip_tests.add("basics/string_strip.py")  # can't print nulls to stdout
+            skip_tests.add("extmod/asyncio_basic2.py")
+            skip_tests.add("extmod/asyncio_cancel_self.py")
+            skip_tests.add("extmod/asyncio_current_task.py")
+            skip_tests.add("extmod/asyncio_exception.py")
+            skip_tests.add("extmod/asyncio_gather_finished_early.py")
+            skip_tests.add("extmod/asyncio_get_event_loop.py")
+            skip_tests.add("extmod/asyncio_heaplock.py")
+            skip_tests.add("extmod/asyncio_loop_stop.py")
+            skip_tests.add("extmod/asyncio_new_event_loop.py")
+            skip_tests.add("extmod/asyncio_threadsafeflag.py")
+            skip_tests.add("extmod/asyncio_wait_for_fwd.py")
             skip_tests.add("extmod/binascii_a2b_base64.py")
             skip_tests.add("extmod/re_stack_overflow.py")
             skip_tests.add("extmod/time_res.py")

tools/mpremote/mpremote/commands.py

@@ -6,6 +6,7 @@ import serial.tools.list_ports
 
 from .transport import TransportError
 from .transport_serial import SerialTransport, stdout_write_bytes
+from .mapfs import make_mapfs
 
 
 class CommandError(Exception):
@@ -259,3 +260,46 @@ def do_rtc(state, args):
         _do_execbuffer(state, "import machine; machine.RTC().datetime({})".format(timetuple), True)
     else:
         _do_execbuffer(state, "import machine; print(machine.RTC().datetime())", True)
+
+
+def do_deploy_mapfs(state, args):
+    state.ensure_raw_repl()
+    state.did_action()
+
+    # Detect the mapfs and get its associated device.
+    state.transport.exec("import vfs; mapfs = vfs.mount('/mapfs')")
+    if state.transport.eval("mapfs") == b"None":
+        print("/mapfs does not exist on device")
+        sys.exit(1)
+    if state.transport.eval("mapfs.device()") == b"None":
+        print("/mapfs does not have an associated device")
+        sys.exit(1)
+    state.transport.exec("dev=mapfs.device()")
+    block_count = int(str(state.transport.eval("dev.ioctl(4,0)"), "ascii"))
+    block_size = int(str(state.transport.eval("dev.ioctl(5,0)"), "ascii"))
+    print(
+        f"/mapfs is a block device of size {block_count}*{block_size}={block_count * block_size} bytes"
+    )
+
+    # Create the mapfs filesystem.
+    mapfs = make_mapfs(args.path[0], "/")
+    print(f"Image size is {len(mapfs)} bytes")
+
+    if len(mapfs) > block_count * block_size:
+        print("/mapfs is too small for image")
+        sys.exit(1)
+
+    # Deploy the mapfs filesystem to the device.
+    state.transport.exec(f"buf=bytearray({block_size})")
+    for block in range(0, (len(mapfs) + block_size - 1) // block_size):
+        mapfs_block = mapfs[block * block_size : (block + 1) * block_size]
+        off = 0
+        while off < len(mapfs_block):
+            l = min(len(mapfs_block) - off, 256)
+            state.transport.exec(f"buf[{off}:{off+l}]=" + repr(mapfs_block[off : off + l]))
+            off += l
+        print(f"\rWriting block {block}", end="")
+        state.transport.exec(f"dev.writeblocks({block},buf)")
+
+    print()
+    print("Image deployed")

tools/mpremote/mpremote/main.py

@@ -36,6 +36,7 @@ from .commands import (
     do_resume,
     do_rtc,
     do_soft_reset,
+    do_deploy_mapfs,
 )
 from .mip import do_mip
 from .repl import do_repl
@@ -219,6 +220,12 @@ def argparse_mip():
     return cmd_parser
 
 
+def argparse_deploy_mapfs():
+    cmd_parser = argparse.ArgumentParser(description="deploy a directory to /mapfs on the device")
+    cmd_parser.add_argument("path", nargs=1, help="path to directory to deploy")
+    return cmd_parser
+
+
 def argparse_none(description):
     return lambda: argparse.ArgumentParser(description=description)
 
@@ -293,6 +300,10 @@ _COMMANDS = {
         do_version,
         argparse_none("print version and exit"),
     ),
+    "deploy-mapfs": (
+        do_deploy_mapfs,
+        argparse_deploy_mapfs,
+    ),
 }
 
 # Additional commands aliases.

tools/mpremote/mpremote/mapfs.py

@@ -0,0 +1,110 @@
+# MIT license; Copyright (c) 2022 Damien P. George
+
+import struct, sys, os
+
+
+class VfsMapWriter:
+    MAGIC = b"MF"
+
+    def __init__(self):
+        self.filename = None
+        self.data = bytearray()
+        self.data += VfsMapWriter.MAGIC
+
+    def finalise(self):
+        self.data += b"\x00\x00"
+
+    def open(self, filename, attr):
+        assert self.filename is None
+        assert attr == "wb"
+        self.filename = filename
+        self.filedata = b""
+        return self
+
+    def mkdir(self, dir):
+        assert self.filename is None
+        self.data += struct.pack("<H", 0x8000 | len(dir))
+        self.data += bytes(dir, "ascii")
+
+    def close(self):
+        assert self.filename
+        self.data += struct.pack("<HI", len(self.filename), len(self.filedata))
+        self.data += bytes(self.filename, "ascii")
+        self.data += self.filedata
+        self.filename = None
+
+    def write(self, data):
+        assert self.filename
+        self.filedata += data
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, a, b, c):
+        self.close()
+
+
+def copy_recursively(vfs, src_dir, dest_dir):
+    if dest_dir == "/":
+        dest_dir = ""
+    assert src_dir.endswith("/")
+    assert len(dest_dir) == 0 or dest_dir.endswith("/")
+    DIR = 1 << 14
+    for name in os.listdir(src_dir):
+        if name in (".", ".."):
+            continue
+        src_name = src_dir + name
+        dest_name = dest_dir + name
+        st = os.stat(src_name)
+        if st[0] & DIR:
+            print(" -", dest_name + "/")
+            vfs.mkdir(dest_name)
+            copy_recursively(vfs, src_name + "/", dest_name + "/")
+        else:
+            print(" -", dest_name)
+            with open(src_name, "rb") as src, vfs.open(dest_name, "wb") as dest:
+                dest.write(src.read())
+
+
+def make_mapfs(src_dir, dest_dir):
+    if not src_dir.endswith("/"):
+        src_dir += "/"
+
+    vfs = VfsMapWriter()
+
+    # Build the filesystem recursively.
+    print("Building mapfs filesystem, source directory: {}".format(src_dir))
+    try:
+        copy_recursively(vfs, src_dir, dest_dir)
+    except OSError as er:
+        if er.args[0] == 28:  # ENOSPC
+            print("Error: not enough space on filesystem", file=sys.stderr)
+            sys.exit(1)
+        else:
+            print("Error: OSError {}".format(er.args[0]), file=sys.stderr)
+            sys.exit(1)
+
+    vfs.finalise()
+
+    return vfs.data
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: {} <dir>".format(sys.argv[0]), file=sys.stderr)
+        sys.exit(1)
+
+    # Parse arguments.
+    dir = sys.argv[1]
+
+    mapfs = make_mapfs(dir, "/")
+
+    # Save the block device data.
+    output = dir.rstrip("/") + ".mapfs"
+    print("Writing filesystem image to {}".format(output))
+    with open(output, "wb") as f:
+        f.write(mapfs)
+
+
+if __name__ == "__main__":
+    main()