Merge 3035029363 into d11ca092f7

In the case where an OUT control transfer triggers with wLength==0 (i.e.
all data sent in the SETUP phase, and no additional data phase) the
callbacks were previously implemented to return b"" (i.e. an empty buffer
for the data phase).

However this didn't actually work as intended because b"" can't provide a
RW buffer (needed for OUT transfers with a data phase to write data into),
so actually the endpoint would stall.

The symptom was often that the device process the request (if processing
it in the SETUP phase when all information was already available), but the
host sees the endpoint stall and eventually returns an error.

This commit changes the behaviour so returning True from the SETUP phase of
a control transfer queues a zero length status response.

Signed-off-by: Angus Gratton <angus@redyak.com.au>

docs/library/machine.USBDevice.rst

@@ -130,15 +130,25 @@ Methods
 
       Second argument is a memoryview to read the USB control request
       data for this stage. The memoryview is only valid until the
-      callback function returns.
+      callback function returns. Data in this memoryview will be the same
+      across each of the three stages of a single transfer.
+
+      A successful transfer consists of this callback being called in sequence
+      for the three stages. Generally speaking, if a device wants to do
+      something in response to a control request then it's best to wait until
+      the ACK stage to confirm the host controller completed the transfer as
+      expected.
 
       The callback should return one of the following values:
 
-      - ``False`` to stall the endpoint and reject the transfer.
+      - ``False`` to stall the endpoint and reject the transfer. It won't
+        proceed to any remaining stages.
       - ``True`` to continue the transfer to the next stage.
-      - A buffer object to provide data for this stage of the transfer.
-        This should be a writable buffer for an ``OUT`` direction transfer, or a
-        readable buffer with data for an ``IN`` direction transfer.
+      - A buffer object can be returned at the SETUP stage when the transfer
+        will send or receive additional data. Typically this is the case when
+        the ``wLength`` field in the request has a non-zero value. This should
+        be a writable buffer for an ``OUT`` direction transfer, or a readable
+        buffer with data for an ``IN`` direction transfer.
 
     - ``xfer_cb`` - This callback is called whenever a non-control
       transfer submitted by calling :func:`USBDevice.submit_xfer` completes.

extmod/extmod.cmake

@@ -57,6 +57,8 @@ set(MICROPY_SOURCE_EXTMOD
     ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c
     ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_lfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_map.c
+    ${MICROPY_EXTMOD_DIR}/vfs_map_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_posix.c
     ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_reader.c

extmod/extmod.mk

@@ -59,6 +59,8 @@ SRC_EXTMOD_C += \
 	extmod/vfs_fat_diskio.c \
 	extmod/vfs_fat_file.c \
 	extmod/vfs_lfs.c \
+	extmod/vfs_map.c \
+	extmod/vfs_map_file.c \
 	extmod/vfs_posix.c \
 	extmod/vfs_posix_file.c \
 	extmod/vfs_reader.c \

extmod/modvfs.c

@@ -31,6 +31,7 @@
 #include "extmod/vfs.h"
 #include "extmod/vfs_fat.h"
 #include "extmod/vfs_lfs.h"
+#include "extmod/vfs_map.h"
 #include "extmod/vfs_posix.h"
 
 #if !MICROPY_VFS
@@ -51,6 +52,9 @@ static const mp_rom_map_elem_t vfs_module_globals_table[] = {
     #if MICROPY_VFS_LFS2
     { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) },
     #endif
+    #if MICROPY_VFS_MAP
+    { MP_ROM_QSTR(MP_QSTR_VfsMap), MP_ROM_PTR(&mp_type_vfs_map) },
+    #endif
     #if MICROPY_VFS_POSIX
     { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) },
     #endif

extmod/vfs.c

@@ -208,6 +208,17 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args
         { MP_QSTR_mkfs, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_FALSE} },
     };
 
+    if (n_args == 1) {
+        // query mount table
+        const char *path = mp_obj_str_get_str(pos_args[0]);
+        for (mp_vfs_mount_t *vfs = MP_STATE_VM(vfs_mount_table); vfs != NULL; vfs = vfs->next) {
+            if (strncmp(path, vfs->str, vfs->len) == 0 && path[vfs->len] == '\0') {
+                return vfs->obj;
+            }
+        }
+        return mp_const_none;
+    }
+
     // parse args
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all(n_args - 2, pos_args + 2, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
@@ -262,7 +273,7 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args
 
     return mp_const_none;
 }
-MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 2, mp_vfs_mount);
+MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 1, mp_vfs_mount);
 
 mp_obj_t mp_vfs_umount(mp_obj_t mnt_in) {
     // remove vfs from the mount table

extmod/vfs.h

@@ -55,6 +55,7 @@
 // At the moment the VFS protocol just has import_stat, but could be extended to other methods
 typedef struct _mp_vfs_proto_t {
     mp_import_stat_t (*import_stat)(void *self, const char *path);
+    size_t (*memmap)(void *self, const char *path, const void **ptr_out);
 } mp_vfs_proto_t;
 
 typedef struct _mp_vfs_blockdev_t {

extmod/vfs_map.c

@@ -0,0 +1,325 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "py/runtime.h"
+#include "py/mperrno.h"
+#include "extmod/vfs.h"
+#include "extmod/vfs_map.h"
+
+#if MICROPY_VFS_MAP
+
+#define MAGIC_LEN (2)
+#define MAGIC_BYTE0 ('M')
+#define MAGIC_BYTE1 ('F')
+
+#define GET_LE16(p) ((p)[0] | (p)[1] << 8)
+#define GET_LE32(p) ((p)[0] | (p)[1] << 8 | (p)[2] << 16 | (p)[3] << 24)
+
+struct _mp_obj_vfs_map_t {
+    mp_obj_base_t base;
+    mp_obj_t memory;
+    mp_obj_t device;
+    const uint8_t *filesystem;
+};
+
+mp_import_stat_t mp_vfs_map_search_filesystem(mp_obj_vfs_map_t *self, const char *path, size_t *size_out, const uint8_t **data_out) {
+    if (!(self->filesystem[0] == MAGIC_BYTE0 && self->filesystem[1] == MAGIC_BYTE1)) {
+        return MP_IMPORT_STAT_NO_EXIST;
+    }
+    if (path[0] == '/') {
+        ++path;
+    }
+    size_t path_len = strlen(path);
+    const uint8_t *fs = self->filesystem + MAGIC_LEN;
+    for (;;) {
+        uint16_t nlen = GET_LE16(fs);
+        fs += 2;
+        if (nlen == 0) {
+            return MP_IMPORT_STAT_NO_EXIST;
+        }
+        uint32_t flen;
+        mp_import_stat_t type;
+        if (nlen & 0x8000) {
+            // A directory.
+            nlen &= 0x7fff;
+            flen = 0;
+            type = MP_IMPORT_STAT_DIR;
+        } else {
+            // A file.
+            flen = GET_LE32(fs);
+            fs += 4;
+            type = MP_IMPORT_STAT_FILE;
+        }
+        if (path_len == nlen && memcmp(path, fs, path_len) == 0) {
+            if (size_out != NULL) {
+                *size_out = flen;
+                *data_out = fs + nlen;
+            }
+            return type;
+        }
+        fs += nlen + flen;
+    }
+}
+
+static inline const char *vfs_map_get_path_str(mp_obj_vfs_map_t *self, mp_obj_t path) {
+    return mp_obj_str_get_str(path);
+}
+
+static mp_obj_t vfs_map_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
+    mp_arg_check_num(n_args, n_kw, 1, 2, false);
+
+    mp_obj_vfs_map_t *self = m_new_obj(mp_obj_vfs_map_t);
+    self->base.type = type;
+    self->memory = args[0];
+    self->device = n_args == 1 ? mp_const_none : args[1];
+
+    mp_buffer_info_t bufinfo;
+    if (mp_get_buffer(self->memory, &bufinfo, MP_BUFFER_READ)) {
+        self->filesystem = bufinfo.buf;
+    } else {
+        self->filesystem = (const uint8_t *)(uintptr_t)mp_obj_get_int(self->memory);
+    }
+
+    return MP_OBJ_FROM_PTR(self);
+}
+
+static mp_obj_t vfs_map_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) {
+    (void)self_in;
+    (void)readonly;
+    if (mp_obj_is_true(mkfs)) {
+        mp_raise_OSError(MP_EPERM);
+    }
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_3(vfs_map_mount_obj, vfs_map_mount);
+
+static mp_obj_t vfs_map_umount(mp_obj_t self_in) {
+    (void)self_in;
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(vfs_map_umount_obj, vfs_map_umount);
+
+// mp_vfs_map_file_open is implemented in vfs_map_file.c.
+static MP_DEFINE_CONST_FUN_OBJ_3(vfs_map_open_obj, mp_vfs_map_file_open);
+
+static mp_obj_t vfs_map_chdir(mp_obj_t self_in, mp_obj_t path_in) {
+    (void)self_in;
+    (void)path_in;
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_chdir_obj, vfs_map_chdir);
+
+typedef struct _vfs_map_ilistdir_it_t {
+    mp_obj_base_t base;
+    mp_fun_1_t iternext;
+    bool is_str;
+    const char *path;
+    const uint8_t *index;
+} vfs_map_ilistdir_it_t;
+
+static mp_obj_t vfs_map_ilistdir_it_iternext(mp_obj_t self_in) {
+    vfs_map_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in);
+
+    if (self->index == NULL) {
+        return MP_OBJ_STOP_ITERATION;
+    }
+
+    const char *path = self->path;
+    if (path[0] == '/' || (path[0] == '.' && path[1] == '\0')) {
+        ++path;
+    }
+    size_t path_len = strlen(path);
+
+    for (;;) {
+        uint16_t nlen = GET_LE16(self->index);
+        self->index += 2;
+        if (nlen == 0) {
+            self->index = NULL;
+            return MP_OBJ_STOP_ITERATION;
+        }
+
+        uint32_t flen;
+        uint32_t type;
+        if (nlen & 0x8000) {
+            // dir
+            nlen &= 0x7fff;
+            flen = 0;
+            type = MP_S_IFDIR;
+        } else {
+            // file
+            flen = GET_LE32(self->index);
+            self->index += 4;
+            type = MP_S_IFREG;
+        }
+
+        const uint8_t *nstr = self->index;
+        self->index += nlen + flen;
+
+        if ((path_len == 0 && memchr(nstr, '/', nlen) == NULL)
+            || (path_len < nlen && nstr[path_len] == '/' && memcmp(path, nstr, path_len) == 0 && memchr(nstr + path_len + 1, '/', nlen - path_len - 1) == NULL)) {
+            // Make 4-tuple with info about this entry: (name, attr, inode, size)
+            mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(4, NULL));
+
+            if (path_len > 0) {
+                nstr += path_len + 1;
+                nlen -= path_len + 1;
+            }
+
+            if (self->is_str) {
+                t->items[0] = mp_obj_new_str((const char *)nstr, nlen);
+            } else {
+                t->items[0] = mp_obj_new_bytes(nstr, nlen);
+            }
+
+            t->items[1] = MP_OBJ_NEW_SMALL_INT(type);
+            t->items[2] = MP_OBJ_NEW_SMALL_INT(0);
+            t->items[3] = mp_obj_new_int(flen);
+
+            return MP_OBJ_FROM_PTR(t);
+        }
+    }
+}
+
+static mp_obj_t vfs_map_ilistdir(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+    vfs_map_ilistdir_it_t *iter = m_new_obj(vfs_map_ilistdir_it_t);
+    iter->base.type = &mp_type_polymorph_iter;
+    iter->iternext = vfs_map_ilistdir_it_iternext;
+    iter->is_str = mp_obj_get_type(path_in) == &mp_type_str;
+    iter->path = mp_obj_str_get_str(path_in);
+    if (!(self->filesystem[0] == MAGIC_BYTE0 && self->filesystem[1] == MAGIC_BYTE1)) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    if (iter->path[0] == '\0'
+        || (iter->path[0] == '.' && iter->path[1] == '\0')
+        || (iter->path[0] == '/' && iter->path[1] == '\0')) {
+        // pass
+    } else if (mp_vfs_map_search_filesystem(self, iter->path, NULL, NULL) != MP_IMPORT_STAT_DIR) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    iter->index = self->filesystem + MAGIC_LEN;
+    return MP_OBJ_FROM_PTR(iter);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_ilistdir_obj, vfs_map_ilistdir);
+
+static mp_obj_t vfs_map_stat(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+    const char *path = vfs_map_get_path_str(self, path_in);
+    size_t file_size;
+    const uint8_t *file_data;
+    mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &file_size, &file_data);
+    if (stat == MP_IMPORT_STAT_NO_EXIST) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
+    t->items[0] = MP_OBJ_NEW_SMALL_INT(stat == MP_IMPORT_STAT_FILE ? MP_S_IFREG : MP_S_IFDIR); // st_mode
+    t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino
+    t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // st_dev
+    t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // st_nlink
+    t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid
+    t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid
+    t->items[6] = MP_OBJ_NEW_SMALL_INT(file_size); // st_size
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime
+    return MP_OBJ_FROM_PTR(t);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_stat_obj, vfs_map_stat);
+
+static mp_obj_t vfs_map_statvfs(mp_obj_t self_in, mp_obj_t path_in) {
+    (void)self_in;
+    (void)path_in;
+    mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
+    t->items[0] = MP_OBJ_NEW_SMALL_INT(0); // f_bsize
+    t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // f_frsize
+    t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // f_blocks
+    t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // f_bfree
+    t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // f_bavail
+    t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // f_files
+    t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(65535); // f_namemax
+    return MP_OBJ_FROM_PTR(t);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_statvfs_obj, vfs_map_statvfs);
+
+static mp_obj_t vfs_map_device(mp_obj_t self_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+    return self->device;
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(vfs_map_device_obj, vfs_map_device);
+
+static const mp_rom_map_elem_t vfs_map_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_map_mount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&vfs_map_umount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_map_open_obj) },
+
+    { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_map_chdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_map_ilistdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_map_stat_obj) },
+    { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_map_statvfs_obj) },
+
+    { MP_ROM_QSTR(MP_QSTR_device), MP_ROM_PTR(&vfs_map_device_obj) },
+};
+static MP_DEFINE_CONST_DICT(vfs_map_locals_dict, vfs_map_locals_dict_table);
+
+static mp_import_stat_t mp_vfs_map_import_stat(void *self_in, const char *path) {
+    mp_obj_vfs_map_t *self = self_in;
+    return mp_vfs_map_search_filesystem(self, path, NULL, NULL);
+}
+
+static size_t mp_vfs_map_memmap(void *self_in, const char *path, const void **ptr_out) {
+    mp_obj_vfs_map_t *self = self_in;
+    size_t size;
+    const uint8_t *data;
+    mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &size, &data);
+    if (stat == MP_IMPORT_STAT_FILE) {
+        *ptr_out = data;
+        return size;
+    } else {
+        *ptr_out = NULL;
+        return 0;
+    }
+}
+
+static const mp_vfs_proto_t vfs_map_proto = {
+    .import_stat = mp_vfs_map_import_stat,
+    .memmap = mp_vfs_map_memmap,
+};
+
+MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_map,
+    MP_QSTR_VfsMap,
+    MP_TYPE_FLAG_NONE,
+    make_new, vfs_map_make_new,
+    protocol, &vfs_map_proto,
+    locals_dict, &vfs_map_locals_dict
+    );
+
+#endif // MICROPY_VFS_MAP

extmod/vfs_map.h

@@ -0,0 +1,39 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_EXTMOD_VFS_MAP_H
+#define MICROPY_INCLUDED_EXTMOD_VFS_MAP_H
+
+#include "py/builtin.h"
+#include "py/obj.h"
+
+typedef struct _mp_obj_vfs_map_t mp_obj_vfs_map_t;
+
+extern const mp_obj_type_t mp_type_vfs_map;
+
+mp_import_stat_t mp_vfs_map_search_filesystem(mp_obj_vfs_map_t *self, const char *path, size_t *size_out, const uint8_t **data_out);
+mp_obj_t mp_vfs_map_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in);
+
+#endif // MICROPY_INCLUDED_EXTMOD_VFS_MAP_H

extmod/vfs_map_file.c

@@ -0,0 +1,202 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "py/reader.h"
+#include "py/runtime.h"
+#include "py/stream.h"
+#include "extmod/vfs_map.h"
+
+#if MICROPY_VFS_MAP
+
+typedef struct _mp_obj_vfs_map_file_t {
+    mp_obj_base_t base;
+    size_t file_size;
+    size_t file_offset;
+    const uint8_t *file_data;
+} mp_obj_vfs_map_file_t;
+
+static const mp_obj_type_t mp_type_vfs_map_fileio;
+static const mp_obj_type_t mp_type_vfs_map_textio;
+
+static uintptr_t mp_vfs_map_reader_readbyte(void *data) {
+    mp_obj_vfs_map_file_t *self = data;
+    if (self->file_offset >= self->file_size) {
+        return MP_READER_EOF;
+    }
+    return self->file_data[self->file_offset++];
+}
+
+static intptr_t mp_vfs_map_reader_ioctl(void *data, uintptr_t request, uintptr_t arg) {
+    mp_obj_vfs_map_file_t *self = data;
+
+    if (request == MP_READER_CLOSE) {
+        return 0;
+    } else if (request == MP_READER_MEMMAP) {
+        mp_reader_ioctl_memmap_t *memmap = (mp_reader_ioctl_memmap_t *)arg;
+        memmap->ptr = self->file_data + self->file_offset;
+        self->file_offset += memmap->len;
+        return 0;
+    }
+
+    return -MP_EINVAL;
+}
+
+mp_obj_t mp_vfs_map_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) {
+    mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in);
+
+    const char *mode_s = mp_obj_str_get_str(mode_in);
+    const mp_obj_type_t *type = &mp_type_vfs_map_textio;
+    while (*mode_s) {
+        switch (*mode_s++) {
+            case 'r':
+                break;
+            case 'w':
+            case 'a':
+            case '+':
+                mp_raise_OSError(MP_EROFS);
+            case 'b':
+                type = &mp_type_vfs_map_fileio;
+                break;
+            case 't':
+                type = &mp_type_vfs_map_textio;
+                break;
+        }
+    }
+
+    mp_obj_vfs_map_file_t *o = m_new_obj(mp_obj_vfs_map_file_t);
+    o->base.type = type;
+    o->file_offset = 0;
+
+    const char *path = mp_obj_str_get_str(path_in);
+    mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &o->file_size, &o->file_data);
+    if (stat == MP_IMPORT_STAT_NO_EXIST) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+static mp_obj_t vfs_map_file___exit__(size_t n_args, const mp_obj_t *args) {
+    (void)n_args;
+    return mp_stream_close(args[0]);
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(vfs_map_file___exit___obj, 4, 4, vfs_map_file___exit__);
+
+static mp_uint_t vfs_map_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
+    mp_obj_vfs_map_file_t *self = MP_OBJ_TO_PTR(o_in);
+    size_t remain = self->file_size - self->file_offset;
+    if (size > remain) {
+        size = remain;
+    }
+    memcpy(buf, self->file_data + self->file_offset, size);
+    self->file_offset += size;
+    return size;
+}
+
+static mp_uint_t vfs_map_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) {
+    mp_obj_vfs_map_file_t *self = MP_OBJ_TO_PTR(o_in);
+
+    switch (request) {
+        case MP_STREAM_FLUSH:
+            return 0;
+        case MP_STREAM_SEEK: {
+            struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg;
+            if (s->whence == 0) { // SEEK_SET
+                self->file_offset = MIN(self->file_size, (size_t)s->offset);
+            } else if (s->whence == 1) { // SEEK_CUR
+                self->file_offset = MIN(self->file_size, self->file_offset + s->offset);
+            } else { // SEEK_END
+                self->file_offset = self->file_size;
+            }
+            s->offset = self->file_offset;
+            return 0;
+        }
+        case MP_STREAM_CLOSE:
+            return 0;
+        case MP_STREAM_POLL: {
+            mp_uint_t ret = 0;
+            if (arg & MP_STREAM_POLL_RD) {
+                ret |= MP_STREAM_POLL_RD;
+            }
+            return ret;
+        }
+        case MP_STREAM_INIT_READER: {
+            mp_reader_t *reader = (mp_reader_t *)arg;
+            reader->data = self;
+            reader->readbyte = mp_vfs_map_reader_readbyte;
+            reader->ioctl = mp_vfs_map_reader_ioctl;
+            return 0;
+        }
+        default:
+            *errcode = MP_EINVAL;
+            return MP_STREAM_ERROR;
+    }
+}
+
+static const mp_rom_map_elem_t vfs_map_rawfile_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) },
+    { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) },
+    { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) },
+    { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) },
+    { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
+    { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) },
+    { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_map_file___exit___obj) },
+};
+static MP_DEFINE_CONST_DICT(vfs_map_rawfile_locals_dict, vfs_map_rawfile_locals_dict_table);
+
+static const mp_stream_p_t vfs_map_fileio_stream_p = {
+    .read = vfs_map_file_read,
+    .ioctl = vfs_map_file_ioctl,
+};
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_map_fileio,
+    MP_QSTR_FileIO,
+    MP_TYPE_FLAG_ITER_IS_STREAM,
+    protocol, &vfs_map_fileio_stream_p,
+    locals_dict, &vfs_map_rawfile_locals_dict
+    );
+
+static const mp_stream_p_t vfs_map_textio_stream_p = {
+    .read = vfs_map_file_read,
+    .ioctl = vfs_map_file_ioctl,
+    .is_text = true,
+};
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_map_textio,
+    MP_QSTR_TextIOWrapper,
+    MP_TYPE_FLAG_ITER_IS_STREAM,
+    protocol, &vfs_map_textio_stream_p,
+    locals_dict, &vfs_map_rawfile_locals_dict
+    );
+
+#endif // MICROPY_VFS_MAP

extmod/vfs_reader.c

@@ -49,7 +49,7 @@ typedef struct _mp_reader_vfs_t {
     byte buf[];
 } mp_reader_vfs_t;
 
-static mp_uint_t mp_reader_vfs_readbyte(void *data) {
+static uintptr_t mp_reader_vfs_readbyte(void *data) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
     if (reader->bufpos >= reader->buflen) {
         if (reader->buflen < reader->bufsize) {
@@ -70,13 +70,39 @@ static mp_uint_t mp_reader_vfs_readbyte(void *data) {
     return reader->buf[reader->bufpos++];
 }
 
-static void mp_reader_vfs_close(void *data) {
+static intptr_t mp_reader_vfs_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
-    mp_stream_close(reader->file);
-    m_del_obj(mp_reader_vfs_t, reader);
+
+    if (request == MP_READER_CLOSE) {
+        mp_stream_close(reader->file);
+        m_del_obj(mp_reader_vfs_t, reader);
+        return 0;
+    }
+
+    return -MP_EINVAL;
 }
 
 void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
+    #if MICROPY_VFS_MAP
+    const char *path_out;
+    mp_vfs_mount_t *vfs = mp_vfs_lookup_path(qstr_str(filename), &path_out);
+    if (!(vfs == MP_VFS_NONE || vfs == MP_VFS_ROOT)) {
+        // If the mounted object has the VFS protocol, call its memmap helper.
+        const mp_obj_type_t *type = mp_obj_get_type(vfs->obj);
+        if (MP_OBJ_TYPE_HAS_SLOT(type, protocol)) {
+            const mp_vfs_proto_t *proto = MP_OBJ_TYPE_GET_SLOT(type, protocol);
+            if (proto->memmap != NULL) {
+                const void *data;
+                size_t size = proto->memmap(MP_OBJ_TO_PTR(vfs->obj), path_out, &data);
+                if (data != NULL) {
+                    mp_reader_new_mem(reader, data, size, (size_t)-1);
+                    return;
+                }
+            }
+        }
+    }
+    #endif
+
     mp_obj_t args[2] = {
         MP_OBJ_NEW_QSTR(filename),
         MP_OBJ_NEW_QSTR(MP_QSTR_rb),
@@ -85,6 +111,15 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
 
     const mp_stream_p_t *stream_p = mp_get_stream(file);
     int errcode = 0;
+
+    #if 0 && MICROPY_VFS_MAP
+    // Check if the stream can initialising a reader itself.
+    mp_uint_t reader_ret = stream_p->ioctl(file, MP_STREAM_INIT_READER, (uintptr_t)reader, &errcode);
+    if (reader_ret == 0) {
+        return;
+    }
+    #endif
+
     mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode);
     if (bufsize == MP_STREAM_ERROR || bufsize == 0) {
         // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl
@@ -98,13 +133,14 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
     rf->file = file;
     rf->bufsize = bufsize;
     rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+
     if (errcode != 0) {
         mp_raise_OSError(errcode);
     }
     rf->bufpos = 0;
     reader->data = rf;
     reader->readbyte = mp_reader_vfs_readbyte;
-    reader->close = mp_reader_vfs_close;
+    reader->ioctl = mp_reader_vfs_ioctl;
 }
 
 #endif // MICROPY_READER_VFS

ports/esp32/esp32_partition.c

@@ -265,6 +265,16 @@ static MP_DEFINE_CONST_FUN_OBJ_1(esp32_partition_mark_app_valid_cancel_rollback_
 static MP_DEFINE_CONST_CLASSMETHOD_OBJ(esp32_partition_mark_app_valid_cancel_rollback_obj,
     MP_ROM_PTR(&esp32_partition_mark_app_valid_cancel_rollback_fun_obj));
 
+static mp_obj_t esp32_partition_mmap(mp_obj_t self_in) {
+    esp32_partition_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    const void *ptr;
+    esp_partition_mmap_handle_t handle;
+    // TODO need to esp_partition_munmap(handle) on soft reset
+    check_esp_err(esp_partition_mmap(self->part, 0, self->part->size, ESP_PARTITION_MMAP_DATA, &ptr, &handle));
+    return mp_obj_new_memoryview('B', self->part->size, (void *)ptr);
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(esp32_partition_mmap_obj, esp32_partition_mmap);
+
 static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_find), MP_ROM_PTR(&esp32_partition_find_obj) },
 
@@ -276,6 +286,7 @@ static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_set_boot), MP_ROM_PTR(&esp32_partition_set_boot_obj) },
     { MP_ROM_QSTR(MP_QSTR_mark_app_valid_cancel_rollback), MP_ROM_PTR(&esp32_partition_mark_app_valid_cancel_rollback_obj) },
     { MP_ROM_QSTR(MP_QSTR_get_next_update), MP_ROM_PTR(&esp32_partition_get_next_update_obj) },
+    { MP_ROM_QSTR(MP_QSTR_mmap), MP_ROM_PTR(&esp32_partition_mmap_obj) },
 
     { MP_ROM_QSTR(MP_QSTR_BOOT), MP_ROM_INT(ESP32_PARTITION_BOOT) },
     { MP_ROM_QSTR(MP_QSTR_RUNNING), MP_ROM_INT(ESP32_PARTITION_RUNNING) },

ports/esp32/modules/_boot.py

@@ -1,7 +1,14 @@
 import gc
 import vfs
+from esp32 import Partition
 from flashbdev import bdev
 
+if mapfs := Partition.find(Partition.TYPE_DATA, label="mapfs"):
+    import sys
+
+    sys.path.insert(0, "/mapfs")
+del mapfs
+
 try:
     if bdev:
         vfs.mount(bdev, "/")

ports/esp32/mpconfigport.h

@@ -71,6 +71,7 @@
 #define MICROPY_USE_INTERNAL_PRINTF         (0) // ESP32 SDK requires its own printf
 #define MICROPY_SCHEDULER_DEPTH             (8)
 #define MICROPY_VFS                         (1)
+#define MICROPY_VFS_MAP                     (1)
 
 // control over Python builtins
 #define MICROPY_PY_STR_BYTES_CMP_WARN       (1)

ports/esp32/partitions-4MiB.csv

@@ -3,5 +3,6 @@
 # Name,   Type, SubType, Offset,  Size, Flags
 nvs,      data, nvs,     0x9000,  0x6000,
 phy_init, data, phy,     0xf000,  0x1000,
-factory,  app,  factory, 0x10000, 0x1F0000,
+factory,  app,  factory, 0x10000, 0x1D0000,
+mapfs,    data, 0x8f,    0x1E0000, 0x20000,
 vfs,      data, fat,     0x200000, 0x200000,

ports/esp8266/lexerstr32.c

@@ -26,6 +26,7 @@
  */
 
 #include "py/lexer.h"
+#include "py/mperrno.h"
 
 #if MICROPY_ENABLE_COMPILER
 
@@ -35,7 +36,7 @@ typedef struct _mp_lexer_str32_buf_t {
     uint8_t byte_off;
 } mp_lexer_str32_buf_t;
 
-static mp_uint_t str32_buf_next_byte(void *sb_in) {
+static uintptr_t str32_buf_next_byte(void *sb_in) {
     mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t *)sb_in;
     byte c = sb->val & 0xff;
     if (c == 0) {
@@ -52,9 +53,13 @@ static mp_uint_t str32_buf_next_byte(void *sb_in) {
     return c;
 }
 
-static void str32_buf_free(void *sb_in) {
+static intptr_t str32_buf_free(void *sb_in, uintptr_t request, uintptr_t arg) {
     mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t *)sb_in;
-    m_del_obj(mp_lexer_str32_buf_t, sb);
+    if (request == MP_READER_CLOSE) {
+        m_del_obj(mp_lexer_str32_buf_t, sb);
+        return 0;
+    }
+    return -MP_EINVAL;
 }
 
 mp_lexer_t *mp_lexer_new_from_str32(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len) {

ports/nrf/modules/os/microbitfs.c

@@ -458,11 +458,18 @@ static mp_uint_t microbit_file_write(mp_obj_t obj, const void *buf, mp_uint_t si
     return size;
 }
 
-static void microbit_file_close(file_descriptor_obj *fd) {
-    if (fd->writable) {
-        flash_write_byte((uint32_t)&(file_system_chunks[fd->start_chunk].header.end_offset), fd->seek_offset);
+static intptr_t microbit_file_ioctl(void *fd_in, uintptr_t request, uintptr_t arg) {
+    file_descriptor_obj *fd = fd_in;
+
+    if (request == MP_READER_CLOSE) {
+        if (fd->writable) {
+            flash_write_byte((uint32_t)&(file_system_chunks[fd->start_chunk].header.end_offset), fd->seek_offset);
+        }
+        fd->open = false;
+        return 0;
     }
-    fd->open = false;
+
+    return -MP_EINVAL;
 }
 
 static mp_obj_t microbit_file_list(void) {
@@ -495,7 +502,7 @@ static mp_obj_t microbit_file_size(mp_obj_t filename) {
     return mp_obj_new_int(len);
 }
 
-static mp_uint_t file_read_byte(file_descriptor_obj *fd) {
+static uintptr_t file_read_byte(file_descriptor_obj *fd) {
     if (file_system_chunks[fd->seek_chunk].next_chunk == UNUSED_CHUNK) {
         uint8_t end_offset = file_system_chunks[fd->start_chunk].header.end_offset;
         if (end_offset == UNUSED_CHUNK || fd->seek_offset == end_offset) {
@@ -516,8 +523,8 @@ mp_lexer_t *os_mbfs_new_reader(const char *filename) {
     }
     mp_reader_t reader;
     reader.data = fd;
-    reader.readbyte = (mp_uint_t(*)(void*))file_read_byte;
-    reader.close = (void(*)(void*))microbit_file_close; // no-op
+    reader.readbyte = (uintptr_t(*)(void*))file_read_byte;
+    reader.close = microbit_file_ioctl;
     return mp_lexer_new(qstr_from_str(filename), reader);
 }
 
@@ -538,7 +545,7 @@ static MP_DEFINE_CONST_FUN_OBJ_1(os_mbfs_file_name_obj, os_mbfs_file_name);
 
 static mp_obj_t os_mbfs_file_close(mp_obj_t self) {
     file_descriptor_obj *fd = (file_descriptor_obj*)self;
-    microbit_file_close(fd);
+    microbit_file_ioctl(fd, MP_READER_CLOSE, 0);
     return mp_const_none;
 }
 static MP_DEFINE_CONST_FUN_OBJ_1(os_mbfs_file_close_obj, os_mbfs_file_close);

ports/rp2/modules/_boot.py

@@ -1,7 +1,13 @@
 import vfs
+import sys
 import machine, rp2
 
 
+mapfs = rp2.Flash("mapfs")
+vfs.mount(vfs.VfsMap(mapfs.ioctl(0x100, 0), mapfs), "/mapfs")
+sys.path.insert(0, "/mapfs")
+
+
 # Try to mount the filesystem, and format the flash if it doesn't exist.
 # Note: the flash requires the programming size to be aligned to 256 bytes.
 bdev = rp2.Flash()
@@ -12,4 +18,4 @@ except:
     fs = vfs.VfsLfs2(bdev, progsize=256)
 vfs.mount(fs, "/")
 
-del vfs, bdev, fs
+del vfs, sys, bdev, fs, mapfs

ports/rp2/mpconfigport.h

@@ -151,6 +151,7 @@
 #define MICROPY_VFS                             (1)
 #define MICROPY_VFS_LFS2                        (1)
 #define MICROPY_VFS_FAT                         (1)
+#define MICROPY_VFS_MAP                         (1)
 #define MICROPY_SSL_MBEDTLS                     (1)
 #define MICROPY_PY_LWIP_SOCK_RAW                (MICROPY_PY_LWIP)
 

ports/rp2/rp2_flash.c

@@ -28,6 +28,7 @@
 
 #include "py/mphal.h"
 #include "py/runtime.h"
+#include "py/mperrno.h"
 #include "extmod/vfs.h"
 #include "modrp2.h"
 #include "hardware/flash.h"
@@ -44,6 +45,9 @@ static_assert(MICROPY_HW_FLASH_STORAGE_BYTES % 4096 == 0, "Flash storage size mu
 #define MICROPY_HW_FLASH_STORAGE_BASE (PICO_FLASH_SIZE_BYTES - MICROPY_HW_FLASH_STORAGE_BYTES)
 #endif
 
+#define MICROPY_HW_MAPFS_BASE (512 * 1024) // leave 512k for firmware...
+#define MICROPY_HW_MAPFS_BYTES (MICROPY_HW_FLASH_STORAGE_BASE - MICROPY_HW_MAPFS_BASE)
+
 static_assert(MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big");
 static_assert(MICROPY_HW_FLASH_STORAGE_BASE + MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big");
 
@@ -53,6 +57,12 @@ typedef struct _rp2_flash_obj_t {
     uint32_t flash_size;
 } rp2_flash_obj_t;
 
+static rp2_flash_obj_t rp2_flash_mapfs_obj = {
+    .base = { &rp2_flash_type },
+    .flash_base = MICROPY_HW_MAPFS_BASE,
+    .flash_size = MICROPY_HW_MAPFS_BYTES,
+};
+
 static rp2_flash_obj_t rp2_flash_obj = {
     .base = { &rp2_flash_type },
     .flash_base = MICROPY_HW_FLASH_STORAGE_BASE,
@@ -87,6 +97,15 @@ static void end_critical_flash_section(uint32_t state) {
 }
 
 static mp_obj_t rp2_flash_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) {
+    if (n_args == 1 && n_kw == 0 && mp_obj_is_str(all_args[0])) {
+        if (mp_obj_str_get_qstr(all_args[0]) == MP_QSTR_mapfs) {
+            if (MICROPY_HW_MAPFS_BYTES <= 0) {
+                mp_raise_OSError(MP_ENODEV);
+            }
+            return MP_OBJ_FROM_PTR(&rp2_flash_mapfs_obj);
+        }
+    }
+
     // Parse arguments
     enum { ARG_start, ARG_len };
     static const mp_arg_t allowed_args[] = {
@@ -190,6 +209,8 @@ static mp_obj_t rp2_flash_ioctl(mp_obj_t self_in, mp_obj_t cmd_in, mp_obj_t arg_
             // TODO check return value
             return MP_OBJ_NEW_SMALL_INT(0);
         }
+        case 0x100: // mmap
+            return mp_obj_new_int(XIP_BASE + self->flash_base);
         default:
             return mp_const_none;
     }

ports/stm32/boardctrl.h

@@ -61,6 +61,10 @@
 #define MICROPY_BOARD_TOP_SOFT_RESET_LOOP boardctrl_top_soft_reset_loop
 #endif
 
+#ifndef MICROPY_BOARD_FROZEN_BOOT_FILE
+#define MICROPY_BOARD_FROZEN_BOOT_FILE "_boot.py"
+#endif
+
 #ifndef MICROPY_BOARD_RUN_BOOT_PY
 #define MICROPY_BOARD_RUN_BOOT_PY boardctrl_run_boot_py
 #endif

ports/stm32/boards/NUCLEO_F091RC/mpconfigboard.mk

@@ -8,7 +8,7 @@ MICROPY_VFS_FAT = 0
 MICROPY_VFS_LFS1 ?= 1
 
 # Don't include default frozen modules because MCU is tight on flash space
-FROZEN_MANIFEST ?=
+FROZEN_MANIFEST ?= boards/manifest_minimal.py
 
 # LTO reduces final binary size, may be slower to build depending on gcc version and hardware
 LTO ?= 1

ports/stm32/boards/PYBD_SF2/f722_qspi.ld

@@ -20,7 +20,8 @@ MEMORY
 {
     FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 512K
     FLASH_ISR (rx)  : ORIGIN = 0x08000000, LENGTH = 32K     /* sectors 0,1 */
-    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 480K    /* sectors 2-7 */
+    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 352K    /* sectors 2-6 */
+    FLASH_MAPFS (rx): ORIGIN = 0x08060000, LENGTH = 128K    /* sector 7 */
     FLASH_EXT (rx)  : ORIGIN = 0x90000000, LENGTH = 2048K   /* external QSPI */
     RAM (rwx)       : ORIGIN = 0x20000000, LENGTH = 256K    /* DTCM+SRAM1+SRAM2 */
 }
@@ -40,6 +41,9 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM);
 _heap_start = _ebss; /* heap starts just after statically allocated memory */
 _heap_end = _sstack;
 
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);
+
 /* Define output sections */
 SECTIONS
 {
@@ -51,6 +55,10 @@ SECTIONS
         *lib/mynewt-nimble/*(.text* .rodata*)
         *lib/cyw43-driver/*(.rodata.w4343*_combined)
         *drivers/cyw43/*(.rodata.cyw43_btfw_*)
+        *lib/oofatfs/*(.text* .rodata*)
+        *lib/littlefs/*(.text* .rodata*)
+        *lib/lwip/*(.text* .rodata*)
+        *extmod/*(.text* .rodata*)
         . = ALIGN(4);
     } >FLASH_EXT
 }

ports/stm32/boards/PYBD_SF6/f767.ld

@@ -18,7 +18,8 @@ MEMORY
 {
     FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 2048K
     FLASH_ISR (rx)  : ORIGIN = 0x08000000, LENGTH = 32K     /* sector 0, 32K */
-    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 2016K   /* sectors 1-11 3x32K 1*128K 7*256K */
+    FLASH_APP (rx)  : ORIGIN = 0x08008000, LENGTH = 1504K   /* sectors 1-11 3x32K 1*128K 7*256K */
+    FLASH_MAPFS (rx): ORIGIN = 0x08180000, LENGTH = 512K
     FLASH_EXT (rx)  : ORIGIN = 0x90000000, LENGTH = 2048K   /* external QSPI */
     RAM (rwx)       : ORIGIN = 0x20000000, LENGTH = 512K    /* DTCM=128k, SRAM1=368K, SRAM2=16K */
 }
@@ -38,4 +39,7 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM);
 _heap_start = _ebss; /* heap starts just after statically allocated memory */
 _heap_end = _sstack;
 
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);
+
 INCLUDE common_bl.ld

ports/stm32/boards/_boot.py

@@ -0,0 +1,8 @@
+import vfs, sys, pyb
+
+
+mapfs = pyb.Flash("mapfs")
+vfs.mount(vfs.VfsMap(mapfs.ioctl(0x100, 0), mapfs), "/mapfs")
+sys.path.insert(0, "/mapfs")
+
+del vfs, sys, pyb, mapfs

ports/stm32/boards/manifest.py

@@ -1,3 +1,4 @@
+freeze(".", "_boot.py", opt=3)
 include("$(MPY_DIR)/extmod/asyncio")
 
 require("dht")

ports/stm32/boards/manifest_minimal.py

@@ -0,0 +1 @@
+freeze(".", "_boot.py", opt=3)

ports/stm32/boards/stm32f091xc.ld

@@ -5,7 +5,8 @@
 /* Specify the memory areas */
 MEMORY
 {
-    FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 236K /* sectors 0-117 */
+    FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 226K /* sectors 0-112 */
+    FLASH_MAPFS (rx): ORIGIN = 0x08036000, LENGTH = 10K /* sectors 113-117 */
     FLASH_FS (r)    : ORIGIN = 0x0803B000, LENGTH = 20K /* sectors 118-127 */
     RAM (xrw)       : ORIGIN = 0x20000000, LENGTH = 32K
 }
@@ -32,3 +33,6 @@ _heap_end = _sstack;
 
 _micropy_hw_internal_flash_storage_start = ORIGIN(FLASH_FS);
 _micropy_hw_internal_flash_storage_end   = ORIGIN(FLASH_FS) + LENGTH(FLASH_FS);
+
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);

ports/stm32/boards/stm32f405.ld

@@ -8,7 +8,8 @@ MEMORY
     FLASH (rx)      : ORIGIN = 0x08000000, LENGTH = 1024K /* entire flash */
     FLASH_ISR (rx)  : ORIGIN = 0x08000000, LENGTH = 16K /* sector 0 */
     FLASH_FS (rx)   : ORIGIN = 0x08004000, LENGTH = 112K /* sectors 1,2,3,4 are for filesystem */
-    FLASH_TEXT (rx) : ORIGIN = 0x08020000, LENGTH = 896K /* sectors 5,6,7,8,9,10,11 */
+    FLASH_TEXT (rx) : ORIGIN = 0x08020000, LENGTH = 640K /* sectors 5,6,7,8,9 */
+    FLASH_MAPFS (rx): ORIGIN = 0x080c0000, LENGTH = 256K /* sectors 10,11 */
     CCMRAM (xrw)    : ORIGIN = 0x10000000, LENGTH = 64K
     RAM (xrw)       : ORIGIN = 0x20000000, LENGTH = 128K
 }
@@ -33,3 +34,6 @@ _micropy_hw_internal_flash_storage_ram_cache_start = ORIGIN(CCMRAM);
 _micropy_hw_internal_flash_storage_ram_cache_end = ORIGIN(CCMRAM) + LENGTH(CCMRAM);
 _micropy_hw_internal_flash_storage_start = ORIGIN(FLASH_FS);
 _micropy_hw_internal_flash_storage_end = ORIGIN(FLASH_FS) + LENGTH(FLASH_FS);
+
+_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS);
+_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS);

ports/stm32/mpconfigport.h

@@ -155,6 +155,8 @@
 #define MICROPY_PY_ONEWIRE          (1)
 #endif
 
+#define MICROPY_VFS_MAP             (1)
+
 // fatfs configuration used in ffconf.h
 #define MICROPY_FATFS_ENABLE_LFN       (1)
 #define MICROPY_FATFS_LFN_CODE_PAGE    437 /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */

ports/stm32/qstrdefsport.h

@@ -32,6 +32,7 @@ Q(/flash)
 Q(/flash/lib)
 Q(/sd)
 Q(/sd/lib)
+Q(/mapfs)
 
 // For os.sep
 Q(/)

ports/stm32/storage.c

@@ -259,6 +259,8 @@ int storage_readblocks_ext(uint8_t *dest, uint32_t block, uint32_t offset, uint3
 }
 #endif
 
+static struct _pyb_mapfs_obj_t pyb_mapfs_obj;
+
 typedef struct _pyb_flash_obj_t {
     mp_obj_base_t base;
     uint32_t start; // in bytes
@@ -289,6 +291,11 @@ static mp_obj_t pyb_flash_make_new(const mp_obj_type_t *type, size_t n_args, siz
         { MP_QSTR_start, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = -1} },
         { MP_QSTR_len,   MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = -1} },
     };
+
+    if (n_args == 1 && mp_obj_is_str(all_args[0]) && mp_obj_str_get_qstr(all_args[0]) == MP_QSTR_mapfs) {
+        return MP_OBJ_FROM_PTR(&pyb_mapfs_obj);
+    }
+
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
@@ -475,3 +482,103 @@ void pyb_flash_init_vfs(fs_user_mount_t *vfs) {
 }
 
 #endif
+
+/******************************************************************************/
+// mapfs partition
+//
+// TODO: this doesn't properly implement the block device protocol because it
+// has potentially large sectors but pretends they are small.
+// writing to the start of a sector will erase that whole sector first.
+
+#include "flash.h"
+
+#define MICROPY_HW_MAPFS_BASE (uintptr_t)(&_micropy_hw_mapfs_start)
+#define MICROPY_HW_MAPFS_BYTES (uintptr_t)(&_micropy_hw_mapfs_end - &_micropy_hw_mapfs_start)
+#define MAPFS_BLOCK_SIZE_BYTES (256) // can be anything bigger than alignment requirements of hardware writes
+
+typedef struct _pyb_mapfs_obj_t {
+    mp_obj_base_t base;
+    // uint32_t flash_base;
+    // uint32_t flash_size;
+} pyb_mapfs_obj_t;
+
+extern uint8_t _micropy_hw_mapfs_start;
+extern uint8_t _micropy_hw_mapfs_end;
+
+static const mp_obj_type_t pyb_mapfs_type;
+
+static pyb_mapfs_obj_t pyb_mapfs_obj = {
+    .base = { &pyb_mapfs_type },
+    // .flash_base = MICROPY_HW_MAPFS_BASE,
+    // .flash_size = MICROPY_HW_MAPFS_BYTES,
+};
+
+static mp_obj_t pyb_mapfs_readblocks(size_t n_args, const mp_obj_t *args) {
+    // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(args[0]);
+    uint32_t offset = mp_obj_get_int(args[1]) * MAPFS_BLOCK_SIZE_BYTES;
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_WRITE);
+    memcpy(bufinfo.buf, (void *)(MICROPY_HW_MAPFS_BASE + offset), bufinfo.len);
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_mapfs_readblocks_obj, 3, 3, pyb_mapfs_readblocks);
+
+static mp_obj_t pyb_mapfs_writeblocks(size_t n_args, const mp_obj_t *args) {
+    // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(args[0]);
+    uint32_t offset = mp_obj_get_int(args[1]) * MAPFS_BLOCK_SIZE_BYTES;
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_READ);
+    uint32_t dest = MICROPY_HW_MAPFS_BASE + offset;
+
+    // Compute start and end address of the sector being written.
+    uint32_t sector_size = 0;
+    uint32_t sector_start = 0;
+    int ret = flash_get_sector_info(dest, &sector_start, &sector_size);
+    if (ret < 0) {
+        return MP_OBJ_NEW_SMALL_INT(ret);
+    }
+
+    // Erase sector if writing to the start of it.
+    if (dest == sector_start) {
+        ret = flash_erase(sector_start);
+        if (ret != 0) {
+            return MP_OBJ_NEW_SMALL_INT(ret);
+        }
+    }
+
+    flash_write(dest, bufinfo.buf, bufinfo.len / 4);
+    // TODO check return value
+
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_mapfs_writeblocks_obj, 3, 4, pyb_mapfs_writeblocks);
+
+static mp_obj_t pyb_mapfs_ioctl(mp_obj_t self_in, mp_obj_t cmd_in, mp_obj_t arg_in) {
+    // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    mp_int_t cmd = mp_obj_get_int(cmd_in);
+    switch (cmd) {
+        case MP_BLOCKDEV_IOCTL_BLOCK_COUNT:
+            return MP_OBJ_NEW_SMALL_INT(MICROPY_HW_MAPFS_BYTES / MAPFS_BLOCK_SIZE_BYTES);
+        case MP_BLOCKDEV_IOCTL_BLOCK_SIZE:
+            return MP_OBJ_NEW_SMALL_INT(MAPFS_BLOCK_SIZE_BYTES);
+        case 0x100: // mmap
+            return mp_obj_new_int(MICROPY_HW_MAPFS_BASE);
+        default:
+            return mp_const_none;
+    }
+}
+static MP_DEFINE_CONST_FUN_OBJ_3(pyb_mapfs_ioctl_obj, pyb_mapfs_ioctl);
+
+static const mp_rom_map_elem_t pyb_mapfs_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_readblocks), MP_ROM_PTR(&pyb_mapfs_readblocks_obj) },
+    { MP_ROM_QSTR(MP_QSTR_writeblocks), MP_ROM_PTR(&pyb_mapfs_writeblocks_obj) },
+    { MP_ROM_QSTR(MP_QSTR_ioctl), MP_ROM_PTR(&pyb_mapfs_ioctl_obj) },
+};
+static MP_DEFINE_CONST_DICT(pyb_mapfs_locals_dict, pyb_mapfs_locals_dict_table);
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    pyb_mapfs_type,
+    MP_QSTR_Flash,
+    MP_TYPE_FLAG_NONE,
+    locals_dict, &pyb_mapfs_locals_dict
+    );

ports/unix/variants/mpconfigvariant_common.h

@@ -122,3 +122,5 @@
 #define MICROPY_PY_MACHINE             (1)
 #define MICROPY_PY_MACHINE_PULSE       (1)
 #define MICROPY_PY_MACHINE_PIN_BASE    (1)
+
+#define MICROPY_VFS_MAP                (1)

py/lexer.c

@@ -908,7 +908,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
 
 void mp_lexer_free(mp_lexer_t *lex) {
     if (lex) {
-        lex->reader.close(lex->reader.data);
+        lex->reader.ioctl(lex->reader.data, MP_READER_CLOSE, 0);
         vstr_clear(&lex->vstr);
         #if MICROPY_PY_FSTRINGS
         vstr_clear(&lex->fstring_args);

py/persistentcode.c

@@ -160,6 +160,18 @@ static size_t read_uint(mp_reader_t *reader) {
     return unum;
 }
 
+#if MICROPY_VFS_MAP
+static inline const uint8_t *map_try_read_bytes(mp_reader_t *reader, size_t len) {
+    mp_reader_ioctl_memmap_t memmap;
+    memmap.len = len;
+    intptr_t ret = reader->ioctl(reader->data, MP_READER_MEMMAP, (uintptr_t)&memmap);
+    if (ret < 0) {
+        return NULL;
+    }
+    return memmap.ptr;
+}
+#endif
+
 static qstr load_qstr(mp_reader_t *reader) {
     size_t len = read_uint(reader);
     if (len & 1) {
@@ -167,6 +179,12 @@ static qstr load_qstr(mp_reader_t *reader) {
         return len >> 1;
     }
     len >>= 1;
+    #if MICROPY_VFS_MAP
+    const uint8_t *memmap = map_try_read_bytes(reader, len + 1);
+    if (memmap != NULL) {
+        return qstr_from_strn_static((const char *)memmap, len);
+    }
+    #endif
     char *str = m_new(char, len);
     read_bytes(reader, (byte *)str, len);
     read_byte(reader); // read and discard null terminator
@@ -175,6 +193,22 @@ static qstr load_qstr(mp_reader_t *reader) {
     return qst;
 }
 
+#if MICROPY_VFS_MAP
+static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *data, size_t len) {
+    if (type == &mp_type_str) {
+        qstr q = qstr_find_strn((const char *)data, len);
+        if (q != MP_QSTRnull) {
+            return MP_OBJ_NEW_QSTR(q);
+        }
+    }
+    mp_obj_str_t *o = m_new_obj(mp_obj_str_t);
+    o->base.type = type;
+    o->len = len;
+    o->hash = qstr_compute_hash(data, len);
+    o->data = data;
+    return MP_OBJ_FROM_PTR(o);
+}
+#endif
 static mp_obj_t load_obj(mp_reader_t *reader) {
     byte obj_type = read_byte(reader);
     #if MICROPY_EMIT_MACHINE_CODE
@@ -192,6 +226,7 @@ static mp_obj_t load_obj(mp_reader_t *reader) {
         return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj);
     } else {
         size_t len = read_uint(reader);
+
         if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) {
             read_byte(reader); // skip null terminator
             return mp_const_empty_bytes;
@@ -202,11 +237,25 @@ static mp_obj_t load_obj(mp_reader_t *reader) {
             }
             return MP_OBJ_FROM_PTR(tuple);
         }
+
+
+        const uint8_t *memmap = NULL;
         vstr_t vstr;
-        vstr_init_len(&vstr, len);
-        read_bytes(reader, (byte *)vstr.buf, len);
+        #if MICROPY_VFS_MAP
+        memmap = map_try_read_bytes(reader, len);
+        #endif
+        if (memmap == NULL) {
+            vstr_init_len(&vstr, len);
+            read_bytes(reader, (byte *)vstr.buf, len);
+        }
         if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) {
             read_byte(reader); // skip null terminator
+            #if MICROPY_VFS_MAP
+            if (memmap != NULL) {
+                const mp_obj_type_t *t = obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes;
+                return mp_obj_new_str_static(t, memmap, len);
+            }
+            #endif
             if (obj_type == MP_PERSISTENT_OBJ_STR) {
                 return mp_obj_new_str_from_utf8_vstr(&vstr);
             } else {
@@ -243,10 +292,17 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
     #endif
 
     if (kind == MP_CODE_BYTECODE) {
-        // Allocate memory for the bytecode
-        fun_data = m_new(uint8_t, fun_data_len);
-        // Load bytecode
-        read_bytes(reader, fun_data, fun_data_len);
+        #if MICROPY_VFS_MAP
+        // Try to reference memory-mapped data for the bytecode.
+        fun_data = (uint8_t *)map_try_read_bytes(reader, fun_data_len);
+        #endif
+
+        if (fun_data == NULL) {
+            // Allocate memory for the bytecode.
+            fun_data = m_new(uint8_t, fun_data_len);
+            // Load bytecode.
+            read_bytes(reader, fun_data, fun_data_len);
+        }
 
     #if MICROPY_EMIT_MACHINE_CODE
     } else {
@@ -393,9 +449,14 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
     return rc;
 }
 
+static void reader_close(void *reader_in) {
+    mp_reader_t *reader = reader_in;
+    reader->ioctl(reader->data, MP_READER_CLOSE, 0);
+}
+
 void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) {
     // Set exception handler to close the reader if an exception is raised.
-    MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader->close, reader->data);
+    MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader_close, reader);
     nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback);
 
     byte header[4];

py/qstr.c

@@ -309,7 +309,7 @@ qstr qstr_from_str(const char *str) {
     return qstr_from_strn(str, strlen(str));
 }
 
-qstr qstr_from_strn(const char *str, size_t len) {
+static qstr qstr_from_strn_helper(const char *str, size_t len, bool is_static) {
     QSTR_ENTER();
     qstr q = qstr_find_strn(str, len);
     if (q == 0) {
@@ -321,6 +321,11 @@ qstr qstr_from_strn(const char *str, size_t len) {
             mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("name too long"));
         }
 
+        if (is_static) {
+            assert(str[len] == '\0');
+            goto add;
+        }
+
         // compute number of bytes needed to intern this string
         size_t n_bytes = len + 1;
 
@@ -364,12 +369,23 @@ qstr qstr_from_strn(const char *str, size_t len) {
         // store the interned strings' data
         memcpy(q_ptr, str, len);
         q_ptr[len] = '\0';
-        q = qstr_add(len, q_ptr);
+        str = q_ptr;
+
+    add:
+        q = qstr_add(len, str);
     }
     QSTR_EXIT();
     return q;
 }
 
+qstr qstr_from_strn(const char *str, size_t len) {
+    return qstr_from_strn_helper(str, len, false);
+}
+
+qstr qstr_from_strn_static(const char *str, size_t len) {
+    return qstr_from_strn_helper(str, len, true);
+}
+
 mp_uint_t qstr_hash(qstr q) {
     const qstr_pool_t *pool = find_qstr(&q);
     #if MICROPY_QSTR_BYTES_IN_HASH

py/qstr.h

@@ -101,6 +101,7 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if
 
 qstr qstr_from_str(const char *str);
 qstr qstr_from_strn(const char *str, size_t len);
+qstr qstr_from_strn_static(const char *str, size_t len);
 
 mp_uint_t qstr_hash(qstr q);
 const char *qstr_str(qstr q);

py/reader.c

@@ -39,7 +39,7 @@ typedef struct _mp_reader_mem_t {
     const byte *end;
 } mp_reader_mem_t;
 
-static mp_uint_t mp_reader_mem_readbyte(void *data) {
+static uintptr_t mp_reader_mem_readbyte(void *data) {
     mp_reader_mem_t *reader = (mp_reader_mem_t *)data;
     if (reader->cur < reader->end) {
         return *reader->cur++;
@@ -48,12 +48,23 @@ static mp_uint_t mp_reader_mem_readbyte(void *data) {
     }
 }
 
-static void mp_reader_mem_close(void *data) {
+static intptr_t mp_reader_mem_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_mem_t *reader = (mp_reader_mem_t *)data;
-    if (reader->free_len > 0) {
-        m_del(char, (char *)reader->beg, reader->free_len);
+
+    if (request == MP_READER_CLOSE) {
+        if (reader->free_len > 0 && reader->free_len != (size_t)-1) {
+            m_del(char, (char *)reader->beg, reader->free_len);
+        }
+        m_del_obj(mp_reader_mem_t, reader);
+        return 0;
+    } else if (request == MP_READER_MEMMAP && reader->free_len == (size_t)-1) {
+        mp_reader_ioctl_memmap_t *memmap = (mp_reader_ioctl_memmap_t *)arg;
+        memmap->ptr = reader->cur;
+        reader->cur += memmap->len;
+        return 0;
     }
-    m_del_obj(mp_reader_mem_t, reader);
+
+    return -MP_EINVAL;
 }
 
 void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) {
@@ -64,7 +75,7 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t
     rm->end = buf + len;
     reader->data = rm;
     reader->readbyte = mp_reader_mem_readbyte;
-    reader->close = mp_reader_mem_close;
+    reader->ioctl = mp_reader_mem_ioctl;
 }
 
 #if MICROPY_READER_POSIX
@@ -81,7 +92,7 @@ typedef struct _mp_reader_posix_t {
     byte buf[20];
 } mp_reader_posix_t;
 
-static mp_uint_t mp_reader_posix_readbyte(void *data) {
+static uintptr_t mp_reader_posix_readbyte(void *data) {
     mp_reader_posix_t *reader = (mp_reader_posix_t *)data;
     if (reader->pos >= reader->len) {
         if (reader->len == 0) {
@@ -101,14 +112,20 @@ static mp_uint_t mp_reader_posix_readbyte(void *data) {
     return reader->buf[reader->pos++];
 }
 
-static void mp_reader_posix_close(void *data) {
+static intptr_t mp_reader_posix_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_posix_t *reader = (mp_reader_posix_t *)data;
-    if (reader->close_fd) {
-        MP_THREAD_GIL_EXIT();
-        close(reader->fd);
-        MP_THREAD_GIL_ENTER();
+
+    if (request == MP_READER_CLOSE) {
+        if (reader->close_fd) {
+            MP_THREAD_GIL_EXIT();
+            close(reader->fd);
+            MP_THREAD_GIL_ENTER();
+        }
+        m_del_obj(mp_reader_posix_t, reader);
+        return 0;
     }
-    m_del_obj(mp_reader_posix_t, reader);
+
+    return -MP_EINVAL;
 }
 
 void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) {
@@ -129,7 +146,7 @@ void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) {
     rp->pos = 0;
     reader->data = rp;
     reader->readbyte = mp_reader_posix_readbyte;
-    reader->close = mp_reader_posix_close;
+    reader->ioctl = mp_reader_posix_ioctl;
 }
 
 #if !MICROPY_VFS_POSIX

py/reader.h

@@ -28,15 +28,30 @@
 
 #include "py/obj.h"
 
-// the readbyte function must return the next byte in the input stream
-// it must return MP_READER_EOF if end of stream
-// it can be called again after returning MP_READER_EOF, and in that case must return MP_READER_EOF
-#define MP_READER_EOF ((mp_uint_t)(-1))
+#define MP_READER_EOF ((uintptr_t)(-1))
+
+// Reader ioctl request codes.
+#define MP_READER_CLOSE (1)
+#define MP_READER_MEMMAP (2)
+
+// Used as arg for MP_READER_MEMMAP ioctl request.
+typedef struct _mp_reader_ioctl_memmap_t {
+    size_t len;
+    const uint8_t *ptr;
+} mp_reader_ioctl_memmap_t;
 
 typedef struct _mp_reader_t {
+    // Pointer to the context of this reader, passed as the first argument to the methods below.
     void *data;
-    mp_uint_t (*readbyte)(void *data);
-    void (*close)(void *data);
+
+    // The readbyte function must return the next byte in the input stream.
+    // It must return MP_READER_EOF if end of stream.  It can be called again after returning
+    // MP_READER_EOF, and in that case must return MP_READER_EOF.
+    uintptr_t (*readbyte)(void *data);
+
+    // Ioctl method for performing various control actions.
+    // On error it must return a negative errno value.
+    intptr_t (*ioctl)(void *data, uintptr_t request, uintptr_t arg);
 } mp_reader_t;
 
 void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len);

py/stream.h

@@ -44,6 +44,7 @@
 #define MP_STREAM_SET_DATA_OPTS (9)  // Set data/message options
 #define MP_STREAM_GET_FILENO    (10) // Get fileno of underlying file
 #define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file
+#define MP_STREAM_INIT_READER   (12) // Initialise a mp_reader_t for this stream
 
 // These poll ioctl values are compatible with Linux
 #define MP_STREAM_POLL_RD       (0x0001)

shared/runtime/pyexec.c

@@ -34,6 +34,7 @@
 #include "py/repl.h"
 #include "py/gc.h"
 #include "py/frozenmod.h"
+#include "py/mperrno.h"
 #include "py/mphal.h"
 #if MICROPY_HW_ENABLE_USB
 #include "irq.h"
@@ -135,7 +136,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input
 
         if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) {
             const mp_reader_t *reader = source;
-            reader->close(reader->data);
+            reader->ioctl(reader->data, MP_READER_CLOSE, 0);
         }
 
         // print EOF after normal output
@@ -201,7 +202,7 @@ typedef struct _mp_reader_stdin_t {
     uint16_t window_remain;
 } mp_reader_stdin_t;
 
-static mp_uint_t mp_reader_stdin_readbyte(void *data) {
+static uintptr_t mp_reader_stdin_readbyte(void *data) {
     mp_reader_stdin_t *reader = (mp_reader_stdin_t *)data;
 
     if (reader->eof) {
@@ -233,18 +234,24 @@ static mp_uint_t mp_reader_stdin_readbyte(void *data) {
     return c;
 }
 
-static void mp_reader_stdin_close(void *data) {
+static intptr_t mp_reader_stdin_ioctl(void *data, uintptr_t request, uintptr_t arg) {
     mp_reader_stdin_t *reader = (mp_reader_stdin_t *)data;
-    if (!reader->eof) {
-        reader->eof = true;
-        mp_hal_stdout_tx_strn("\x04", 1); // indicate end to host
-        for (;;) {
-            int c = mp_hal_stdin_rx_chr();
-            if (c == CHAR_CTRL_C || c == CHAR_CTRL_D) {
-                break;
+
+    if (request == MP_READER_CLOSE) {
+        if (!reader->eof) {
+            reader->eof = true;
+            mp_hal_stdout_tx_strn("\x04", 1); // indicate end to host
+            for (;;) {
+                int c = mp_hal_stdin_rx_chr();
+                if (c == CHAR_CTRL_C || c == CHAR_CTRL_D) {
+                    break;
+                }
             }
         }
+        return 0;
     }
+
+    return -MP_EINVAL;
 }
 
 static void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_stdin, uint16_t buf_max) {
@@ -260,7 +267,7 @@ static void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s
     reader_stdin->window_remain = window;
     reader->data = reader_stdin;
     reader->readbyte = mp_reader_stdin_readbyte;
-    reader->close = mp_reader_stdin_close;
+    reader->ioctl = mp_reader_stdin_ioctl;
 }
 
 static int do_reader_stdin(int c) {

shared/tinyusb/mp_usbd_runtime.c

@@ -295,6 +295,7 @@ static bool runtime_dev_control_xfer_cb(uint8_t rhport, uint8_t stage, tusb_cont
     mp_obj_usb_device_t *usbd = MP_OBJ_TO_PTR(MP_STATE_VM(usbd));
     tusb_dir_t dir = request->bmRequestType_bit.direction;
     mp_buffer_info_t buf_info;
+    bool result;
 
     if (!usbd) {
         return false;
@@ -319,7 +320,7 @@ static bool runtime_dev_control_xfer_cb(uint8_t rhport, uint8_t stage, tusb_cont
 
     // Check if callback returned any data to submit
     if (mp_get_buffer(cb_res, &buf_info, dir == TUSB_DIR_IN ? MP_BUFFER_READ : MP_BUFFER_RW)) {
-        bool result = tud_control_xfer(USBD_RHPORT,
+        result = tud_control_xfer(USBD_RHPORT,
             request,
             buf_info.buf,
             buf_info.len);
@@ -328,17 +329,21 @@ static bool runtime_dev_control_xfer_cb(uint8_t rhport, uint8_t stage, tusb_cont
             // Keep buffer object alive until the transfer completes
             usbd->xfer_data[0][dir] = cb_res;
         }
-
-        return result;
     } else {
         // Expect True or False to stall or continue
+        result = mp_obj_is_true(cb_res);
 
-        if (stage == CONTROL_STAGE_ACK) {
+        if (stage == CONTROL_STAGE_SETUP && result) {
+            // If no additional data but callback says to continue transfer then
+            // queue a status response.
+            tud_control_status(rhport, request);
+        } else if (stage == CONTROL_STAGE_ACK) {
             // Allow data to be GCed once it's no longer in use
             usbd->xfer_data[0][dir] = mp_const_none;
         }
-        return mp_obj_is_true(cb_res);
     }
+
+    return result;
 }
 
 static bool runtime_dev_xfer_cb(uint8_t rhport, uint8_t ep_addr, xfer_result_t result, uint32_t xferred_bytes) {

tools/mpremote/mpremote/commands.py

@@ -6,6 +6,7 @@ import serial.tools.list_ports
 
 from .transport import TransportError
 from .transport_serial import SerialTransport, stdout_write_bytes
+from .mapfs import make_mapfs
 
 
 class CommandError(Exception):
@@ -259,3 +260,46 @@ def do_rtc(state, args):
         _do_execbuffer(state, "import machine; machine.RTC().datetime({})".format(timetuple), True)
     else:
         _do_execbuffer(state, "import machine; print(machine.RTC().datetime())", True)
+
+
+def do_deploy_mapfs(state, args):
+    state.ensure_raw_repl()
+    state.did_action()
+
+    # Detect the mapfs and get its associated device.
+    state.transport.exec("import vfs; mapfs = vfs.mount('/mapfs')")
+    if state.transport.eval("mapfs") == b"None":
+        print("/mapfs does not exist on device")
+        sys.exit(1)
+    if state.transport.eval("mapfs.device()") == b"None":
+        print("/mapfs does not have an associated device")
+        sys.exit(1)
+    state.transport.exec("dev=mapfs.device()")
+    block_count = int(str(state.transport.eval("dev.ioctl(4,0)"), "ascii"))
+    block_size = int(str(state.transport.eval("dev.ioctl(5,0)"), "ascii"))
+    print(
+        f"/mapfs is a block device of size {block_count}*{block_size}={block_count * block_size} bytes"
+    )
+
+    # Create the mapfs filesystem.
+    mapfs = make_mapfs(args.path[0], "/")
+    print(f"Image size is {len(mapfs)} bytes")
+
+    if len(mapfs) > block_count * block_size:
+        print("/mapfs is too small for image")
+        sys.exit(1)
+
+    # Deploy the mapfs filesystem to the device.
+    state.transport.exec(f"buf=bytearray({block_size})")
+    for block in range(0, (len(mapfs) + block_size - 1) // block_size):
+        mapfs_block = mapfs[block * block_size : (block + 1) * block_size]
+        off = 0
+        while off < len(mapfs_block):
+            l = min(len(mapfs_block) - off, 256)
+            state.transport.exec(f"buf[{off}:{off+l}]=" + repr(mapfs_block[off : off + l]))
+            off += l
+        print(f"\rWriting block {block}", end="")
+        state.transport.exec(f"dev.writeblocks({block},buf)")
+
+    print()
+    print("Image deployed")

tools/mpremote/mpremote/main.py

@@ -36,6 +36,7 @@ from .commands import (
     do_resume,
     do_rtc,
     do_soft_reset,
+    do_deploy_mapfs,
 )
 from .mip import do_mip
 from .repl import do_repl
@@ -219,6 +220,12 @@ def argparse_mip():
     return cmd_parser
 
 
+def argparse_deploy_mapfs():
+    cmd_parser = argparse.ArgumentParser(description="deploy a directory to /mapfs on the device")
+    cmd_parser.add_argument("path", nargs=1, help="path to directory to deploy")
+    return cmd_parser
+
+
 def argparse_none(description):
     return lambda: argparse.ArgumentParser(description=description)
 
@@ -293,6 +300,10 @@ _COMMANDS = {
         do_version,
         argparse_none("print version and exit"),
     ),
+    "deploy-mapfs": (
+        do_deploy_mapfs,
+        argparse_deploy_mapfs,
+    ),
 }
 
 # Additional commands aliases.

tools/mpremote/mpremote/mapfs.py

@@ -0,0 +1,110 @@
+# MIT license; Copyright (c) 2022 Damien P. George
+
+import struct, sys, os
+
+
+class VfsMapWriter:
+    MAGIC = b"MF"
+
+    def __init__(self):
+        self.filename = None
+        self.data = bytearray()
+        self.data += VfsMapWriter.MAGIC
+
+    def finalise(self):
+        self.data += b"\x00\x00"
+
+    def open(self, filename, attr):
+        assert self.filename is None
+        assert attr == "wb"
+        self.filename = filename
+        self.filedata = b""
+        return self
+
+    def mkdir(self, dir):
+        assert self.filename is None
+        self.data += struct.pack("<H", 0x8000 | len(dir))
+        self.data += bytes(dir, "ascii")
+
+    def close(self):
+        assert self.filename
+        self.data += struct.pack("<HI", len(self.filename), len(self.filedata))
+        self.data += bytes(self.filename, "ascii")
+        self.data += self.filedata
+        self.filename = None
+
+    def write(self, data):
+        assert self.filename
+        self.filedata += data
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, a, b, c):
+        self.close()
+
+
+def copy_recursively(vfs, src_dir, dest_dir):
+    if dest_dir == "/":
+        dest_dir = ""
+    assert src_dir.endswith("/")
+    assert len(dest_dir) == 0 or dest_dir.endswith("/")
+    DIR = 1 << 14
+    for name in os.listdir(src_dir):
+        if name in (".", ".."):
+            continue
+        src_name = src_dir + name
+        dest_name = dest_dir + name
+        st = os.stat(src_name)
+        if st[0] & DIR:
+            print(" -", dest_name + "/")
+            vfs.mkdir(dest_name)
+            copy_recursively(vfs, src_name + "/", dest_name + "/")
+        else:
+            print(" -", dest_name)
+            with open(src_name, "rb") as src, vfs.open(dest_name, "wb") as dest:
+                dest.write(src.read())
+
+
+def make_mapfs(src_dir, dest_dir):
+    if not src_dir.endswith("/"):
+        src_dir += "/"
+
+    vfs = VfsMapWriter()
+
+    # Build the filesystem recursively.
+    print("Building mapfs filesystem, source directory: {}".format(src_dir))
+    try:
+        copy_recursively(vfs, src_dir, dest_dir)
+    except OSError as er:
+        if er.args[0] == 28:  # ENOSPC
+            print("Error: not enough space on filesystem", file=sys.stderr)
+            sys.exit(1)
+        else:
+            print("Error: OSError {}".format(er.args[0]), file=sys.stderr)
+            sys.exit(1)
+
+    vfs.finalise()
+
+    return vfs.data
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: {} <dir>".format(sys.argv[0]), file=sys.stderr)
+        sys.exit(1)
+
+    # Parse arguments.
+    dir = sys.argv[1]
+
+    mapfs = make_mapfs(dir, "/")
+
+    # Save the block device data.
+    output = dir.rstrip("/") + ".mapfs"
+    print("Writing filesystem image to {}".format(output))
+    with open(output, "wb") as f:
+        f.write(mapfs)
+
+
+if __name__ == "__main__":
+    main()