extmod/vfs_lfs: Add mtime support to littlefs files.

This commit adds support for modification time of files on littlefs v2
filesystems, using file attributes.  For some background see issue #6114.

Features/properties of this implementation:
- Only supported on littlefs2 (not littlefs1).
- Uses littlefs2's general file attributes to store the timestamp.
- The timestamp is 64-bits and stores nanoseconds since 1970/1/1 (if the
  range to the year 2554 is not enough then additional bits can be added to
  this timestamp by adding another file attribute).
- mtime is enabled by default but can be disabled in the constructor, eg:
  uos.mount(uos.VfsLfs2(bdev, mtime=False), '/flash')
- It's fully backwards compatible, existing littlefs2 filesystems will work
  without reformatting and timestamps will be added transparently to
  existing files (once they are opened for writing).
- Files without timestamps will open correctly, and stat will just return 0
  for their timestamp.
- mtime can be disabled or enabled each mount time and timestamps will only
  be updated if mtime is enabled (otherwise they will be untouched).

Signed-off-by: Damien George <damien@micropython.org>

docs/library/uos.rst

@@ -178,7 +178,7 @@ represented by VFS classes.
 
         Build a FAT filesystem on *block_dev*.
 
-.. class:: VfsLfs1(block_dev)
+.. class:: VfsLfs1(block_dev, readsize=32, progsize=32, lookahead=32)
 
     Create a filesystem object that uses the `littlefs v1 filesystem format`_.
     Storage of the littlefs filesystem is provided by *block_dev*, which must
@@ -187,23 +187,31 @@ represented by VFS classes.
 
     See :ref:`filesystem` for more information.
 
-    .. staticmethod:: mkfs(block_dev)
+    .. staticmethod:: mkfs(block_dev, readsize=32, progsize=32, lookahead=32)
 
         Build a Lfs1 filesystem on *block_dev*.
 
     .. note:: There are reports of littlefs v1 failing in certain situations,
               for details see `littlefs issue 347`_.
 
-.. class:: VfsLfs2(block_dev)
+.. class:: VfsLfs2(block_dev, readsize=32, progsize=32, lookahead=32, mtime=True)
 
     Create a filesystem object that uses the `littlefs v2 filesystem format`_.
     Storage of the littlefs filesystem is provided by *block_dev*, which must
     support the :ref:`extended interface <block-device-interface>`.
     Objects created by this constructor can be mounted using :func:`mount`.
 
+    The *mtime* argument enables modification timestamps for files, stored using
+    littlefs attributes.  This option can be disabled or enabled differently each
+    mount time and timestamps will only be added or updated if *mtime* is enabled,
+    otherwise the timestamps will remain untouched.  Littlefs v2 filesystems without
+    timestamps will work without reformatting and timestamps will be added
+    transparently to existing files once they are opened for writing.  When *mtime*
+    is enabled `uos.stat` on files without timestamps will return 0 for the timestamp.
+
     See :ref:`filesystem` for more information.
 
-    .. staticmethod:: mkfs(block_dev)
+    .. staticmethod:: mkfs(block_dev, readsize=32, progsize=32, lookahead=32)
 
         Build a Lfs2 filesystem on *block_dev*.
 

extmod/vfs_lfs.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2019 Damien P. George
+ * Copyright (c) 2019-2020 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -25,18 +25,20 @@
  */
 
 #include "py/runtime.h"
+#include "py/mphal.h"
 #include "extmod/vfs.h"
 #include "extmod/vfs_lfs.h"
 
 #if MICROPY_VFS && (MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2)
 
-enum { LFS_MAKE_ARG_bdev, LFS_MAKE_ARG_readsize, LFS_MAKE_ARG_progsize, LFS_MAKE_ARG_lookahead };
+enum { LFS_MAKE_ARG_bdev, LFS_MAKE_ARG_readsize, LFS_MAKE_ARG_progsize, LFS_MAKE_ARG_lookahead, LFS_MAKE_ARG_mtime };
 
 static const mp_arg_t lfs_make_allowed_args[] = {
     { MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ },
     { MP_QSTR_readsize, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 32} },
     { MP_QSTR_progsize, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 32} },
     { MP_QSTR_lookahead, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 32} },
+    { MP_QSTR_mtime, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
 };
 
 #if MICROPY_VFS_LFS1
@@ -98,9 +100,13 @@ mp_obj_t mp_vfs_lfs1_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode
 #define MP_TYPE_VFS_LFSx mp_type_vfs_lfs2
 #define MP_TYPE_VFS_LFSx_(s) mp_type_vfs_lfs2##s
 
+// Attribute ids for lfs2_attr.type.
+#define LFS_ATTR_MTIME (1) // 64-bit little endian, nanoseconds since 1970/1/1
+
 typedef struct _mp_obj_vfs_lfs2_t {
     mp_obj_base_t base;
     mp_vfs_blockdev_t blockdev;
+    bool enable_mtime;
     vstr_t cur_dir;
     struct lfs2_config config;
     lfs2_t lfs;
@@ -109,14 +115,25 @@ typedef struct _mp_obj_vfs_lfs2_t {
 typedef struct _mp_obj_vfs_lfs2_file_t {
     mp_obj_base_t base;
     mp_obj_vfs_lfs2_t *vfs;
+    uint8_t mtime[8];
     lfs2_file_t file;
     struct lfs2_file_config cfg;
+    struct lfs2_attr attrs[1];
     uint8_t file_buffer[0];
 } mp_obj_vfs_lfs2_file_t;
 
 const char *mp_vfs_lfs2_make_path(mp_obj_vfs_lfs2_t *self, mp_obj_t path_in);
 mp_obj_t mp_vfs_lfs2_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in);
 
+STATIC void lfs_get_mtime(uint8_t buf[8]) {
+    uint64_t ns = mp_hal_time_ns();
+    // Store "ns" to "buf" in little-endian format (essentially htole64).
+    for (size_t i = 0; i < 8; ++i) {
+        buf[i] = ns;
+        ns >>= 8;
+    }
+}
+
 #include "extmod/vfs_lfsx.c"
 #include "extmod/vfs_lfsx_file.c"
 

extmod/vfs_lfsx.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2019 Damien P. George
+ * Copyright (c) 2019-2020 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -34,6 +34,7 @@
 #include "py/objstr.h"
 #include "py/mperrno.h"
 #include "extmod/vfs.h"
+#include "lib/timeutils/timeutils.h"
 
 STATIC int MP_VFS_LFSx(dev_ioctl)(const struct LFSx_API (config) * c, int cmd, int arg, bool must_return_int) {
     mp_obj_t ret = mp_vfs_blockdev_ioctl(c->context, cmd, arg);
@@ -120,6 +121,9 @@ STATIC mp_obj_t MP_VFS_LFSx(make_new)(const mp_obj_type_t * type, size_t n_args,
     self->base.type = type;
     vstr_init(&self->cur_dir, 16);
     vstr_add_byte(&self->cur_dir, '/');
+    #if LFS_BUILD_VERSION == 2
+    self->enable_mtime = args[LFS_MAKE_ARG_mtime].u_bool;
+    #endif
     MP_VFS_LFSx(init_config)(self, args[LFS_MAKE_ARG_bdev].u_obj,
         args[LFS_MAKE_ARG_readsize].u_int, args[LFS_MAKE_ARG_progsize].u_int, args[LFS_MAKE_ARG_lookahead].u_int);
     int ret = LFSx_API(mount)(&self->lfs, &self->config);
@@ -352,6 +356,19 @@ STATIC mp_obj_t MP_VFS_LFSx(stat)(mp_obj_t self_in, mp_obj_t path_in) {
         mp_raise_OSError(-ret);
     }
 
+    mp_uint_t mtime = 0;
+    #if LFS_BUILD_VERSION == 2
+    uint8_t mtime_buf[8];
+    lfs2_ssize_t sz = lfs2_getattr(&self->lfs, path, LFS_ATTR_MTIME, &mtime_buf, sizeof(mtime_buf));
+    if (sz == sizeof(mtime_buf)) {
+        uint64_t ns = 0;
+        for (size_t i = sizeof(mtime_buf); i > 0; --i) {
+            ns = ns << 8 | mtime_buf[i - 1];
+        }
+        mtime = timeutils_seconds_since_2000_from_nanoseconds_since_1970(ns);
+    }
+    #endif
+
     mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
     t->items[0] = MP_OBJ_NEW_SMALL_INT(info.type == LFSx_MACRO(_TYPE_REG) ? MP_S_IFREG : MP_S_IFDIR); // st_mode
     t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino
@@ -360,9 +377,9 @@ STATIC mp_obj_t MP_VFS_LFSx(stat)(mp_obj_t self_in, mp_obj_t path_in) {
     t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid
     t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid
     t->items[6] = mp_obj_new_int_from_uint(info.size); // st_size
-    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime
-    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime
-    t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(mtime); // st_atime
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(mtime); // st_mtime
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(mtime); // st_ctime
 
     return MP_OBJ_FROM_PTR(t);
 }

extmod/vfs_lfsx_file.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2019 Damien P. George
+ * Copyright (c) 2019-2020 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -101,6 +101,17 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod
     #endif
     o->cfg.buffer = &o->file_buffer[0];
 
+    #if LFS_BUILD_VERSION == 2
+    if (self->enable_mtime) {
+        lfs_get_mtime(&o->mtime[0]);
+        o->attrs[0].type = LFS_ATTR_MTIME;
+        o->attrs[0].buffer = &o->mtime[0];
+        o->attrs[0].size = sizeof(o->mtime);
+        o->cfg.attrs = &o->attrs[0];
+        o->cfg.attr_count = MP_ARRAY_SIZE(o->attrs);
+    }
+    #endif
+
     const char *path = MP_VFS_LFSx(make_path)(self, path_in);
     int ret = LFSx_API(file_opencfg)(&self->lfs, &o->file, path, flags, &o->cfg);
     if (ret < 0) {
@@ -131,6 +142,11 @@ STATIC mp_uint_t MP_VFS_LFSx(file_read)(mp_obj_t self_in, void *buf, mp_uint_t s
 STATIC mp_uint_t MP_VFS_LFSx(file_write)(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) {
     MP_OBJ_VFS_LFSx_FILE *self = MP_OBJ_TO_PTR(self_in);
     MP_VFS_LFSx(check_open)(self);
+    #if LFS_BUILD_VERSION == 2
+    if (self->vfs->enable_mtime) {
+        lfs_get_mtime(&self->mtime[0]);
+    }
+    #endif
     LFSx_API(ssize_t) sz = LFSx_API(file_write)(&self->vfs->lfs, &self->file, buf, size);
     if (sz < 0) {
         *errcode = -sz;

tests/extmod/vfs_lfs.py

@@ -35,6 +35,11 @@ class RAMBlockDevice:
             return 0
 
 
+def print_stat(st, print_size=True):
+    # don't print times (just check that they have the correct type)
+    print(st[:6], st[6] if print_size else -1, type(st[7]), type(st[8]), type(st[9]))
+
+
 def test(bdev, vfs_class):
     print("test", vfs_class)
 
@@ -69,10 +74,10 @@ def test(bdev, vfs_class):
     vfs.mkdir("testdir")
 
     # stat a file
-    print(vfs.stat("test"))
+    print_stat(vfs.stat("test"))
 
     # stat a dir (size seems to vary on LFS2 so don't print that)
-    print(vfs.stat("testdir")[:6])
+    print_stat(vfs.stat("testdir"), False)
 
     # read
     with vfs.open("test", "r") as f:
@@ -112,8 +117,8 @@ def test(bdev, vfs_class):
 
     # create file in directory to make sure paths are relative
     vfs.open("test2", "w").close()
-    print(vfs.stat("test2"))
-    print(vfs.stat("/testdir/test2"))
+    print_stat(vfs.stat("test2"))
+    print_stat(vfs.stat("/testdir/test2"))
     vfs.remove("test2")
 
     # chdir back to root and remove testdir

tests/extmod/vfs_lfs.py.exp

@@ -7,8 +7,8 @@ test <class 'VfsLfs1'>
 [('test', 32768, 0, 8), ('testdir', 16384, 0, 0)]
 []
 [('test', 32768, 0, 8)]
-(32768, 0, 0, 0, 0, 0, 8, 0, 0, 0)
-(16384, 0, 0, 0, 0, 0)
+(32768, 0, 0, 0, 0, 0) 8 <class 'int'> <class 'int'> <class 'int'>
+(16384, 0, 0, 0, 0, 0) -1 <class 'int'> <class 'int'> <class 'int'>
 littlefs
 data length: 4096
 write 0
@@ -22,8 +22,8 @@ write 3
 [('test', 32768, 0, 8), ('testdir', 16384, 0, 0)]
 /
 /testdir
-(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
+(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
 /
 /testdir
 /
@@ -44,8 +44,8 @@ test <class 'VfsLfs2'>
 [('testdir', 16384, 0, 0), ('test', 32768, 0, 8)]
 []
 [('test', 32768, 0, 8)]
-(32768, 0, 0, 0, 0, 0, 8, 0, 0, 0)
-(16384, 0, 0, 0, 0, 0)
+(32768, 0, 0, 0, 0, 0) 8 <class 'int'> <class 'int'> <class 'int'>
+(16384, 0, 0, 0, 0, 0) -1 <class 'int'> <class 'int'> <class 'int'>
 littlefs
 data length: 4096
 write 0
@@ -59,8 +59,8 @@ write 3
 [('test', 32768, 0, 8), ('testdir', 16384, 0, 0)]
 /
 /testdir
-(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-(32768, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
+(32768, 0, 0, 0, 0, 0) 0 <class 'int'> <class 'int'> <class 'int'>
 /
 /testdir
 /

tests/extmod/vfs_lfs_mtime.py

@@ -0,0 +1,98 @@
+# Test for VfsLfs using a RAM device, mtime feature
+
+try:
+    import utime, uos
+
+    utime.sleep
+    uos.VfsLfs2
+except (ImportError, AttributeError):
+    print("SKIP")
+    raise SystemExit
+
+
+class RAMBlockDevice:
+    ERASE_BLOCK_SIZE = 1024
+
+    def __init__(self, blocks):
+        self.data = bytearray(blocks * self.ERASE_BLOCK_SIZE)
+
+    def readblocks(self, block, buf, off):
+        addr = block * self.ERASE_BLOCK_SIZE + off
+        for i in range(len(buf)):
+            buf[i] = self.data[addr + i]
+
+    def writeblocks(self, block, buf, off):
+        addr = block * self.ERASE_BLOCK_SIZE + off
+        for i in range(len(buf)):
+            self.data[addr + i] = buf[i]
+
+    def ioctl(self, op, arg):
+        if op == 4:  # block count
+            return len(self.data) // self.ERASE_BLOCK_SIZE
+        if op == 5:  # block size
+            return self.ERASE_BLOCK_SIZE
+        if op == 6:  # erase block
+            return 0
+
+
+def test(bdev, vfs_class):
+    print("test", vfs_class)
+
+    # Initial format of block device.
+    vfs_class.mkfs(bdev)
+
+    # construction
+    print("mtime=True")
+    vfs = vfs_class(bdev, mtime=True)
+
+    # Create an empty file, should have a timestamp.
+    vfs.open("test1", "wt").close()
+
+    # Wait 1 second so mtime will increase by at least 1.
+    utime.sleep(1)
+
+    # Create another empty file, should have a timestamp.
+    vfs.open("test2", "wt").close()
+
+    # Stat the files and check that test1 is older than test2.
+    stat1 = vfs.stat("test1")
+    stat2 = vfs.stat("test2")
+    print(stat1[8] != 0, stat2[8] != 0)
+    print(stat1[8] < stat2[8])
+
+    # Wait 1 second so mtime will increase by at least 1.
+    utime.sleep(1)
+
+    # Open test1 for reading and ensure mtime did not change.
+    vfs.open("test1", "rt").close()
+    print(vfs.stat("test1") == stat1)
+
+    # Open test1 for writing and ensure mtime increased from the previous value.
+    vfs.open("test1", "wt").close()
+    stat1_old = stat1
+    stat1 = vfs.stat("test1")
+    print(stat1_old[8] < stat1[8])
+
+    # Unmount.
+    vfs.umount()
+
+    # Check that remounting with mtime=False can read the timestamps.
+    print("mtime=False")
+    vfs = vfs_class(bdev, mtime=False)
+    print(vfs.stat("test1") == stat1)
+    print(vfs.stat("test2") == stat2)
+    f = vfs.open("test1", "wt")
+    f.close()
+    print(vfs.stat("test1") == stat1)
+    vfs.umount()
+
+    # Check that remounting with mtime=True still has the timestamps.
+    print("mtime=True")
+    vfs = vfs_class(bdev, mtime=True)
+    print(vfs.stat("test1") == stat1)
+    print(vfs.stat("test2") == stat2)
+    vfs.umount()
+
+
+bdev = RAMBlockDevice(30)
+test(bdev, uos.VfsLfs2)

tests/extmod/vfs_lfs_mtime.py.exp

@@ -0,0 +1,13 @@
+test <class 'VfsLfs2'>
+mtime=True
+True True
+True
+True
+True
+mtime=False
+True
+True
+True
+mtime=True
+True
+True