Flash: filesystem now automatically synchronised. _dirty flag implemented.

bdevice.py

@@ -65,6 +65,9 @@ class BlockDevice:
         return self.readwrite(start, buf, True)
 
     # IOCTL protocol.
+    def sync(self):  # Nothing to do for unbuffered devices. Subclass overrides.
+        return 0
+
     def readblocks(self, blocknum, buf, offset=0):
         self.readwrite(offset + (blocknum << self._nbits), buf, True)
 
@@ -72,7 +75,9 @@ class BlockDevice:
         offset = 0 if offset is None else offset
         self.readwrite(offset + (blocknum << self._nbits), buf, False)
 
-    def ioctl(self, op, arg):
+    def ioctl(self, op, arg):  # ioctl calls: see extmod/vfs.h
+        if op == 3:  # SYNCHRONISE
+            return self.sync()
         if op == 4:  # BP_IOCTL_SEC_COUNT
             return self._a_bytes >> self._nbits
         if op == 5:  # BP_IOCTL_SEC_SIZE
@@ -83,6 +88,7 @@ class BlockDevice:
 # Hardware agnostic base class for flash memory, where a single sector is cached.
 # This minimises RAM usage. Under FAT wear is reduced if you cache at least two
 # sectors. This driver is primarily intended for littlefs which has no such issue.
+# Class assumes erased state is 0xff.
 
 # Subclass must provide these hardware-dependent methods:
 # .rdchip(addr, mvb) Read from chip into memoryview: data guaranteed not to be cached.
@@ -91,6 +97,7 @@ class BlockDevice:
 
 _RDBUFSIZE = const(32)  # Size of read buffer for erasure test
 
+
 class FlashDevice(BlockDevice):
 
     def __init__(self, nbits, nchips, chip_size, sec_size):
@@ -102,7 +109,10 @@ class FlashDevice(BlockDevice):
         self._mvbuf = memoryview(self._buf)
         self._cache = bytearray(sec_size)  # Cache always contains one sector
         self._mvd = memoryview(self._cache)
-        self._acache = 0  # Address in chip of byte 0 of current cached sector
+        self._acache = 0  # Address in chip of byte 0 of current cached sector.
+        # A newly cached sector, or one which has been flushed, will be clean,
+        # so .sync() will do nothing. If cache is modified, dirty will be set.
+        self._dirty = False
 
     def read(self, addr, mvb):
         nbytes = len(mvb)
@@ -126,23 +136,26 @@ class FlashDevice(BlockDevice):
                 self.rdchip(addr + nr, mvb[boff + nr : ])
         return mvb
 
-    def synchronise(self):
+    def sync(self):
-#        print('SYNCHRONISE')
+        if self._dirty:
-        self.flush(self._mvd, self._acache)  # Write out old data
+            self.flush(self._mvd, self._acache)  # Write out old data
+        self._dirty = False
+        return 0
 
-# TODO Performance enhancement: if cache intersects address range, update it first.
+# Performance enhancement: if cache intersects address range, update it first.
-# Currently in this case it would be written twice.
+# Currently in this case it would be written twice. This may be rare.
     def write(self, addr, mvb):
         nbytes = len(mvb)
         acache = self._acache
         boff = 0  # Offset into buf.
         while nbytes:
             if (addr & self._fmask) != acache:
-                self.synchronise()  # Erase sector and write out old data
+                self.sync()  # Erase sector and write out old data
                 self._fill_cache(addr)  # Cache sector which includes addr
             offs = addr & self._cache_mask  # Offset into cache
             npage = min(nbytes, self.sec_size - offs)  # No. of bytes in current sector
             self._mvd[offs : offs + npage] = mvb[boff : boff + npage]
+            self._dirty = True  # Cache contents do not match those of chip
             nbytes -= npage
             boff += npage
             addr += npage
@@ -154,6 +167,7 @@ class FlashDevice(BlockDevice):
         addr &= self._fmask
         self.rdchip(addr, self._mvd)
         self._acache = addr
+        self._dirty = False
 
     def initialise(self):
         self._fill_cache(0)

flash/FLASH.md

@@ -22,6 +22,9 @@ an inevitable price for the large capacity of flash chips.
 FAT and littlefs filesystems are supported but the latter is preferred owing to
 its resilience and wear levelling characteristics.
 
+Byte level access on such large devices probably has few use cases other than
+for facilitating effective hardware tests and diagnostics.
+
 # 2. Connections
 
 Any SPI interface may be used. The table below assumes a Pyboard running SPI(2)
@@ -41,7 +44,7 @@ connected to 3V3 or left unconnected.
 | 8     |  Vcc    | 3V3 | 3V3    |
 
 For multiple chips a separate CS pin must be assigned to each chip: each one
-must be wired to a single chip's CS line. Multiple chips should have 3V3, Gnd,
+being wired to a single chip's CS line. Multiple chips should have 3V3, Gnd,
 SCL, MOSI and MISO lines wired in parallel.
 
 If you use a Pyboard D and power the chips from the 3V3 output you will need
@@ -49,15 +52,15 @@ to enable the voltage rail by issuing:
 ```python
 machine.Pin.board.EN_3V3.value(1)
 ```
-Other platforms may vary.
+Other platforms may vary but the Cypress chips require a 3.3V supply.
 
 ## 2.1 SPI Bus
 
 The devices support baudrates up to 50MHz. In practice MicroPython targets do
 not support such high rates. In testing I found it necessary to specify 5MHz
 otherwise erratic results occurred. This was probably because of my breadboard
-test setup. On a PCB I would hope to run at a sunbstantially higher rate. The
+test setup. I have a PCB in manufacture and hope to run at 20MHz. For now code
-SPI bus is fast: wiring should be short and direct.
+samples specify 5MHz. SPI bus wiring should be short and direct.
 
 # 3. Files
 
@@ -74,7 +77,7 @@ Test scripts assume two chips with CS/ pins wired to Pyboard pins Y4 and Y5.
 
 The driver supports mounting the Flash chips as a filesystem. Initially the
 device will be unformatted so it is necessary to issue code along these lines
-to format the device. Code assumes two devices and also assumes the littlefs
+to format the device. Code assumes two devices and the (recommended) littlefs
 filesystem:
 
 ```python
@@ -82,19 +85,17 @@ import os
 from machine import SPI, Pin
 from flash_spi import FLASH
 cspins = (Pin(Pin.board.Y5, Pin.OUT, value=1), Pin(Pin.board.Y4, Pin.OUT, value=1))
-flash = FLASH(SPI(2, baudrate=20_000_000), cspins)
+flash = FLASH(SPI(2, baudrate=5_000_000), cspins)
 # Format the filesystem
 os.VfsLfs2.mkfs(flash)  # Omit this to mount an existing filesystem
 os.mount(flash,'/fl_ext')
 ```
-The above will reformat a drive with an existing filesystem: to mount an
+The above will reformat a drive with an existing filesystem erasing all files:
-existing filesystem simply omit the commented line.
+to mount an existing filesystem omit the commented line.
 
 Note that, at the outset, you need to decide whether to use the array as a
-mounted filesystem or as a byte array. The filesystem is relatively small but
+mounted filesystem or as a byte array. Most use cases for flash will require a
-has high integrity owing to the hardware longevity. Typical use-cases involve
+filesystem, although byte level reads may be used to debug filesystem issues.
-files which are frequently updated. These include files used for storing Python
-objects serialised using Pickle/ujson or files holding a btree database.
 
 The SPI bus must be instantiated using the `machine` module.
 
@@ -107,7 +108,8 @@ multiple physical devices on a common SPI bus.
 
 This tests each chip in the list of chip select pins - if a chip is detected on
 each chip select line a flash array is instantiated. A `RuntimeError` will be
-raised if a device is not detected on a CS line.
+raised if a device is not detected on a CS line. The test has no effect on
+the array contents.
 
 Arguments:  
  1. `spi` Mandatory. An initialised SPI bus created by `machine`.
@@ -146,7 +148,7 @@ of single byte access:
 from machine import SPI, Pin
 from flash_spi import FLASH
 cspins = (Pin(Pin.board.Y5, Pin.OUT, value=1), Pin(Pin.board.Y4, Pin.OUT, value=1))
-flash = FLASH(SPI(2, baudrate=20_000_000), cspins)
+flash = FLASH(SPI(2, baudrate=5_000_000), cspins)
 flash[2000] = 42
 print(flash[2000])  # Return an integer
 ```
@@ -156,7 +158,7 @@ writing, the size of the slice must match the length of the buffer:
 from machine import SPI, Pin
 from flash_spi import FLASH
 cspins = (Pin(Pin.board.Y5, Pin.OUT, value=1), Pin(Pin.board.Y4, Pin.OUT, value=1))
-flash = FLASH(SPI(2, baudrate=20_000_000), cspins)
+flash = FLASH(SPI(2, baudrate=5_000_000), cspins)
 flash[2000:2002] = bytearray((42, 43))
 print(flash[2000:2002])  # Returns a bytearray
 ```
@@ -178,12 +180,13 @@ advantage when reading of using a pre-allocated buffer. Arguments:
 
 ### 4.1.3 Other methods
 
-#### synchronise
+#### sync
 
-This causes the cached sector to be written to the device. Should be called
+This causes the cached sector to be written to the device. In normal filesystem
-prior to power down. **TODO: check flush/synchronise**
+use this need not be called. If byte-level writes have been performed it should
+be called prior to power down.
 
-#### The len() operator
+#### The len operator
 
 The size of the flash array in bytes may be retrieved by issuing `len(flash)`
 where `flash` is the `FLASH` instance.
@@ -196,7 +199,7 @@ pin does not correspond to a valid chip.
 
 Other than for debugging there is no need to call `scan()`: the constructor
 will throw a `RuntimeError` if it fails to communicate with and correctly
-identify the chip.
+identify each chip.
 
 #### erase
 
@@ -224,17 +227,19 @@ This performs a basic test of single and multi-byte access to chip 0. The test
 reports how many chips can be accessed. Existing array data will be lost. This
 primarily tests the driver: as a hardware test it is not exhaustive.
 
-## 5.2 full_test()
+## 5.2 full_test(count=10)
 
-This is a hardware test. Tests the entire array. Fills each 256 byte page with
+This is a hardware test. Tests the entire array. Creates an array of 256 bytes
-random data, reads it back, and checks the outcome. Existing array data will be
+of random data and writes it to a random address. After synchronising the cache
-lost. **TODO long run time.**
+with the hardware, reads it back, and checks the outcome. Existing array data
+will be lost. The arg determines the number of passes.
 
 ## 5.3 fstest(format=False)
 
-If `True` is passed, formats the flash array as a littlefs filesystem and mounts
+If `True` is passed, formats the flash array as a littlefs filesystem deleting
-the device on `/fl_ext`. If no arg is passed it mounts the array and lists the
+existing contents. In both cases of the arg it mounts the device on `/fl_ext`
-contents. It also prints the outcome of `uos.statvfs` on the array.
+lists the contents of the mountpoint. It also prints the outcome of
+`uos.statvfs` on the mountpoint.
 
 ## 5.4 cptest()
 

flash/flash_spi.py

@@ -148,7 +148,8 @@ class FLASH(FlashDevice):
         pe = (addr & -self._c_bytes) + self._c_bytes  # Byte 0 of next chip
         return min(nbytes, pe - la)
 
-    # Erase sector. Address is start byte address of sector.
+    # Erase sector. Address is start byte address of sector. Optimisation: skip
+    # if sector is already erased.
     def _sector_erase(self, addr):
         if not self.is_empty(addr):
             self._getaddr(addr, 1)

flash/flash_test.py

@@ -132,7 +132,7 @@ def full_test(count=10):
             if sa < (flash._a_bytes - 256):
                 break
         flash[sa:sa + 256] = data
-        flash.synchronise()
+        flash.sync()
         got = flash[sa:sa + 256]
         if got == data:
             print('Pass {} address {:08x} passed'.format(n, sa))
@@ -142,7 +142,7 @@ def full_test(count=10):
             print('Pass {} address {:08x} readback failed.'.format(n, sa))
             sa1 = sa & 0xfff
             print('Bounds {} to {}'.format(sa1, sa1+256))
-#            flash.synchronise()
+#            flash.sync()
             got1 = flash[sa:sa + 256]
             if got1 == data:
                 print('second attempt OK')

i2c/eep_i2c.py

@@ -93,36 +93,30 @@ def test():
 # ***** TEST OF FILESYSTEM MOUNT *****
 def fstest(format=False):
     eep = get_eep()
+    try:
+        uos.umount('/eeprom')
+    except OSError:
+        pass
     # ***** CODE FOR FATFS *****
     #if format:
-        #uos.VfsFat.mkfs(eep)
+        #os.VfsFat.mkfs(eep)
-    #vfs=uos.VfsFat(eep)
-    #try:
-        #uos.mount(vfs,'/eeprom')
-    #except OSError:  # Already mounted
-        #pass
     # ***** CODE FOR LITTLEFS *****
     if format:
         uos.VfsLfs2.mkfs(eep)
+    # General
     try:
         uos.mount(eep,'/eeprom')
-    except OSError:  # Already mounted
+    except OSError:
-        pass
+        raise OSError("Can't mount device: have you formatted it?")
     print('Contents of "/": {}'.format(uos.listdir('/')))
     print('Contents of "/eeprom": {}'.format(uos.listdir('/eeprom')))
     print(uos.statvfs('/eeprom'))
 
-def cptest():
+def cptest():  # Assumes pre-existing filesystem of either type
     eep = get_eep()
     if 'eeprom' in uos.listdir('/'):
         print('Device already mounted.')
     else:
-        #vfs=uos.VfsFat(eep)
-        #try:
-            #uos.mount(vfs,'/eeprom')
-        #except OSError:
-            #print('Fail mounting device. Have you formatted it?')
-            #return
         try:
             uos.mount(eep,'/eeprom')
         except OSError:

spi/eep_spi.py

@@ -97,37 +97,30 @@ def test(stm=False):
 # ***** TEST OF FILESYSTEM MOUNT *****
 def fstest(format=False, stm=False):
     eep = get_eep(stm)
+    try:
+        uos.umount('/eeprom')
+    except OSError:
+        pass
     # ***** CODE FOR FATFS *****
     #if format:
-        #uos.VfsFat.mkfs(eep)
+        #os.VfsFat.mkfs(eep)
-    #vfs=uos.VfsFat(eep)
-    #try:
-        #uos.mount(vfs,'/eeprom')
-    #except OSError:  # Already mounted
-        #pass
     # ***** CODE FOR LITTLEFS *****
     if format:
         uos.VfsLfs2.mkfs(eep)
+    # General
     try:
         uos.mount(eep,'/eeprom')
-    except OSError:  # Already mounted
+    except OSError:
-        pass
+        raise OSError("Can't mount device: have you formatted it?")
     print('Contents of "/": {}'.format(uos.listdir('/')))
     print('Contents of "/eeprom": {}'.format(uos.listdir('/eeprom')))
     print(uos.statvfs('/eeprom'))
 
-def cptest(stm=False):
+def cptest(stm=False):  # Assumes pre-existing filesystem of either type
     eep = get_eep(stm)
     if 'eeprom' in uos.listdir('/'):
         print('Device already mounted.')
     else:
-        #vfs=uos.VfsFat(eep)
-        #try:
-            #uos.mount(vfs,'/eeprom')
-        #except OSError:
-            #print('Fail mounting device. Have you formatted it?')
-            #return
-        #vfs=uos.VfsFat(eep)
         try:
             uos.mount(eep,'/eeprom')
         except OSError: