From 7f997f27465adedc87fb6a392d429fb6fa84a573 Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Mon, 30 Dec 2019 09:51:10 +0000 Subject: [PATCH] Flash: filesystem now automatically synchronised. _dirty flag implemented. --- bdevice.py | 30 +++++++++++++++++------ flash/FLASH.md | 59 ++++++++++++++++++++++++--------------------- flash/flash_spi.py | 3 ++- flash/flash_test.py | 4 +-- i2c/eep_i2c.py | 24 +++++++----------- spi/eep_spi.py | 25 +++++++------------ 6 files changed, 76 insertions(+), 69 deletions(-) diff --git a/bdevice.py b/bdevice.py index fb8caa5..cbdf21e 100644 --- a/bdevice.py +++ b/bdevice.py @@ -65,6 +65,9 @@ class BlockDevice: return self.readwrite(start, buf, True) # IOCTL protocol. + def sync(self): # Nothing to do for unbuffered devices. Subclass overrides. + return 0 + def readblocks(self, blocknum, buf, offset=0): self.readwrite(offset + (blocknum << self._nbits), buf, True) @@ -72,7 +75,9 @@ class BlockDevice: offset = 0 if offset is None else offset self.readwrite(offset + (blocknum << self._nbits), buf, False) - def ioctl(self, op, arg): + def ioctl(self, op, arg): # ioctl calls: see extmod/vfs.h + if op == 3: # SYNCHRONISE + return self.sync() if op == 4: # BP_IOCTL_SEC_COUNT return self._a_bytes >> self._nbits if op == 5: # BP_IOCTL_SEC_SIZE @@ -83,6 +88,7 @@ class BlockDevice: # Hardware agnostic base class for flash memory, where a single sector is cached. # This minimises RAM usage. Under FAT wear is reduced if you cache at least two # sectors. This driver is primarily intended for littlefs which has no such issue. +# Class assumes erased state is 0xff. # Subclass must provide these hardware-dependent methods: # .rdchip(addr, mvb) Read from chip into memoryview: data guaranteed not to be cached. @@ -91,6 +97,7 @@ class BlockDevice: _RDBUFSIZE = const(32) # Size of read buffer for erasure test + class FlashDevice(BlockDevice): def __init__(self, nbits, nchips, chip_size, sec_size): @@ -102,7 +109,10 @@ class FlashDevice(BlockDevice): self._mvbuf = memoryview(self._buf) self._cache = bytearray(sec_size) # Cache always contains one sector self._mvd = memoryview(self._cache) - self._acache = 0 # Address in chip of byte 0 of current cached sector + self._acache = 0 # Address in chip of byte 0 of current cached sector. + # A newly cached sector, or one which has been flushed, will be clean, + # so .sync() will do nothing. If cache is modified, dirty will be set. + self._dirty = False def read(self, addr, mvb): nbytes = len(mvb) @@ -126,23 +136,26 @@ class FlashDevice(BlockDevice): self.rdchip(addr + nr, mvb[boff + nr : ]) return mvb - def synchronise(self): -# print('SYNCHRONISE') - self.flush(self._mvd, self._acache) # Write out old data + def sync(self): + if self._dirty: + self.flush(self._mvd, self._acache) # Write out old data + self._dirty = False + return 0 -# TODO Performance enhancement: if cache intersects address range, update it first. -# Currently in this case it would be written twice. +# Performance enhancement: if cache intersects address range, update it first. +# Currently in this case it would be written twice. This may be rare. def write(self, addr, mvb): nbytes = len(mvb) acache = self._acache boff = 0 # Offset into buf. while nbytes: if (addr & self._fmask) != acache: - self.synchronise() # Erase sector and write out old data + self.sync() # Erase sector and write out old data self._fill_cache(addr) # Cache sector which includes addr offs = addr & self._cache_mask # Offset into cache npage = min(nbytes, self.sec_size - offs) # No. of bytes in current sector self._mvd[offs : offs + npage] = mvb[boff : boff + npage] + self._dirty = True # Cache contents do not match those of chip nbytes -= npage boff += npage addr += npage @@ -154,6 +167,7 @@ class FlashDevice(BlockDevice): addr &= self._fmask self.rdchip(addr, self._mvd) self._acache = addr + self._dirty = False def initialise(self): self._fill_cache(0) diff --git a/flash/FLASH.md b/flash/FLASH.md index 5b083f6..ca543d4 100644 --- a/flash/FLASH.md +++ b/flash/FLASH.md @@ -22,6 +22,9 @@ an inevitable price for the large capacity of flash chips. FAT and littlefs filesystems are supported but the latter is preferred owing to its resilience and wear levelling characteristics. +Byte level access on such large devices probably has few use cases other than +for facilitating effective hardware tests and diagnostics. + # 2. Connections Any SPI interface may be used. The table below assumes a Pyboard running SPI(2) @@ -41,7 +44,7 @@ connected to 3V3 or left unconnected. | 8 | Vcc | 3V3 | 3V3 | For multiple chips a separate CS pin must be assigned to each chip: each one -must be wired to a single chip's CS line. Multiple chips should have 3V3, Gnd, +being wired to a single chip's CS line. Multiple chips should have 3V3, Gnd, SCL, MOSI and MISO lines wired in parallel. If you use a Pyboard D and power the chips from the 3V3 output you will need @@ -49,15 +52,15 @@ to enable the voltage rail by issuing: ```python machine.Pin.board.EN_3V3.value(1) ``` -Other platforms may vary. +Other platforms may vary but the Cypress chips require a 3.3V supply. ## 2.1 SPI Bus The devices support baudrates up to 50MHz. In practice MicroPython targets do not support such high rates. In testing I found it necessary to specify 5MHz otherwise erratic results occurred. This was probably because of my breadboard -test setup. On a PCB I would hope to run at a sunbstantially higher rate. The -SPI bus is fast: wiring should be short and direct. +test setup. I have a PCB in manufacture and hope to run at 20MHz. For now code +samples specify 5MHz. SPI bus wiring should be short and direct. # 3. Files @@ -74,7 +77,7 @@ Test scripts assume two chips with CS/ pins wired to Pyboard pins Y4 and Y5. The driver supports mounting the Flash chips as a filesystem. Initially the device will be unformatted so it is necessary to issue code along these lines -to format the device. Code assumes two devices and also assumes the littlefs +to format the device. Code assumes two devices and the (recommended) littlefs filesystem: ```python @@ -82,19 +85,17 @@ import os from machine import SPI, Pin from flash_spi import FLASH cspins = (Pin(Pin.board.Y5, Pin.OUT, value=1), Pin(Pin.board.Y4, Pin.OUT, value=1)) -flash = FLASH(SPI(2, baudrate=20_000_000), cspins) +flash = FLASH(SPI(2, baudrate=5_000_000), cspins) # Format the filesystem os.VfsLfs2.mkfs(flash) # Omit this to mount an existing filesystem os.mount(flash,'/fl_ext') ``` -The above will reformat a drive with an existing filesystem: to mount an -existing filesystem simply omit the commented line. +The above will reformat a drive with an existing filesystem erasing all files: +to mount an existing filesystem omit the commented line. Note that, at the outset, you need to decide whether to use the array as a -mounted filesystem or as a byte array. The filesystem is relatively small but -has high integrity owing to the hardware longevity. Typical use-cases involve -files which are frequently updated. These include files used for storing Python -objects serialised using Pickle/ujson or files holding a btree database. +mounted filesystem or as a byte array. Most use cases for flash will require a +filesystem, although byte level reads may be used to debug filesystem issues. The SPI bus must be instantiated using the `machine` module. @@ -107,7 +108,8 @@ multiple physical devices on a common SPI bus. This tests each chip in the list of chip select pins - if a chip is detected on each chip select line a flash array is instantiated. A `RuntimeError` will be -raised if a device is not detected on a CS line. +raised if a device is not detected on a CS line. The test has no effect on +the array contents. Arguments: 1. `spi` Mandatory. An initialised SPI bus created by `machine`. @@ -146,7 +148,7 @@ of single byte access: from machine import SPI, Pin from flash_spi import FLASH cspins = (Pin(Pin.board.Y5, Pin.OUT, value=1), Pin(Pin.board.Y4, Pin.OUT, value=1)) -flash = FLASH(SPI(2, baudrate=20_000_000), cspins) +flash = FLASH(SPI(2, baudrate=5_000_000), cspins) flash[2000] = 42 print(flash[2000]) # Return an integer ``` @@ -156,7 +158,7 @@ writing, the size of the slice must match the length of the buffer: from machine import SPI, Pin from flash_spi import FLASH cspins = (Pin(Pin.board.Y5, Pin.OUT, value=1), Pin(Pin.board.Y4, Pin.OUT, value=1)) -flash = FLASH(SPI(2, baudrate=20_000_000), cspins) +flash = FLASH(SPI(2, baudrate=5_000_000), cspins) flash[2000:2002] = bytearray((42, 43)) print(flash[2000:2002]) # Returns a bytearray ``` @@ -178,12 +180,13 @@ advantage when reading of using a pre-allocated buffer. Arguments: ### 4.1.3 Other methods -#### synchronise +#### sync -This causes the cached sector to be written to the device. Should be called -prior to power down. **TODO: check flush/synchronise** +This causes the cached sector to be written to the device. In normal filesystem +use this need not be called. If byte-level writes have been performed it should +be called prior to power down. -#### The len() operator +#### The len operator The size of the flash array in bytes may be retrieved by issuing `len(flash)` where `flash` is the `FLASH` instance. @@ -196,7 +199,7 @@ pin does not correspond to a valid chip. Other than for debugging there is no need to call `scan()`: the constructor will throw a `RuntimeError` if it fails to communicate with and correctly -identify the chip. +identify each chip. #### erase @@ -224,17 +227,19 @@ This performs a basic test of single and multi-byte access to chip 0. The test reports how many chips can be accessed. Existing array data will be lost. This primarily tests the driver: as a hardware test it is not exhaustive. -## 5.2 full_test() +## 5.2 full_test(count=10) -This is a hardware test. Tests the entire array. Fills each 256 byte page with -random data, reads it back, and checks the outcome. Existing array data will be -lost. **TODO long run time.** +This is a hardware test. Tests the entire array. Creates an array of 256 bytes +of random data and writes it to a random address. After synchronising the cache +with the hardware, reads it back, and checks the outcome. Existing array data +will be lost. The arg determines the number of passes. ## 5.3 fstest(format=False) -If `True` is passed, formats the flash array as a littlefs filesystem and mounts -the device on `/fl_ext`. If no arg is passed it mounts the array and lists the -contents. It also prints the outcome of `uos.statvfs` on the array. +If `True` is passed, formats the flash array as a littlefs filesystem deleting +existing contents. In both cases of the arg it mounts the device on `/fl_ext` +lists the contents of the mountpoint. It also prints the outcome of +`uos.statvfs` on the mountpoint. ## 5.4 cptest() diff --git a/flash/flash_spi.py b/flash/flash_spi.py index 506703f..bf5c3c2 100644 --- a/flash/flash_spi.py +++ b/flash/flash_spi.py @@ -148,7 +148,8 @@ class FLASH(FlashDevice): pe = (addr & -self._c_bytes) + self._c_bytes # Byte 0 of next chip return min(nbytes, pe - la) - # Erase sector. Address is start byte address of sector. + # Erase sector. Address is start byte address of sector. Optimisation: skip + # if sector is already erased. def _sector_erase(self, addr): if not self.is_empty(addr): self._getaddr(addr, 1) diff --git a/flash/flash_test.py b/flash/flash_test.py index 97adfb3..df7078e 100644 --- a/flash/flash_test.py +++ b/flash/flash_test.py @@ -132,7 +132,7 @@ def full_test(count=10): if sa < (flash._a_bytes - 256): break flash[sa:sa + 256] = data - flash.synchronise() + flash.sync() got = flash[sa:sa + 256] if got == data: print('Pass {} address {:08x} passed'.format(n, sa)) @@ -142,7 +142,7 @@ def full_test(count=10): print('Pass {} address {:08x} readback failed.'.format(n, sa)) sa1 = sa & 0xfff print('Bounds {} to {}'.format(sa1, sa1+256)) -# flash.synchronise() +# flash.sync() got1 = flash[sa:sa + 256] if got1 == data: print('second attempt OK') diff --git a/i2c/eep_i2c.py b/i2c/eep_i2c.py index 7d2088e..20854c7 100644 --- a/i2c/eep_i2c.py +++ b/i2c/eep_i2c.py @@ -93,36 +93,30 @@ def test(): # ***** TEST OF FILESYSTEM MOUNT ***** def fstest(format=False): eep = get_eep() + try: + uos.umount('/eeprom') + except OSError: + pass # ***** CODE FOR FATFS ***** #if format: - #uos.VfsFat.mkfs(eep) - #vfs=uos.VfsFat(eep) - #try: - #uos.mount(vfs,'/eeprom') - #except OSError: # Already mounted - #pass + #os.VfsFat.mkfs(eep) # ***** CODE FOR LITTLEFS ***** if format: uos.VfsLfs2.mkfs(eep) + # General try: uos.mount(eep,'/eeprom') - except OSError: # Already mounted - pass + except OSError: + raise OSError("Can't mount device: have you formatted it?") print('Contents of "/": {}'.format(uos.listdir('/'))) print('Contents of "/eeprom": {}'.format(uos.listdir('/eeprom'))) print(uos.statvfs('/eeprom')) -def cptest(): +def cptest(): # Assumes pre-existing filesystem of either type eep = get_eep() if 'eeprom' in uos.listdir('/'): print('Device already mounted.') else: - #vfs=uos.VfsFat(eep) - #try: - #uos.mount(vfs,'/eeprom') - #except OSError: - #print('Fail mounting device. Have you formatted it?') - #return try: uos.mount(eep,'/eeprom') except OSError: diff --git a/spi/eep_spi.py b/spi/eep_spi.py index 4bf5e97..0cfdbff 100644 --- a/spi/eep_spi.py +++ b/spi/eep_spi.py @@ -97,37 +97,30 @@ def test(stm=False): # ***** TEST OF FILESYSTEM MOUNT ***** def fstest(format=False, stm=False): eep = get_eep(stm) + try: + uos.umount('/eeprom') + except OSError: + pass # ***** CODE FOR FATFS ***** #if format: - #uos.VfsFat.mkfs(eep) - #vfs=uos.VfsFat(eep) - #try: - #uos.mount(vfs,'/eeprom') - #except OSError: # Already mounted - #pass + #os.VfsFat.mkfs(eep) # ***** CODE FOR LITTLEFS ***** if format: uos.VfsLfs2.mkfs(eep) + # General try: uos.mount(eep,'/eeprom') - except OSError: # Already mounted - pass + except OSError: + raise OSError("Can't mount device: have you formatted it?") print('Contents of "/": {}'.format(uos.listdir('/'))) print('Contents of "/eeprom": {}'.format(uos.listdir('/eeprom'))) print(uos.statvfs('/eeprom')) -def cptest(stm=False): +def cptest(stm=False): # Assumes pre-existing filesystem of either type eep = get_eep(stm) if 'eeprom' in uos.listdir('/'): print('Device already mounted.') else: - #vfs=uos.VfsFat(eep) - #try: - #uos.mount(vfs,'/eeprom') - #except OSError: - #print('Fail mounting device. Have you formatted it?') - #return - #vfs=uos.VfsFat(eep) try: uos.mount(eep,'/eeprom') except OSError: