diff --git a/docs/reference/mpyfiles.rst b/docs/reference/mpyfiles.rst index fcb4996565..d93a86383e 100644 --- a/docs/reference/mpyfiles.rst +++ b/docs/reference/mpyfiles.rst @@ -27,6 +27,11 @@ Compatibility is based on the following: * Version of the .mpy file: the version of the file must match the version supported by the system loading it. +* Sub-version of the .mpy file: if the .mpy file contains native machine code + then the sub-version of the file must match the version support by the + system loading it. Otherwise, if there is no native machine code in the .mpy + file, then the sub-version is ignored when loading. + * Small integer bits: the .mpy file will require a minimum number of bits in a small integer and the system loading it must support at least this many bits. @@ -55,6 +60,7 @@ If importing an .mpy file fails then try the following: 'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp', 'xtensa', 'xtensawin'][sys_mpy >> 10] print('mpy version:', sys_mpy & 0xff) + print('mpy sub-version:', sys_mpy >> 8 & 3) print('mpy flags:', end='') if arch: print(' -march=' + arch, end='') diff --git a/mpy-cross/main.c b/mpy-cross/main.c index f3ffff61fd..4975c8ddb2 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -228,7 +228,7 @@ MP_NOINLINE int main_(int argc, char **argv) { a += 1; } else if (strcmp(argv[a], "--version") == 0) { printf("MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE - "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "\n"); + "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "." MP_STRINGIFY(MPY_SUB_VERSION) "\n"); return 0; } else if (strcmp(argv[a], "-v") == 0) { mp_verbose_flag++; diff --git a/py/persistentcode.c b/py/persistentcode.c index 1b1741f261..de84c17359 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -393,14 +393,14 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) { byte header[4]; read_bytes(reader, header, sizeof(header)); + byte arch = MPY_FEATURE_DECODE_ARCH(header[2]); if (header[0] != 'M' || header[1] != MPY_VERSION - || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS + || (arch != MP_NATIVE_ARCH_NONE && MPY_FEATURE_DECODE_SUB_VERSION(header[2]) != MPY_SUB_VERSION) || header[3] > MP_SMALL_INT_BITS) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) { - byte arch = MPY_FEATURE_DECODE_ARCH(header[2]); if (!MPY_FEATURE_ARCH_TEST(arch)) { if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) { // On supported ports this can be resolved by enabling feature, eg @@ -596,7 +596,7 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { byte header[4] = { 'M', MPY_VERSION, - MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC), + MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION), #if MICROPY_DYNAMIC_COMPILER mp_dynamic_compiler.small_int_bits, #else diff --git a/py/persistentcode.h b/py/persistentcode.h index 29ccce4a3d..e664358737 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -30,24 +30,23 @@ #include "py/reader.h" #include "py/emitglue.h" -// The current version of .mpy files +// The current version of .mpy files. A bytecode-only .mpy file can be loaded +// as long as MPY_VERSION matches, but a native .mpy (i.e. one with an arch +// set) must also match MPY_SUB_VERSION. This allows 3 additional updates to +// the native ABI per bytecode revision. #define MPY_VERSION 6 +#define MPY_SUB_VERSION 1 -// Macros to encode/decode flags to/from the feature byte -#define MPY_FEATURE_ENCODE_FLAGS(flags) (flags) -#define MPY_FEATURE_DECODE_FLAGS(feat) ((feat) & 3) +// Macros to encode/decode sub-version to/from the feature byte. This replaces +// the bits previously used to encode the flags (map caching and unicode) +// which are no longer used starting at .mpy version 6. +#define MPY_FEATURE_ENCODE_SUB_VERSION(version) (version) +#define MPY_FEATURE_DECODE_SUB_VERSION(feat) ((feat) & 3) // Macros to encode/decode native architecture to/from the feature byte #define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2) #define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2) -// The feature flag bits encode the compile-time config options that affect -// the generate bytecode. Note: no longer used. -// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE and MICROPY_PY_BUILTINS_STR_UNICODE). -#define MPY_FEATURE_FLAGS (0) -// This is a version of the flags that can be configured at runtime. -#define MPY_FEATURE_FLAGS_DYNAMIC (0) - // Define the host architecture #if MICROPY_EMIT_X86 #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86) @@ -82,7 +81,7 @@ // 16-bit little-endian integer with the second and third bytes of supported .mpy files #define MPY_FILE_HEADER_INT (MPY_VERSION \ - | (MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8) + | (MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8) enum { MP_NATIVE_ARCH_NONE = 0, diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py index 449371ddac..73e20694cc 100644 --- a/tests/micropython/import_mpy_native.py +++ b/tests/micropython/import_mpy_native.py @@ -11,7 +11,8 @@ except (ImportError, AttributeError): raise SystemExit mpy_arch = usys.implementation._mpy >> 8 -if mpy_arch == 0: +if mpy_arch >> 2 == 0: + # This system does not support .mpy files containing native code print("SKIP") raise SystemExit @@ -54,8 +55,8 @@ class UserFS: valid_header = bytes([77, 6, mpy_arch, 31]) # fmt: off user_files = { - # bad architecture - '/mod0.mpy': b'M\x06\xfc\x1f', + # bad architecture (mpy_arch needed for sub-version) + '/mod0.mpy': bytes([77, 6, 0xfc | mpy_arch, 31]), # test loading of viper and asm '/mod1.mpy': valid_header + ( diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index bc8dcafdc7..e18720fb31 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -46,24 +46,26 @@ class UserFS: # Pre-compiled examples/natmod/features0 example for various architectures, keyed -# by the required value of sys.implementation._mpy. +# by the required value of sys.implementation._mpy (without sub-version). features0_file_contents = { # -march=x64 - 0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', + 0x806: b'M\x06\x09\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', # -march=armv6m - 0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", + 0x1006: b"M\x06\x11\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", } # Populate armv7m-derived archs based on armv6m. for arch in (0x1406, 0x1806, 0x1C06, 0x2006): features0_file_contents[arch] = features0_file_contents[0x1006] -if sys.implementation._mpy not in features0_file_contents: +# Check that a .mpy exists for the target (ignore sub-version in lookup). +sys_implementation_mpy = sys.implementation._mpy & ~(3 << 8) +if sys_implementation_mpy not in features0_file_contents: print("SKIP") raise SystemExit # These are the test .mpy files. -user_files = {"/features0.mpy": features0_file_contents[sys.implementation._mpy]} +user_files = {"/features0.mpy": features0_file_contents[sys_implementation_mpy]} # Create and mount a user filesystem. uos.mount(UserFS(user_files), "/userfs") diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 31212fd5bd..8b644c137f 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -88,6 +88,7 @@ class FreezeError(Exception): class Config: MPY_VERSION = 6 + MPY_SUB_VERSION = 1 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -1335,6 +1336,9 @@ def read_mpy(filename): feature_byte = header[2] mpy_native_arch = feature_byte >> 2 if mpy_native_arch != MP_NATIVE_ARCH_NONE: + mpy_sub_version = feature_byte & 3 + if mpy_sub_version != config.MPY_SUB_VERSION: + raise MPYReadError(filename, "incompatible .mpy sub-version") if config.native_arch == MP_NATIVE_ARCH_NONE: config.native_arch = mpy_native_arch elif config.native_arch != mpy_native_arch: @@ -1658,7 +1662,9 @@ def merge_mpy(compiled_modules, output_file): else: main_cm_idx = None for idx, cm in enumerate(compiled_modules): - if cm.header[2]: + feature_byte = cm.header[2] + mpy_native_arch = feature_byte >> 2 + if mpy_native_arch: # Must use qstr_table and obj_table from this raw_code if main_cm_idx is not None: raise Exception("can't merge files when more than one contains native code") @@ -1670,7 +1676,7 @@ def merge_mpy(compiled_modules, output_file): header = bytearray(4) header[0] = ord("M") header[1] = config.MPY_VERSION - header[2] = config.native_arch << 2 + header[2] = config.native_arch << 2 | config.MPY_SUB_VERSION header[3] = config.mp_small_int_bits merged_mpy.extend(header) diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 09ea90dcd1..05e70709ad 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -36,6 +36,7 @@ import makeqstrdata as qstrutil # MicroPython constants MPY_VERSION = 6 +MPY_SUB_VERSION = 1 MP_CODE_BYTECODE = 2 MP_CODE_NATIVE_VIPER = 4 MP_NATIVE_ARCH_X86 = 1 @@ -917,7 +918,11 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.open(fmpy) # MPY: header - out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) + out.write_bytes( + bytearray( + [ord("M"), MPY_VERSION, env.arch.mpy_feature | MPY_SUB_VERSION, MP_SMALL_INT_BITS] + ) + ) # MPY: n_qstr out.write_uint(1 + len(native_qstr_vals))