From d94141e1473aebae0d3c63aeaa8397651ad6fa01 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Sat, 17 Sep 2022 23:57:12 +1000 Subject: [PATCH] py/persistentcode: Introduce .mpy sub-version. The intent is to allow us to make breaking changes to the native ABI (e.g. changes to dynruntime.h) without needing the bytecode version to increment. With this commit the two bits previously used for the feature flags (but now unused as of .mpy version 6) encode a sub-version. A bytecode-only .mpy file can be loaded as long as MPY_VERSION matches, but a native .mpy (i.e. one with an arch set) must also match MPY_SUB_VERSION. This allows 3 additional updates to the native ABI per bytecode revision. The sub-version is set to 1 because the previous commits that changed the layout of mp_obj_type_t have changed the native ABI. Signed-off-by: Jim Mussared Signed-off-by: Damien George --- docs/reference/mpyfiles.rst | 6 ++++++ mpy-cross/main.c | 2 +- py/persistentcode.c | 6 +++--- py/persistentcode.h | 23 +++++++++++------------ tests/micropython/import_mpy_native.py | 7 ++++--- tests/micropython/import_mpy_native_gc.py | 12 +++++++----- tools/mpy-tool.py | 10 ++++++++-- tools/mpy_ld.py | 7 ++++++- 8 files changed, 46 insertions(+), 27 deletions(-) diff --git a/docs/reference/mpyfiles.rst b/docs/reference/mpyfiles.rst index fcb4996565..d93a86383e 100644 --- a/docs/reference/mpyfiles.rst +++ b/docs/reference/mpyfiles.rst @@ -27,6 +27,11 @@ Compatibility is based on the following: * Version of the .mpy file: the version of the file must match the version supported by the system loading it. +* Sub-version of the .mpy file: if the .mpy file contains native machine code + then the sub-version of the file must match the version support by the + system loading it. Otherwise, if there is no native machine code in the .mpy + file, then the sub-version is ignored when loading. + * Small integer bits: the .mpy file will require a minimum number of bits in a small integer and the system loading it must support at least this many bits. @@ -55,6 +60,7 @@ If importing an .mpy file fails then try the following: 'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp', 'xtensa', 'xtensawin'][sys_mpy >> 10] print('mpy version:', sys_mpy & 0xff) + print('mpy sub-version:', sys_mpy >> 8 & 3) print('mpy flags:', end='') if arch: print(' -march=' + arch, end='') diff --git a/mpy-cross/main.c b/mpy-cross/main.c index f3ffff61fd..4975c8ddb2 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -228,7 +228,7 @@ MP_NOINLINE int main_(int argc, char **argv) { a += 1; } else if (strcmp(argv[a], "--version") == 0) { printf("MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE - "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "\n"); + "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "." MP_STRINGIFY(MPY_SUB_VERSION) "\n"); return 0; } else if (strcmp(argv[a], "-v") == 0) { mp_verbose_flag++; diff --git a/py/persistentcode.c b/py/persistentcode.c index 1b1741f261..de84c17359 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -393,14 +393,14 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) { byte header[4]; read_bytes(reader, header, sizeof(header)); + byte arch = MPY_FEATURE_DECODE_ARCH(header[2]); if (header[0] != 'M' || header[1] != MPY_VERSION - || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS + || (arch != MP_NATIVE_ARCH_NONE && MPY_FEATURE_DECODE_SUB_VERSION(header[2]) != MPY_SUB_VERSION) || header[3] > MP_SMALL_INT_BITS) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) { - byte arch = MPY_FEATURE_DECODE_ARCH(header[2]); if (!MPY_FEATURE_ARCH_TEST(arch)) { if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) { // On supported ports this can be resolved by enabling feature, eg @@ -596,7 +596,7 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { byte header[4] = { 'M', MPY_VERSION, - MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC), + MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION), #if MICROPY_DYNAMIC_COMPILER mp_dynamic_compiler.small_int_bits, #else diff --git a/py/persistentcode.h b/py/persistentcode.h index 29ccce4a3d..e664358737 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -30,24 +30,23 @@ #include "py/reader.h" #include "py/emitglue.h" -// The current version of .mpy files +// The current version of .mpy files. A bytecode-only .mpy file can be loaded +// as long as MPY_VERSION matches, but a native .mpy (i.e. one with an arch +// set) must also match MPY_SUB_VERSION. This allows 3 additional updates to +// the native ABI per bytecode revision. #define MPY_VERSION 6 +#define MPY_SUB_VERSION 1 -// Macros to encode/decode flags to/from the feature byte -#define MPY_FEATURE_ENCODE_FLAGS(flags) (flags) -#define MPY_FEATURE_DECODE_FLAGS(feat) ((feat) & 3) +// Macros to encode/decode sub-version to/from the feature byte. This replaces +// the bits previously used to encode the flags (map caching and unicode) +// which are no longer used starting at .mpy version 6. +#define MPY_FEATURE_ENCODE_SUB_VERSION(version) (version) +#define MPY_FEATURE_DECODE_SUB_VERSION(feat) ((feat) & 3) // Macros to encode/decode native architecture to/from the feature byte #define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2) #define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2) -// The feature flag bits encode the compile-time config options that affect -// the generate bytecode. Note: no longer used. -// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE and MICROPY_PY_BUILTINS_STR_UNICODE). -#define MPY_FEATURE_FLAGS (0) -// This is a version of the flags that can be configured at runtime. -#define MPY_FEATURE_FLAGS_DYNAMIC (0) - // Define the host architecture #if MICROPY_EMIT_X86 #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86) @@ -82,7 +81,7 @@ // 16-bit little-endian integer with the second and third bytes of supported .mpy files #define MPY_FILE_HEADER_INT (MPY_VERSION \ - | (MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8) + | (MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8) enum { MP_NATIVE_ARCH_NONE = 0, diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py index 449371ddac..73e20694cc 100644 --- a/tests/micropython/import_mpy_native.py +++ b/tests/micropython/import_mpy_native.py @@ -11,7 +11,8 @@ except (ImportError, AttributeError): raise SystemExit mpy_arch = usys.implementation._mpy >> 8 -if mpy_arch == 0: +if mpy_arch >> 2 == 0: + # This system does not support .mpy files containing native code print("SKIP") raise SystemExit @@ -54,8 +55,8 @@ class UserFS: valid_header = bytes([77, 6, mpy_arch, 31]) # fmt: off user_files = { - # bad architecture - '/mod0.mpy': b'M\x06\xfc\x1f', + # bad architecture (mpy_arch needed for sub-version) + '/mod0.mpy': bytes([77, 6, 0xfc | mpy_arch, 31]), # test loading of viper and asm '/mod1.mpy': valid_header + ( diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index bc8dcafdc7..e18720fb31 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -46,24 +46,26 @@ class UserFS: # Pre-compiled examples/natmod/features0 example for various architectures, keyed -# by the required value of sys.implementation._mpy. +# by the required value of sys.implementation._mpy (without sub-version). features0_file_contents = { # -march=x64 - 0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', + 0x806: b'M\x06\x09\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', # -march=armv6m - 0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", + 0x1006: b"M\x06\x11\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", } # Populate armv7m-derived archs based on armv6m. for arch in (0x1406, 0x1806, 0x1C06, 0x2006): features0_file_contents[arch] = features0_file_contents[0x1006] -if sys.implementation._mpy not in features0_file_contents: +# Check that a .mpy exists for the target (ignore sub-version in lookup). +sys_implementation_mpy = sys.implementation._mpy & ~(3 << 8) +if sys_implementation_mpy not in features0_file_contents: print("SKIP") raise SystemExit # These are the test .mpy files. -user_files = {"/features0.mpy": features0_file_contents[sys.implementation._mpy]} +user_files = {"/features0.mpy": features0_file_contents[sys_implementation_mpy]} # Create and mount a user filesystem. uos.mount(UserFS(user_files), "/userfs") diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 31212fd5bd..8b644c137f 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -88,6 +88,7 @@ class FreezeError(Exception): class Config: MPY_VERSION = 6 + MPY_SUB_VERSION = 1 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -1335,6 +1336,9 @@ def read_mpy(filename): feature_byte = header[2] mpy_native_arch = feature_byte >> 2 if mpy_native_arch != MP_NATIVE_ARCH_NONE: + mpy_sub_version = feature_byte & 3 + if mpy_sub_version != config.MPY_SUB_VERSION: + raise MPYReadError(filename, "incompatible .mpy sub-version") if config.native_arch == MP_NATIVE_ARCH_NONE: config.native_arch = mpy_native_arch elif config.native_arch != mpy_native_arch: @@ -1658,7 +1662,9 @@ def merge_mpy(compiled_modules, output_file): else: main_cm_idx = None for idx, cm in enumerate(compiled_modules): - if cm.header[2]: + feature_byte = cm.header[2] + mpy_native_arch = feature_byte >> 2 + if mpy_native_arch: # Must use qstr_table and obj_table from this raw_code if main_cm_idx is not None: raise Exception("can't merge files when more than one contains native code") @@ -1670,7 +1676,7 @@ def merge_mpy(compiled_modules, output_file): header = bytearray(4) header[0] = ord("M") header[1] = config.MPY_VERSION - header[2] = config.native_arch << 2 + header[2] = config.native_arch << 2 | config.MPY_SUB_VERSION header[3] = config.mp_small_int_bits merged_mpy.extend(header) diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 09ea90dcd1..05e70709ad 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -36,6 +36,7 @@ import makeqstrdata as qstrutil # MicroPython constants MPY_VERSION = 6 +MPY_SUB_VERSION = 1 MP_CODE_BYTECODE = 2 MP_CODE_NATIVE_VIPER = 4 MP_NATIVE_ARCH_X86 = 1 @@ -917,7 +918,11 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.open(fmpy) # MPY: header - out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) + out.write_bytes( + bytearray( + [ord("M"), MPY_VERSION, env.arch.mpy_feature | MPY_SUB_VERSION, MP_SMALL_INT_BITS] + ) + ) # MPY: n_qstr out.write_uint(1 + len(native_qstr_vals))