From f2040bfc7ee033e48acef9f289790f3b4e6b74e5 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 22 Oct 2021 22:22:47 +1100 Subject: [PATCH] py: Rework bytecode and .mpy file format to be mostly static data. Background: .mpy files are precompiled .py files, built using mpy-cross, that contain compiled bytecode functions (and can also contain machine code). The benefit of using an .mpy file over a .py file is that they are faster to import and take less memory when importing. They are also smaller on disk. But the real benefit of .mpy files comes when they are frozen into the firmware. This is done by loading the .mpy file during compilation of the firmware and turning it into a set of big C data structures (the job of mpy-tool.py), which are then compiled and downloaded into the ROM of a device. These C data structures can be executed in-place, ie directly from ROM. This makes importing even faster because there is very little to do, and also means such frozen modules take up much less RAM (because their bytecode stays in ROM). The downside of frozen code is that it requires recompiling and reflashing the entire firmware. This can be a big barrier to entry, slows down development time, and makes it harder to do OTA updates of frozen code (because the whole firmware must be updated). This commit attempts to solve this problem by providing a solution that sits between loading .mpy files into RAM and freezing them into the firmware. The .mpy file format has been reworked so that it consists of data and bytecode which is mostly static and ready to run in-place. If these new .mpy files are located in flash/ROM which is memory addressable, the .mpy file can be executed (mostly) in-place. With this approach there is still a small amount of unpacking and linking of the .mpy file that needs to be done when it's imported, but it's still much better than loading an .mpy from disk into RAM (although not as good as freezing .mpy files into the firmware). The main trick to make static .mpy files is to adjust the bytecode so any qstrs that it references now go through a lookup table to convert from local qstr number in the module to global qstr number in the firmware. That means the bytecode does not need linking/rewriting of qstrs when it's loaded. Instead only a small qstr table needs to be built (and put in RAM) at import time. This means the bytecode itself is static/constant and can be used directly if it's in addressable memory. Also the qstr string data in the .mpy file, and some constant object data, can be used directly. Note that the qstr table is global to the module (ie not per function). In more detail, in the VM what used to be (schematically): qst = DECODE_QSTR_VALUE; is now (schematically): idx = DECODE_QSTR_INDEX; qst = qstr_table[idx]; That allows the bytecode to be fixed at compile time and not need relinking/rewriting of the qstr values. Only qstr_table needs to be linked when the .mpy is loaded. Incidentally, this helps to reduce the size of bytecode because what used to be 2-byte qstr values in the bytecode are now (mostly) 1-byte indices. If the module uses the same qstr more than two times then the bytecode is smaller than before. The following changes are measured for this commit compared to the previous (the baseline): - average 7%-9% reduction in size of .mpy files - frozen code size is reduced by about 5%-7% - importing .py files uses about 5% less RAM in total - importing .mpy files uses about 4% less RAM in total - importing .py and .mpy files takes about the same time as before The qstr indirection in the bytecode has only a small impact on VM performance. For stm32 on PYBv1.0 the performance change of this commit is: diff of scores (higher is better) N=100 M=100 baseline -> this-commit diff diff% (error%) bm_chaos.py 371.07 -> 357.39 : -13.68 = -3.687% (+/-0.02%) bm_fannkuch.py 78.72 -> 77.49 : -1.23 = -1.563% (+/-0.01%) bm_fft.py 2591.73 -> 2539.28 : -52.45 = -2.024% (+/-0.00%) bm_float.py 6034.93 -> 5908.30 : -126.63 = -2.098% (+/-0.01%) bm_hexiom.py 48.96 -> 47.93 : -1.03 = -2.104% (+/-0.00%) bm_nqueens.py 4510.63 -> 4459.94 : -50.69 = -1.124% (+/-0.00%) bm_pidigits.py 650.28 -> 644.96 : -5.32 = -0.818% (+/-0.23%) core_import_mpy_multi.py 564.77 -> 581.49 : +16.72 = +2.960% (+/-0.01%) core_import_mpy_single.py 68.67 -> 67.16 : -1.51 = -2.199% (+/-0.01%) core_qstr.py 64.16 -> 64.12 : -0.04 = -0.062% (+/-0.00%) core_yield_from.py 362.58 -> 354.50 : -8.08 = -2.228% (+/-0.00%) misc_aes.py 429.69 -> 405.59 : -24.10 = -5.609% (+/-0.01%) misc_mandel.py 3485.13 -> 3416.51 : -68.62 = -1.969% (+/-0.00%) misc_pystone.py 2496.53 -> 2405.56 : -90.97 = -3.644% (+/-0.01%) misc_raytrace.py 381.47 -> 374.01 : -7.46 = -1.956% (+/-0.01%) viper_call0.py 576.73 -> 572.49 : -4.24 = -0.735% (+/-0.04%) viper_call1a.py 550.37 -> 546.21 : -4.16 = -0.756% (+/-0.09%) viper_call1b.py 438.23 -> 435.68 : -2.55 = -0.582% (+/-0.06%) viper_call1c.py 442.84 -> 440.04 : -2.80 = -0.632% (+/-0.08%) viper_call2a.py 536.31 -> 532.35 : -3.96 = -0.738% (+/-0.06%) viper_call2b.py 382.34 -> 377.07 : -5.27 = -1.378% (+/-0.03%) And for unix on x64: diff of scores (higher is better) N=2000 M=2000 baseline -> this-commit diff diff% (error%) bm_chaos.py 13594.20 -> 13073.84 : -520.36 = -3.828% (+/-5.44%) bm_fannkuch.py 60.63 -> 59.58 : -1.05 = -1.732% (+/-3.01%) bm_fft.py 112009.15 -> 111603.32 : -405.83 = -0.362% (+/-4.03%) bm_float.py 246202.55 -> 247923.81 : +1721.26 = +0.699% (+/-2.79%) bm_hexiom.py 615.65 -> 617.21 : +1.56 = +0.253% (+/-1.64%) bm_nqueens.py 215807.95 -> 215600.96 : -206.99 = -0.096% (+/-3.52%) bm_pidigits.py 8246.74 -> 8422.82 : +176.08 = +2.135% (+/-3.64%) misc_aes.py 16133.00 -> 16452.74 : +319.74 = +1.982% (+/-1.50%) misc_mandel.py 128146.69 -> 130796.43 : +2649.74 = +2.068% (+/-3.18%) misc_pystone.py 83811.49 -> 83124.85 : -686.64 = -0.819% (+/-1.03%) misc_raytrace.py 21688.02 -> 21385.10 : -302.92 = -1.397% (+/-3.20%) The code size change is (firmware with a lot of frozen code benefits the most): bare-arm: +396 +0.697% minimal x86: +1595 +0.979% [incl +32(data)] unix x64: +2408 +0.470% [incl +800(data)] unix nanbox: +1396 +0.309% [incl -96(data)] stm32: -1256 -0.318% PYBV10 cc3200: +288 +0.157% esp8266: -260 -0.037% GENERIC esp32: -216 -0.014% GENERIC[incl -1072(data)] nrf: +116 +0.067% pca10040 rp2: -664 -0.135% PICO samd: +844 +0.607% ADAFRUIT_ITSYBITSY_M4_EXPRESS As part of this change the .mpy file format version is bumped to version 6. And mpy-tool.py has been improved to provide a good visualisation of the contents of .mpy files. In summary: this commit changes the bytecode to use qstr indirection, and reworks the .mpy file format to be simpler and allow .mpy files to be executed in-place. Performance is not impacted too much. Eventually it will be possible to store such .mpy files in a linear, read-only, memory- mappable filesystem so they can be executed from flash/ROM. This will essentially be able to replace frozen code for most applications. Signed-off-by: Damien George --- mpy-cross/main.c | 5 +- ports/unix/coverage.c | 4 + py/asmbase.c | 3 +- py/asmbase.h | 2 +- py/bc.c | 69 +- py/bc.h | 84 +- py/builtinevex.c | 2 +- py/builtinimport.c | 28 +- py/compile.c | 171 ++- py/compile.h | 3 +- py/dynruntime.h | 10 +- py/emit.h | 50 +- py/emitbc.c | 156 +-- py/emitcommon.c | 11 + py/emitglue.c | 61 +- py/emitglue.h | 26 +- py/emitnative.c | 263 ++-- py/emitnx86.c | 2 +- py/frozenmod.c | 2 +- py/mpconfig.h | 8 + py/nativeglue.c | 6 +- py/nativeglue.h | 12 +- py/obj.h | 4 - py/objfun.c | 50 +- py/objfun.h | 10 +- py/objgenerator.c | 18 +- py/objmodule.c | 9 +- py/persistentcode.c | 558 +++----- py/persistentcode.h | 12 +- py/profile.c | 47 +- py/profile.h | 4 +- py/qstr.h | 1 + py/scope.c | 3 +- py/scope.h | 3 +- py/showbc.c | 73 +- py/vm.c | 73 +- shared/runtime/pyexec.c | 6 +- tests/cmdline/cmd_parsetree.py.exp | 74 +- tests/cmdline/cmd_showbc.py.exp | 762 ++++++----- tests/cmdline/cmd_verbose.py.exp | 18 +- tests/micropython/import_mpy_invalid.py | 1 - tests/micropython/import_mpy_invalid.py.exp | 1 - tests/micropython/import_mpy_native_gc.py | 8 +- tests/micropython/import_mpy_native_x64.py | 63 +- tests/perf_bench/core_import_mpy_multi.py | 2 +- tests/perf_bench/core_import_mpy_single.py | 2 +- tools/mpy-tool.py | 1326 +++++++++++++------ tools/mpy_ld.py | 35 +- 48 files changed, 2388 insertions(+), 1753 deletions(-) diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 7218921b31..a2daf7b22f 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -72,7 +72,8 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha #endif mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT); - mp_raw_code_t *rc = mp_compile_to_raw_code(&parse_tree, source_name, false); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + mp_compiled_module_t cm = mp_compile_to_raw_code(&parse_tree, source_name, false, ctx); vstr_t vstr; vstr_init(&vstr, 16); @@ -83,7 +84,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha } else { vstr_add_str(&vstr, output_file); } - mp_raw_code_save_file(rc, vstr_null_terminated_str(&vstr)); + mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); vstr_clear(&vstr); nlr_pop(); diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 6b00cdfef2..a855237200 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -2,6 +2,7 @@ #include #include "py/obj.h" +#include "py/objfun.h" #include "py/objstr.h" #include "py/runtime.h" #include "py/gc.h" @@ -449,7 +450,10 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# VM\n"); // call mp_execute_bytecode with invalide bytecode (should raise NotImplementedError) + mp_module_context_t context; mp_obj_fun_bc_t fun_bc; + fun_bc.context = &context; + fun_bc.child_table = NULL; fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, mp_obj_t, 1); code_state->fun_bc = &fun_bc; diff --git a/py/asmbase.c b/py/asmbase.c index 344e03e7a7..4a3fd089cb 100644 --- a/py/asmbase.c +++ b/py/asmbase.c @@ -61,7 +61,8 @@ void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { // all functions must go through this one to emit bytes // if as->pass < MP_ASM_PASS_EMIT, then this function just counts the number // of bytes needed and returns NULL, and callers should not store any data -uint8_t *mp_asm_base_get_cur_to_write_bytes(mp_asm_base_t *as, size_t num_bytes_to_write) { +uint8_t *mp_asm_base_get_cur_to_write_bytes(void *as_in, size_t num_bytes_to_write) { + mp_asm_base_t *as = as_in; uint8_t *c = NULL; if (as->pass == MP_ASM_PASS_EMIT) { assert(as->code_offset + num_bytes_to_write <= as->code_size); diff --git a/py/asmbase.h b/py/asmbase.h index 24c3af8679..960be7685f 100644 --- a/py/asmbase.h +++ b/py/asmbase.h @@ -45,7 +45,7 @@ typedef struct _mp_asm_base_t { void mp_asm_base_init(mp_asm_base_t *as, size_t max_num_labels); void mp_asm_base_deinit(mp_asm_base_t *as, bool free_code); void mp_asm_base_start_pass(mp_asm_base_t *as, int pass); -uint8_t *mp_asm_base_get_cur_to_write_bytes(mp_asm_base_t *as, size_t num_bytes_to_write); +uint8_t *mp_asm_base_get_cur_to_write_bytes(void *as, size_t num_bytes_to_write); void mp_asm_base_label_assign(mp_asm_base_t *as, size_t label); void mp_asm_base_align(mp_asm_base_t *as, unsigned int align); void mp_asm_base_data(mp_asm_base_t *as, unsigned int bytesize, uintptr_t val); diff --git a/py/bc.c b/py/bc.c index 69b6739e8f..b98df39e28 100644 --- a/py/bc.c +++ b/py/bc.c @@ -29,9 +29,9 @@ #include #include -#include "py/runtime.h" #include "py/bc0.h" #include "py/bc.h" +#include "py/objfun.h" #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) @@ -40,7 +40,23 @@ #define DEBUG_printf(...) (void)0 #endif -#if !MICROPY_PERSISTENT_CODE +void mp_encode_uint(void *env, mp_encode_uint_allocator_t allocator, mp_uint_t val) { + // We store each 7 bits in a separate byte, and that's how many bytes needed + byte buf[MP_ENCODE_UINT_MAX_BYTES]; + byte *p = buf + sizeof(buf); + // We encode in little-ending order, but store in big-endian, to help decoding + do { + *--p = val & 0x7f; + val >>= 7; + } while (val != 0); + byte *c = allocator(env, buf + sizeof(buf) - p); + if (c != NULL) { + while (p != buf + sizeof(buf) - 1) { + *c++ = *p++ | 0x80; + } + *c = *p; + } +} mp_uint_t mp_decode_uint(const byte **ptr) { mp_uint_t unum = 0; @@ -72,8 +88,6 @@ const byte *mp_decode_uint_skip(const byte *ptr) { return ptr; } -#endif - STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE // generic message, used also for other argument issues @@ -107,8 +121,8 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // On entry code_state should be allocated somewhere (stack/heap) and // contain the following valid entries: // - code_state->fun_bc should contain a pointer to the function object -// - code_state->ip should contain the offset in bytes from the pointer -// code_state->fun_bc->bytecode to the entry n_state (0 for bytecode, non-zero for native) +// - code_state->ip should contain a pointer to the beginning of the prelude +// - code_state->n_state should be the number of objects in the local state void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { // This function is pretty complicated. It's main aim is to be efficient in speed and RAM // usage for the common case of positional only args. @@ -116,9 +130,6 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // get the function object that we want to set up (could be bytecode or native code) mp_obj_fun_bc_t *self = code_state->fun_bc; - // ip comes in as an offset into bytecode, so turn it into a true pointer - code_state->ip = self->bytecode + (size_t)code_state->ip; - #if MICROPY_STACKLESS code_state->prev = NULL; #endif @@ -134,6 +145,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // Decode prelude size_t n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args; MP_BC_PRELUDE_SIG_DECODE_INTO(code_state->ip, n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args); + MP_BC_PRELUDE_SIZE_DECODE(code_state->ip); (void)n_state_unused; (void)n_exc_stack_unused; @@ -194,14 +206,20 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw *var_pos_kw_args = dict; } - // get pointer to arg_names array - const mp_obj_t *arg_names = (const mp_obj_t *)self->const_table; - for (size_t i = 0; i < n_kw; i++) { // the keys in kwargs are expected to be qstr objects mp_obj_t wanted_arg_name = kwargs[2 * i]; + + // get pointer to arg_names array + const uint8_t *arg_names = code_state->ip; + arg_names = mp_decode_uint_skip(arg_names); + for (size_t j = 0; j < n_pos_args + n_kwonly_args; j++) { - if (wanted_arg_name == arg_names[j]) { + qstr arg_qstr = mp_decode_uint(&arg_names); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + arg_qstr = self->context->constants.qstr_table[arg_qstr]; + #endif + if (wanted_arg_name == MP_OBJ_NEW_QSTR(arg_qstr)) { if (code_state->state[n_state - 1 - j] != MP_OBJ_NULL) { mp_raise_msg_varg(&mp_type_TypeError, MP_ERROR_TEXT("function got multiple values for argument '%q'"), MP_OBJ_QSTR_VALUE(wanted_arg_name)); @@ -248,17 +266,25 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // Check that all mandatory keyword args are specified // Fill in default kw args if we have them + const uint8_t *arg_names = mp_decode_uint_skip(code_state->ip); + for (size_t i = 0; i < n_pos_args; i++) { + arg_names = mp_decode_uint_skip(arg_names); + } for (size_t i = 0; i < n_kwonly_args; i++) { + qstr arg_qstr = mp_decode_uint(&arg_names); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + arg_qstr = self->context->constants.qstr_table[arg_qstr]; + #endif if (code_state->state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { mp_map_elem_t *elem = NULL; if ((scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) { - elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, arg_names[n_pos_args + i], MP_MAP_LOOKUP); + elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, MP_OBJ_NEW_QSTR(arg_qstr), MP_MAP_LOOKUP); } if (elem != NULL) { code_state->state[n_state - 1 - n_pos_args - i] = elem->value; } else { mp_raise_msg_varg(&mp_type_TypeError, - MP_ERROR_TEXT("function missing required keyword argument '%q'"), MP_OBJ_QSTR_VALUE(arg_names[n_pos_args + i])); + MP_ERROR_TEXT("function missing required keyword argument '%q'"), arg_qstr); } } } @@ -273,12 +299,8 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw } } - // read the size part of the prelude - const byte *ip = code_state->ip; - MP_BC_PRELUDE_SIZE_DECODE(ip); - - // jump over code info (source file and line-number mapping) - ip += n_info; + // jump over code info (source file, argument names and line-number mapping) + const uint8_t *ip = code_state->ip + n_info; // bytecode prelude: initialise closed over variables for (; n_cell; --n_cell) { @@ -287,11 +309,6 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw mp_obj_new_cell(code_state->state[n_state - 1 - local_num]); } - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - ip = MP_ALIGN(ip, sizeof(mp_uint_t)); - #endif - // now that we skipped over the prelude, set the ip for the VM code_state->ip = ip; diff --git a/py/bc.h b/py/bc.h index ef5afeae16..7d761e30e6 100644 --- a/py/bc.h +++ b/py/bc.h @@ -28,7 +28,6 @@ #define MICROPY_INCLUDED_PY_BC_H #include "py/runtime.h" -#include "py/objfun.h" // bytecode layout: // @@ -50,7 +49,9 @@ // // source info section: // simple_name : var qstr -// source_file : var qstr +// argname0 : var qstr +// ... : var qstr +// argnameN : var qstr N = num_pos_args + num_kwonly_args - 1 // // // closure section: @@ -58,19 +59,16 @@ // ... : byte // local_numN : byte N = n_cells-1 // -// only needed if bytecode contains pointers -// // // // // constant table layout: // -// argname0 : obj (qstr) -// ... : obj (qstr) -// argnameN : obj (qstr) N = num_pos_args + num_kwonly_args // const0 : obj // constN : obj +#define MP_ENCODE_UINT_MAX_BYTES ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) + #define MP_BC_PRELUDE_SIG_ENCODE(S, E, scope, out_byte, out_env) \ do { \ /*// Get values to store in prelude */ \ @@ -182,9 +180,9 @@ typedef struct _mp_bytecode_prelude_t { uint n_pos_args; uint n_kwonly_args; uint n_def_pos_args; - qstr qstr_block_name; - qstr qstr_source_file; + qstr qstr_block_name_idx; const byte *line_info; + const byte *line_info_top; const byte *opcodes; } mp_bytecode_prelude_t; @@ -198,12 +196,46 @@ typedef struct _mp_exc_stack_t { mp_obj_base_t *prev_exc; } mp_exc_stack_t; +// Constants associated with a module, to interface bytecode with runtime. +typedef struct _mp_module_constants_t { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + qstr_short_t *qstr_table; + #else + qstr source_file; + #endif + mp_obj_t *obj_table; +} mp_module_constants_t; + +// State associated with a module. +typedef struct _mp_module_context_t { + mp_obj_module_t module; + mp_module_constants_t constants; +} mp_module_context_t; + +// Outer level struct defining a compiled module. +typedef struct _mp_compiled_module_t { + const mp_module_context_t *context; + const struct _mp_raw_code_t *rc; + #if MICROPY_PERSISTENT_CODE_SAVE + bool has_native; + size_t n_qstr; + size_t n_obj; + #endif +} mp_compiled_module_t; + +// Outer level struct defining a frozen module. +typedef struct _mp_frozen_module_t { + const mp_module_constants_t constants; + const struct _mp_raw_code_t *rc; +} mp_frozen_module_t; + +// State for an executing function. typedef struct _mp_code_state_t { // The fun_bc entry points to the underlying function object that is being executed. // It is needed to access the start of bytecode and the const_table. // It is also needed to prevent the GC from reclaiming the bytecode during execution, // because the ip pointer below will always point to the interior of the bytecode. - mp_obj_fun_bc_t *fun_bc; + struct _mp_obj_fun_bc_t *fun_bc; const byte *ip; mp_obj_t *sp; uint16_t n_state; @@ -222,6 +254,10 @@ typedef struct _mp_code_state_t { // mp_exc_stack_t exc_state[0]; } mp_code_state_t; +// Allocator may return NULL, in which case data is not stored (can be used to compute size). +typedef uint8_t *(*mp_encode_uint_allocator_t)(void *env, size_t nbytes); + +void mp_encode_uint(void *env, mp_encode_uint_allocator_t allocator, mp_uint_t val); mp_uint_t mp_decode_uint(const byte **ptr); mp_uint_t mp_decode_uint_value(const byte *ptr); const byte *mp_decode_uint_skip(const byte *ptr); @@ -229,10 +265,10 @@ const byte *mp_decode_uint_skip(const byte *ptr); mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_uint_t *const_table); -void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_uint_t *const_table); +void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_module_constants_t *cm); +void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_module_constants_t *cm); const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip); -#define mp_bytecode_print_inst(print, code, const_table) mp_bytecode_print2(print, code, 1, const_table) +#define mp_bytecode_print_inst(print, code, x_table) mp_bytecode_print2(print, code, 1, x_table) // Helper macros to access pointer with least significant bits holding flags #define MP_TAGPTR_PTR(x) ((void *)((uintptr_t)(x) & ~((uintptr_t)3))) @@ -246,10 +282,26 @@ uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint); #endif -static inline size_t mp_bytecode_get_source_line(const byte *line_info, size_t bc_offset) { +static inline void mp_module_context_alloc_tables(mp_module_context_t *context, size_t n_qstr, size_t n_obj) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + size_t nq = (n_qstr * sizeof(qstr_short_t) + sizeof(mp_uint_t) - 1) / sizeof(mp_uint_t); + size_t no = n_obj; + mp_uint_t *mem = m_new(mp_uint_t, nq + no); + context->constants.qstr_table = (void *)(mem); + context->constants.obj_table = (void *)(mem + nq); + #else + if (n_obj == 0) { + context->constants.obj_table = NULL; + } else { + context->constants.obj_table = m_new(mp_obj_t, n_obj); + } + #endif +} + +static inline size_t mp_bytecode_get_source_line(const byte *line_info, const byte *line_info_top, size_t bc_offset) { size_t source_line = 1; - size_t c; - while ((c = *line_info)) { + while (line_info < line_info_top) { + size_t c = *line_info; size_t b, l; if ((c & 0x80) == 0) { // 0b0LLBBBBB encoding diff --git a/py/builtinevex.c b/py/builtinevex.c index 800a20223a..4e1c6a66d1 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -54,7 +54,7 @@ STATIC mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj // the correct one if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc)) { mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(self->module_fun); - fun_bc->globals = globals; + ((mp_module_context_t *)fun_bc->context)->module.globals = globals; } // execute code diff --git a/py/builtinimport.c b/py/builtinimport.c index 3e336633d9..094959f97d 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -146,28 +146,28 @@ STATIC mp_import_stat_t stat_top_level_dir_or_file(qstr mod_name, vstr_t *dest) } #if MICROPY_MODULE_FROZEN_STR || MICROPY_ENABLE_COMPILER -STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex) { +STATIC void do_load_from_lexer(mp_module_context_t *context, mp_lexer_t *lex) { #if MICROPY_PY___FILE__ qstr source_name = lex->source_name; - mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); + mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); #endif // parse, compile and execute the module in its context - mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_obj_dict_t *mod_globals = context->module.globals; mp_parse_compile_execute(lex, MP_PARSE_FILE_INPUT, mod_globals, mod_globals); } #endif #if (MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD) || MICROPY_MODULE_FROZEN_MPY -STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, const char *source_name) { +STATIC void do_execute_raw_code(mp_module_context_t *context, const mp_raw_code_t *rc, const mp_module_context_t *mc, const char *source_name) { (void)source_name; #if MICROPY_PY___FILE__ - mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name))); + mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name))); #endif // execute the module in its context - mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_obj_dict_t *mod_globals = context->module.globals; // save context mp_obj_dict_t *volatile old_globals = mp_globals_get(); @@ -179,7 +179,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_obj_t module_fun = mp_make_function_from_raw_code(raw_code, MP_OBJ_NULL, MP_OBJ_NULL); + mp_obj_t module_fun = mp_make_function_from_raw_code(rc, mc, NULL); mp_call_function_0(module_fun); // finish nlr block, restore context @@ -195,7 +195,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co } #endif -STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { +STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) { #if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) const char *file_str = vstr_null_terminated_str(file); #endif @@ -222,7 +222,9 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // its data) in the list of frozen files, execute it. #if MICROPY_MODULE_FROZEN_MPY if (frozen_type == MP_FROZEN_MPY) { - do_execute_raw_code(module_obj, modref, file_str + frozen_path_prefix_len); + const mp_frozen_module_t *frozen = modref; + module_obj->constants = frozen->constants; + do_execute_raw_code(module_obj, frozen->rc, module_obj, file_str + frozen_path_prefix_len); return; } #endif @@ -234,8 +236,8 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // the correct format and, if so, load and execute the file. #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD if (file_str[file->len - 3] == 'm') { - mp_raw_code_t *raw_code = mp_raw_code_load_file(file_str); - do_execute_raw_code(module_obj, raw_code, file_str); + mp_compiled_module_t cm = mp_raw_code_load_file(file_str, module_obj); + do_execute_raw_code(module_obj, cm.rc, cm.context, file_str); return; } #endif @@ -434,7 +436,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, size_t orig_path_len = path->len; vstr_add_str(path, PATH_SEP_CHAR "__init__.py"); if (stat_file_py_or_mpy(path) == MP_IMPORT_STAT_FILE) { - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); } else { // No-op. Nothing to load. // mp_warning("%s is imported as namespace package", vstr_str(&path)); @@ -443,7 +445,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, path->len = orig_path_len; } else { // MP_IMPORT_STAT_FILE // File -- execute "path.(m)py". - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); // Note: This should be the last component in the import path. If // there are remaining components then it's an ImportError // because the current path(the module that was just loaded) is diff --git a/py/compile.c b/py/compile.c index 8ebcc2289f..92736e22e5 100644 --- a/py/compile.c +++ b/py/compile.c @@ -35,6 +35,7 @@ #include "py/compile.h" #include "py/runtime.h" #include "py/asmbase.h" +#include "py/nativeglue.h" #include "py/persistentcode.h" #if MICROPY_ENABLE_COMPILER @@ -88,7 +89,7 @@ typedef enum { #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER #define NATIVE_EMITTER(f) emit_native_table[mp_dynamic_compiler.native_arch]->emit_##f -#define NATIVE_EMITTER_TABLE emit_native_table[mp_dynamic_compiler.native_arch] +#define NATIVE_EMITTER_TABLE (emit_native_table[mp_dynamic_compiler.native_arch]) STATIC const emit_method_table_t *emit_native_table[] = { NULL, @@ -121,7 +122,7 @@ STATIC const emit_method_table_t *emit_native_table[] = { #else #error "unknown native emitter" #endif -#define NATIVE_EMITTER_TABLE &NATIVE_EMITTER(method_table) +#define NATIVE_EMITTER_TABLE (&NATIVE_EMITTER(method_table)) #endif #if MICROPY_EMIT_INLINE_ASM && MICROPY_DYNAMIC_COMPILER @@ -162,8 +163,6 @@ STATIC const emit_inline_asm_method_table_t *emit_asm_table[] = { // elements in this struct are ordered to make it compact typedef struct _compiler_t { - qstr source_file; - uint8_t is_repl; uint8_t pass; // holds enum type pass_kind_t uint8_t have_star; @@ -185,7 +184,10 @@ typedef struct _compiler_t { scope_t *scope_head; scope_t *scope_cur; + mp_emit_common_t emit_common; + emit_t *emit; // current emitter + emit_t *emit_bc; #if NEED_METHOD_TABLE const emit_method_table_t *emit_method_table; // current emit method table #endif @@ -196,6 +198,72 @@ typedef struct _compiler_t { #endif } compiler_t; +/******************************************************************************/ +// mp_emit_common_t helper functions +// These are defined here so they can be inlined, to reduce code size. + +STATIC void mp_emit_common_init(mp_emit_common_t *emit, qstr source_file) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + mp_map_init(&emit->qstr_map, 1); + + // add the source file as the first entry in the qstr table + mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(source_file), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + elem->value = MP_OBJ_NEW_SMALL_INT(0); + #endif +} + +STATIC void mp_emit_common_start_pass(mp_emit_common_t *emit, pass_kind_t pass) { + emit->pass = pass; + if (pass == MP_PASS_STACK_SIZE) { + emit->ct_cur_obj_base = emit->ct_cur_obj; + } else if (pass > MP_PASS_STACK_SIZE) { + emit->ct_cur_obj = emit->ct_cur_obj_base; + } + if (pass == MP_PASS_EMIT) { + if (emit->ct_cur_child == 0) { + emit->children = NULL; + } else { + emit->children = m_new0(mp_raw_code_t *, emit->ct_cur_child); + } + } + emit->ct_cur_child = 0; +} + +STATIC void mp_emit_common_finalise(mp_emit_common_t *emit, bool has_native_code) { + emit->ct_cur_obj += has_native_code; // allocate an additional slot for &mp_fun_table + emit->const_table = m_new0(mp_uint_t, emit->ct_cur_obj); + emit->ct_cur_obj = has_native_code; // reserve slot 0 for &mp_fun_table + #if MICROPY_EMIT_NATIVE + if (has_native_code) { + // store mp_fun_table pointer at the start of the constant table + emit->const_table[0] = (mp_uint_t)(uintptr_t)&mp_fun_table; + } + #endif +} + +STATIC void mp_emit_common_populate_module_context(mp_emit_common_t *emit, qstr source_file, mp_module_context_t *context) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + size_t qstr_map_used = emit->qstr_map.used; + mp_module_context_alloc_tables(context, qstr_map_used, emit->ct_cur_obj); + for (size_t i = 0; i < emit->qstr_map.alloc; ++i) { + if (mp_map_slot_is_filled(&emit->qstr_map, i)) { + size_t idx = MP_OBJ_SMALL_INT_VALUE(emit->qstr_map.table[i].value); + qstr qst = MP_OBJ_QSTR_VALUE(emit->qstr_map.table[i].key); + context->constants.qstr_table[idx] = qst; + } + } + #else + mp_module_context_alloc_tables(context, 0, emit->ct_cur_obj); + context->constants.source_file = source_file; + #endif + + if (emit->ct_cur_obj > 0) { + memcpy(context->constants.obj_table, emit->const_table, emit->ct_cur_obj * sizeof(mp_uint_t)); + } +} + +/******************************************************************************/ + STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) { // if the line of the error is unknown then try to update it from the pn if (comp->compile_error_line == 0 && MP_PARSE_NODE_IS_STRUCT(pn)) { @@ -246,7 +314,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) { } STATIC scope_t *scope_new_and_link(compiler_t *comp, scope_kind_t kind, mp_parse_node_t pn, uint emit_options) { - scope_t *scope = scope_new(kind, pn, comp->source_file, emit_options); + scope_t *scope = scope_new(kind, pn, emit_options); scope->parent = comp->scope_cur; scope->next = NULL; if (comp->scope_head == NULL) { @@ -290,7 +358,8 @@ STATIC void compile_generic_all_nodes(compiler_t *comp, mp_parse_node_struct_t * STATIC void compile_load_id(compiler_t *comp, qstr qst) { if (comp->pass == MP_PASS_SCOPE) { mp_emit_common_get_id_for_load(comp->scope_cur, qst); - } else { + } + { #if NEED_METHOD_TABLE mp_emit_common_id_op(comp->emit, &comp->emit_method_table->load_id, comp->scope_cur, qst); #else @@ -302,7 +371,8 @@ STATIC void compile_load_id(compiler_t *comp, qstr qst) { STATIC void compile_store_id(compiler_t *comp, qstr qst) { if (comp->pass == MP_PASS_SCOPE) { mp_emit_common_get_id_for_modification(comp->scope_cur, qst); - } else { + } + { #if NEED_METHOD_TABLE mp_emit_common_id_op(comp->emit, &comp->emit_method_table->store_id, comp->scope_cur, qst); #else @@ -314,7 +384,8 @@ STATIC void compile_store_id(compiler_t *comp, qstr qst) { STATIC void compile_delete_id(compiler_t *comp, qstr qst) { if (comp->pass == MP_PASS_SCOPE) { mp_emit_common_get_id_for_modification(comp->scope_cur, qst); - } else { + } + { #if NEED_METHOD_TABLE mp_emit_common_id_op(comp->emit, &comp->emit_method_table->delete_id, comp->scope_cur, qst); #else @@ -819,7 +890,7 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid micropython decorator")); } - #if MICROPY_DYNAMIC_COMPILER + #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) { if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) { compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid arch")); @@ -2082,6 +2153,7 @@ STATIC void compile_lambdef(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_namedexpr_helper(compiler_t *comp, mp_parse_node_t pn_name, mp_parse_node_t pn_expr) { if (!MP_PARSE_NODE_IS_ID(pn_name)) { compile_syntax_error(comp, (mp_parse_node_t)pn_name, MP_ERROR_TEXT("can't assign to expression")); + return; // because pn_name is not a valid qstr (in compile_store_id below) } compile_node(comp, pn_expr); EMIT(dup_top); @@ -2814,6 +2886,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn // comes before a star, so counts as a positional parameter comp->scope_cur->num_pos_args += 1; } + mp_emit_common_use_qstr(&comp->emit_common, param_name); } else { assert(MP_PARSE_NODE_IS_STRUCT(pn)); pns = (mp_parse_node_struct_t *)pn; @@ -2827,6 +2900,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn // comes before a star, so counts as a positional parameter comp->scope_cur->num_pos_args += 1; } + mp_emit_common_use_qstr(&comp->emit_common, param_name); } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == pn_star) { if (comp->have_star) { // more than one star @@ -2976,6 +3050,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { comp->pass = pass; comp->scope_cur = scope; comp->next_label = 0; + mp_emit_common_start_pass(&comp->emit_common, pass); EMIT_ARG(start_pass, pass, scope); reserve_labels_for_native(comp, 6); // used by native's start_pass @@ -2984,6 +3059,14 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { // they will be computed in this first pass scope->stack_size = 0; scope->exc_stack_size = 0; + + #if MICROPY_EMIT_NATIVE + if (scope->emit_options == MP_EMIT_OPT_NATIVE_PYTHON || scope->emit_options == MP_EMIT_OPT_VIPER) { + // allow native code to perfom basic tasks during the pass scope + // note: the first argument passed here is mp_emit_common_t, not the native emitter context + NATIVE_EMITTER_TABLE->start_pass((void *)&comp->emit_common, comp->pass, scope); + } + #endif } // compile @@ -3064,6 +3147,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { if (comp->pass == MP_PASS_SCOPE) { scope_find_or_add_id(comp->scope_cur, qstr_arg, ID_INFO_KIND_LOCAL); scope->num_pos_args = 1; + mp_emit_common_use_qstr(&comp->emit_common, MP_QSTR__star_); } // Set the source line number for the start of the comprehension @@ -3296,9 +3380,10 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind f, mp_asm_base_get_code_size((mp_asm_base_t *)comp->emit_inline_asm), NULL, #if MICROPY_PERSISTENT_CODE_SAVE - 0, 0, 0, 0, NULL, + 0, + 0, 0, NULL, #endif - comp->scope_cur->num_pos_args, 0, type_sig); + 0, comp->scope_cur->num_pos_args, type_sig); } } @@ -3310,7 +3395,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } #endif -STATIC void scope_compute_things(scope_t *scope) { +STATIC void scope_compute_things(scope_t *scope, mp_emit_common_t *emit_common) { // in MicroPython we put the *x parameter after all other parameters (except **y) if (scope->scope_flags & MP_SCOPE_FLAG_VARARGS) { id_info_t *id_param = NULL; @@ -3399,6 +3484,7 @@ STATIC void scope_compute_things(scope_t *scope) { } scope->num_pos_args += num_free; // free vars are counted as params for passing them into the function scope->num_locals += num_free; + mp_emit_common_use_qstr(emit_common, MP_QSTR__star_); } } } @@ -3406,15 +3492,15 @@ STATIC void scope_compute_things(scope_t *scope) { #if !MICROPY_PERSISTENT_CODE_SAVE STATIC #endif -mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *context) { // put compiler state on the stack, it's relatively small compiler_t comp_state = {0}; compiler_t *comp = &comp_state; - comp->source_file = source_file; comp->is_repl = is_repl; comp->break_label = INVALID_LABEL; comp->continue_label = INVALID_LABEL; + mp_emit_common_init(&comp->emit_common, source_file); // create the module scope #if MICROPY_EMIT_NATIVE @@ -3425,10 +3511,11 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f scope_t *module_scope = scope_new_and_link(comp, SCOPE_MODULE, parse_tree->root, emit_opt); // create standard emitter; it's used at least for MP_PASS_SCOPE - emit_t *emit_bc = emit_bc_new(); + emit_t *emit_bc = emit_bc_new(&comp->emit_common); - // compile pass 1 + // compile MP_PASS_SCOPE comp->emit = emit_bc; + comp->emit_bc = emit_bc; #if MICROPY_EMIT_NATIVE comp->emit_method_table = &emit_bc_method_table; #endif @@ -3458,14 +3545,24 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f } // compute some things related to scope and identifiers + bool has_native_code = false; for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) { - scope_compute_things(s); + #if MICROPY_EMIT_NATIVE + if (s->emit_options == MP_EMIT_OPT_NATIVE_PYTHON || s->emit_options == MP_EMIT_OPT_VIPER) { + has_native_code = true; + } + #endif + + scope_compute_things(s, &comp->emit_common); } // set max number of labels now that it's calculated emit_bc_set_max_num_labels(emit_bc, max_num_labels); - // compile pass 2 and 3 + // finalise and allocate the constant table + mp_emit_common_finalise(&comp->emit_common, has_native_code); + + // compile MP_PASS_STACK_SIZE, MP_PASS_CODE_SIZE, MP_PASS_EMIT #if MICROPY_EMIT_NATIVE emit_t *emit_native = NULL; #endif @@ -3505,7 +3602,7 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f case MP_EMIT_OPT_NATIVE_PYTHON: case MP_EMIT_OPT_VIPER: if (emit_native == NULL) { - emit_native = NATIVE_EMITTER(new)(&comp->compile_error, &comp->next_label, max_num_labels); + emit_native = NATIVE_EMITTER(new)(&comp->emit_common, &comp->compile_error, &comp->next_label, max_num_labels); } comp->emit_method_table = NATIVE_EMITTER_TABLE; comp->emit = emit_native; @@ -3540,10 +3637,33 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f // number for the start of this scope compile_error_set_line(comp, comp->scope_cur->pn); // add a traceback to the exception using relevant source info - mp_obj_exception_add_traceback(comp->compile_error, comp->source_file, + mp_obj_exception_add_traceback(comp->compile_error, source_file, comp->compile_error_line, comp->scope_cur->simple_name); } + // construct the global qstr/const table for this module + mp_compiled_module_t cm; + cm.rc = module_scope->raw_code; + cm.context = context; + #if MICROPY_PERSISTENT_CODE_SAVE + cm.has_native = has_native_code; + cm.n_qstr = comp->emit_common.qstr_map.used; + cm.n_obj = comp->emit_common.ct_cur_obj; + #endif + if (comp->compile_error == MP_OBJ_NULL) { + mp_emit_common_populate_module_context(&comp->emit_common, source_file, context); + + #if MICROPY_DEBUG_PRINTERS + // now that the module context is valid, the raw codes can be printed + if (mp_verbose_flag >= 2) { + for (scope_t *s = comp->scope_head; s != NULL; s = s->next) { + mp_raw_code_t *rc = s->raw_code; + mp_bytecode_print(&mp_plat_print, rc, rc->fun_data, rc->fun_data_len, &cm.context->constants); + } + } + #endif + } + // free the emitters emit_bc_free(emit_bc); @@ -3562,7 +3682,6 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f mp_parse_tree_clear(parse_tree); // free the scopes - mp_raw_code_t *outer_raw_code = module_scope->raw_code; for (scope_t *s = module_scope; s;) { scope_t *next = s->next; scope_free(s); @@ -3571,15 +3690,17 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f if (comp->compile_error != MP_OBJ_NULL) { nlr_raise(comp->compile_error); - } else { - return outer_raw_code; } + + return cm; } mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { - mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, is_repl); + mp_module_context_t *context = m_new_obj(mp_module_context_t); + context->module.globals = mp_globals_get(); + mp_compiled_module_t cm = mp_compile_to_raw_code(parse_tree, source_file, is_repl, context); // return function that executes the outer module - return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); + return mp_make_function_from_raw_code(cm.rc, cm.context, NULL); } #endif // MICROPY_ENABLE_COMPILER diff --git a/py/compile.h b/py/compile.h index 1ad1f5e9cd..ae87bf2a04 100644 --- a/py/compile.h +++ b/py/compile.h @@ -32,11 +32,12 @@ // the compiler will raise an exception if an error occurred // the compiler will clear the parse tree before it returns +// mp_globals_get() will be used for the context mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); #if MICROPY_PERSISTENT_CODE_SAVE // this has the same semantics as mp_compile -mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *globals); #endif // this is implemented in runtime.c diff --git a/py/dynruntime.h b/py/dynruntime.h index fdb91ed37e..b9dca60d13 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -30,6 +30,7 @@ // MicroPython runtime API defined in py/obj.h and py/runtime.h. #include "py/nativeglue.h" +#include "py/objfun.h" #include "py/objstr.h" #include "py/objtype.h" @@ -177,8 +178,8 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { #define mp_unary_op(op, obj) (mp_fun_table.unary_op((op), (obj))) #define mp_binary_op(op, lhs, rhs) (mp_fun_table.binary_op((op), (lhs), (rhs))) -#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \ - (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args))) +#define mp_make_function_from_raw_code(rc, context, def_args) \ + (mp_fun_table.make_function_from_raw_code((rc), (context), (def_args))) #define mp_call_function_n_kw(fun, n_args, n_kw, args) \ (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args)) @@ -187,11 +188,10 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw)))) #define MP_DYNRUNTIME_INIT_ENTRY \ - mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \ + mp_obj_t old_globals = mp_fun_table.swap_globals(self->context->module.globals); \ mp_raw_code_t rc; \ rc.kind = MP_CODE_NATIVE_VIPER; \ rc.scope_flags = 0; \ - rc.const_table = (void *)self->const_table; \ (void)rc; #define MP_DYNRUNTIME_INIT_EXIT \ @@ -199,7 +199,7 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { return mp_const_none; #define MP_DYNRUNTIME_MAKE_FUNCTION(f) \ - (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL)) + (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), self->context, NULL)) #define mp_import_name(name, fromlist, level) \ (mp_fun_table.import_name((name), (fromlist), (level))) diff --git a/py/emit.h b/py/emit.h index 13bd3e9b2e..6f3593a0e8 100644 --- a/py/emit.h +++ b/py/emit.h @@ -92,6 +92,18 @@ typedef enum { typedef struct _emit_t emit_t; +typedef struct _mp_emit_common_t { + pass_kind_t pass; + uint16_t ct_cur_obj_base; + uint16_t ct_cur_obj; + uint16_t ct_cur_child; + mp_uint_t *const_table; + mp_raw_code_t **children; + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + mp_map_t qstr_map; + #endif +} mp_emit_common_t; + typedef struct _mp_emit_method_table_id_ops_t { void (*local)(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void (*global)(emit_t *emit, qstr qst, int kind); @@ -99,7 +111,7 @@ typedef struct _mp_emit_method_table_id_ops_t { typedef struct _emit_method_table_t { #if MICROPY_DYNAMIC_COMPILER - emit_t *(*emit_new)(mp_obj_t * error_slot, uint *label_slot, mp_uint_t max_num_labels); + emit_t *(*emit_new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); void (*emit_free)(emit_t *emit); #endif @@ -161,6 +173,28 @@ typedef struct _emit_method_table_t { void (*end_except_handler)(emit_t *emit); } emit_method_table_t; +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst); +#else +static inline qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) { + return qst; +} +#endif + +static inline size_t mp_emit_common_alloc_const_obj(mp_emit_common_t *emit, mp_obj_t obj) { + if (emit->pass == MP_PASS_EMIT) { + emit->const_table[emit->ct_cur_obj] = (mp_uint_t)obj; + } + return emit->ct_cur_obj++; +} + +static inline size_t mp_emit_common_alloc_const_child(mp_emit_common_t *emit, mp_raw_code_t *rc) { + if (emit->pass == MP_PASS_EMIT) { + emit->children[emit->ct_cur_child] = rc; + } + return emit->ct_cur_child++; +} + static inline void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst) { scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT); } @@ -180,13 +214,13 @@ extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops; extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops; extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_delete_id_ops; -emit_t *emit_bc_new(void); -emit_t *emit_native_x64_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_x86_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_thumb_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_arm_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_xtensa_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_xtensawin_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_bc_new(mp_emit_common_t *emit_common); +emit_t *emit_native_x64_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_x86_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_thumb_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_arm_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_xtensa_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_xtensawin_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); void emit_bc_set_max_num_labels(emit_t *emit, mp_uint_t max_num_labels); diff --git a/py/emitbc.c b/py/emitbc.c index ca74046033..c04701ca78 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -36,8 +36,7 @@ #if MICROPY_ENABLE_COMPILER -#define BYTES_FOR_INT ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) -#define DUMMY_DATA_SIZE (BYTES_FOR_INT) +#define DUMMY_DATA_SIZE (MP_ENCODE_UINT_MAX_BYTES) struct _emit_t { // Accessed as mp_obj_t, so must be aligned as such, and we rely on the @@ -50,6 +49,7 @@ struct _emit_t { int stack_size; + mp_emit_common_t *emit_common; scope_t *scope; mp_uint_t last_source_line_offset; @@ -66,17 +66,11 @@ struct _emit_t { size_t n_info; size_t n_cell; - - #if MICROPY_PERSISTENT_CODE - uint16_t ct_cur_obj; - uint16_t ct_num_obj; - uint16_t ct_cur_raw_code; - #endif - mp_uint_t *const_table; }; -emit_t *emit_bc_new(void) { +emit_t *emit_bc_new(mp_emit_common_t *emit_common) { emit_t *emit = m_new0(emit_t, 1); + emit->emit_common = emit_common; return emit; } @@ -90,26 +84,9 @@ void emit_bc_free(emit_t *emit) { m_del_obj(emit_t, emit); } -typedef byte *(*emit_allocator_t)(emit_t *emit, int nbytes); - -STATIC void emit_write_uint(emit_t *emit, emit_allocator_t allocator, mp_uint_t val) { - // We store each 7 bits in a separate byte, and that's how many bytes needed - byte buf[BYTES_FOR_INT]; - byte *p = buf + sizeof(buf); - // We encode in little-ending order, but store in big-endian, to help decoding - do { - *--p = val & 0x7f; - val >>= 7; - } while (val != 0); - byte *c = allocator(emit, buf + sizeof(buf) - p); - while (p != buf + sizeof(buf) - 1) { - *c++ = *p++ | 0x80; - } - *c = *p; -} - // all functions must go through this one to emit code info -STATIC byte *emit_get_cur_to_write_code_info(emit_t *emit, int num_bytes_to_write) { +STATIC uint8_t *emit_get_cur_to_write_code_info(void *emit_in, size_t num_bytes_to_write) { + emit_t *emit = emit_in; if (emit->pass < MP_PASS_EMIT) { emit->code_info_offset += num_bytes_to_write; return emit->dummy_data; @@ -126,14 +103,7 @@ STATIC void emit_write_code_info_byte(emit_t *emit, byte val) { } STATIC void emit_write_code_info_qstr(emit_t *emit, qstr qst) { - #if MICROPY_PERSISTENT_CODE - assert((qst >> 16) == 0); - byte *c = emit_get_cur_to_write_code_info(emit, 2); - c[0] = qst; - c[1] = qst >> 8; - #else - emit_write_uint(emit, emit_get_cur_to_write_code_info, qst); - #endif + mp_encode_uint(emit, emit_get_cur_to_write_code_info, mp_emit_common_use_qstr(emit->emit_common, qst)); } #if MICROPY_ENABLE_SOURCE_LINE @@ -166,7 +136,8 @@ STATIC void emit_write_code_info_bytes_lines(emit_t *emit, mp_uint_t bytes_to_sk #endif // all functions must go through this one to emit byte code -STATIC byte *emit_get_cur_to_write_bytecode(emit_t *emit, int num_bytes_to_write) { +STATIC uint8_t *emit_get_cur_to_write_bytecode(void *emit_in, size_t num_bytes_to_write) { + emit_t *emit = emit_in; if (emit->pass < MP_PASS_EMIT) { emit->bytecode_offset += num_bytes_to_write; return emit->dummy_data; @@ -189,12 +160,12 @@ STATIC void emit_write_bytecode_byte(emit_t *emit, int stack_adj, byte b1) { c[0] = b1; } -// Similar to emit_write_bytecode_uint(), just some extra handling to encode sign +// Similar to mp_encode_uint(), just some extra handling to encode sign STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, mp_int_t num) { emit_write_bytecode_byte(emit, stack_adj, b1); // We store each 7 bits in a separate byte, and that's how many bytes needed - byte buf[BYTES_FOR_INT]; + byte buf[MP_ENCODE_UINT_MAX_BYTES]; byte *p = buf + sizeof(buf); // We encode in little-ending order, but store in big-endian, to help decoding do { @@ -218,61 +189,25 @@ STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, m STATIC void emit_write_bytecode_byte_uint(emit_t *emit, int stack_adj, byte b, mp_uint_t val) { emit_write_bytecode_byte(emit, stack_adj, b); - emit_write_uint(emit, emit_get_cur_to_write_bytecode, val); + mp_encode_uint(emit, emit_get_cur_to_write_bytecode, val); } -#if MICROPY_PERSISTENT_CODE -STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n, mp_uint_t c) { - if (emit->pass == MP_PASS_EMIT) { - emit->const_table[n] = c; - } +STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n) { emit_write_bytecode_byte_uint(emit, stack_adj, b, n); } -#endif STATIC void emit_write_bytecode_byte_qstr(emit_t *emit, int stack_adj, byte b, qstr qst) { - #if MICROPY_PERSISTENT_CODE - assert((qst >> 16) == 0); - mp_emit_bc_adjust_stack_size(emit, stack_adj); - byte *c = emit_get_cur_to_write_bytecode(emit, 3); - c[0] = b; - c[1] = qst; - c[2] = qst >> 8; - #else - emit_write_bytecode_byte_uint(emit, stack_adj, b, qst); - #endif + emit_write_bytecode_byte_uint(emit, stack_adj, b, mp_emit_common_use_qstr(emit->emit_common, qst)); } STATIC void emit_write_bytecode_byte_obj(emit_t *emit, int stack_adj, byte b, mp_obj_t obj) { - #if MICROPY_PERSISTENT_CODE emit_write_bytecode_byte_const(emit, stack_adj, b, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_cur_obj++, (mp_uint_t)obj); - #else - // aligns the pointer so it is friendly to GC - emit_write_bytecode_byte(emit, stack_adj, b); - emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(mp_obj_t)); - mp_obj_t *c = (mp_obj_t *)emit_get_cur_to_write_bytecode(emit, sizeof(mp_obj_t)); - // Verify thar c is already uint-aligned - assert(c == MP_ALIGN(c, sizeof(mp_obj_t))); - *c = obj; - #endif + mp_emit_common_alloc_const_obj(emit->emit_common, obj)); } -STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) { - #if MICROPY_PERSISTENT_CODE +STATIC void emit_write_bytecode_byte_child(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) { emit_write_bytecode_byte_const(emit, stack_adj, b, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_num_obj + emit->ct_cur_raw_code++, (mp_uint_t)(uintptr_t)rc); - #else - // aligns the pointer so it is friendly to GC - emit_write_bytecode_byte(emit, stack_adj, b); - emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(void *)); - void **c = (void **)emit_get_cur_to_write_bytecode(emit, sizeof(void *)); - // Verify thar c is already uint-aligned - assert(c == MP_ALIGN(c, sizeof(void *))); - *c = rc; - #endif + mp_emit_common_alloc_const_child(emit->emit_common, rc)); #if MICROPY_PY_SYS_SETTRACE rc->line_of_definition = emit->last_source_line; #endif @@ -343,27 +278,19 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { } // Write number of cells and size of the source code info - if (pass >= MP_PASS_CODE_SIZE) { - MP_BC_PRELUDE_SIZE_ENCODE(emit->n_info, emit->n_cell, emit_write_code_info_byte, emit); + if (emit->pass >= MP_PASS_CODE_SIZE) { + size_t n_info = emit->n_info; + size_t n_cell = emit->n_cell; + MP_BC_PRELUDE_SIZE_ENCODE(n_info, n_cell, emit_write_code_info_byte, emit); } emit->n_info = emit->code_info_offset; - // Write the name and source file of this function. + // Write the name of this function. emit_write_code_info_qstr(emit, scope->simple_name); - emit_write_code_info_qstr(emit, scope->source_file); - - #if MICROPY_PERSISTENT_CODE - emit->ct_cur_obj = 0; - emit->ct_cur_raw_code = 0; - #endif - - if (pass == MP_PASS_EMIT) { - // Write argument names (needed to resolve positional args passed as - // keywords). We store them as full word-sized objects for efficient access - // in mp_setup_code_state this is the start of the prelude and is guaranteed - // to be aligned on a word boundary. + // Write argument names, needed to resolve positional args passed as keywords. + { // For a given argument position (indexed by i) we need to find the // corresponding id_info which is a parameter, as it has the correct // qstr name to use as the argument name. Note that it's not a simple @@ -383,7 +310,7 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { break; } } - emit->const_table[i] = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); + emit_write_code_info_qstr(emit, qst); } } } @@ -396,8 +323,6 @@ void mp_emit_bc_end_pass(emit_t *emit) { // check stack is back to zero size assert(emit->stack_size == 0); - emit_write_code_info_byte(emit, 0); // end of line number info - // Calculate size of source code info section emit->n_info = emit->code_info_offset - emit->n_info; @@ -412,39 +337,20 @@ void mp_emit_bc_end_pass(emit_t *emit) { } } - #if MICROPY_PERSISTENT_CODE - assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->ct_num_obj == emit->ct_cur_obj)); - emit->ct_num_obj = emit->ct_cur_obj; - #endif - if (emit->pass == MP_PASS_CODE_SIZE) { - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - emit->code_info_offset = (size_t)MP_ALIGN(emit->code_info_offset, sizeof(mp_uint_t)); - #endif - // calculate size of total code-info + bytecode, in bytes emit->code_info_size = emit->code_info_offset; emit->bytecode_size = emit->bytecode_offset; emit->code_base = m_new0(byte, emit->code_info_size + emit->bytecode_size); - #if MICROPY_PERSISTENT_CODE - emit->const_table = m_new0(mp_uint_t, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_cur_obj + emit->ct_cur_raw_code); - #else - emit->const_table = m_new0(mp_uint_t, - emit->scope->num_pos_args + emit->scope->num_kwonly_args); - #endif - } else if (emit->pass == MP_PASS_EMIT) { mp_emit_glue_assign_bytecode(emit->scope->raw_code, emit->code_base, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS emit->code_info_size + emit->bytecode_size, #endif - emit->const_table, + emit->emit_common->children, #if MICROPY_PERSISTENT_CODE_SAVE - emit->ct_cur_obj, emit->ct_cur_raw_code, + emit->emit_common->ct_cur_child, #endif emit->scope->scope_flags); } @@ -783,21 +689,21 @@ void mp_emit_bc_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right) { void mp_emit_bc_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { if (n_pos_defaults == 0 && n_kw_defaults == 0) { - emit_write_bytecode_byte_raw_code(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code); + emit_write_bytecode_byte_child(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code); } else { - emit_write_bytecode_byte_raw_code(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code); + emit_write_bytecode_byte_child(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code); } } void mp_emit_bc_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { if (n_pos_defaults == 0 && n_kw_defaults == 0) { int stack_adj = -n_closed_over + 1; - emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code); + emit_write_bytecode_byte_child(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code); emit_write_bytecode_raw_byte(emit, n_closed_over); } else { assert(n_closed_over <= 255); int stack_adj = -2 - (mp_int_t)n_closed_over + 1; - emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code); + emit_write_bytecode_byte_child(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code); emit_write_bytecode_raw_byte(emit, n_closed_over); } } diff --git a/py/emitcommon.c b/py/emitcommon.c index 791bf398ab..c1780d2db9 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -30,6 +30,17 @@ #if MICROPY_ENABLE_COMPILER +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) { + mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + if (elem->value == MP_OBJ_NULL) { + assert(emit->pass == MP_PASS_SCOPE); + elem->value = MP_OBJ_NEW_SMALL_INT(emit->qstr_map.used - 1); + } + return MP_OBJ_SMALL_INT_VALUE(elem->value); +} +#endif + void mp_emit_common_get_id_for_modification(scope_t *scope, qstr qst) { // name adding/lookup id_info_t *id = scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT); diff --git a/py/emitglue.c b/py/emitglue.c index 09b48682ff..cd902838af 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -34,6 +34,7 @@ #include "py/emitglue.h" #include "py/runtime0.h" #include "py/bc.h" +#include "py/objfun.h" #include "py/profile.h" #if MICROPY_DEBUG_VERBOSE // print debugging info @@ -63,20 +64,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS size_t len, #endif - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t n_obj, uint16_t n_raw_code, + size_t n_children, #endif mp_uint_t scope_flags) { rc->kind = MP_CODE_BYTECODE; rc->scope_flags = scope_flags; rc->fun_data = code; - rc->const_table = const_table; - #if MICROPY_PERSISTENT_CODE_SAVE + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS rc->fun_data_len = len; - rc->n_obj = n_obj; - rc->n_raw_code = n_raw_code; + #endif + rc->children = children; + + #if MICROPY_PERSISTENT_CODE_SAVE + rc->n_children = n_children; #endif #if MICROPY_PY_SYS_SETTRACE @@ -85,26 +88,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #endif #ifdef DEBUG_PRINT - #if !MICROPY_DEBUG_PRINTERS + #if !(MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS) const size_t len = 0; #endif DEBUG_printf("assign byte code: code=%p len=" UINT_FMT " flags=%x\n", code, len, (uint)scope_flags); #endif - #if MICROPY_DEBUG_PRINTERS - if (mp_verbose_flag >= 2) { - mp_bytecode_print(&mp_plat_print, rc, code, len, const_table); - } - #endif } #if MICROPY_EMIT_MACHINE_CODE -void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, +void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE + size_t n_children, uint16_t prelude_offset, - uint16_t n_obj, uint16_t n_raw_code, uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link, #endif - mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) { + mp_uint_t scope_flags, mp_uint_t n_pos_args, mp_uint_t type_sig) { assert(kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER || kind == MP_CODE_NATIVE_ASM); @@ -135,20 +134,24 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void rc->kind = kind; rc->scope_flags = scope_flags; - rc->n_pos_args = n_pos_args; rc->fun_data = fun_data; - rc->const_table = const_table; - rc->type_sig = type_sig; + + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + rc->fun_data_len = fun_len; + #endif + rc->children = children; #if MICROPY_PERSISTENT_CODE_SAVE - rc->fun_data_len = fun_len; + rc->n_children = n_children; rc->prelude_offset = prelude_offset; - rc->n_obj = n_obj; - rc->n_raw_code = n_raw_code; rc->n_qstr = n_qstr; rc->qstr_link = qstr_link; #endif + // These two entries are only needed for MP_CODE_NATIVE_ASM. + rc->n_pos_args = n_pos_args; + rc->type_sig = type_sig; + #ifdef DEBUG_PRINT DEBUG_printf("assign native: kind=%d fun=%p len=" UINT_FMT " n_pos_args=" UINT_FMT " flags=%x\n", kind, fun_data, fun_len, n_pos_args, (uint)scope_flags); for (mp_uint_t i = 0; i < fun_len; i++) { @@ -170,15 +173,15 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void } #endif -mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args) { +mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, const mp_obj_t *def_args) { DEBUG_OP_printf("make_function_from_raw_code %p\n", rc); assert(rc != NULL); // def_args must be MP_OBJ_NULL or a tuple - assert(def_args == MP_OBJ_NULL || mp_obj_is_type(def_args, &mp_type_tuple)); + assert(def_args == NULL || def_args[0] == MP_OBJ_NULL || mp_obj_is_type(def_args[0], &mp_type_tuple)); // def_kw_args must be MP_OBJ_NULL or a dict - assert(def_kw_args == MP_OBJ_NULL || mp_obj_is_type(def_kw_args, &mp_type_dict)); + assert(def_args == NULL || def_args[1] == MP_OBJ_NULL || mp_obj_is_type(def_args[1], &mp_type_dict)); // make the function, depending on the raw code kind mp_obj_t fun; @@ -186,7 +189,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar #if MICROPY_EMIT_NATIVE case MP_CODE_NATIVE_PY: case MP_CODE_NATIVE_VIPER: - fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->fun_data, rc->const_table); + fun = mp_obj_new_fun_native(def_args, rc->fun_data, context, rc->children); // Check for a generator function, and if so change the type of the object if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_gen_wrap; @@ -201,7 +204,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar default: // rc->kind should always be set and BYTECODE is the only remaining case assert(rc->kind == MP_CODE_BYTECODE); - fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->fun_data, rc->const_table); + fun = mp_obj_new_fun_bc(def_args, rc->fun_data, context, rc->children); // check for generator functions and if so change the type of the object if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_gen_wrap; @@ -218,16 +221,16 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar return fun; } -mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args) { +mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, mp_uint_t n_closed_over, const mp_obj_t *args) { DEBUG_OP_printf("make_closure_from_raw_code %p " UINT_FMT " %p\n", rc, n_closed_over, args); // make function object mp_obj_t ffun; if (n_closed_over & 0x100) { // default positional and keyword args given - ffun = mp_make_function_from_raw_code(rc, args[0], args[1]); + ffun = mp_make_function_from_raw_code(rc, context, args); } else { // default positional and keyword args not given - ffun = mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); + ffun = mp_make_function_from_raw_code(rc, context, NULL); } // wrap function in closure object return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2)); diff --git a/py/emitglue.h b/py/emitglue.h index a5411dc2e2..9a22a93218 100644 --- a/py/emitglue.h +++ b/py/emitglue.h @@ -54,16 +54,20 @@ typedef struct _mp_qstr_link_entry_t { uint16_t qst; } mp_qstr_link_entry_t; +// compiled bytecode: instance in RAM, referenced by outer scope, usually freed after first (and only) use +// mpy file: instance in RAM, created when .mpy file is loaded (same comments as above) +// frozen: instance in ROM typedef struct _mp_raw_code_t { mp_uint_t kind : 3; // of type mp_raw_code_kind_t mp_uint_t scope_flags : 7; mp_uint_t n_pos_args : 11; const void *fun_data; - const mp_uint_t *const_table; + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t fun_data_len; // so mp_raw_code_save and mp_bytecode_print work + #endif + struct _mp_raw_code_t **children; #if MICROPY_PERSISTENT_CODE_SAVE - size_t fun_data_len; - uint16_t n_obj; - uint16_t n_raw_code; + size_t n_children; #if MICROPY_PY_SYS_SETTRACE mp_bytecode_prelude_t prelude; // line_of_definition is a Python source line where the raw_code was @@ -89,22 +93,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS size_t len, #endif - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t n_obj, uint16_t n_raw_code, + size_t n_children, #endif mp_uint_t scope_flags); void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE + size_t n_children, uint16_t prelude_offset, - uint16_t n_obj, uint16_t n_raw_code, uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link, #endif - mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig); + mp_uint_t scope_flags, mp_uint_t n_pos_args, mp_uint_t type_sig); -mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args); -mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args); +mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, const mp_obj_t *def_args); +mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, mp_uint_t n_closed_over, const mp_obj_t *args); #endif // MICROPY_INCLUDED_PY_EMITGLUE_H diff --git a/py/emitnative.c b/py/emitnative.c index 6504f37765..ca34e89f64 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -48,6 +48,7 @@ #include "py/emit.h" #include "py/nativeglue.h" +#include "py/objfun.h" #include "py/objstr.h" #if MICROPY_DEBUG_VERBOSE // print debugging info @@ -92,9 +93,13 @@ #define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)) #define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t)) #define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_GLOBALS (offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_CONTEXT (offsetof(mp_obj_fun_bc_t, context) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_CHILD_TABLE (offsetof(mp_obj_fun_bc_t, child_table) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_CONST_TABLE (offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_OBJ_TABLE (offsetof(mp_module_context_t, constants.obj_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_GLOBALS (offsetof(mp_module_context_t, module.globals) / sizeof(uintptr_t)) +#define INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE (0) // If not already defined, set parent args to same as child call registers #ifndef REG_PARENT_RET @@ -205,6 +210,7 @@ typedef struct _exc_stack_entry_t { } exc_stack_entry_t; struct _emit_t { + mp_emit_common_t *emit_common; mp_obj_t *error_slot; uint *label_slot; uint exit_label; @@ -225,18 +231,17 @@ struct _emit_t { exc_stack_entry_t *exc_stack; int prelude_offset; + #if N_PRELUDE_AS_BYTES_OBJ + size_t prelude_const_table_offset; + #endif int start_offset; int n_state; uint16_t code_state_start; uint16_t stack_start; int stack_size; + uint16_t n_info; uint16_t n_cell; - uint16_t const_table_cur_obj; - uint16_t const_table_num_obj; - uint16_t const_table_cur_raw_code; - mp_uint_t *const_table; - #if MICROPY_PERSISTENT_CODE_SAVE uint16_t qstr_link_cur; mp_qstr_link_entry_t *qstr_link; @@ -255,8 +260,9 @@ STATIC void emit_native_global_exc_entry(emit_t *emit); STATIC void emit_native_global_exc_exit(emit_t *emit); STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj); -emit_t *EXPORT_FUN(new)(mp_obj_t * error_slot, uint *label_slot, mp_uint_t max_num_labels) { +emit_t *EXPORT_FUN(new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels) { emit_t *emit = m_new0(emit_t, 1); + emit->emit_common = emit_common; emit->error_slot = error_slot; emit->label_slot = label_slot; emit->stack_info_alloc = 8; @@ -340,30 +346,22 @@ STATIC void emit_native_mov_reg_qstr_obj(emit_t *emit, int reg_dest, qstr qst) { emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ } while (false) -#define emit_native_mov_state_imm_fix_u16_via(emit, local_num, imm, reg_temp) \ - do { \ - ASM_MOV_REG_IMM_FIX_U16((emit)->as, (reg_temp), (imm)); \ - emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ - } while (false) - -#define emit_native_mov_state_imm_fix_word_via(emit, local_num, imm, reg_temp) \ - do { \ - ASM_MOV_REG_IMM_FIX_WORD((emit)->as, (reg_temp), (imm)); \ - emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ - } while (false) - STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { DEBUG_printf("start_pass(pass=%u, scope=%p)\n", pass, scope); + if (pass == MP_PASS_SCOPE) { + // Note: the first argument passed here is mp_emit_common_t, not the native emitter context + #if N_PRELUDE_AS_BYTES_OBJ + if (scope->emit_options == MP_EMIT_OPT_NATIVE_PYTHON) { + mp_emit_common_alloc_const_obj((mp_emit_common_t *)emit, mp_const_none); + } + #endif + return; + } + emit->pass = pass; emit->do_viper_types = scope->emit_options == MP_EMIT_OPT_VIPER; emit->stack_size = 0; - #if N_PRELUDE_AS_BYTES_OBJ - emit->const_table_cur_obj = emit->do_viper_types ? 0 : 1; // reserve first obj for prelude bytes obj - #else - emit->const_table_cur_obj = 0; - #endif - emit->const_table_cur_raw_code = 0; #if MICROPY_PERSISTENT_CODE_SAVE emit->qstr_link_cur = 0; #endif @@ -455,8 +453,9 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, 0); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE); // Store function object (passed as first arg) to stack if needed if (NEED_FUN_OBJ(emit)) { @@ -514,7 +513,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->stack_start = SIZEOF_CODE_STATE; #if N_PRELUDE_AS_BYTES_OBJ // Load index of prelude bytes object in const_table - mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)(emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1)); + mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_const_table_offset); #else mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_offset); #endif @@ -536,8 +535,9 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, emit->scope->num_pos_args + emit->scope->num_kwonly_args); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE); } else { // The locals and stack start after the code_state structure emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE; @@ -555,22 +555,33 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE); // Set code_state.fun_bc ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1); - // Set code_state.ip (offset from start of this function to prelude info) + // Set code_state.ip, a pointer to the beginning of the prelude + // Need to use some locals for this, so assert that they are available for use + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_1); + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_2); + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_3); + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_4); int code_state_ip_local = emit->code_state_start + OFFSETOF_CODE_STATE_IP; #if N_PRELUDE_AS_BYTES_OBJ - // Prelude is a bytes object in const_table; store ip = prelude->data - fun_bc->bytecode - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1); + // Prelude is a bytes object in const_table[prelude_const_table_offset]. + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->prelude_const_table_offset); ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, offsetof(mp_obj_str_t, data) / sizeof(uintptr_t)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE); - ASM_SUB_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1); - emit_native_mov_state_reg(emit, code_state_ip_local, REG_LOCAL_3); #else + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_1); + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_2); + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_3); + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_4); + // Prelude is at the end of the machine code + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE); if (emit->pass == MP_PASS_CODE_SIZE) { // Commit to the encoding size based on the value of prelude_offset in this pass. // By using 32768 as the cut-off it is highly unlikely that prelude_offset will @@ -579,14 +590,16 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop } if (emit->prelude_offset_uses_u16_encoding) { assert(emit->prelude_offset <= 65535); - emit_native_mov_state_imm_fix_u16_via(emit, code_state_ip_local, emit->prelude_offset, REG_PARENT_ARG_1); + ASM_MOV_REG_IMM_FIX_U16((emit)->as, REG_LOCAL_2, emit->prelude_offset); } else { - emit_native_mov_state_imm_fix_word_via(emit, code_state_ip_local, emit->prelude_offset, REG_PARENT_ARG_1); + ASM_MOV_REG_IMM_FIX_WORD((emit)->as, REG_LOCAL_2, emit->prelude_offset); } + ASM_ADD_REG_REG(emit->as, REG_LOCAL_3, REG_LOCAL_2); #endif + emit_native_mov_state_reg(emit, code_state_ip_local, REG_LOCAL_3); // Set code_state.n_state (only works on little endian targets due to n_state being uint16_t) - emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t), emit->n_state, REG_ARG_1); + emit_native_mov_state_imm_via(emit, emit->code_state_start + OFFSETOF_CODE_STATE_N_STATE, emit->n_state, REG_ARG_1); // Put address of code_state into first arg ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, emit->code_state_start); @@ -628,30 +641,17 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->local_vtype[id->local_num] = VTYPE_PYOBJ; } } - - if (pass == MP_PASS_EMIT) { - // write argument names as qstr objects - // see comment in corresponding part of emitbc.c about the logic here - for (int i = 0; i < scope->num_pos_args + scope->num_kwonly_args; i++) { - qstr qst = MP_QSTR__star_; - for (int j = 0; j < scope->id_info_len; ++j) { - id_info_t *id = &scope->id_info[j]; - if ((id->flags & ID_FLAG_IS_PARAM) && id->local_num == i) { - qst = id->qst; - break; - } - } - emit->const_table[i] = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); - } - } } - } static inline void emit_native_write_code_info_byte(emit_t *emit, byte val) { mp_asm_base_data(&emit->as->base, 1, val); } +static inline void emit_native_write_code_info_qstr(emit_t *emit, qstr qst) { + mp_encode_uint(&emit->as->base, mp_asm_base_get_cur_to_write_bytes, mp_emit_common_use_qstr(emit->emit_common, qst)); +} + STATIC void emit_native_end_pass(emit_t *emit) { emit_native_global_exc_exit(emit); @@ -662,21 +662,25 @@ STATIC void emit_native_end_pass(emit_t *emit) { size_t n_exc_stack = 0; // exc-stack not needed for native code MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, emit->scope, emit_native_write_code_info_byte, emit); - #if MICROPY_PERSISTENT_CODE - size_t n_info = 4; - #else - size_t n_info = 1; - #endif - MP_BC_PRELUDE_SIZE_ENCODE(n_info, emit->n_cell, emit_native_write_code_info_byte, emit); + size_t n_info = emit->n_info; + size_t n_cell = emit->n_cell; + MP_BC_PRELUDE_SIZE_ENCODE(n_info, n_cell, emit_native_write_code_info_byte, emit); - #if MICROPY_PERSISTENT_CODE - mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name); - mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name >> 8); - mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file); - mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file >> 8); - #else - mp_asm_base_data(&emit->as->base, 1, 1); - #endif + // bytecode prelude: source info (function and argument qstrs) + size_t info_start = mp_asm_base_get_code_pos(&emit->as->base); + emit_native_write_code_info_qstr(emit, emit->scope->simple_name); + for (int i = 0; i < emit->scope->num_pos_args + emit->scope->num_kwonly_args; i++) { + qstr qst = MP_QSTR__star_; + for (int j = 0; j < emit->scope->id_info_len; ++j) { + id_info_t *id = &emit->scope->id_info[j]; + if ((id->flags & ID_FLAG_IS_PARAM) && id->local_num == i) { + qst = id->qst; + break; + } + } + emit_native_write_code_info_qstr(emit, qst); + } + emit->n_info = mp_asm_base_get_code_pos(&emit->as->base) - info_start; // bytecode prelude: initialise closed over variables size_t cell_start = mp_asm_base_get_code_pos(&emit->as->base); @@ -690,13 +694,14 @@ STATIC void emit_native_end_pass(emit_t *emit) { emit->n_cell = mp_asm_base_get_code_pos(&emit->as->base) - cell_start; #if N_PRELUDE_AS_BYTES_OBJ - // Prelude bytes object is after qstr arg names and mp_fun_table - size_t table_off = emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1; + // Create the prelude as a bytes object, and store it in the constant table + mp_obj_t prelude = mp_const_none; if (emit->pass == MP_PASS_EMIT) { void *buf = emit->as->base.code_base + emit->prelude_offset; size_t n = emit->as->base.code_offset - emit->prelude_offset; - emit->const_table[table_off] = (uintptr_t)mp_obj_new_bytes(buf, n); + prelude = mp_obj_new_bytes(buf, n); } + emit->prelude_const_table_offset = mp_emit_common_alloc_const_obj(emit->emit_common, prelude); #endif } @@ -706,31 +711,15 @@ STATIC void emit_native_end_pass(emit_t *emit) { assert(emit->stack_size == 0); assert(emit->exc_stack_size == 0); - // Deal with const table accounting - assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->const_table_num_obj == emit->const_table_cur_obj)); - emit->const_table_num_obj = emit->const_table_cur_obj; + #if MICROPY_PERSISTENT_CODE_SAVE + // Allocate qstr_link table if needed if (emit->pass == MP_PASS_CODE_SIZE) { - size_t const_table_alloc = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code; - size_t nqstr = 0; - if (!emit->do_viper_types) { - // Add room for qstr names of arguments - nqstr = emit->scope->num_pos_args + emit->scope->num_kwonly_args; - const_table_alloc += nqstr; - } - emit->const_table = m_new(mp_uint_t, const_table_alloc); - #if !MICROPY_DYNAMIC_COMPILER - // Store mp_fun_table pointer just after qstrs - // (but in dynamic-compiler mode eliminate dependency on mp_fun_table) - emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)&mp_fun_table; - #endif - - #if MICROPY_PERSISTENT_CODE_SAVE size_t qstr_link_alloc = emit->qstr_link_cur; if (qstr_link_alloc > 0) { emit->qstr_link = m_new(mp_qstr_link_entry_t, qstr_link_alloc); } - #endif } + #endif if (emit->pass == MP_PASS_EMIT) { void *f = mp_asm_base_get_code(&emit->as->base); @@ -738,13 +727,14 @@ STATIC void emit_native_end_pass(emit_t *emit) { mp_emit_glue_assign_native(emit->scope->raw_code, emit->do_viper_types ? MP_CODE_NATIVE_VIPER : MP_CODE_NATIVE_PY, - f, f_len, emit->const_table, + f, f_len, + emit->emit_common->children, #if MICROPY_PERSISTENT_CODE_SAVE + emit->emit_common->ct_cur_child, emit->prelude_offset, - emit->const_table_cur_obj, emit->const_table_cur_raw_code, emit->qstr_link_cur, emit->qstr_link, #endif - emit->scope->num_pos_args, emit->scope->scope_flags, 0); + emit->scope->scope_flags, 0, 0); } } @@ -1137,29 +1127,19 @@ STATIC exc_stack_entry_t *emit_native_pop_exc_stack(emit_t *emit) { return e; } -STATIC void emit_load_reg_with_ptr(emit_t *emit, int reg, mp_uint_t ptr, size_t table_off) { - if (!emit->do_viper_types) { - // Skip qstr names of arguments - table_off += emit->scope->num_pos_args + emit->scope->num_kwonly_args; - } - if (emit->pass == MP_PASS_EMIT) { - emit->const_table[table_off] = ptr; - } +STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) { + size_t table_off = mp_emit_common_alloc_const_obj(emit->emit_common, obj); emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off); } -STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) { - // First entry is for mp_fun_table - size_t table_off = 1 + emit->const_table_cur_obj++; - emit_load_reg_with_ptr(emit, reg, (mp_uint_t)obj, table_off); -} - -STATIC void emit_load_reg_with_raw_code(emit_t *emit, int reg, mp_raw_code_t *rc) { - // First entry is for mp_fun_table, then constant objects - size_t table_off = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code++; - emit_load_reg_with_ptr(emit, reg, (mp_uint_t)rc, table_off); +STATIC void emit_load_reg_with_child(emit_t *emit, int reg, mp_raw_code_t *rc) { + size_t table_off = mp_emit_common_alloc_const_child(emit->emit_common, rc); + emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CHILD_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off); } STATIC void emit_native_label_assign(emit_t *emit, mp_uint_t l) { @@ -1203,7 +1183,8 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { if (!(emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR)) { // Set new globals emit_native_mov_reg_state(emit, REG_ARG_1, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_GLOBALS); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_MODULE_CONTEXT_GLOBALS); emit_call(emit, MP_F_NATIVE_SWAP_GLOBALS); // Save old globals (or NULL if globals didn't change) @@ -1254,7 +1235,8 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { #if N_NLR_SETJMP // Reload REG_FUN_TABLE, since it may be clobbered by longjmp emit_native_mov_reg_state(emit, REG_LOCAL_1, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, emit->scope->num_pos_args + emit->scope->num_kwonly_args); #endif ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, LOCAL_IDX_EXC_HANDLER_PC(emit)); @@ -1385,11 +1367,7 @@ STATIC void emit_native_import(emit_t *emit, qstr qst, int kind) { STATIC void emit_native_load_const_tok(emit_t *emit, mp_token_kind_t tok) { DEBUG_printf("load_const_tok(tok=%u)\n", tok); if (tok == MP_TOKEN_ELLIPSIS) { - #if MICROPY_PERSISTENT_CODE_SAVE emit_native_load_const_obj(emit, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj)); - #else - emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj)); - #endif } else { emit_native_pre(emit); if (tok == MP_TOKEN_KW_NONE) { @@ -2682,33 +2660,46 @@ STATIC void emit_native_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_ri STATIC void emit_native_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { // call runtime, with type info for args, or don't support dict/default params, or only support Python objects for them emit_native_pre(emit); + emit_native_mov_reg_state(emit, REG_ARG_2, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_2, REG_ARG_2, OFFSETOF_OBJ_FUN_BC_CONTEXT); if (n_pos_defaults == 0 && n_kw_defaults == 0) { need_reg_all(emit); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_REG_IMM(emit->as, REG_ARG_3, (mp_uint_t)MP_OBJ_NULL); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, 0); } else { - vtype_kind_t vtype_def_tuple, vtype_def_dict; - emit_pre_pop_reg_reg(emit, &vtype_def_dict, REG_ARG_3, &vtype_def_tuple, REG_ARG_2); - assert(vtype_def_tuple == VTYPE_PYOBJ); - assert(vtype_def_dict == VTYPE_PYOBJ); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, 2); need_reg_all(emit); } - emit_load_reg_with_raw_code(emit, REG_ARG_1, scope->raw_code); + emit_load_reg_with_child(emit, REG_ARG_1, scope->raw_code); ASM_CALL_IND(emit->as, MP_F_MAKE_FUNCTION_FROM_RAW_CODE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } STATIC void emit_native_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { + // make function emit_native_pre(emit); + emit_native_mov_reg_state(emit, REG_ARG_2, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_2, REG_ARG_2, OFFSETOF_OBJ_FUN_BC_CONTEXT); if (n_pos_defaults == 0 && n_kw_defaults == 0) { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); + need_reg_all(emit); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, 0); } else { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over + 2); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0x100 | n_closed_over); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, 2 + n_closed_over); + adjust_stack(emit, 2 + n_closed_over); + need_reg_all(emit); } - emit_load_reg_with_raw_code(emit, REG_ARG_1, scope->raw_code); - ASM_CALL_IND(emit->as, MP_F_MAKE_CLOSURE_FROM_RAW_CODE); + emit_load_reg_with_child(emit, REG_ARG_1, scope->raw_code); + ASM_CALL_IND(emit->as, MP_F_MAKE_FUNCTION_FROM_RAW_CODE); + + // make closure + #if REG_ARG_1 != REG_RET + ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_RET); + #endif + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); + if (n_pos_defaults != 0 || n_kw_defaults != 0) { + adjust_stack(emit, -2); + } + ASM_CALL_IND(emit->as, MP_F_NEW_CLOSURE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } diff --git a/py/emitnx86.c b/py/emitnx86.c index f0553f0682..d990ef5a9e 100644 --- a/py/emitnx86.c +++ b/py/emitnx86.c @@ -56,7 +56,7 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { [MP_F_UNPACK_EX] = 3, [MP_F_DELETE_NAME] = 1, [MP_F_DELETE_GLOBAL] = 1, - [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, + [MP_F_NEW_CLOSURE] = 3, [MP_F_ARG_CHECK_NUM_SIG] = 3, [MP_F_SETUP_CODE_STATE] = 4, [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, diff --git a/py/frozenmod.c b/py/frozenmod.c index 6cb68d1ec0..61c2f20aa1 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -57,7 +57,7 @@ extern const char mp_frozen_str_content[]; #include "py/emitglue.h" -extern const mp_raw_code_t *const mp_frozen_mpy_content[]; +extern const mp_frozen_module_t *const mp_frozen_mpy_content[]; #endif // MICROPY_MODULE_FROZEN_MPY diff --git a/py/mpconfig.h b/py/mpconfig.h index 47779a67de..5edff69dfd 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -328,6 +328,14 @@ #define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY) #endif +// Whether bytecode uses a qstr_table to map internal qstr indices in the bytecode +// to global qstr values in the runtime (behaviour when feature is enabled), or +// just stores global qstr values directly in the bytecode. This must be enabled +// if MICROPY_PERSISTENT_CODE is enabled. +#ifndef MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +#define MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE (MICROPY_PERSISTENT_CODE) +#endif + // Whether to emit x64 native code #ifndef MICROPY_EMIT_X64 #define MICROPY_EMIT_X64 (0) diff --git a/py/nativeglue.c b/py/nativeglue.c index 30e5b40061..aed6fecddf 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -300,7 +300,7 @@ const mp_fun_table_t mp_fun_table = { mp_unpack_ex, mp_delete_name, mp_delete_global, - mp_make_closure_from_raw_code, + mp_obj_new_closure, mp_arg_check_num_sig, mp_setup_code_state, mp_small_int_floor_divide, @@ -344,4 +344,8 @@ const mp_fun_table_t mp_fun_table = { &mp_stream_write_obj, }; +#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER + +const int mp_fun_table; + #endif // MICROPY_EMIT_NATIVE diff --git a/py/nativeglue.h b/py/nativeglue.h index 9d9a97b9e7..49ce665f2d 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -75,7 +75,7 @@ typedef enum { MP_F_UNPACK_EX, MP_F_DELETE_NAME, MP_F_DELETE_GLOBAL, - MP_F_MAKE_CLOSURE_FROM_RAW_CODE, + MP_F_NEW_CLOSURE, MP_F_ARG_CHECK_NUM_SIG, MP_F_SETUP_CODE_STATE, MP_F_SMALL_INT_FLOOR_DIVIDE, @@ -112,7 +112,7 @@ typedef struct _mp_fun_table_t { void (*set_store)(mp_obj_t self_in, mp_obj_t item); mp_obj_t (*list_append)(mp_obj_t self_in, mp_obj_t arg); mp_obj_t (*dict_store)(mp_obj_t self_in, mp_obj_t key, mp_obj_t value); - mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args); + mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, const mp_module_context_t *cm, const mp_obj_t *def_args); mp_obj_t (*call_function_n_kw)(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args); mp_obj_t (*call_method_n_kw)(size_t n_args, size_t n_kw, const mp_obj_t *args); mp_obj_t (*call_method_n_kw_var)(bool have_self, size_t n_args_n_kw, const mp_obj_t *args); @@ -129,7 +129,7 @@ typedef struct _mp_fun_table_t { void (*unpack_ex)(mp_obj_t seq, size_t num, mp_obj_t *items); void (*delete_name)(qstr qst); void (*delete_global)(qstr qst); - mp_obj_t (*make_closure_from_raw_code)(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args); + mp_obj_t (*new_closure)(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed); void (*arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig); void (*setup_code_state)(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); mp_int_t (*small_int_floor_divide)(mp_int_t num, mp_int_t denom); @@ -172,6 +172,12 @@ typedef struct _mp_fun_table_t { const mp_obj_fun_builtin_var_t *stream_write_obj; } mp_fun_table_t; +#if (MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME extern const mp_fun_table_t mp_fun_table; +#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER +// In dynamic-compiler mode eliminate dependency on entries in mp_fun_table. +// This only needs to be an independent pointer, content doesn't matter. +extern const int mp_fun_table; +#endif #endif // MICROPY_INCLUDED_PY_NATIVEGLUE_H diff --git a/py/obj.h b/py/obj.h index b52ee0e2f4..08a35ee6f8 100644 --- a/py/obj.h +++ b/py/obj.h @@ -781,9 +781,6 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_err #ifdef va_start mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list arg); // same fmt restrictions as above #endif -mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table); -mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table); -mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig); mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); mp_obj_t mp_obj_new_closure(mp_obj_t fun, size_t n_closed, const mp_obj_t *closed); mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); @@ -992,7 +989,6 @@ typedef struct _mp_obj_fun_builtin_var_t { } mp_obj_fun_builtin_var_t; qstr mp_obj_fun_get_name(mp_const_obj_t fun); -qstr mp_obj_code_get_name(const byte *code_info); mp_obj_t mp_identity(mp_obj_t self); MP_DECLARE_CONST_FUN_OBJ_1(mp_identity_obj); diff --git a/py/objfun.c b/py/objfun.c index 85f531c88b..3542cc0e3f 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -143,13 +143,13 @@ const mp_obj_type_t mp_type_fun_builtin_var = { /******************************************************************************/ /* byte code functions */ -qstr mp_obj_code_get_name(const byte *code_info) { +STATIC qstr mp_obj_code_get_name(const mp_obj_fun_bc_t *fun, const byte *code_info) { MP_BC_PRELUDE_SIZE_DECODE(code_info); - #if MICROPY_PERSISTENT_CODE - return code_info[0] | (code_info[1] << 8); - #else - return mp_decode_uint_value(code_info); + mp_uint_t name = mp_decode_uint_value(code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + name = fun->context->constants.qstr_table[name]; #endif + return name; } #if MICROPY_EMIT_NATIVE @@ -167,7 +167,7 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { const byte *bc = fun->bytecode; MP_BC_PRELUDE_SIG_DECODE(bc); - return mp_obj_code_get_name(bc); + return mp_obj_code_get_name(fun, bc); } #if MICROPY_CPYTHON_COMPAT @@ -209,7 +209,7 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { #define INIT_CODESTATE(code_state, _fun_bc, _n_state, n_args, n_kw, args) \ code_state->fun_bc = _fun_bc; \ - code_state->ip = 0; \ + code_state->ip = _fun_bc->bytecode; \ code_state->n_state = _n_state; \ mp_setup_code_state(code_state, n_args, n_kw, args); \ code_state->old_globals = mp_globals_get(); @@ -240,7 +240,7 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(self->globals); + mp_globals_set(self->context->module.globals); return code_state; } @@ -285,7 +285,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(self->globals); + mp_globals_set(self->context->module.globals); mp_vm_return_kind_t vm_return_kind = mp_execute_bytecode(code_state, MP_OBJ_NULL); mp_globals_set(code_state->old_globals); @@ -358,7 +358,7 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } if (attr == MP_QSTR___globals__) { mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - dest[0] = MP_OBJ_FROM_PTR(self->globals); + dest[0] = MP_OBJ_FROM_PTR(self->context->module.globals); } } #endif @@ -377,25 +377,29 @@ const mp_obj_type_t mp_type_fun_bc = { #endif }; -mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args_in, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table) { +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *context, struct _mp_raw_code_t *const *child_table) { size_t n_def_args = 0; size_t n_extra_args = 0; - mp_obj_tuple_t *def_args = MP_OBJ_TO_PTR(def_args_in); - if (def_args_in != MP_OBJ_NULL) { - assert(mp_obj_is_type(def_args_in, &mp_type_tuple)); - n_def_args = def_args->len; - n_extra_args = def_args->len; + mp_obj_tuple_t *def_pos_args = NULL; + mp_obj_t def_kw_args = MP_OBJ_NULL; + if (def_args != NULL && def_args[0] != MP_OBJ_NULL) { + assert(mp_obj_is_type(def_args[0], &mp_type_tuple)); + def_pos_args = MP_OBJ_TO_PTR(def_args[0]); + n_def_args = def_pos_args->len; + n_extra_args = def_pos_args->len; } - if (def_kw_args != MP_OBJ_NULL) { + if (def_args != NULL && def_args[1] != MP_OBJ_NULL) { + assert(mp_obj_is_type(def_args[1], &mp_type_dict)); + def_kw_args = def_args[1]; n_extra_args += 1; } mp_obj_fun_bc_t *o = m_new_obj_var(mp_obj_fun_bc_t, mp_obj_t, n_extra_args); o->base.type = &mp_type_fun_bc; - o->globals = mp_globals_get(); o->bytecode = code; - o->const_table = const_table; - if (def_args != NULL) { - memcpy(o->extra_args, def_args->items, n_def_args * sizeof(mp_obj_t)); + o->context = context; + o->child_table = child_table; + if (def_pos_args != NULL) { + memcpy(o->extra_args, def_pos_args->items, n_def_args * sizeof(mp_obj_t)); } if (def_kw_args != MP_OBJ_NULL) { o->extra_args[n_def_args] = def_kw_args; @@ -423,8 +427,8 @@ STATIC const mp_obj_type_t mp_type_fun_native = { .unary_op = mp_generic_unary_op, }; -mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table) { - mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args_in, def_kw_args, (const byte *)fun_data, const_table)); +mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { + mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); o->base.type = &mp_type_fun_native; return MP_OBJ_FROM_PTR(o); } diff --git a/py/objfun.h b/py/objfun.h index 771bf31a95..9de15b8841 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -26,13 +26,14 @@ #ifndef MICROPY_INCLUDED_PY_OBJFUN_H #define MICROPY_INCLUDED_PY_OBJFUN_H +#include "py/bc.h" #include "py/obj.h" typedef struct _mp_obj_fun_bc_t { mp_obj_base_t base; - mp_obj_dict_t *globals; // the context within which this function was defined - const byte *bytecode; // bytecode for the function - const mp_uint_t *const_table; // constant table + const mp_module_context_t *context; // context within which this function was defined + struct _mp_raw_code_t *const *child_table; // table of children + const byte *bytecode; // bytecode for the function #if MICROPY_PY_SYS_SETTRACE const struct _mp_raw_code_t *rc; #endif @@ -42,6 +43,9 @@ typedef struct _mp_obj_fun_bc_t { mp_obj_t extra_args[]; } mp_obj_fun_bc_t; +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig); void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); #endif // MICROPY_INCLUDED_PY_OBJFUN_H diff --git a/py/objgenerator.c b/py/objgenerator.c index 784310092e..cbe79e66a5 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -65,7 +65,7 @@ STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; - o->code_state.ip = 0; + o->code_state.ip = self_fun->bytecode; o->code_state.n_state = n_state; mp_setup_code_state(&o->code_state, n_args, n_kw, args); return MP_OBJ_FROM_PTR(o); @@ -91,14 +91,18 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k // The state for a native generating function is held in the same struct as a bytecode function mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in); - // Determine start of prelude, and extract n_state from it + // Determine start of prelude. uintptr_t prelude_offset = ((uintptr_t *)self_fun->bytecode)[0]; #if MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ // Prelude is in bytes object in const_table, at index prelude_offset - mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->const_table[prelude_offset]); - prelude_offset = (const byte *)prelude_bytes->data - self_fun->bytecode; + mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->context->constants.obj_table[prelude_offset]); + const uint8_t *prelude_ptr = prelude_bytes->data; + #else + const uint8_t *prelude_ptr = self_fun->bytecode + prelude_offset; #endif - const uint8_t *ip = self_fun->bytecode + prelude_offset; + + // Extract n_state from the prelude. + const uint8_t *ip = prelude_ptr; size_t n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args; MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args); size_t n_exc_stack = 0; @@ -111,7 +115,7 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k // Parse the input arguments and set up the code state o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; - o->code_state.ip = (const byte *)prelude_offset; + o->code_state.ip = prelude_ptr; o->code_state.n_state = n_state; mp_setup_code_state(&o->code_state, n_args, n_kw, args); @@ -184,7 +188,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ // Set up the correct globals context for the generator and execute it self->code_state.old_globals = mp_globals_get(); - mp_globals_set(self->code_state.fun_bc->globals); + mp_globals_set(self->code_state.fun_bc->context->module.globals); mp_vm_return_kind_t ret_kind; diff --git a/py/objmodule.c b/py/objmodule.c index 9be4bad92c..4b04f7ca9c 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -112,6 +112,7 @@ const mp_obj_type_t mp_type_module = { .attr = module_attr, }; +#include "py/bc.h" mp_obj_t mp_obj_new_module(qstr module_name) { mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; mp_map_elem_t *el = mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); @@ -122,12 +123,12 @@ mp_obj_t mp_obj_new_module(qstr module_name) { } // create new module object - mp_obj_module_t *o = m_new_obj(mp_obj_module_t); - o->base.type = &mp_type_module; - o->globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); + mp_module_context_t *o = m_new_obj(mp_module_context_t); + o->module.base.type = &mp_type_module; + o->module.globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); // store __name__ entry in the module - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->module.globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); // store the new module into the slot in the global dict holding all modules el->value = MP_OBJ_FROM_PTR(o); diff --git a/py/persistentcode.c b/py/persistentcode.c index ac523990c1..b473f18308 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -63,57 +63,6 @@ STATIC int mp_small_int_bits(void) { } #endif -#define QSTR_WINDOW_SIZE (32) - -typedef struct _qstr_window_t { - uint16_t idx; // indexes the head of the window - uint16_t window[QSTR_WINDOW_SIZE]; -} qstr_window_t; - -// Push a qstr to the head of the window, and the tail qstr is overwritten -STATIC void qstr_window_push(qstr_window_t *qw, qstr qst) { - qw->idx = (qw->idx + 1) % QSTR_WINDOW_SIZE; - qw->window[qw->idx] = qst; -} - -// Pull an existing qstr from within the window to the head of the window -STATIC qstr qstr_window_pull(qstr_window_t *qw, size_t idx) { - qstr qst = qw->window[idx]; - if (idx > qw->idx) { - memmove(&qw->window[idx], &qw->window[idx + 1], (QSTR_WINDOW_SIZE - idx - 1) * sizeof(uint16_t)); - qw->window[QSTR_WINDOW_SIZE - 1] = qw->window[0]; - idx = 0; - } - memmove(&qw->window[idx], &qw->window[idx + 1], (qw->idx - idx) * sizeof(uint16_t)); - qw->window[qw->idx] = qst; - return qst; -} - -#if MICROPY_PERSISTENT_CODE_LOAD - -// Access a qstr at the given index, relative to the head of the window (0=head) -STATIC qstr qstr_window_access(qstr_window_t *qw, size_t idx) { - return qstr_window_pull(qw, (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE); -} - -#endif - -#if MICROPY_PERSISTENT_CODE_SAVE - -// Insert a qstr at the head of the window, either by pulling an existing one or pushing a new one -STATIC size_t qstr_window_insert(qstr_window_t *qw, qstr qst) { - for (size_t idx = 0; idx < QSTR_WINDOW_SIZE; ++idx) { - if (qw->window[idx] == qst) { - qstr_window_pull(qw, idx); - return (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE; - } - } - qstr_window_push(qw, qst); - return QSTR_WINDOW_SIZE; -} - -#endif - typedef struct _bytecode_prelude_t { uint n_state; uint n_exc_stack; @@ -124,23 +73,6 @@ typedef struct _bytecode_prelude_t { uint code_info_size; } bytecode_prelude_t; -// ip will point to start of opcodes -// return value will point to simple_name, source_file qstrs -STATIC byte *extract_prelude(const byte **ip, bytecode_prelude_t *prelude) { - MP_BC_PRELUDE_SIG_DECODE(*ip); - prelude->n_state = n_state; - prelude->n_exc_stack = n_exc_stack; - prelude->scope_flags = scope_flags; - prelude->n_pos_args = n_pos_args; - prelude->n_kwonly_args = n_kwonly_args; - prelude->n_def_pos_args = n_def_pos_args; - MP_BC_PRELUDE_SIZE_DECODE(*ip); - byte *ip_info = (byte *)*ip; - *ip += n_info; - *ip += n_cell; - return ip_info; -} - #endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #if MICROPY_PERSISTENT_CODE_LOAD @@ -148,13 +80,14 @@ STATIC byte *extract_prelude(const byte **ip, bytecode_prelude_t *prelude) { #include "py/parsenum.h" STATIC int read_byte(mp_reader_t *reader); -STATIC size_t read_uint(mp_reader_t *reader, byte **out); +STATIC size_t read_uint(mp_reader_t *reader); #if MICROPY_EMIT_MACHINE_CODE typedef struct _reloc_info_t { mp_reader_t *reader; - mp_uint_t *const_table; + uint8_t *rodata; + uint8_t *bss; } reloc_info_t; #if MICROPY_EMIT_THUMB @@ -197,13 +130,13 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { while ((op = read_byte(ri->reader)) != 0xff) { if (op & 1) { // Point to new location to make adjustments - size_t addr = read_uint(ri->reader, NULL); + size_t addr = read_uint(ri->reader); if ((addr & 1) == 0) { // Point to somewhere in text addr_to_adjust = &((uintptr_t *)text)[addr >> 1]; } else { // Point to somewhere in rodata - addr_to_adjust = &((uintptr_t *)ri->const_table[1])[addr >> 1]; + addr_to_adjust = &((uintptr_t *)ri->rodata)[addr >> 1]; } } op >>= 1; @@ -212,15 +145,18 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { if (op <= 5) { if (op & 1) { // Read in number of adjustments to make - n = read_uint(ri->reader, NULL); + n = read_uint(ri->reader); } op >>= 1; if (op == 0) { // Destination is text dest = reloc_text; + } else if (op == 1) { + // Destination is rodata + dest = (uintptr_t)ri->rodata; } else { - // Destination is rodata (op=1) or bss (op=1 if no rodata, else op=2) - dest = ri->const_table[op]; + // Destination is bss + dest = (uintptr_t)ri->bss; } } else if (op == 6) { // Destination is mp_fun_table itself @@ -247,14 +183,10 @@ STATIC void read_bytes(mp_reader_t *reader, byte *buf, size_t len) { } } -STATIC size_t read_uint(mp_reader_t *reader, byte **out) { +STATIC size_t read_uint(mp_reader_t *reader) { size_t unum = 0; for (;;) { byte b = reader->readbyte(reader->data); - if (out != NULL) { - **out = b; - ++*out; - } unum = (unum << 7) | (b & 0x7f); if ((b & 0x80) == 0) { break; @@ -263,35 +195,41 @@ STATIC size_t read_uint(mp_reader_t *reader, byte **out) { return unum; } -STATIC qstr load_qstr(mp_reader_t *reader, qstr_window_t *qw) { - size_t len = read_uint(reader, NULL); - if (len == 0) { - // static qstr - return read_byte(reader); - } +STATIC qstr load_qstr(mp_reader_t *reader) { + size_t len = read_uint(reader); if (len & 1) { - // qstr in window - return qstr_window_access(qw, len >> 1); + // static qstr + return len >> 1; } len >>= 1; char *str = m_new(char, len); read_bytes(reader, (byte *)str, len); + read_byte(reader); // read and discard null terminator qstr qst = qstr_from_strn(str, len); m_del(char, str, len); - qstr_window_push(qw, qst); return qst; } STATIC mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); + #if MICROPY_EMIT_MACHINE_CODE + if (obj_type == 't') { + return MP_OBJ_FROM_PTR(&mp_fun_table); + } else + #endif if (obj_type == 'e') { return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); } else { - size_t len = read_uint(reader, NULL); + size_t len = read_uint(reader); + if (len == 0 && obj_type == 'b') { + read_byte(reader); // skip null terminator + return mp_const_empty_bytes; + } vstr_t vstr; vstr_init_len(&vstr, len); read_bytes(reader, (byte *)vstr.buf, len); if (obj_type == 's' || obj_type == 'b') { + read_byte(reader); // skip null terminator return mp_obj_new_str_from_vstr(obj_type == 's' ? &mp_type_str : &mp_type_bytes, &vstr); } else if (obj_type == 'i') { return mp_parse_num_integer(vstr.buf, vstr.len, 10, NULL); @@ -302,58 +240,12 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) { } } -STATIC void load_prelude_qstrs(mp_reader_t *reader, qstr_window_t *qw, byte *ip) { - qstr simple_name = load_qstr(reader, qw); - ip[0] = simple_name; - ip[1] = simple_name >> 8; - qstr source_file = load_qstr(reader, qw); - ip[2] = source_file; - ip[3] = source_file >> 8; -} - -STATIC void load_prelude(mp_reader_t *reader, qstr_window_t *qw, byte **ip, bytecode_prelude_t *prelude) { - // Read in the prelude header - byte *ip_read = *ip; - read_uint(reader, &ip_read); // read in n_state/etc (is effectively a var-uint) - read_uint(reader, &ip_read); // read in n_info/n_cell (is effectively a var-uint) - - // Prelude header has been read into *ip, now decode and extract values from it - extract_prelude((const byte **)ip, prelude); - - // Load qstrs in prelude - load_prelude_qstrs(reader, qw, ip_read); - ip_read += 4; - - // Read remaining code info - read_bytes(reader, ip_read, *ip - ip_read); -} - -STATIC void load_bytecode(mp_reader_t *reader, qstr_window_t *qw, byte *ip, byte *ip_top) { - while (ip < ip_top) { - *ip = read_byte(reader); - size_t sz; - uint f = mp_opcode_format(ip, &sz, false); - ++ip; - --sz; - if (f == MP_BC_FORMAT_QSTR) { - qstr qst = load_qstr(reader, qw); - *ip++ = qst; - *ip++ = qst >> 8; - sz -= 2; - } else if (f == MP_BC_FORMAT_VAR_UINT) { - while ((*ip++ = read_byte(reader)) & 0x80) { - } - } - read_bytes(reader, ip, sz); - ip += sz; - } -} - -STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { +STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { // Load function kind and data length - size_t kind_len = read_uint(reader, NULL); + size_t kind_len = read_uint(reader); int kind = (kind_len & 3) + MP_CODE_BYTECODE; - size_t fun_data_len = kind_len >> 2; + bool has_children = !!(kind_len & 4); + size_t fun_data_len = kind_len >> 3; #if !MICROPY_EMIT_MACHINE_CODE if (kind != MP_CODE_BYTECODE) { @@ -362,23 +254,18 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { #endif uint8_t *fun_data = NULL; - bytecode_prelude_t prelude = {0}; #if MICROPY_EMIT_MACHINE_CODE size_t prelude_offset = 0; - mp_uint_t type_sig = 0; - size_t n_qstr_link = 0; + mp_uint_t native_scope_flags = 0; + mp_uint_t native_n_pos_args = 0; + mp_uint_t native_type_sig = 0; #endif if (kind == MP_CODE_BYTECODE) { // Allocate memory for the bytecode fun_data = m_new(uint8_t, fun_data_len); - - // Load prelude - byte *ip = fun_data; - load_prelude(reader, qw, &ip, &prelude); - // Load bytecode - load_bytecode(reader, qw, ip, fun_data + fun_data_len); + read_bytes(reader, fun_data, fun_data_len); #if MICROPY_EMIT_MACHINE_CODE } else { @@ -389,10 +276,10 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) { // Parse qstr link table and link native code - n_qstr_link = read_uint(reader, NULL); + size_t n_qstr_link = read_uint(reader); for (size_t i = 0; i < n_qstr_link; ++i) { - size_t off = read_uint(reader, NULL); - qstr qst = load_qstr(reader, qw); + size_t off = read_uint(reader); + qstr qst = load_qstr(reader); uint8_t *dest = fun_data + (off >> 2); if ((off & 3) == 0) { // Generic 16-bit link @@ -409,113 +296,92 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { } if (kind == MP_CODE_NATIVE_PY) { - // Extract prelude for later use - prelude_offset = read_uint(reader, NULL); + // Read prelude offset within fun_data, and extract scope flags. + prelude_offset = read_uint(reader); const byte *ip = fun_data + prelude_offset; - byte *ip_info = extract_prelude(&ip, &prelude); - // Load qstrs in prelude - load_prelude_qstrs(reader, qw, ip_info); + MP_BC_PRELUDE_SIG_DECODE(ip); + native_scope_flags = scope_flags; } else { - // Load basic scope info for viper and asm - prelude.scope_flags = read_uint(reader, NULL); - prelude.n_pos_args = 0; - prelude.n_kwonly_args = 0; + // Load basic scope info for viper and asm. + native_scope_flags = read_uint(reader); if (kind == MP_CODE_NATIVE_ASM) { - prelude.n_pos_args = read_uint(reader, NULL); - type_sig = read_uint(reader, NULL); + native_n_pos_args = read_uint(reader); + native_type_sig = read_uint(reader); } } #endif } - size_t n_obj = 0; - size_t n_raw_code = 0; - mp_uint_t *const_table = NULL; + size_t n_children = 0; + mp_raw_code_t **children = NULL; - if (kind != MP_CODE_NATIVE_ASM) { - // Load constant table for bytecode, native and viper - - // Number of entries in constant table - n_obj = read_uint(reader, NULL); - n_raw_code = read_uint(reader, NULL); - - // Allocate constant table - size_t n_alloc = prelude.n_pos_args + prelude.n_kwonly_args + n_obj + n_raw_code; - #if MICROPY_EMIT_MACHINE_CODE - if (kind != MP_CODE_BYTECODE) { - ++n_alloc; // additional entry for mp_fun_table - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { - ++n_alloc; // additional entry for rodata - } - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) { - ++n_alloc; // additional entry for BSS - } - } - #endif - - const_table = m_new(mp_uint_t, n_alloc); - mp_uint_t *ct = const_table; - - // Load function argument names (initial entries in const_table) - // (viper has n_pos_args=n_kwonly_args=0 so doesn't load any qstrs here) - for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { - *ct++ = (mp_uint_t)MP_OBJ_NEW_QSTR(load_qstr(reader, qw)); + #if MICROPY_EMIT_MACHINE_CODE + // Load optional BSS/rodata for viper. + uint8_t *rodata = NULL; + uint8_t *bss = NULL; + if (kind == MP_CODE_NATIVE_VIPER) { + size_t rodata_size = 0; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { + rodata_size = read_uint(reader); } - #if MICROPY_EMIT_MACHINE_CODE - if (kind != MP_CODE_BYTECODE) { - // Populate mp_fun_table entry - *ct++ = (mp_uint_t)(uintptr_t)&mp_fun_table; + size_t bss_size = 0; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERBSS) { + bss_size = read_uint(reader); + } - // Allocate and load rodata if needed - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { - size_t size = read_uint(reader, NULL); - uint8_t *rodata = m_new(uint8_t, size); - read_bytes(reader, rodata, size); - *ct++ = (uintptr_t)rodata; + if (rodata_size + bss_size != 0) { + bss_size = (uintptr_t)MP_ALIGN(bss_size, sizeof(uintptr_t)); + uint8_t *data = m_new0(uint8_t, bss_size + rodata_size); + bss = data; + rodata = bss + bss_size; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { + read_bytes(reader, rodata, rodata_size); } - // Allocate BSS if needed - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) { - size_t size = read_uint(reader, NULL); - uint8_t *bss = m_new0(uint8_t, size); - *ct++ = (uintptr_t)bss; - } + // Viper code with BSS/rodata should not have any children. + // Reuse the children pointer to reference the BSS/rodata + // memory so that it is not reclaimed by the GC. + assert(!has_children); + children = (void *)data; } - #endif + } + #endif - // Load constant objects and raw code children - for (size_t i = 0; i < n_obj; ++i) { - *ct++ = (mp_uint_t)load_obj(reader); - } - for (size_t i = 0; i < n_raw_code; ++i) { - *ct++ = (mp_uint_t)(uintptr_t)load_raw_code(reader, qw); + // Load children if any. + if (has_children) { + n_children = read_uint(reader); + children = m_new(mp_raw_code_t *, n_children); + for (size_t i = 0; i < n_children; ++i) { + children[i] = load_raw_code(reader); } } // Create raw_code and return it mp_raw_code_t *rc = mp_emit_glue_new_raw_code(); if (kind == MP_CODE_BYTECODE) { + const byte *ip = fun_data; + MP_BC_PRELUDE_SIG_DECODE(ip); // Assign bytecode to raw code object mp_emit_glue_assign_bytecode(rc, fun_data, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS fun_data_len, #endif - const_table, + children, #if MICROPY_PERSISTENT_CODE_SAVE - n_obj, n_raw_code, + n_children, #endif - prelude.scope_flags); + scope_flags); #if MICROPY_EMIT_MACHINE_CODE } else { // Relocate and commit code to executable address space - reloc_info_t ri = {reader, const_table}; + reloc_info_t ri = {reader, rodata, bss}; #if defined(MP_PLAT_COMMIT_EXEC) - void *opt_ri = (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; + void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri); #else - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { #if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE // If native code needs relocations then it's not guaranteed that a pointer to // the head of `buf` (containing the machine code) will be retained for the GC @@ -534,26 +400,27 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { // Assign native code to raw code object mp_emit_glue_assign_native(rc, kind, - fun_data, fun_data_len, const_table, + fun_data, fun_data_len, + children, #if MICROPY_PERSISTENT_CODE_SAVE + n_children, prelude_offset, - n_obj, n_raw_code, - n_qstr_link, NULL, + 0, NULL, #endif - prelude.n_pos_args, prelude.scope_flags, type_sig); + native_scope_flags, native_n_pos_args, native_type_sig + ); #endif } return rc; } -mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) { byte header[4]; read_bytes(reader, header, sizeof(header)); if (header[0] != 'M' || header[1] != MPY_VERSION || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS - || header[3] > mp_small_int_bits() - || read_uint(reader, NULL) > QSTR_WINDOW_SIZE) { + || header[3] > mp_small_int_bits()) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) { @@ -562,25 +429,49 @@ mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy arch")); } } - qstr_window_t qw; - qw.idx = 0; - mp_raw_code_t *rc = load_raw_code(reader, &qw); + + size_t n_qstr = read_uint(reader); + size_t n_obj = read_uint(reader); + mp_module_context_alloc_tables(context, n_qstr, n_obj); + + // Load qstrs. + for (size_t i = 0; i < n_qstr; ++i) { + context->constants.qstr_table[i] = load_qstr(reader); + } + + // Load constant objects. + for (size_t i = 0; i < n_obj; ++i) { + context->constants.obj_table[i] = load_obj(reader); + } + + // Load top-level module. + mp_compiled_module_t cm2; + cm2.rc = load_raw_code(reader); + cm2.context = context; + + #if MICROPY_PERSISTENT_CODE_SAVE + cm2.has_native = MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE; + cm2.n_qstr = n_qstr; + cm2.n_obj = n_obj; + #endif + reader->close(reader->data); - return rc; + + return cm2; } -mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len) { +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *context) { mp_reader_t reader; mp_reader_new_mem(&reader, buf, len, 0); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, context); } #if MICROPY_HAS_FILE_READER -mp_raw_code_t *mp_raw_code_load_file(const char *filename) { +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *context) { mp_reader_t reader; mp_reader_new_file(&reader, filename); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, context); } #endif // MICROPY_HAS_FILE_READER @@ -607,26 +498,25 @@ STATIC void mp_print_uint(mp_print_t *print, size_t n) { print->print_strn(print->data, (char *)p, buf + sizeof(buf) - p); } -STATIC void save_qstr(mp_print_t *print, qstr_window_t *qw, qstr qst) { +STATIC void save_qstr(mp_print_t *print, qstr qst) { if (qst <= QSTR_LAST_STATIC) { // encode static qstr - byte buf[2] = {0, qst & 0xff}; - mp_print_bytes(print, buf, 2); - return; - } - size_t idx = qstr_window_insert(qw, qst); - if (idx < QSTR_WINDOW_SIZE) { - // qstr found in window, encode index to it - mp_print_uint(print, idx << 1 | 1); + mp_print_uint(print, qst << 1 | 1); return; } size_t len; const byte *str = qstr_data(qst, &len); mp_print_uint(print, len << 1); - mp_print_bytes(print, str, len); + mp_print_bytes(print, str, len + 1); // +1 to store null terminator } STATIC void save_obj(mp_print_t *print, mp_obj_t o) { + #if MICROPY_EMIT_MACHINE_CODE + if (o == MP_OBJ_FROM_PTR(&mp_fun_table)) { + byte obj_type = 't'; + mp_print_bytes(print, &obj_type, 1); + } else + #endif if (mp_obj_is_str_or_bytes(o)) { byte obj_type; if (mp_obj_is_str(o)) { @@ -638,7 +528,7 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) { const char *str = mp_obj_str_get_data(o, &len); mp_print_bytes(print, &obj_type, 1); mp_print_uint(print, len); - mp_print_bytes(print, (const byte *)str, len); + mp_print_bytes(print, (const byte *)str, len + 1); // +1 to store null terminator } else if (MP_OBJ_TO_PTR(o) == &mp_const_ellipsis_obj) { byte obj_type = 'e'; mp_print_bytes(print, &obj_type, 1); @@ -667,136 +557,45 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) { } } -STATIC void save_prelude_qstrs(mp_print_t *print, qstr_window_t *qw, const byte *ip) { - save_qstr(print, qw, ip[0] | (ip[1] << 8)); // simple_name - save_qstr(print, qw, ip[2] | (ip[3] << 8)); // source_file -} - -STATIC void save_bytecode(mp_print_t *print, qstr_window_t *qw, const byte *ip, const byte *ip_top) { - while (ip < ip_top) { - size_t sz; - uint f = mp_opcode_format(ip, &sz, true); - if (f == MP_BC_FORMAT_QSTR) { - mp_print_bytes(print, ip, 1); - qstr qst = ip[1] | (ip[2] << 8); - save_qstr(print, qw, qst); - ip += 3; - sz -= 3; - } - mp_print_bytes(print, ip, sz); - ip += sz; - } -} - -STATIC void save_raw_code(mp_print_t *print, mp_raw_code_t *rc, qstr_window_t *qstr_window) { +STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { // Save function kind and data length - mp_print_uint(print, (rc->fun_data_len << 2) | (rc->kind - MP_CODE_BYTECODE)); + mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE)); - bytecode_prelude_t prelude; + // Save function code. + mp_print_bytes(print, rc->fun_data, rc->fun_data_len); - if (rc->kind == MP_CODE_BYTECODE) { - // Extract prelude - const byte *ip = rc->fun_data; - const byte *ip_info = extract_prelude(&ip, &prelude); - - // Save prelude - mp_print_bytes(print, rc->fun_data, ip_info - (const byte *)rc->fun_data); - save_prelude_qstrs(print, qstr_window, ip_info); - ip_info += 4; - mp_print_bytes(print, ip_info, ip - ip_info); - - // Save bytecode - const byte *ip_top = (const byte *)rc->fun_data + rc->fun_data_len; - save_bytecode(print, qstr_window, ip, ip_top); #if MICROPY_EMIT_MACHINE_CODE - } else { - // Save native code - mp_print_bytes(print, rc->fun_data, rc->fun_data_len); - - if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) { - // Save qstr link table for native code - mp_print_uint(print, rc->n_qstr); - for (size_t i = 0; i < rc->n_qstr; ++i) { - mp_print_uint(print, rc->qstr_link[i].off); - save_qstr(print, qstr_window, rc->qstr_link[i].qst); - } + if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) { + // Save qstr link table for native code + mp_print_uint(print, rc->n_qstr); + for (size_t i = 0; i < rc->n_qstr; ++i) { + mp_print_uint(print, rc->qstr_link[i].off); + save_qstr(print, rc->qstr_link[i].qst); } + } - if (rc->kind == MP_CODE_NATIVE_PY) { - // Save prelude size - mp_print_uint(print, rc->prelude_offset); - - // Extract prelude and save qstrs in prelude - const byte *ip = (const byte *)rc->fun_data + rc->prelude_offset; - const byte *ip_info = extract_prelude(&ip, &prelude); - save_prelude_qstrs(print, qstr_window, ip_info); - } else { - // Save basic scope info for viper and asm - mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG); - prelude.n_pos_args = 0; - prelude.n_kwonly_args = 0; - if (rc->kind == MP_CODE_NATIVE_ASM) { - mp_print_uint(print, rc->n_pos_args); - mp_print_uint(print, rc->type_sig); - } + if (rc->kind == MP_CODE_NATIVE_PY) { + // Save prelude size + mp_print_uint(print, rc->prelude_offset); + } else if (rc->kind == MP_CODE_NATIVE_VIPER || rc->kind == MP_CODE_NATIVE_ASM) { + // Save basic scope info for viper and asm + mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG); + if (rc->kind == MP_CODE_NATIVE_ASM) { + mp_print_uint(print, rc->n_pos_args); + mp_print_uint(print, rc->type_sig); } + } #endif - } - if (rc->kind != MP_CODE_NATIVE_ASM) { - // Save constant table for bytecode, native and viper - - // Number of entries in constant table - mp_print_uint(print, rc->n_obj); - mp_print_uint(print, rc->n_raw_code); - - const mp_uint_t *const_table = rc->const_table; - - // Save function argument names (initial entries in const_table) - // (viper has n_pos_args=n_kwonly_args=0 so doesn't save any qstrs here) - for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { - mp_obj_t o = (mp_obj_t)*const_table++; - save_qstr(print, qstr_window, MP_OBJ_QSTR_VALUE(o)); - } - - if (rc->kind != MP_CODE_BYTECODE) { - // Skip saving mp_fun_table entry - ++const_table; - } - - // Save constant objects and raw code children - for (size_t i = 0; i < rc->n_obj; ++i) { - save_obj(print, (mp_obj_t)*const_table++); - } - for (size_t i = 0; i < rc->n_raw_code; ++i) { - save_raw_code(print, (mp_raw_code_t *)(uintptr_t)*const_table++, qstr_window); + if (rc->n_children) { + mp_print_uint(print, rc->n_children); + for (size_t i = 0; i < rc->n_children; ++i) { + save_raw_code(print, rc->children[i]); } } } -STATIC bool mp_raw_code_has_native(mp_raw_code_t *rc) { - if (rc->kind != MP_CODE_BYTECODE) { - return true; - } - - const byte *ip = rc->fun_data; - bytecode_prelude_t prelude; - extract_prelude(&ip, &prelude); - - const mp_uint_t *const_table = rc->const_table - + prelude.n_pos_args + prelude.n_kwonly_args - + rc->n_obj; - - for (size_t i = 0; i < rc->n_raw_code; ++i) { - if (mp_raw_code_has_native((mp_raw_code_t *)(uintptr_t)*const_table++)) { - return true; - } - } - - return false; -} - -void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { +void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { // header contains: // byte 'M' // byte version @@ -813,16 +612,27 @@ void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { mp_small_int_bits(), #endif }; - if (mp_raw_code_has_native(rc)) { + if (cm->has_native) { header[2] |= MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC); } mp_print_bytes(print, header, sizeof(header)); - mp_print_uint(print, QSTR_WINDOW_SIZE); - qstr_window_t qw; - qw.idx = 0; - memset(qw.window, 0, sizeof(qw.window)); - save_raw_code(print, rc, &qw); + // Number of entries in constant table. + mp_print_uint(print, cm->n_qstr); + mp_print_uint(print, cm->n_obj); + + // Save qstrs. + for (size_t i = 0; i < cm->n_qstr; ++i) { + save_qstr(print, cm->context->constants.qstr_table[i]); + } + + // Save constant objects. + for (size_t i = 0; i < cm->n_obj; ++i) { + save_obj(print, (mp_obj_t)cm->context->constants.obj_table[i]); + } + + // Save outer raw code, which will save all its child raw codes. + save_raw_code(print, cm->rc); } #if MICROPY_PERSISTENT_CODE_SAVE_FILE @@ -839,12 +649,12 @@ STATIC void fd_print_strn(void *env, const char *str, size_t len) { (void)ret; } -void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename) { +void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename) { MP_THREAD_GIL_EXIT(); int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); MP_THREAD_GIL_ENTER(); mp_print_t fd_print = {(void *)(intptr_t)fd, fd_print_strn}; - mp_raw_code_save(rc, &fd_print); + mp_raw_code_save(cm, &fd_print); MP_THREAD_GIL_EXIT(); close(fd); MP_THREAD_GIL_ENTER(); diff --git a/py/persistentcode.h b/py/persistentcode.h index 94fc3bf2d3..1991ba26ff 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -31,7 +31,7 @@ #include "py/emitglue.h" // The current version of .mpy files -#define MPY_VERSION 5 +#define MPY_VERSION 6 // Macros to encode/decode flags to/from the feature byte #define MPY_FEATURE_ENCODE_FLAGS(flags) (flags) @@ -102,12 +102,12 @@ enum { MP_NATIVE_ARCH_XTENSAWIN, }; -mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader); -mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len); -mp_raw_code_t *mp_raw_code_load_file(const char *filename); +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *ctx); -void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print); -void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename); +void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); +void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename); void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text); diff --git a/py/profile.c b/py/profile.c index d0ac99e074..4e23e9eac4 100644 --- a/py/profile.c +++ b/py/profile.c @@ -27,14 +27,16 @@ #include "py/profile.h" #include "py/bc0.h" #include "py/gc.h" +#include "py/objfun.h" #if MICROPY_PY_SYS_SETTRACE #define prof_trace_cb MP_STATE_THREAD(prof_trace_callback) +#define QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) STATIC uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { const mp_bytecode_prelude_t *prelude = &rc->prelude; - return mp_bytecode_get_source_line(prelude->line_info, bc); + return mp_bytecode_get_source_line(prelude->line_info, prelude->line_info_top, bc); } void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude) { @@ -50,13 +52,14 @@ void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelud MP_BC_PRELUDE_SIZE_DECODE(ip); - prelude->line_info = ip + 4; + prelude->line_info_top = ip + n_info; prelude->opcodes = ip + n_info + n_cell; - qstr block_name = ip[0] | (ip[1] << 8); - qstr source_file = ip[2] | (ip[3] << 8); - prelude->qstr_block_name = block_name; - prelude->qstr_source_file = source_file; + prelude->qstr_block_name_idx = mp_decode_uint_value(ip); + for (size_t i = 0; i < 1 + n_pos_args + n_kwonly_args; ++i) { + ip = mp_decode_uint_skip(ip); + } + prelude->line_info = ip; } /******************************************************************************/ @@ -69,22 +72,19 @@ STATIC void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t k const mp_bytecode_prelude_t *prelude = &rc->prelude; mp_printf(print, "", - prelude->qstr_block_name, + QSTR_MAP(o->context, prelude->qstr_block_name_idx), o, - prelude->qstr_source_file, + QSTR_MAP(o->context, 0), rc->line_of_definition ); } -STATIC mp_obj_tuple_t *code_consts(const mp_raw_code_t *rc) { - const mp_bytecode_prelude_t *prelude = &rc->prelude; - int start = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj; - int stop = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj + rc->n_raw_code; - mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(stop - start + 1, NULL)); +STATIC mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { + mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); size_t const_no = 0; - for (int i = start; i < stop; ++i) { - mp_obj_t code = mp_obj_new_code((const mp_raw_code_t *)MP_OBJ_TO_PTR(rc->const_table[i])); + for (size_t i = 0; i < rc->n_children; ++i) { + mp_obj_t code = mp_obj_new_code(context, rc->children[i]); if (code == MP_OBJ_NULL) { m_malloc_fail(sizeof(mp_obj_code_t)); } @@ -149,16 +149,16 @@ STATIC void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { ); break; case MP_QSTR_co_consts: - dest[0] = MP_OBJ_FROM_PTR(code_consts(rc)); + dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); break; case MP_QSTR_co_filename: - dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_source_file); + dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, 0)); break; case MP_QSTR_co_firstlineno: dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); break; case MP_QSTR_co_name: - dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_block_name); + dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, prelude->qstr_block_name_idx)); break; case MP_QSTR_co_names: dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); @@ -180,12 +180,13 @@ const mp_obj_type_t mp_type_settrace_codeobj = { .attr = code_attr, }; -mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc) { +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc) { mp_obj_code_t *o = m_new_obj_maybe(mp_obj_code_t); if (o == NULL) { return MP_OBJ_NULL; } o->base.type = &mp_type_settrace_codeobj; + o->context = context; o->rc = rc; o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? o->lnotab = MP_OBJ_NULL; @@ -204,9 +205,9 @@ STATIC void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_printf(print, "", frame, - prelude->qstr_source_file, + QSTR_MAP(code->context, 0), frame->lineno, - prelude->qstr_block_name + QSTR_MAP(code->context, prelude->qstr_block_name_idx) ); } @@ -229,7 +230,7 @@ STATIC void frame_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { dest[0] = MP_OBJ_FROM_PTR(o->code); break; case MP_QSTR_f_globals: - dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->globals); + dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->context->module.globals); break; case MP_QSTR_f_lasti: dest[0] = MP_OBJ_NEW_SMALL_INT(o->lasti); @@ -258,7 +259,7 @@ mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state) { return MP_OBJ_NULL; } - mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->rc)); + mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc)); if (code == NULL) { return MP_OBJ_NULL; } diff --git a/py/profile.h b/py/profile.h index 64e207d04f..7f3f914034 100644 --- a/py/profile.h +++ b/py/profile.h @@ -34,7 +34,9 @@ #define mp_prof_is_executing MP_STATE_THREAD(prof_callback_is_executing) typedef struct _mp_obj_code_t { + // TODO this was 4 words mp_obj_base_t base; + const mp_module_context_t *context; const mp_raw_code_t *rc; mp_obj_dict_t *dict_locals; mp_obj_t lnotab; @@ -53,7 +55,7 @@ typedef struct _mp_obj_frame_t { void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude); -mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc); +mp_obj_t mp_obj_new_code(const mp_module_context_t *mc, const mp_raw_code_t *rc); mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state); // This is the implementation for the sys.settrace diff --git a/py/qstr.h b/py/qstr.h index ded105760c..fa634f90b0 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -46,6 +46,7 @@ enum { }; typedef size_t qstr; +typedef uint16_t qstr_short_t; #if MICROPY_QSTR_BYTES_IN_HASH == 1 typedef uint8_t qstr_hash_t; diff --git a/py/scope.c b/py/scope.c index 98e02fb53f..8fc0943289 100644 --- a/py/scope.c +++ b/py/scope.c @@ -40,7 +40,7 @@ STATIC const uint8_t scope_simple_name_table[] = { [SCOPE_GEN_EXPR] = MP_QSTR__lt_genexpr_gt_, }; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options) { +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, mp_uint_t emit_options) { // Make sure those qstrs indeed fit in an uint8_t. MP_STATIC_ASSERT(MP_QSTR__lt_module_gt_ <= UINT8_MAX); MP_STATIC_ASSERT(MP_QSTR__lt_lambda_gt_ <= UINT8_MAX); @@ -52,7 +52,6 @@ scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_u scope_t *scope = m_new0(scope_t, 1); scope->kind = kind; scope->pn = pn; - scope->source_file = source_file; if (kind == SCOPE_FUNCTION || kind == SCOPE_CLASS) { assert(MP_PARSE_NODE_IS_STRUCT(pn)); scope->simple_name = MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t *)pn)->nodes[0]); diff --git a/py/scope.h b/py/scope.h index edf164c4ad..5006deadea 100644 --- a/py/scope.h +++ b/py/scope.h @@ -75,7 +75,6 @@ typedef struct _scope_t { struct _scope_t *next; mp_parse_node_t pn; mp_raw_code_t *raw_code; - uint16_t source_file; // a qstr uint16_t simple_name; // a qstr uint16_t scope_flags; // see runtime0.h uint16_t emit_options; // see emitglue.h @@ -90,7 +89,7 @@ typedef struct _scope_t { id_info_t *id_info; } scope_t; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options); +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, mp_uint_t emit_options); void scope_free(scope_t *scope); id_info_t *scope_find_or_add_id(scope_t *scope, qstr qstr, id_info_kind_t kind); id_info_t *scope_find(scope_t *scope, qstr qstr); diff --git a/py/showbc.c b/py/showbc.c index da8077eccd..1300ac1fa3 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -41,43 +41,32 @@ #define DECODE_ULABEL do { unum = (ip[0] | (ip[1] << 8)); ip += 2; } while (0) #define DECODE_SLABEL do { unum = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2; } while (0) -#if MICROPY_PERSISTENT_CODE +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE #define DECODE_QSTR \ - qst = ip[0] | ip[1] << 8; \ - ip += 2; -#define DECODE_PTR \ DECODE_UINT; \ - unum = mp_showbc_const_table[unum] -#define DECODE_OBJ \ - DECODE_UINT; \ - unum = mp_showbc_const_table[unum] + qst = mp_showbc_constants->qstr_table[unum] #else -#define DECODE_QSTR { \ - qst = 0; \ - do { \ - qst = (qst << 7) + (*ip & 0x7f); \ - } while ((*ip++ & 0x80) != 0); \ -} -#define DECODE_PTR do { \ - ip = (byte *)MP_ALIGN(ip, sizeof(void *)); \ - unum = (uintptr_t)*(void **)ip; \ - ip += sizeof(void *); \ -} while (0) -#define DECODE_OBJ do { \ - ip = (byte *)MP_ALIGN(ip, sizeof(mp_obj_t)); \ - unum = (mp_uint_t)*(mp_obj_t *)ip; \ - ip += sizeof(mp_obj_t); \ -} while (0) +#define DECODE_QSTR \ + DECODE_UINT; \ + qst = unum; #endif -const byte *mp_showbc_code_start; -const mp_uint_t *mp_showbc_const_table; +#define DECODE_PTR \ + DECODE_UINT; -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_uint_t *const_table) { +#define DECODE_OBJ \ + DECODE_UINT; \ + unum = (mp_uint_t)mp_showbc_constants->obj_table[unum] + +const byte * mp_showbc_code_start; +const mp_module_constants_t *mp_showbc_constants; + +#include "py/emitglue.h" +void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_module_constants_t *cm) { mp_showbc_code_start = ip; // Decode prelude @@ -85,16 +74,15 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i MP_BC_PRELUDE_SIZE_DECODE(ip); const byte *code_info = ip; - #if MICROPY_PERSISTENT_CODE - qstr block_name = code_info[0] | (code_info[1] << 8); - qstr source_file = code_info[2] | (code_info[3] << 8); - code_info += 4; - #else qstr block_name = mp_decode_uint(&code_info); - qstr source_file = mp_decode_uint(&code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + block_name = cm->qstr_table[block_name]; + qstr source_file = cm->qstr_table[0]; + #else + qstr source_file = cm->source_file; #endif - mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p " UINT_FMT " bytes)\n", - qstr_str(source_file), qstr_str(block_name), descr, mp_showbc_code_start, len); + mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p " UINT_FMT " bytes)\n",// rct=%p\n", + qstr_str(source_file), qstr_str(block_name), descr, mp_showbc_code_start, len);// , ((mp_raw_code_t*)descr)->children); // raw bytecode dump size_t prelude_size = ip - mp_showbc_code_start + n_info + n_cell; @@ -111,7 +99,11 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i // bytecode prelude: arg names (as qstr objects) mp_printf(print, "arg names:"); for (mp_uint_t i = 0; i < n_pos_args + n_kwonly_args; i++) { - mp_printf(print, " %s", qstr_str(MP_OBJ_QSTR_VALUE(const_table[i]))); + qstr qst = mp_decode_uint(&code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + qst = cm->qstr_table[qst]; + #endif + mp_printf(print, " %s", qstr_str(qst)); } mp_printf(print, "\n"); @@ -120,6 +112,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i // skip over code_info ip += n_info; + const byte *line_info_top = ip; // bytecode prelude: initialise closed over variables for (size_t i = 0; i < n_cell; ++i) { @@ -132,7 +125,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i mp_int_t bc = 0; mp_uint_t source_line = 1; mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); - for (const byte *ci = code_info; *ci;) { + for (const byte *ci = code_info; ci < line_info_top;) { if ((ci[0] & 0x80) == 0) { // 0b0LLBBBBB encoding bc += ci[0] & 0x1f; @@ -147,7 +140,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); } } - mp_bytecode_print2(print, ip, len - prelude_size, const_table); + mp_bytecode_print2(print, ip, len - prelude_size, cm); } const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { @@ -535,9 +528,9 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { return ip; } -void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_uint_t *const_table) { +void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_module_constants_t *cm) { mp_showbc_code_start = ip; - mp_showbc_const_table = const_table; + mp_showbc_constants = cm; while (ip < len + mp_showbc_code_start) { mp_printf(print, "%02u ", (uint)(ip - mp_showbc_code_start)); ip = mp_bytecode_print_str(print, ip); diff --git a/py/vm.c b/py/vm.c index 0289bcfae5..b762c92bf2 100644 --- a/py/vm.c +++ b/py/vm.c @@ -31,9 +31,9 @@ #include "py/emitglue.h" #include "py/objtype.h" +#include "py/objfun.h" #include "py/runtime.h" #include "py/bc0.h" -#include "py/bc.h" #include "py/profile.h" // *FORMAT-OFF* @@ -44,7 +44,7 @@ #else #define TRACE_PREFIX mp_printf(&mp_plat_print, "sp=%d ", (int)(sp - &code_state->state[0] + 1)) #endif -#define TRACE(ip) TRACE_PREFIX; mp_bytecode_print2(&mp_plat_print, ip, 1, code_state->fun_bc->const_table); +#define TRACE(ip) TRACE_PREFIX; mp_bytecode_print2(&mp_plat_print, ip, 1, &code_state->fun_bc->context->constants); #else #define TRACE(ip) #endif @@ -64,35 +64,28 @@ #define DECODE_ULABEL size_t ulab = (ip[0] | (ip[1] << 8)); ip += 2 #define DECODE_SLABEL size_t slab = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2 -#if MICROPY_PERSISTENT_CODE +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE #define DECODE_QSTR \ - qstr qst = ip[0] | ip[1] << 8; \ - ip += 2; -#define DECODE_PTR \ DECODE_UINT; \ - void *ptr = (void*)(uintptr_t)code_state->fun_bc->const_table[unum] -#define DECODE_OBJ \ - DECODE_UINT; \ - mp_obj_t obj = (mp_obj_t)code_state->fun_bc->const_table[unum] + qstr qst = qstr_table[unum] #else -#define DECODE_QSTR qstr qst = 0; \ - do { \ - qst = (qst << 7) + (*ip & 0x7f); \ - } while ((*ip++ & 0x80) != 0) -#define DECODE_PTR \ - ip = (byte*)MP_ALIGN(ip, sizeof(void*)); \ - void *ptr = *(void**)ip; \ - ip += sizeof(void*) -#define DECODE_OBJ \ - ip = (byte*)MP_ALIGN(ip, sizeof(mp_obj_t)); \ - mp_obj_t obj = *(mp_obj_t*)ip; \ - ip += sizeof(mp_obj_t) +#define DECODE_QSTR \ + DECODE_UINT; \ + qstr qst = unum; #endif +#define DECODE_PTR \ + DECODE_UINT; \ + void *ptr = (void *)(uintptr_t)code_state->fun_bc->child_table[unum] + +#define DECODE_OBJ \ + DECODE_UINT; \ + mp_obj_t obj = (mp_obj_t)code_state->fun_bc->context->constants.obj_table[unum] + #define PUSH(val) *++sp = (val) #define POP() (*sp--) #define TOP() (*sp) @@ -255,6 +248,9 @@ outer_dispatch_loop: // local variables that are not visible to the exception handler const byte *ip = code_state->ip; mp_obj_t *sp = code_state->sp; + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + const qstr_short_t *qstr_table = code_state->fun_bc->context->constants.qstr_table; + #endif mp_obj_t obj_shared; MICROPY_VM_HOOK_INIT @@ -859,15 +855,15 @@ unwind_jump:; ENTRY(MP_BC_MAKE_FUNCTION): { DECODE_PTR; - PUSH(mp_make_function_from_raw_code(ptr, MP_OBJ_NULL, MP_OBJ_NULL)); + PUSH(mp_make_function_from_raw_code(ptr, code_state->fun_bc->context, NULL)); DISPATCH(); } ENTRY(MP_BC_MAKE_FUNCTION_DEFARGS): { DECODE_PTR; // Stack layout: def_tuple def_dict <- TOS - mp_obj_t def_dict = POP(); - SET_TOP(mp_make_function_from_raw_code(ptr, TOP(), def_dict)); + sp -= 1; + SET_TOP(mp_make_function_from_raw_code(ptr, code_state->fun_bc->context, sp)); DISPATCH(); } @@ -876,7 +872,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: closed_overs <- TOS sp -= n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->context, n_closed_over, sp)); DISPATCH(); } @@ -885,7 +881,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: def_tuple def_dict closed_overs <- TOS sp -= 2 + n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, 0x100 | n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->context, 0x100 | n_closed_over, sp)); DISPATCH(); } @@ -1384,23 +1380,20 @@ unwind_loop: const byte *ip = code_state->fun_bc->bytecode; MP_BC_PRELUDE_SIG_DECODE(ip); MP_BC_PRELUDE_SIZE_DECODE(ip); + const byte *line_info_top = ip + n_info; const byte *bytecode_start = ip + n_info + n_cell; - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - bytecode_start = MP_ALIGN(bytecode_start, sizeof(mp_uint_t)); - #endif size_t bc = code_state->ip - bytecode_start; - #if MICROPY_PERSISTENT_CODE - qstr block_name = ip[0] | (ip[1] << 8); - qstr source_file = ip[2] | (ip[3] << 8); - ip += 4; - #else qstr block_name = mp_decode_uint_value(ip); - ip = mp_decode_uint_skip(ip); - qstr source_file = mp_decode_uint_value(ip); - ip = mp_decode_uint_skip(ip); + for (size_t i = 0; i < 1 + n_pos_args + n_kwonly_args; ++i) { + ip = mp_decode_uint_skip(ip); + } + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + block_name = code_state->fun_bc->context->constants.qstr_table[block_name]; + qstr source_file = code_state->fun_bc->context->constants.qstr_table[0]; + #else + qstr source_file = code_state->fun_bc->context->constants.source_file; #endif - size_t source_line = mp_bytecode_get_source_line(ip, bc); + size_t source_line = mp_bytecode_get_source_line(ip, line_info_top, bc); mp_obj_exception_add_traceback(MP_OBJ_FROM_PTR(nlr.ret_val), source_file, source_line, block_name); } diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 5f05c1da3e..ae6dd770b1 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -83,7 +83,11 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input #if MICROPY_MODULE_FROZEN_MPY if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { // source is a raw_code object, create the function - module_fun = mp_make_function_from_raw_code(source, MP_OBJ_NULL, MP_OBJ_NULL); + const mp_frozen_module_t *frozen = source; + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + ctx->constants = frozen->constants; + module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL); } else #endif { diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index bee4fc99d1..ac16f6d88d 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -39,50 +39,52 @@ [ 13] \(rule\|arglist\)(164) (n=1) id(b) ---------------- -File cmdline/cmd_parsetree.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_parsetree.py, code block '' (descriptor: \.\+, bytecode @\.\+ 64 bytes) +Raw bytecode (code_info_size=13, bytecode_size=51): + 20 16 01 60 28 23 23 24 24 24 24 24 25 2a 00 5f + 4b 05 00 16 02 42 f8 7f 51 16 03 10 04 16 05 23 + 00 16 06 23 01 16 07 23 02 16 08 23 03 16 09 22 + 80 7b 16 0a 23 04 14 0b 11 05 36 01 16 0c 51 63 arg names: (N_STATE 5) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=4 - bc=9 line=5 - bc=12 line=6 - bc=16 line=7 - bc=22 line=8 - bc=27 line=9 - bc=32 line=10 - bc=37 line=11 - bc=42 line=12 - bc=48 line=13 + bc=8 line=5 + bc=11 line=6 + bc=14 line=7 + bc=18 line=8 + bc=22 line=9 + bc=26 line=10 + bc=30 line=11 + bc=34 line=12 + bc=39 line=13 00 BUILD_TUPLE 0 02 GET_ITER_STACK -03 FOR_ITER 12 +03 FOR_ITER 11 06 STORE_NAME i -09 JUMP 3 -12 LOAD_CONST_NONE -13 STORE_NAME a -16 LOAD_CONST_STRING 'str' -19 STORE_NAME b -22 LOAD_CONST_OBJ \.\+ -24 STORE_NAME c -27 LOAD_CONST_OBJ \.\+ -29 STORE_NAME d -32 LOAD_CONST_OBJ \.\+ -34 STORE_NAME e -37 LOAD_CONST_OBJ \.\+ -39 STORE_NAME f -42 LOAD_CONST_SMALL_INT 123 -45 STORE_NAME g -48 LOAD_CONST_OBJ \.\+ -50 LOAD_METHOD format -53 LOAD_NAME b -56 CALL_METHOD n=1 nkw=0 -58 STORE_NAME h -61 LOAD_CONST_NONE -62 RETURN_VALUE +08 JUMP 3 +11 LOAD_CONST_NONE +12 STORE_NAME a +14 LOAD_CONST_STRING 'str' +16 STORE_NAME b +18 LOAD_CONST_OBJ \.\+='a very long str that will not be interned' +20 STORE_NAME c +22 LOAD_CONST_OBJ \.\+=b'bytes' +24 STORE_NAME d +26 LOAD_CONST_OBJ \.\+=b'a very long bytes that will not be interned' +28 STORE_NAME e +30 LOAD_CONST_OBJ \.\+=123456789012345678901234567890 +32 STORE_NAME f +34 LOAD_CONST_SMALL_INT 123 +37 STORE_NAME g +39 LOAD_CONST_OBJ \.\+="fstring: '{}'" +41 LOAD_METHOD format +43 LOAD_NAME b +45 CALL_METHOD n=1 nkw=0 +47 STORE_NAME h +49 LOAD_CONST_NONE +50 RETURN_VALUE mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ GC: total: \\d\+, used: \\d\+, free: \\d\+ diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 22712b79ee..031820fcd9 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -1,51 +1,144 @@ -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 63 bytes) +Raw bytecode (code_info_size=18, bytecode_size=45): + 10 20 01 60 20 84 7d 64 60 88 07 64 60 69 20 62 + 64 20 32 00 16 02 32 01 16 02 81 2a 01 53 33 02 + 16 02 32 03 16 02 54 32 04 10 03 34 02 16 03 19 + 03 32 05 16 02 80 10 04 2a 01 1b 05 69 51 63 arg names: (N_STATE 3) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=160 + bc=0 line=4 + bc=0 line=5 + bc=4 line=130 + bc=8 line=133 + bc=8 line=136 + bc=16 line=143 + bc=20 line=146 + bc=20 line=149 + bc=29 line=152 + bc=29 line=153 + bc=31 line=156 + bc=35 line=159 + bc=35 line=160 00 MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ BUILD_TUPLE 1 -\\d\+ LOAD_NULL -\\d\+ MAKE_FUNCTION_DEFARGS \.\+ -\\d\+ STORE_NAME f -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_BUILD_CLASS -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ LOAD_CONST_STRING 'Class' -\\d\+ CALL_FUNCTION n=2 nkw=0 -\\d\+ STORE_NAME Class -\\d\+ DELETE_NAME Class -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_STRING '*' -\\d\+ BUILD_TUPLE 1 -\\d\+ IMPORT_NAME 'sys' -\\d\+ IMPORT_STAR -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): +02 STORE_NAME f +04 MAKE_FUNCTION \.\+ +06 STORE_NAME f +08 LOAD_CONST_SMALL_INT 1 +09 BUILD_TUPLE 1 +11 LOAD_NULL +12 MAKE_FUNCTION_DEFARGS \.\+ +14 STORE_NAME f +16 MAKE_FUNCTION \.\+ +18 STORE_NAME f +20 LOAD_BUILD_CLASS +21 MAKE_FUNCTION \.\+ +23 LOAD_CONST_STRING 'Class' +25 CALL_FUNCTION n=2 nkw=0 +27 STORE_NAME Class +29 DELETE_NAME Class +31 MAKE_FUNCTION \.\+ +33 STORE_NAME f +35 LOAD_CONST_SMALL_INT 0 +36 LOAD_CONST_STRING '*' +38 BUILD_TUPLE 1 +40 IMPORT_NAME 'sys' +42 IMPORT_STAR +43 LOAD_CONST_NONE +44 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 48\[24\] bytes) +Raw bytecode (code_info_size=8\[46\], bytecode_size=398): + a8 12 9\[bf\] 03 02 60 60 26 22 24 64 22 26 25 25 24 + 26 23 63 22 22 25 23 23 31 6d 25 65 25 25 69 68 + 26 65 27 6a 62 20 23 62 2a 29 69 24 25 28 67 26 ######## -\.\+rg names: +\.\+81 63 +arg names: (N_STATE 22) (N_EXC_STACK 2) (INIT_CELL 14) (INIT_CELL 15) (INIT_CELL 16) bc=0 line=1 + bc=0 line=4 + bc=0 line=7 + bc=6 line=8 + bc=8 line=9 + bc=12 line=10 + bc=16 line=13 + bc=18 line=14 + bc=24 line=15 + bc=29 line=16 + bc=34 line=17 + bc=38 line=18 + bc=44 line=19 + bc=47 line=20 + bc=50 line=23 + bc=52 line=24 + bc=54 line=25 + bc=59 line=26 + bc=62 line=27 + bc=65 line=28 + bc=82 line=29 + bc=95 line=32 + bc=100 line=33 + bc=105 line=36 + bc=110 line=37 + bc=115 line=38 + bc=124 line=41 + bc=132 line=44 + bc=138 line=45 + bc=143 line=48 + bc=150 line=49 + bc=160 line=52 + bc=162 line=55 + bc=162 line=56 + bc=165 line=57 + bc=167 line=60 + bc=177 line=61 + bc=186 line=62 + bc=195 line=65 + bc=199 line=66 + bc=204 line=67 + bc=212 line=68 + bc=219 line=71 + bc=225 line=72 + bc=232 line=73 + bc=242 line=74 + bc=250 line=77 + bc=254 line=78 + bc=260 line=80 + bc=263 line=81 + bc=266 line=82 + bc=273 line=83 + bc=276 line=84 + bc=283 line=85 + bc=289 line=88 + bc=296 line=89 + bc=301 line=92 + bc=307 line=93 + bc=310 line=94 ######## - bc=\\d\+ line=127 + bc=321 line=96 + bc=329 line=98 + bc=332 line=99 + bc=335 line=100 + bc=338 line=101 +######## + bc=346 line=103 + bc=354 line=106 + bc=359 line=107 + bc=365 line=110 + bc=368 line=111 + bc=374 line=114 + bc=374 line=117 + bc=379 line=118 + bc=391 line=121 + bc=391 line=122 + bc=392 line=123 + bc=394 line=126 + bc=396 line=127 00 LOAD_CONST_NONE 01 LOAD_CONST_FALSE 02 BINARY_OP 27 __add__ @@ -80,247 +173,251 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 42 STORE_MAP 43 STORE_FAST 3 44 LOAD_CONST_STRING 'a' -47 STORE_FAST 4 -48 LOAD_CONST_OBJ \.\+ -\\d\+ STORE_FAST 5 -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ STORE_FAST 6 -\\d\+ LOAD_CONST_SMALL_INT 2 -\\d\+ STORE_FAST 7 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -58 BINARY_OP 27 __add__ -\\d\+ STORE_FAST 8 -\\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 1 __neg__ -\\d\+ STORE_FAST 9 -\\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 3 -\\d\+ STORE_FAST 10 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ DUP_TOP -\\d\+ ROT_THREE -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP_IF_FALSE_OR_POP \\d\+ -\\d\+ LOAD_FAST 1 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP \\d\+ -\\d\+ ROT_TWO -\\d\+ POP_TOP -\\d\+ STORE_FAST 10 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP_IF_FALSE_OR_POP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 1 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ UNARY_OP 3 -\\d\+ STORE_FAST 10 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_ATTR c -\\d\+ STORE_FAST 11 -\\d\+ LOAD_FAST 11 -\\d\+ LOAD_DEREF 14 -\\d\+ STORE_ATTR c -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_SUBSCR -\\d\+ STORE_FAST 12 -\\d\+ LOAD_FAST 12 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ STORE_SUBSCR -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ DUP_TOP_TWO -\\d\+ LOAD_SUBSCR -\\d\+ LOAD_FAST 12 -\\d\+ BINARY_OP 14 __iadd__ -\\d\+ ROT_THREE -\\d\+ STORE_SUBSCR -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_NONE -\\d\+ LOAD_CONST_NONE -\\d\+ BUILD_SLICE 2 -\\d\+ LOAD_SUBSCR -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 1 -\\d\+ UNPACK_SEQUENCE 2 -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ UNPACK_EX 1 -\\d\+ STORE_FAST 0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ ROT_TWO -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ LOAD_FAST 1 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ ROT_THREE -\\d\+ ROT_TWO -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ STORE_FAST 1 -\\d\+ DELETE_FAST 0 -\\d\+ LOAD_FAST 0 -\\d\+ STORE_GLOBAL gl -\\d\+ DELETE_GLOBAL gl -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ GET_ITER -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 0 -\\d\+ CALL_FUNCTION n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_CONST_STRING 'b' -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_FUNCTION n=0 nkw=1 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_NULL -\\d\+ CALL_FUNCTION_VAR_KW n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ CALL_METHOD n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_METHOD n=1 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_CONST_STRING 'c' -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_METHOD n=0 nkw=1 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_FAST 1 -\\d\+ LOAD_NULL -\\d\+ CALL_METHOD_VAR_KW n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_FALSE \\d\+ -\\d\+ LOAD_DEREF 16 -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ LOAD_GLOBAL y -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_FALSE \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ JUMP_IF_TRUE_OR_POP \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ GET_ITER_STACK -\\d\+ FOR_ITER \\d\+ -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 1 -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ SETUP_FINALLY \\d\+ -\\d\+ SETUP_EXCEPT \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ END_FINALLY -\\d\+ LOAD_CONST_NONE -\\d\+ LOAD_FAST 1 -\\d\+ POP_TOP -\\d\+ END_FINALLY -\\d\+ JUMP \\d\+ -\\d\+ SETUP_EXCEPT \\d\+ -\\d\+ UNWIND_JUMP \\d\+ 1 -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ END_FINALLY -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ SETUP_WITH \\d\+ -\\d\+ POP_TOP -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_CONST_NONE -\\d\+ WITH_CLEANUP -\\d\+ END_FINALLY -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ STORE_DEREF 16 -\\d\+ LOAD_FAST_N 16 -\\d\+ MAKE_CLOSURE \.\+ 1 -\\d\+ STORE_FAST 13 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_NONE -\\d\+ IMPORT_NAME 'a' -\\d\+ STORE_FAST 0 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_STRING 'b' -\\d\+ BUILD_TUPLE 1 -\\d\+ IMPORT_NAME 'a' -\\d\+ IMPORT_FROM 'b' -\\d\+ STORE_DEREF 14 -\\d\+ POP_TOP -\\d\+ RAISE_LAST -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ RAISE_OBJ -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+rg names: +46 STORE_FAST 4 +47 LOAD_CONST_OBJ \.\+=b'a' +49 STORE_FAST 5 +50 LOAD_CONST_SMALL_INT 1 +51 STORE_FAST 6 +52 LOAD_CONST_SMALL_INT 2 +53 STORE_FAST 7 +54 LOAD_FAST 0 +55 LOAD_DEREF 14 +57 BINARY_OP 27 __add__ +58 STORE_FAST 8 +59 LOAD_FAST 0 +60 UNARY_OP 1 __neg__ +61 STORE_FAST 9 +62 LOAD_FAST 0 +63 UNARY_OP 3 +64 STORE_FAST 10 +65 LOAD_FAST 0 +66 LOAD_DEREF 14 +68 DUP_TOP +69 ROT_THREE +70 BINARY_OP 2 __eq__ +71 JUMP_IF_FALSE_OR_POP 79 +74 LOAD_FAST 1 +75 BINARY_OP 2 __eq__ +76 JUMP 81 +79 ROT_TWO +80 POP_TOP +81 STORE_FAST 10 +82 LOAD_FAST 0 +83 LOAD_DEREF 14 +85 BINARY_OP 2 __eq__ +86 JUMP_IF_FALSE_OR_POP 93 +89 LOAD_DEREF 14 +91 LOAD_FAST 1 +92 BINARY_OP 2 __eq__ +93 UNARY_OP 3 +94 STORE_FAST 10 +95 LOAD_DEREF 14 +97 LOAD_ATTR c +99 STORE_FAST 11 +100 LOAD_FAST 11 +101 LOAD_DEREF 14 +103 STORE_ATTR c +105 LOAD_DEREF 14 +107 LOAD_CONST_SMALL_INT 0 +108 LOAD_SUBSCR +109 STORE_FAST 12 +110 LOAD_FAST 12 +111 LOAD_DEREF 14 +113 LOAD_CONST_SMALL_INT 0 +114 STORE_SUBSCR +115 LOAD_DEREF 14 +117 LOAD_CONST_SMALL_INT 0 +118 DUP_TOP_TWO +119 LOAD_SUBSCR +120 LOAD_FAST 12 +121 BINARY_OP 14 __iadd__ +122 ROT_THREE +123 STORE_SUBSCR +124 LOAD_DEREF 14 +126 LOAD_CONST_NONE +127 LOAD_CONST_NONE +128 BUILD_SLICE 2 +130 LOAD_SUBSCR +131 STORE_FAST 0 +132 LOAD_FAST 1 +133 UNPACK_SEQUENCE 2 +135 STORE_FAST 0 +136 STORE_DEREF 14 +138 LOAD_FAST 0 +139 UNPACK_EX 1 +141 STORE_FAST 0 +142 STORE_FAST 0 +143 LOAD_DEREF 14 +145 LOAD_FAST 0 +146 ROT_TWO +147 STORE_FAST 0 +148 STORE_DEREF 14 +150 LOAD_FAST 1 +151 LOAD_DEREF 14 +153 LOAD_FAST 0 +154 ROT_THREE +155 ROT_TWO +156 STORE_FAST 0 +157 STORE_DEREF 14 +159 STORE_FAST 1 +160 DELETE_FAST 0 +162 LOAD_FAST 0 +163 STORE_GLOBAL gl +165 DELETE_GLOBAL gl +167 LOAD_FAST 14 +168 LOAD_FAST 15 +169 MAKE_CLOSURE \.\+ 2 +172 LOAD_FAST 2 +173 GET_ITER +174 CALL_FUNCTION n=1 nkw=0 +176 STORE_FAST 0 +177 LOAD_FAST 14 +178 LOAD_FAST 15 +179 MAKE_CLOSURE \.\+ 2 +182 LOAD_FAST 2 +183 CALL_FUNCTION n=1 nkw=0 +185 STORE_FAST 0 +186 LOAD_FAST 14 +187 LOAD_FAST 15 +188 MAKE_CLOSURE \.\+ 2 +191 LOAD_FAST 2 +192 CALL_FUNCTION n=1 nkw=0 +194 STORE_FAST 0 +195 LOAD_FAST 0 +196 CALL_FUNCTION n=0 nkw=0 +198 POP_TOP +199 LOAD_FAST 0 +200 LOAD_CONST_SMALL_INT 1 +201 CALL_FUNCTION n=1 nkw=0 +203 POP_TOP +204 LOAD_FAST 0 +205 LOAD_CONST_STRING 'b' +207 LOAD_CONST_SMALL_INT 1 +208 CALL_FUNCTION n=0 nkw=1 +211 POP_TOP +212 LOAD_FAST 0 +213 LOAD_DEREF 14 +215 LOAD_NULL +216 CALL_FUNCTION_VAR_KW n=0 nkw=0 +218 POP_TOP +219 LOAD_FAST 0 +220 LOAD_METHOD b +222 CALL_METHOD n=0 nkw=0 +224 POP_TOP +225 LOAD_FAST 0 +226 LOAD_METHOD b +228 LOAD_CONST_SMALL_INT 1 +229 CALL_METHOD n=1 nkw=0 +231 POP_TOP +232 LOAD_FAST 0 +233 LOAD_METHOD b +235 LOAD_CONST_STRING 'c' +237 LOAD_CONST_SMALL_INT 1 +238 CALL_METHOD n=0 nkw=1 +241 POP_TOP +242 LOAD_FAST 0 +243 LOAD_METHOD b +245 LOAD_FAST 1 +246 LOAD_NULL +247 CALL_METHOD_VAR_KW n=0 nkw=0 +249 POP_TOP +250 LOAD_FAST 0 +251 POP_JUMP_IF_FALSE 260 +254 LOAD_DEREF 16 +256 POP_TOP +257 JUMP 263 +260 LOAD_GLOBAL y +262 POP_TOP +263 JUMP 269 +266 LOAD_DEREF 14 +268 POP_TOP +269 LOAD_FAST 0 +270 POP_JUMP_IF_TRUE 266 +273 JUMP 279 +276 LOAD_DEREF 14 +278 POP_TOP +279 LOAD_FAST 0 +280 POP_JUMP_IF_FALSE 276 +283 LOAD_FAST 0 +284 JUMP_IF_TRUE_OR_POP 288 +287 LOAD_FAST 0 +288 STORE_FAST 0 +289 LOAD_DEREF 14 +291 GET_ITER_STACK +292 FOR_ITER 301 +295 STORE_FAST 0 +296 LOAD_FAST 1 +297 POP_TOP +298 JUMP 292 +301 SETUP_FINALLY 329 +304 SETUP_EXCEPT 320 +307 JUMP 313 +310 JUMP 317 +313 LOAD_FAST 0 +314 POP_JUMP_IF_TRUE 310 +317 POP_EXCEPT_JUMP 328 +320 POP_TOP +321 LOAD_DEREF 14 +323 POP_TOP +324 POP_EXCEPT_JUMP 328 +327 END_FINALLY +328 LOAD_CONST_NONE +329 LOAD_FAST 1 +330 POP_TOP +331 END_FINALLY +332 JUMP 350 +335 SETUP_EXCEPT 345 +338 UNWIND_JUMP 354 1 +342 POP_EXCEPT_JUMP 350 +345 POP_TOP +346 POP_EXCEPT_JUMP 350 +349 END_FINALLY +350 LOAD_FAST 0 +351 POP_JUMP_IF_TRUE 335 +354 LOAD_FAST 0 +355 SETUP_WITH 363 +358 POP_TOP +359 LOAD_DEREF 14 +361 POP_TOP +362 LOAD_CONST_NONE +363 WITH_CLEANUP +364 END_FINALLY +365 LOAD_CONST_SMALL_INT 1 +366 STORE_DEREF 16 +368 LOAD_FAST_N 16 +370 MAKE_CLOSURE \.\+ 1 +373 STORE_FAST 13 +374 LOAD_CONST_SMALL_INT 0 +375 LOAD_CONST_NONE +376 IMPORT_NAME 'a' +378 STORE_FAST 0 +379 LOAD_CONST_SMALL_INT 0 +380 LOAD_CONST_STRING 'b' +382 BUILD_TUPLE 1 +384 IMPORT_NAME 'a' +386 IMPORT_FROM 'b' +388 STORE_DEREF 14 +390 POP_TOP +391 RAISE_LAST +392 LOAD_CONST_SMALL_INT 1 +393 RAISE_OBJ +394 LOAD_CONST_NONE +395 RETURN_VALUE +396 LOAD_CONST_SMALL_INT 1 +397 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 59 bytes) +Raw bytecode (code_info_size=8, bytecode_size=51): + a8 10 0a 02 80 82 34 38 81 57 c0 57 c1 57 c2 57 + c3 57 c4 57 c5 57 c6 57 c7 57 c8 c9 82 57 ca 57 + cb 57 cc 57 cd 57 ce 57 cf 57 26 10 57 26 11 57 + 26 12 26 13 b9 24 13 f2 59 51 63 +arg names: (N_STATE 22) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=133 + bc=0 line=131 + bc=20 line=132 + bc=44 line=133 00 LOAD_CONST_SMALL_INT 1 01 DUP_TOP 02 STORE_FAST 0 @@ -367,28 +464,29 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 48 POP_TOP 49 LOAD_CONST_NONE 50 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +Raw bytecode (code_info_size=9, bytecode_size=11): + a1 01 0b 02 06 80 88 40 00 82 2a 01 53 b0 21 00 + 01 c1 51 63 arg names: a (N_STATE 5) (N_EXC_STACK 0) (INIT_CELL 0) -######## - bc=\\d\+ line=139 + bc=0 line=1 + bc=0 line=137 + bc=0 line=139 00 LOAD_CONST_SMALL_INT 2 01 BUILD_TUPLE 1 03 LOAD_NULL 04 LOAD_FAST 0 05 MAKE_CLOSURE_DEFARGS \.\+ 1 -\\d\+ STORE_FAST 1 -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +08 STORE_FAST 1 +09 LOAD_CONST_NONE +10 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 21 bytes) +Raw bytecode (code_info_size=8, bytecode_size=13): + 88 40 0a 02 80 8f 23 23 51 67 59 81 67 59 81 5e + 51 68 59 51 63 arg names: (N_STATE 2) (N_EXC_STACK 0) @@ -409,49 +507,47 @@ arg names: 10 POP_TOP 11 LOAD_CONST_NONE 12 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ 1\[56\] bytes) +Raw bytecode (code_info_size=\[56\], bytecode_size=10): + 00 \.\+ 11 0c 16 0d 10 03 16 0e 51 63 arg names: (N_STATE 1) (N_EXC_STACK 0) bc=0 line=1 ######## - bc=12 line=150 + bc=8 line=150 00 LOAD_NAME __name__ -03 STORE_NAME __module__ -06 LOAD_CONST_STRING 'Class' -09 STORE_NAME __qualname__ -12 LOAD_CONST_NONE -13 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +02 STORE_NAME __module__ +04 LOAD_CONST_STRING 'Class' +06 STORE_NAME __qualname__ +08 LOAD_CONST_NONE +09 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 18 bytes) +Raw bytecode (code_info_size=6, bytecode_size=12): + 19 08 02 0f 80 9c 12 10 12 11 b0 15 02 36 00 59 + 51 63 arg names: self (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=157 00 LOAD_GLOBAL super -\\d\+ LOAD_GLOBAL __class__ -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_SUPER_METHOD f -\\d\+ CALL_METHOD n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +02 LOAD_GLOBAL __class__ +04 LOAD_FAST 0 +05 LOAD_SUPER_METHOD f +07 CALL_METHOD n=0 nkw=0 +09 POP_TOP +10 LOAD_CONST_NONE +11 RETURN_VALUE +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 31 bytes) +Raw bytecode (code_info_size=9, bytecode_size=22): + c3 40 0c 12 04 04 04 80 3b 53 b2 53 53 4b 0d 00 + c3 25 01 44 f7 7f 25 00 67 59 42 f0 7f 51 63 arg names: * * * (N_STATE 9) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=60 -######## 00 LOAD_NULL 01 LOAD_FAST 2 02 LOAD_NULL @@ -466,16 +562,15 @@ arg names: * * * 17 JUMP 4 20 LOAD_CONST_NONE 21 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 29 bytes) +Raw bytecode (code_info_size=8, bytecode_size=21): + 4b 0c 14 04 04 04 80 3c 2b 00 b2 5f 4b 0d 00 c3 + 25 01 44 f7 7f 25 00 2f 14 42 f0 7f 63 arg names: * * * (N_STATE 10) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=61 -######## 00 BUILD_LIST 0 02 LOAD_FAST 2 03 GET_ITER_STACK @@ -487,15 +582,15 @@ arg names: * * * 15 STORE_COMP 20 17 JUMP 4 20 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 31 bytes) +Raw bytecode (code_info_size=8, bytecode_size=23): + 53 0c 15 04 04 04 80 3d 2c 00 b2 5f 4b 0f 00 c3 + 25 01 44 f7 7f 25 00 25 00 2f 19 42 ee 7f 63 arg names: * * * (N_STATE 11) (N_EXC_STACK 0) bc=0 line=1 -######## + bc=0 line=62 00 BUILD_MAP 0 02 LOAD_FAST 2 03 GET_ITER_STACK @@ -508,16 +603,17 @@ arg names: * * * 17 STORE_COMP 25 19 JUMP 4 22 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +Raw bytecode (code_info_size=8, bytecode_size=12): + 19 0c 16 04 80 6f 25 23 25 00 81 f2 c1 81 27 00 + 29 00 51 63 arg names: * (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=114 + bc=0 line=112 + bc=5 line=113 + bc=8 line=114 00 LOAD_DEREF 0 02 LOAD_CONST_SMALL_INT 1 03 BINARY_OP 27 __add__ @@ -527,16 +623,14 @@ arg names: * 08 DELETE_DEREF 0 10 LOAD_CONST_NONE 11 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 13 bytes) +Raw bytecode (code_info_size=8, bytecode_size=5): + 9a 01 0a 02 04 09 80 8b b1 25 00 f2 63 arg names: * b (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=140 + bc=0 line=140 00 LOAD_FAST 1 01 LOAD_DEREF 0 03 BINARY_OP 27 __add__ diff --git a/tests/cmdline/cmd_verbose.py.exp b/tests/cmdline/cmd_verbose.py.exp index 0edd050c22..ae833dbec8 100644 --- a/tests/cmdline/cmd_verbose.py.exp +++ b/tests/cmdline/cmd_verbose.py.exp @@ -1,19 +1,17 @@ -File cmdline/cmd_verbose.py, code block '' (descriptor: \.\+, bytecode \.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): - 08 \.\+ -######## -\.\+63 +File cmdline/cmd_verbose.py, code block '' (descriptor: \.\+, bytecode @\.\+ 12 bytes) +Raw bytecode (code_info_size=4, bytecode_size=8): + 08 04 01 40 11 02 81 34 01 59 51 63 arg names: (N_STATE 2) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=3 00 LOAD_NAME print -03 LOAD_CONST_SMALL_INT 1 -04 CALL_FUNCTION n=1 nkw=0 -06 POP_TOP -07 LOAD_CONST_NONE -08 RETURN_VALUE +02 LOAD_CONST_SMALL_INT 1 +03 CALL_FUNCTION n=1 nkw=0 +05 POP_TOP +06 LOAD_CONST_NONE +07 RETURN_VALUE 1 mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ diff --git a/tests/micropython/import_mpy_invalid.py b/tests/micropython/import_mpy_invalid.py index 02fd4b1254..b02312a7a8 100644 --- a/tests/micropython/import_mpy_invalid.py +++ b/tests/micropython/import_mpy_invalid.py @@ -49,7 +49,6 @@ user_files = { "/mod0.mpy": b"", # empty file "/mod1.mpy": b"M", # too short header "/mod2.mpy": b"M\x00\x00\x00", # bad version - "/mod3.mpy": b"M\x00\x00\x00\x7f", # qstr window too large } # create and mount a user filesystem diff --git a/tests/micropython/import_mpy_invalid.py.exp b/tests/micropython/import_mpy_invalid.py.exp index ea748ff6c1..1727ea1cea 100644 --- a/tests/micropython/import_mpy_invalid.py.exp +++ b/tests/micropython/import_mpy_invalid.py.exp @@ -1,4 +1,3 @@ mod0 ValueError incompatible .mpy file mod1 ValueError incompatible .mpy file mod2 ValueError incompatible .mpy file -mod3 ValueError incompatible .mpy file diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index f982a4ee17..6f37b15ab7 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -49,14 +49,14 @@ class UserFS: # by the required value of sys.implementation.mpy. features0_file_contents = { # -march=x64 - 0xA05: b'M\x05\x0a\x1f \x84b\xe9/\x00\x00\x00SH\x8b\x1ds\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dA\x00\x00\x00H\x8b\x7f\x08L\x8bc(A\xff\xd4H\x8d5\x1f\x00\x00\x00H\x89\xc5H\x8b\x05-\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x84@\x12factorial\x10\x00\x00\r \x01"\x9f\x1c\x01\x1e\xff', + 0xA06: b'M\x06\n\x1f\x01\x004build/features0.native.mpy\x00\x8aB\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x85\x00\x12factorial\x00\x10\r$\x01&\x9f \x01"\xff', # -march=armv7m - 0x1605: b"M\x05\x16\x1f \x84\x12\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\tN\tK~D\xf4X@hgi\xb8G\x05F\x07K\x07I\xf2XyD\x10\x88ck\x98G(F\xb8G h\xf8\xbd6\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial\x10\x00\x00\r<\x01>\x9f8\x01:\xff", + 0x1606: b"M\x06\x16\x1f\x01\x004build/features0.native.mpy\x00\x88B\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfn\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05F\x07K\x08I\xf2XyD\x10\x88ck\x98G(F\xb8G h\xf8\xbd\x00\xbf:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x84\x00\x00\x00\x00\x00\x00\x00\x01\x84\x10\x12factorial\x00\x10\r>\x01@\x9f:\x01<\xff", } # Populate other armv7m-derived archs based on armv7m. -for arch in (0x1A05, 0x1E05, 0x2205): - features0_file_contents[arch] = features0_file_contents[0x1605] +for arch in (0x1A06, 0x1E06, 0x2206): + features0_file_contents[arch] = features0_file_contents[0x1606] if sys.implementation.mpy not in features0_file_contents: print("SKIP") diff --git a/tests/micropython/import_mpy_native_x64.py b/tests/micropython/import_mpy_native_x64.py index ff0142a9cf..3a25df1866 100644 --- a/tests/micropython/import_mpy_native_x64.py +++ b/tests/micropython/import_mpy_native_x64.py @@ -52,49 +52,62 @@ class UserFS: # fmt: off user_files = { # bad architecture - '/mod0.mpy': b'M\x05\xfe\x00\x10', + '/mod0.mpy': b'M\x06\xfe\x00\x10', # test loading of viper and asm '/mod1.mpy': ( - b'M\x05\x0a\x1f\x20' # header + b'M\x06\x0a\x1f' # header - b'\x20' # n bytes, bytecode - b'\x00\x08\x02m\x02m' # prelude + b'\x02' # n_qstr + b'\x00' # n_obj + + b'\x0emod1.py\x00' # qstr0 = "mod1.py" + b'\x0aouter\x00' # qstr1 = "outer" + + b'\x2c' # 5 bytes, have children, bytecode + b'\x00\x02' # prelude + b'\x01' # simple name (qstr index) b'\x51' # LOAD_CONST_NONE b'\x63' # RETURN_VALUE - b'\x00\x02' # n_obj, n_raw_code + b'\x02' # 2 children - b'\x22' # n bytes, viper code - b'\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00' # qstr0 - b'\x01\x0c\x0aprint' # n_qstr, qstr0 - b'\x00\x00\x00' # scope_flags, n_obj, n_raw_code + b'\x42' # 8 bytes, no children, viper code + b'\x00\x00\x00\x00\x00\x00' # dummy machine code + b'\x00\x00' # slot for qstr0 + b'\x01\x0c\x0aprint\x00' # n_qstr=1, qstr0 + b'\x00' # scope_flags - b'\x23' # n bytes, asm code - b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00\x00' # scope_flags, n_pos_args, type_sig + b'\x43' # 8 bytes, no children, asm code + b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code + b'\x00\x00\x00' # scope_flags, n_pos_args, type_sig ), # test loading viper with additional scope flags and relocation '/mod2.mpy': ( - b'M\x05\x0a\x1f\x20' # header + b'M\x06\x0a\x1f' # header - b'\x20' # n bytes, bytecode - b'\x00\x08\x02m\x02m' # prelude + b'\x02' # n_qstr + b'\x00' # n_obj + + b'\x0emod2.py\x00' # qstr0 = "mod2.py" + b'\x0aouter\x00' # qstr1 = "outer" + + b'\x2c' # 5 bytes, have children, bytecode + b'\x00\x02' # prelude + b'\x01' # simple name (qstr index) b'\x51' # LOAD_CONST_NONE b'\x63' # RETURN_VALUE - b'\x00\x01' # n_obj, n_raw_code + b'\x01' # 1 child - b'\x12' # n bytes(=4), viper code - b'\x00\x00\x00\x00' # dummy machine code - b'\x00' # n_qstr - b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC - b'\x00\x00' # n_obj, n_raw_code - b'\x06rodata' # rodata, 6 bytes - b'\x04' # bss, 4 bytes - b'\x03\x01\x00' # dummy relocation of rodata + b'\x22' # 4 bytes, no children, viper code + b'\x00\x00\x00\x00' # dummy machine code + b'\x00' # n_qstr=0 + b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC + b'\x06\x04' # rodata=6 bytes, bss=4 bytes + b'rodata' # rodata content + b'\x03\x01\x00' # dummy relocation of rodata ), } # fmt: on diff --git a/tests/perf_bench/core_import_mpy_multi.py b/tests/perf_bench/core_import_mpy_multi.py index 0da466206b..74caf20a38 100644 --- a/tests/perf_bench/core_import_mpy_multi.py +++ b/tests/perf_bench/core_import_mpy_multi.py @@ -23,7 +23,7 @@ def f(): x = ("const tuple", None, False, True, 1, 2, 3) result = 123 """ -file_data = b'M\x05\x02\x1f \x81\x0c\x10\x12\x00\x07\x0etest.py\x8b\x07e`\x00T2\x00\x10\x02A4\x02\x16\x012\x01\x16\x02f"\x80{\x16\x0cresultQc\x00\x02\x81\x1c\x00\x10\x05\x07,EE\x00\x11\x00\x17\x16\x00\x16\x10\x03\x16\x00\x1a2\x00\x16\x00\x112\x01\x16\x00\xa42\x02\x16\x00}Qc\x00\x03<\x1a\x0c\x00\x11\x03@\x00\xb1\xb0\x18\x06argQc\x00\x00\x00\x89\x01,\t\x0e\x00\xa4\x03` \x00Qc\x00\x00\x00\x89,\t\x0e\x00}\x01``\x00Qc\x00\x00\x00\x89\x81l8\x16\t\x03\x80\x08/,##\x00\x12\x00{\x12\x00\x97\x12\x00B\x12\x00K*\x04Y\x12\x00$\x12\x007\x12\x006*\x03Y#\x00\xc0#\x01\xc0#\x02QPR\x81\x82\x83*\x07\xc0Qc\x03\x00s\x1cthis will be a string objectb\x1bthis will be a bytes objects\x0bconst tuple' +file_data = b'M\x06\x02\x1f\x14\x03\x0etest.py\x00\x0f\x02A\x00\x02f\x00\x0cresult\x00/-5#\x82I\x81{\x81w\x82/\x81\x05\x81\x17Iom\x82\x13\x06arg\x00s\x1cthis will be a string object\x00b\x1bthis will be a bytes object\x00s\x0bconst tuple\x00\x81\\\x10\n\x01\x89\x07d`T2\x00\x10\x024\x02\x16\x022\x01\x16\x03"\x80{\x16\x04Qc\x02\x81d\x00\x08\x02(DD\x11\x05\x16\x06\x10\x02\x16\x072\x00\x16\x082\x01\x16\t2\x02\x16\nQc\x03`\x1a\x08\x08\x12\x13@\xb1\xb0\x18\x13Qc@\t\x08\t\x12` Qc@\t\x08\n\x12``Qc\x83\x008\x0e\x03\x80\x08+)##\x12\x0b\x12\x0c\x12\r\x12\x0e*\x04Y\x12\x0f\x12\x10\x12\x11*\x03Y#\x00\xc0#\x01\xc0#\x02QPR\x81\x82\x83*\x07\xc0Qc' class File(uio.IOBase): diff --git a/tests/perf_bench/core_import_mpy_single.py b/tests/perf_bench/core_import_mpy_single.py index 5ca3584959..1e8a4f531e 100644 --- a/tests/perf_bench/core_import_mpy_single.py +++ b/tests/perf_bench/core_import_mpy_single.py @@ -78,7 +78,7 @@ def f1(): x = ("const tuple 9", None, False, True, 1, 2, 3) result = 123 """ -file_data = b'M\x05\x02\x1f \x81\\\x10\x1a\x00\x07\x0etest.pyk@k@\x85\x18\x85\x1f\x00T2\x00\x10\x04A04\x02\x16\x01T2\x01\x10\x04A14\x02\x16\x012\x02\x16\x04f02\x03\x16\x04f1"\x80{\x16\x0cresultQc\x00\x04\x814\x00\x12\t\x0b,%%%\x00\x11\x00\x17\x16\x00\x16\x10\x03\x16\x00\x1a2\x00\x16\x04a02\x01\x16\x04a12\x02\x16\x04a22\x03\x16\x04a3Qc\x00\x04(\t\x0c\x07\x0b \x00Qc\x00\x00\x00\x89(\t\x0c\t\x03@\x00Qc\x00\x00\x00\x89(\t\x0c\t\x03`\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03` \x00Qc\x00\x00\x00\x89\x818\x00\x14\x13\x03l`%%%\x00\x11\x00\x17\x16\x00\x16\x10\x03\x16\x00\x1a2\x00\x16\x0b2\x01\x16\x0b2\x02\x16\x0b2\x03\x16\x0bQc\x00\x04,\t\x0e\x07\x0b``\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03\x80\x07\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03\x80\x08\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03\x80\t\x00Qc\x00\x00\x00\x89\x94\x1cPP\x13\x03\x80\x0b8;555222\x1f%\x1f%\x1f"\x1f"\x1f%\x1f%\x1f"\x1f%\x1f"\x1f%)\x1f"//\x00\x12\x00\t\x12\x00\n\x12\x00\x0b\x12\x00\x0c\x12\x00\r\x12\x00\x0e\x12\x00\x0f*\x07Y\x12\x00\x10\x12\x00\x11\x12\x00\x12\x12\x00\x13\x12\x00\x14\x12\x00\x15\x12\x00\x16\x12\x00\x17*\x08Y\x12\x00\x18\x12\x00\x19\x12\x00\x1a\x12\x00\x1b\x12\x00\x1c\x12\x00\x1d*\x06Y\x12\x00\x1e\x12\x00\x1f\x12\x00 \x12\x00!\x12\x00"\x12\x00#*\x06Y\x12\x00$\x12\x00%\x12\x00&\x12\x00\'\x12\x00(\x12\x00)*\x06Y\x12\x00*\x12\x00+\x12\x00,\x12\x00-\x12\x00.*\x05Y\x12\x00/\x12\x000\x12\x001\x12\x002\x12\x003*\x05Y\x12\x004\x12\x005\x12\x006\x12\x007\x12\x008*\x05Y\x12\x009\x12\x00:\x12\x00;\x12\x00<\x12\x00=\x12\x00>\x12\x00?\x12\x00@\x12\x00A\x12\x00B\x12\x00C*\x0bY\x12\x00D\x12\x00E\x12\x00F\x12\x00G\x12\x00H\x12\x00I\x12\x00J\x12\x00K\x12\x00L\x12\x00M\x12\x00N*\x0bY\x12\x00O\x12\x00P\x12\x00Q\x12\x00R\x12\x00S\x12\x00T\x12\x00U\x12\x00V\x12\x00W\x12\x00X*\nY\x12\x00Y\x12\x00Z\x12\x00[\x12\x00\\\x12\x00]\x12\x00^\x12\x00_\x12\x00`\x12\x00a\x12\x00b*\nY\x12\x00c\x12\x00d\x12\x00e\x12\x00f\x12\x00g\x12\x00h\x12\x00i\x12\x00j\x12\x00k\x12\x00l\x12\x00m*\x0bY\x12\x00n\x12\x00o\x12\x00p\x12\x00q\x12\x00r\x12\x00s\x12\x00t\x12\x00u\x12\x00v\x12\x00w\x12\x00x*\x0bY\x12\x00y\x12\x00z\x12\x00{\x12\x00|\x12\x00}\x12\x00~\x12\x00\x7f\x12\x00\x80\x12\x00\x81\x12\x00\x82*\nY\x12\x00\x83\x12\x00\x84\x12\x00\x85\x12\x00\x86\x12\x00\x87\x12\x00\x88\x12\x00\x89\x12\x00\x8a\x12\x00\x8b\x12\x00\x8c\x12\x00\x8d*\x0bY\x12\x00\x8e\x12\x00\x8f\x12\x00\x90\x12\x00\x91\x12\x00\x92\x12\x00\x93\x12\x00\x94\x12\x00\x95\x12\x00\x96\x12\x00\x97*\nY\x12\x00\x98\x12\x00\x99\x12\x00\x9a\x12\x00\x9b\x12\x00\x9c\x12\x00\x9d\x12\x00\x9e\x12\x00\x9f\x12\x00\xa0\x12\x00\xa2\x12\x00\xa3*\x0bY\x12\x00\xa4\x12\x00\xa5*\x02Y\x12\nname0\x12\nname1\x12\nname2\x12\nname3\x12\nname4\x12\nname5\x12\nname6\x12\nname7\x12\nname8\x12\nname9*\nY\x12$quite_a_long_name0\x12$quite_a_long_name1\x12$quite_a_long_name2\x12$quite_a_long_name3*\x04Y\x12$quite_a_long_name4\x12$quite_a_long_name5\x12$quite_a_long_name6\x12$quite_a_long_name7*\x04Y\x12$quite_a_long_name8\x12$quite_a_long_name9\x12&quite_a_long_name10\x12&quite_a_long_name11*\x04YQc\x00\x00\x86H8H?/\x80#####################+++++++++\x00#\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14QPR\x81\x82\x83*\x07\xc0#\x15QPR\x81\x82\x83*\x07\xc0#\x16QPR\x81\x82\x83*\x07\xc0#\x17QPR\x81\x82\x83*\x07\xc0#\x18QPR\x81\x82\x83*\x07\xc0#\x19QPR\x81\x82\x83*\x07\xc0#\x1aQPR\x81\x82\x83*\x07\xc0#\x1bQPR\x81\x82\x83*\x07\xc0#\x1cQPR\x81\x82\x83*\x07\xc0#\x1dQPR\x81\x82\x83*\x07\xc0Qc\x1e\x00s\x1ethis will be a string object 0s\x1ethis will be a string object 1s\x1ethis will be a string object 2s\x1ethis will be a string object 3s\x1ethis will be a string object 4s\x1ethis will be a string object 5s\x1ethis will be a string object 6s\x1ethis will be a string object 7s\x1ethis will be a string object 8s\x1ethis will be a string object 9b\x1dthis will be a bytes object 0b\x1dthis will be a bytes object 1b\x1dthis will be a bytes object 2b\x1dthis will be a bytes object 3b\x1dthis will be a bytes object 4b\x1dthis will be a bytes object 5b\x1dthis will be a bytes object 6b\x1dthis will be a bytes object 7b\x1dthis will be a bytes object 8b\x1dthis will be a bytes object 9s\rconst tuple 0s\rconst tuple 1s\rconst tuple 2s\rconst tuple 3s\rconst tuple 4s\rconst tuple 5s\rconst tuple 6s\rconst tuple 7s\rconst tuple 8s\rconst tuple 9' +file_data = b"M\x06\x02\x1f\x81=\x1e\x0etest.py\x00\x0f\x04A0\x00\x04A1\x00\x04f0\x00\x04f1\x00\x0cresult\x00/-5\x04a0\x00\x04a1\x00\x04a2\x00\x04a3\x00\x13\x15\x17\x19\x1b\x1d\x1f!#%')+1379;=?ACEGIKMOQSUWY[]_acegikmoqsuwy{}\x7f\x81\x01\x81\x03\x81\x05\x81\x07\x81\t\x81\x0b\x81\r\x81\x0f\x81\x11\x81\x13\x81\x15\x81\x17\x81\x19\x81\x1b\x81\x1d\x81\x1f\x81!\x81#\x81%\x81'\x81)\x81+\x81-\x81/\x811\x813\x815\x817\x819\x81;\x81=\x81?\x81A\x81C\x81E\x81G\x81I\x81K\x81M\x81O\x81Q\x81S\x81U\x81W\x81Y\x81[\x81]\x81_\x81a\x81c\x81e\x81g\x81i\x81k\x81m\x81o\x81q\x81s\x81u\x81w\x81y\x81{\x81}\x81\x7f\x82\x01\x82\x03\x82\x05\x82\x07\x82\t\x82\x0b\x82\r\x82\x0f\x82\x11\x82\x13\x82\x15\x82\x17\x82\x19\x82\x1b\x82\x1d\x82\x1f\x82!\x82#\x82%\x82'\x82)\x82+\x82-\x82/\x821\x823\x825\x827\x829\x82;\x82=\x82?\x82A\x82E\x82G\x82I\x82K\nname0\x00\nname1\x00\nname2\x00\nname3\x00\nname4\x00\nname5\x00\nname6\x00\nname7\x00\nname8\x00\nname9\x00$quite_a_long_name0\x00$quite_a_long_name1\x00$quite_a_long_name2\x00$quite_a_long_name3\x00$quite_a_long_name4\x00$quite_a_long_name5\x00$quite_a_long_name6\x00$quite_a_long_name7\x00$quite_a_long_name8\x00$quite_a_long_name9\x00&quite_a_long_name10\x00&quite_a_long_name11\x00s\x1ethis will be a string object 0\x00s\x1ethis will be a string object 1\x00s\x1ethis will be a string object 2\x00s\x1ethis will be a string object 3\x00s\x1ethis will be a string object 4\x00s\x1ethis will be a string object 5\x00s\x1ethis will be a string object 6\x00s\x1ethis will be a string object 7\x00s\x1ethis will be a string object 8\x00s\x1ethis will be a string object 9\x00b\x1dthis will be a bytes object 0\x00b\x1dthis will be a bytes object 1\x00b\x1dthis will be a bytes object 2\x00b\x1dthis will be a bytes object 3\x00b\x1dthis will be a bytes object 4\x00b\x1dthis will be a bytes object 5\x00b\x1dthis will be a bytes object 6\x00b\x1dthis will be a bytes object 7\x00b\x1dthis will be a bytes object 8\x00b\x1dthis will be a bytes object 9\x00s\rconst tuple 0\x00s\rconst tuple 1\x00s\rconst tuple 2\x00s\rconst tuple 3\x00s\rconst tuple 4\x00s\rconst tuple 5\x00s\rconst tuple 6\x00s\rconst tuple 7\x00s\rconst tuple 8\x00s\rconst tuple 9\x00\x82d\x10\x12\x01i@i@\x84\x18\x84\x1fT2\x00\x10\x024\x02\x16\x02T2\x01\x10\x034\x02\x16\x032\x02\x16\x042\x03\x16\x05\"\x80{\x16\x06Qc\x04\x82\x0c\x00\n\x02($$$\x11\x07\x16\x08\x10\x02\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04@\t\x08\n\x81\x0b Qc@\t\x08\x0b\x81\x0b@Qc@\t\x08\x0c\x81\x0b`QcH\t\n\r\x81\x0b` Qc\x82\x14\x00\x0c\x03h`$$$\x11\x07\x16\x08\x10\x03\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04H\t\n\n\x81\x0b``QcH\t\n\x0b\x81\x0b\x80\x07QcH\t\n\x0c\x81\x0b\x80\x08QcH\t\n\r\x81\x0b\x80\tQc\xa08P:\x04\x80\x0b13///---997799<\x1f%\x1f\"\x1f%)\x1f\"//\x12\x0e\x12\x0f\x12\x10\x12\x11\x12\x12\x12\x13\x12\x14*\x07Y\x12\x15\x12\x16\x12\x17\x12\x18\x12\x19\x12\x1a\x12\x08\x12\x07*\x08Y\x12\x1b\x12\x1c\x12\t\x12\x1d\x12\x1e\x12\x1f*\x06Y\x12 \x12!\x12\"\x12#\x12$\x12%*\x06Y\x12&\x12'\x12(\x12)\x12*\x12+*\x06Y\x12,\x12-\x12.\x12/\x120*\x05Y\x121\x122\x123\x124\x125*\x05Y\x126\x127\x128\x129\x12:*\x05Y\x12;\x12<\x12=\x12>\x12?\x12@\x12A\x12B\x12C\x12D\x12E*\x0bY\x12F\x12G\x12H\x12I\x12J\x12K\x12L\x12M\x12N\x12O\x12P*\x0bY\x12Q\x12R\x12S\x12T\x12U\x12V\x12W\x12X\x12Y\x12Z*\nY\x12[\x12\\\x12]\x12^\x12_\x12`\x12a\x12b\x12c\x12d*\nY\x12e\x12f\x12g\x12h\x12i\x12j\x12k\x12l\x12m\x12n\x12o*\x0bY\x12p\x12q\x12r\x12s\x12t\x12u\x12v\x12w\x12x\x12y\x12z*\x0bY\x12{\x12|\x12}\x12~\x12\x7f\x12\x81\x00\x12\x81\x01\x12\x81\x02\x12\x81\x03\x12\x81\x04*\nY\x12\x81\x05\x12\x81\x06\x12\x81\x07\x12\x81\x08\x12\x81\t\x12\x81\n\x12\x81\x0b\x12\x81\x0c\x12\x81\r\x12\x81\x0e\x12\x81\x0f*\x0bY\x12\x81\x10\x12\x81\x11\x12\x81\x12\x12\x81\x13\x12\x81\x14\x12\x81\x15\x12\x81\x16\x12\x81\x17\x12\x81\x18\x12\x81\x19*\nY\x12\x81\x1a\x12\x81\x1b\x12\x81\x1c\x12\x81\x1d\x12\x81\x1e\x12\x81\x1f\x12\x81 \x12\x81!\x12\x81\"\x12\x81#\x12\x81$*\x0bY\x12\x81%\x12\x81&*\x02Y\x12\x81'\x12\x81(\x12\x81)\x12\x81*\x12\x81+\x12\x81,\x12\x81-\x12\x81.\x12\x81/\x12\x810*\nY\x12\x811\x12\x812\x12\x813\x12\x814*\x04Y\x12\x815\x12\x816\x12\x817\x12\x818*\x04Y\x12\x819\x12\x81:\x12\x81;\x12\x81<*\x04YQc\x8cp8@\x05\x80#####################+++++++++#\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14QPR\x81\x82\x83*\x07\xc0#\x15QPR\x81\x82\x83*\x07\xc0#\x16QPR\x81\x82\x83*\x07\xc0#\x17QPR\x81\x82\x83*\x07\xc0#\x18QPR\x81\x82\x83*\x07\xc0#\x19QPR\x81\x82\x83*\x07\xc0#\x1aQPR\x81\x82\x83*\x07\xc0#\x1bQPR\x81\x82\x83*\x07\xc0#\x1cQPR\x81\x82\x83*\x07\xc0#\x1dQPR\x81\x82\x83*\x07\xc0Qc" class File(uio.IOBase): diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 58a4a75c91..da6352dab7 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -29,27 +29,49 @@ from __future__ import print_function import platform if platform.python_version_tuple()[0] == "2": - str_cons = lambda val, enc=None: val + from binascii import hexlify as hexlify_py2 + + str_cons = lambda val, enc=None: str(val) bytes_cons = lambda val, enc=None: bytearray(val) is_str_type = lambda o: type(o) is str is_bytes_type = lambda o: type(o) is bytearray is_int_type = lambda o: type(o) is int or type(o) is long + + def hexlify_to_str(b): + x = hexlify_py2(b) + return ":".join(x[i : i + 2] for i in range(0, len(x), 2)) + else: + from binascii import hexlify + str_cons = str bytes_cons = bytes is_str_type = lambda o: type(o) is str is_bytes_type = lambda o: type(o) is bytes is_int_type = lambda o: type(o) is int + + def hexlify_to_str(b): + return str(hexlify(b, ":"), "ascii") + + # end compatibility code import sys import struct -from collections import namedtuple sys.path.append(sys.path[0] + "/../py") import makeqstrdata as qstrutil +class MPYReadError(Exception): + def __init__(self, filename, msg): + self.filename = filename + self.msg = msg + + def __str__(self): + return "%s: %s" % (self.filename, self.msg) + + class FreezeError(Exception): def __init__(self, rawcode, msg): self.rawcode = rawcode @@ -60,7 +82,7 @@ class FreezeError(Exception): class Config: - MPY_VERSION = 5 + MPY_VERSION = 6 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -82,20 +104,6 @@ for n in qstrutil.static_qstr_list: global_qstrs.append(QStrType(n)) -class QStrWindow: - def __init__(self, size): - self.window = [] - self.size = size - - def push(self, val): - self.window = [val] + self.window[: self.size - 1] - - def access(self, idx): - val = self.window[idx] - self.window = [val] + self.window[:idx] + self.window[idx + 1 :] - return val - - MP_CODE_BYTECODE = 2 MP_CODE_NATIVE_PY = 3 MP_CODE_NATIVE_VIPER = 4 @@ -113,6 +121,10 @@ MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 +MP_SCOPE_FLAG_VIPERRELOC = 0x10 +MP_SCOPE_FLAG_VIPERRODATA = 0x20 +MP_SCOPE_FLAG_VIPERBSS = 0x40 + MP_BC_MASK_EXTRA_BYTE = 0x9E MP_BC_FORMAT_BYTE = 0 @@ -120,11 +132,180 @@ MP_BC_FORMAT_QSTR = 1 MP_BC_FORMAT_VAR_UINT = 2 MP_BC_FORMAT_OFFSET = 3 -# extra byte if caching enabled: -MP_BC_LOAD_NAME = 0x11 -MP_BC_LOAD_GLOBAL = 0x12 -MP_BC_LOAD_ATTR = 0x13 -MP_BC_STORE_ATTR = 0x18 +mp_unary_op_method_name = ( + "__pos__", + "__neg__", + "__invert__", + "", +) + +mp_binary_op_method_name = ( + "__lt__", + "__gt__", + "__eq__", + "__le__", + "__ge__", + "__ne__", + "", + "", + "", + "__ior__", + "__ixor__", + "__iand__", + "__ilshift__", + "__irshift__", + "__iadd__", + "__isub__", + "__imul__", + "__imatmul__", + "__ifloordiv__", + "__itruediv__", + "__imod__", + "__ipow__", + "__or__", + "__xor__", + "__and__", + "__lshift__", + "__rshift__", + "__add__", + "__sub__", + "__mul__", + "__matmul__", + "__floordiv__", + "__truediv__", + "__mod__", + "__pow__", +) + + +class Opcodes: + # fmt: off + # Load, Store, Delete, Import, Make, Build, Unpack, Call, Jump, Exception, For, sTack, Return, Yield, Op + MP_BC_BASE_RESERVED = (0x00) # ---------------- + MP_BC_BASE_QSTR_O = (0x10) # LLLLLLSSSDDII--- + MP_BC_BASE_VINT_E = (0x20) # MMLLLLSSDDBBBBBB + MP_BC_BASE_VINT_O = (0x30) # UUMMCCCC-------- + MP_BC_BASE_JUMP_E = (0x40) # J-JJJJJEEEEF---- + MP_BC_BASE_BYTE_O = (0x50) # LLLLSSDTTTTTEEFF + MP_BC_BASE_BYTE_E = (0x60) # --BREEEYYI------ + MP_BC_LOAD_CONST_SMALL_INT_MULTI = (0x70) # LLLLLLLLLLLLLLLL + # = (0x80) # LLLLLLLLLLLLLLLL + # = (0x90) # LLLLLLLLLLLLLLLL + # = (0xa0) # LLLLLLLLLLLLLLLL + MP_BC_LOAD_FAST_MULTI = (0xb0) # LLLLLLLLLLLLLLLL + MP_BC_STORE_FAST_MULTI = (0xc0) # SSSSSSSSSSSSSSSS + MP_BC_UNARY_OP_MULTI = (0xd0) # OOOOOOO + MP_BC_BINARY_OP_MULTI = (0xd7) # OOOOOOOOO + # = (0xe0) # OOOOOOOOOOOOOOOO + # = (0xf0) # OOOOOOOOOO------ + + MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM = 64 + MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS = 16 + MP_BC_LOAD_FAST_MULTI_NUM = 16 + MP_BC_STORE_FAST_MULTI_NUM = 16 + MP_BC_UNARY_OP_MULTI_NUM = 4 # MP_UNARY_OP_NUM_BYTECODE + MP_BC_BINARY_OP_MULTI_NUM = 35 # MP_BINARY_OP_NUM_BYTECODE + + MP_BC_LOAD_CONST_FALSE = (MP_BC_BASE_BYTE_O + 0x00) + MP_BC_LOAD_CONST_NONE = (MP_BC_BASE_BYTE_O + 0x01) + MP_BC_LOAD_CONST_TRUE = (MP_BC_BASE_BYTE_O + 0x02) + MP_BC_LOAD_CONST_SMALL_INT = (MP_BC_BASE_VINT_E + 0x02) # signed var-int + MP_BC_LOAD_CONST_STRING = (MP_BC_BASE_QSTR_O + 0x00) # qstr + MP_BC_LOAD_CONST_OBJ = (MP_BC_BASE_VINT_E + 0x03) # ptr + MP_BC_LOAD_NULL = (MP_BC_BASE_BYTE_O + 0x03) + + MP_BC_LOAD_FAST_N = (MP_BC_BASE_VINT_E + 0x04) # uint + MP_BC_LOAD_DEREF = (MP_BC_BASE_VINT_E + 0x05) # uint + MP_BC_LOAD_NAME = (MP_BC_BASE_QSTR_O + 0x01) # qstr + MP_BC_LOAD_GLOBAL = (MP_BC_BASE_QSTR_O + 0x02) # qstr + MP_BC_LOAD_ATTR = (MP_BC_BASE_QSTR_O + 0x03) # qstr + MP_BC_LOAD_METHOD = (MP_BC_BASE_QSTR_O + 0x04) # qstr + MP_BC_LOAD_SUPER_METHOD = (MP_BC_BASE_QSTR_O + 0x05) # qstr + MP_BC_LOAD_BUILD_CLASS = (MP_BC_BASE_BYTE_O + 0x04) + MP_BC_LOAD_SUBSCR = (MP_BC_BASE_BYTE_O + 0x05) + + MP_BC_STORE_FAST_N = (MP_BC_BASE_VINT_E + 0x06) # uint + MP_BC_STORE_DEREF = (MP_BC_BASE_VINT_E + 0x07) # uint + MP_BC_STORE_NAME = (MP_BC_BASE_QSTR_O + 0x06) # qstr + MP_BC_STORE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x07) # qstr + MP_BC_STORE_ATTR = (MP_BC_BASE_QSTR_O + 0x08) # qstr + MP_BC_STORE_SUBSCR = (MP_BC_BASE_BYTE_O + 0x06) + + MP_BC_DELETE_FAST = (MP_BC_BASE_VINT_E + 0x08) # uint + MP_BC_DELETE_DEREF = (MP_BC_BASE_VINT_E + 0x09) # uint + MP_BC_DELETE_NAME = (MP_BC_BASE_QSTR_O + 0x09) # qstr + MP_BC_DELETE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x0a) # qstr + + MP_BC_DUP_TOP = (MP_BC_BASE_BYTE_O + 0x07) + MP_BC_DUP_TOP_TWO = (MP_BC_BASE_BYTE_O + 0x08) + MP_BC_POP_TOP = (MP_BC_BASE_BYTE_O + 0x09) + MP_BC_ROT_TWO = (MP_BC_BASE_BYTE_O + 0x0a) + MP_BC_ROT_THREE = (MP_BC_BASE_BYTE_O + 0x0b) + + MP_BC_JUMP = (MP_BC_BASE_JUMP_E + 0x02) # rel byte code offset, 16-bit signed, in excess + MP_BC_POP_JUMP_IF_TRUE = (MP_BC_BASE_JUMP_E + 0x03) # rel byte code offset, 16-bit signed, in excess + MP_BC_POP_JUMP_IF_FALSE = (MP_BC_BASE_JUMP_E + 0x04) # rel byte code offset, 16-bit signed, in excess + MP_BC_JUMP_IF_TRUE_OR_POP = (MP_BC_BASE_JUMP_E + 0x05) # rel byte code offset, 16-bit signed, in excess + MP_BC_JUMP_IF_FALSE_OR_POP = (MP_BC_BASE_JUMP_E + 0x06) # rel byte code offset, 16-bit signed, in excess + MP_BC_UNWIND_JUMP = (MP_BC_BASE_JUMP_E + 0x00) # rel byte code offset, 16-bit signed, in excess; then a byte + MP_BC_SETUP_WITH = (MP_BC_BASE_JUMP_E + 0x07) # rel byte code offset, 16-bit unsigned + MP_BC_SETUP_EXCEPT = (MP_BC_BASE_JUMP_E + 0x08) # rel byte code offset, 16-bit unsigned + MP_BC_SETUP_FINALLY = (MP_BC_BASE_JUMP_E + 0x09) # rel byte code offset, 16-bit unsigned + MP_BC_POP_EXCEPT_JUMP = (MP_BC_BASE_JUMP_E + 0x0a) # rel byte code offset, 16-bit unsigned + MP_BC_FOR_ITER = (MP_BC_BASE_JUMP_E + 0x0b) # rel byte code offset, 16-bit unsigned + MP_BC_WITH_CLEANUP = (MP_BC_BASE_BYTE_O + 0x0c) + MP_BC_END_FINALLY = (MP_BC_BASE_BYTE_O + 0x0d) + MP_BC_GET_ITER = (MP_BC_BASE_BYTE_O + 0x0e) + MP_BC_GET_ITER_STACK = (MP_BC_BASE_BYTE_O + 0x0f) + + MP_BC_BUILD_TUPLE = (MP_BC_BASE_VINT_E + 0x0a) # uint + MP_BC_BUILD_LIST = (MP_BC_BASE_VINT_E + 0x0b) # uint + MP_BC_BUILD_MAP = (MP_BC_BASE_VINT_E + 0x0c) # uint + MP_BC_STORE_MAP = (MP_BC_BASE_BYTE_E + 0x02) + MP_BC_BUILD_SET = (MP_BC_BASE_VINT_E + 0x0d) # uint + MP_BC_BUILD_SLICE = (MP_BC_BASE_VINT_E + 0x0e) # uint + MP_BC_STORE_COMP = (MP_BC_BASE_VINT_E + 0x0f) # uint + MP_BC_UNPACK_SEQUENCE = (MP_BC_BASE_VINT_O + 0x00) # uint + MP_BC_UNPACK_EX = (MP_BC_BASE_VINT_O + 0x01) # uint + + MP_BC_RETURN_VALUE = (MP_BC_BASE_BYTE_E + 0x03) + MP_BC_RAISE_LAST = (MP_BC_BASE_BYTE_E + 0x04) + MP_BC_RAISE_OBJ = (MP_BC_BASE_BYTE_E + 0x05) + MP_BC_RAISE_FROM = (MP_BC_BASE_BYTE_E + 0x06) + MP_BC_YIELD_VALUE = (MP_BC_BASE_BYTE_E + 0x07) + MP_BC_YIELD_FROM = (MP_BC_BASE_BYTE_E + 0x08) + + MP_BC_MAKE_FUNCTION = (MP_BC_BASE_VINT_O + 0x02) # uint + MP_BC_MAKE_FUNCTION_DEFARGS = (MP_BC_BASE_VINT_O + 0x03) # uint + MP_BC_MAKE_CLOSURE = (MP_BC_BASE_VINT_E + 0x00) # uint; extra byte + MP_BC_MAKE_CLOSURE_DEFARGS = (MP_BC_BASE_VINT_E + 0x01) # uint; extra byte + MP_BC_CALL_FUNCTION = (MP_BC_BASE_VINT_O + 0x04) # uint + MP_BC_CALL_FUNCTION_VAR_KW = (MP_BC_BASE_VINT_O + 0x05) # uint + MP_BC_CALL_METHOD = (MP_BC_BASE_VINT_O + 0x06) # uint + MP_BC_CALL_METHOD_VAR_KW = (MP_BC_BASE_VINT_O + 0x07) # uint + + MP_BC_IMPORT_NAME = (MP_BC_BASE_QSTR_O + 0x0b) # qstr + MP_BC_IMPORT_FROM = (MP_BC_BASE_QSTR_O + 0x0c) # qstr + MP_BC_IMPORT_STAR = (MP_BC_BASE_BYTE_E + 0x09) + # fmt: on + + # Create a dict mapping opcode value to opcode name. + mapping = ["unknown" for _ in range(256)] + for op_name in list(locals()): + if op_name.startswith("MP_BC_"): + mapping[locals()[op_name]] = op_name[len("MP_BC_") :] + for i in range(MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM): + name = "LOAD_CONST_SMALL_INT %d" % (i - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS) + mapping[MP_BC_LOAD_CONST_SMALL_INT_MULTI + i] = name + for i in range(MP_BC_LOAD_FAST_MULTI_NUM): + mapping[MP_BC_LOAD_FAST_MULTI + i] = "LOAD_FAST %d" % i + for i in range(MP_BC_STORE_FAST_MULTI_NUM): + mapping[MP_BC_STORE_FAST_MULTI + i] = "STORE_FAST %d" % i + for i in range(MP_BC_UNARY_OP_MULTI_NUM): + mapping[MP_BC_UNARY_OP_MULTI + i] = "UNARY_OP %d %s" % (i, mp_unary_op_method_name[i]) + for i in range(MP_BC_BINARY_OP_MULTI_NUM): + mapping[MP_BC_BINARY_OP_MULTI + i] = "BINARY_OP %d %s" % (i, mp_binary_op_method_name[i]) + # this function mirrors that in py/bc.c def mp_opcode_format(bytecode, ip, count_var_uint): @@ -147,6 +328,26 @@ def mp_opcode_format(bytecode, ip, count_var_uint): return f, ip - ip_start +def mp_opcode_decode(bytecode, ip): + opcode = bytecode[ip] + ip_start = ip + f = (0x000003A4 >> (2 * ((opcode) >> 4))) & 3 + extra_byte = (opcode & MP_BC_MASK_EXTRA_BYTE) == 0 + ip += 1 + arg = 0 + if f in (MP_BC_FORMAT_QSTR, MP_BC_FORMAT_VAR_UINT): + arg = bytecode[ip] & 0x7F + while bytecode[ip] & 0x80 != 0: + ip += 1 + arg = arg << 7 | bytecode[ip] & 0x7F + ip += 1 + elif f == MP_BC_FORMAT_OFFSET: + arg = bytecode[ip] | bytecode[ip + 1] << 8 + ip += 2 + ip += extra_byte + return f, ip - ip_start, arg + + def read_prelude_sig(read_byte): z = read_byte() # xSSSSEAA @@ -201,6 +402,7 @@ def extract_prelude(bytecode, ip): n_kwonly_args, n_def_pos_args, ) = read_prelude_sig(local_read_byte) + n_info, n_cell = read_prelude_size(local_read_byte) ip = ip_ref[0] @@ -208,74 +410,186 @@ def extract_prelude(bytecode, ip): ip = ip2 + n_info + n_cell # ip now points to first opcode # ip2 points to simple_name qstr - return ip, ip2, (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args) + + # Extract simple_name and argument qstrs (var uints). + args = [] + for arg_num in range(1 + n_pos_args + n_kwonly_args): + value = 0 + while True: + b = local_read_byte() + value = (value << 7) | (b & 0x7F) + if b & 0x80 == 0: + break + args.append(value) + + return ( + ip2, + ip, + ip_ref[0], + (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args), + args, + ) class MPFunTable: - pass + def __repr__(self): + return "mp_fun_table" -class RawCode(object): - # a set of all escaped names, to make sure they are unique - escaped_names = set() - - # convert code kind number to string - code_kind_str = { - MP_CODE_BYTECODE: "MP_CODE_BYTECODE", - MP_CODE_NATIVE_PY: "MP_CODE_NATIVE_PY", - MP_CODE_NATIVE_VIPER: "MP_CODE_NATIVE_VIPER", - MP_CODE_NATIVE_ASM: "MP_CODE_NATIVE_ASM", - } - - def __init__(self, code_kind, bytecode, prelude_offset, qstrs, objs, raw_codes): - # set core variables - self.code_kind = code_kind - self.bytecode = bytecode - self.prelude_offset = prelude_offset - self.qstrs = qstrs - self.objs = objs - self.raw_codes = raw_codes - - if self.prelude_offset is None: - # no prelude, assign a dummy simple_name - self.prelude_offset = 0 - self.simple_name = global_qstrs[1] - else: - # extract prelude - self.ip, self.ip2, self.prelude = extract_prelude(self.bytecode, self.prelude_offset) - self.simple_name = self._unpack_qstr(self.ip2) - self.source_file = self._unpack_qstr(self.ip2 + 2) - self.line_info_offset = self.ip2 + 4 +class CompiledModule: + def __init__( + self, + mpy_source_file, + mpy_segments, + header, + qstr_table, + obj_table, + raw_code, + raw_code_file_offset, + escaped_name, + ): + self.mpy_source_file = mpy_source_file + self.mpy_segments = mpy_segments + self.source_file = qstr_table[0] + self.header = header + self.qstr_table = qstr_table + self.obj_table = obj_table + self.raw_code_file_offset = raw_code_file_offset + self.raw_code = raw_code + self.escaped_name = escaped_name def _unpack_qstr(self, ip): qst = self.bytecode[ip] | self.bytecode[ip + 1] << 8 return global_qstrs[qst] - def dump(self): - # dump children first - for rc in self.raw_codes: - rc.freeze("") - # TODO + def hexdump(self): + with open(self.mpy_source_file, "rb") as f: + WIDTH = 16 + COL_OFF = "\033[0m" + COL_TABLE = ( + ("", ""), # META + ("\033[0;31m", "\033[0;91m"), # QSTR + ("\033[0;32m", "\033[0;92m"), # OBJ + ("\033[0;34m", "\033[0;94m"), # CODE + ) + cur_col = "" + cur_col_index = 0 + offset = 0 + segment_index = 0 + while True: + data = bytes_cons(f.read(WIDTH)) + if not data: + break - def freeze_children(self, parent_name): - self.escaped_name = parent_name + self.simple_name.qstr_esc + # Print out the hex dump of this line of data. + line_hex = cur_col + line_chr = cur_col + line_comment = "" + for i in range(len(data)): + # Determine the colour of the data, if any, and the line comment. + while segment_index < len(self.mpy_segments): + if offset + i == self.mpy_segments[segment_index].start: + cur_col = COL_TABLE[self.mpy_segments[segment_index].kind][ + cur_col_index + ] + cur_col_index = 1 - cur_col_index + line_hex += cur_col + line_chr += cur_col + line_comment += " %s%s%s" % ( + cur_col, + self.mpy_segments[segment_index].name, + COL_OFF, + ) + if offset + i == self.mpy_segments[segment_index].end: + cur_col = "" + line_hex += COL_OFF + line_chr += COL_OFF + segment_index += 1 + else: + break - # make sure the escaped name is unique - i = 2 - while self.escaped_name in RawCode.escaped_names: - self.escaped_name = parent_name + self.simple_name.qstr_esc + str(i) - i += 1 - RawCode.escaped_names.add(self.escaped_name) + # Add to the hex part of the line. + if i % 2 == 0: + line_hex += " " + line_hex += "%02x" % data[i] - # emit children first - for rc in self.raw_codes: - rc.freeze(self.escaped_name + "_") + # Add to the characters part of the line. + if 0x20 <= data[i] <= 0x7E: + line_chr += "%s" % chr(data[i]) + else: + line_chr += "." + + # Print out this line. + if cur_col: + line_hex += COL_OFF + line_chr += COL_OFF + pad = " " * ((WIDTH - len(data)) * 5 // 2) + print("%08x:%s%s %s %s" % (offset, line_hex, pad, line_chr, line_comment)) + offset += WIDTH + + def disassemble(self): + print("mpy_source_file:", self.mpy_source_file) + print("source_file:", self.source_file.str) + print("header:", hexlify_to_str(self.header)) + print("qstr_table[%u]:" % len(self.qstr_table)) + for q in self.qstr_table: + print(" %s" % q.str) + print("obj_table:", self.obj_table) + self.raw_code.disassemble() + + def freeze(self, compiled_module_index): + print() + print("/" * 80) + print("// frozen module %s" % self.escaped_name) + print("// - original source file: %s" % self.mpy_source_file) + print("// - frozen file name: %s" % self.source_file.str) + print("// - .mpy header: %s" % ":".join("%02x" % b for b in self.header)) + print() + + self.raw_code.freeze() + print() + + self.freeze_constants() + + print() + print("static const mp_frozen_module_t frozen_module_%s = {" % self.escaped_name) + print(" .constants = {") + if len(self.qstr_table): + print( + " .qstr_table = (qstr_short_t *)&const_qstr_table_data_%s," + % self.escaped_name + ) + else: + print(" .qstr_table = NULL,") + if len(self.obj_table): + print(" .obj_table = (mp_obj_t *)&const_obj_table_data_%s," % self.escaped_name) + else: + print(" .obj_table = NULL,") + print(" },") + print(" .rc = &raw_code_%s," % self.raw_code.escaped_name) + print("};") def freeze_constants(self): + global const_str_content, const_int_content, const_obj_content + + if len(self.qstr_table): + print( + "static const qstr_short_t const_qstr_table_data_%s[%u] = {" + % (self.escaped_name, len(self.qstr_table)) + ) + for q in self.qstr_table: + print(" %s," % q.qstr_id) + print("};") + + if not len(self.obj_table): + return + # generate constant objects - for i, obj in enumerate(self.objs): + print() + print("// constants") + for i, obj in enumerate(self.obj_table): obj_name = "const_obj_%s_%u" % (self.escaped_name, i) - if obj is MPFunTable: + if isinstance(obj, MPFunTable): pass elif obj is Ellipsis: print("#define %s mp_const_ellipsis_obj" % obj_name) @@ -286,7 +600,7 @@ class RawCode(object): else: obj_type = "mp_type_bytes" print( - 'STATIC const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};' + 'static const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};' % ( obj_name, obj_type, @@ -295,6 +609,8 @@ class RawCode(object): "".join(("\\x%02x" % b) for b in obj), ) ) + const_str_content += len(obj) + const_obj_content += 4 * 4 elif is_int_type(obj): if config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_NONE: # TODO check if we can actually fit this long-int into a small-int @@ -316,76 +632,139 @@ class RawCode(object): ndigs = len(digs) digs = ",".join(("%#x" % d) for d in digs) print( - "STATIC const mp_obj_int_t %s = {{&mp_type_int}, " + "static const mp_obj_int_t %s = {{&mp_type_int}, " "{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t*)(const uint%u_t[]){%s}}};" % (obj_name, neg, ndigs, ndigs, bits_per_dig, bits_per_dig, digs) ) + const_int_content += (digs.count(",") + 1) * bits_per_dig // 8 + const_obj_content += 4 * 4 elif type(obj) is float: print( "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" ) print( - "STATIC const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};" + "static const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};" % (obj_name, obj) ) print("#endif") + const_obj_content += 3 * 4 elif type(obj) is complex: print( - "STATIC const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};" + "static const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};" % (obj_name, obj.real, obj.imag) ) else: raise FreezeError(self, "freezing of object %r is not implemented" % (obj,)) - # generate constant table, if it has any entries - const_table_len = len(self.qstrs) + len(self.objs) + len(self.raw_codes) - if const_table_len: - print( - "STATIC const mp_rom_obj_t const_table_data_%s[%u] = {" - % (self.escaped_name, const_table_len) - ) - for qst in self.qstrs: - print(" MP_ROM_QSTR(%s)," % global_qstrs[qst].qstr_id) - for i in range(len(self.objs)): - if self.objs[i] is MPFunTable: - print(" &mp_fun_table,") - elif type(self.objs[i]) is float: - print( - "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" - ) - print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) - print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C") - n = struct.unpack("> 8,") - print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") - print(" ", end="") - for i in range(self.ip2 + 4, self.ip): - print(" 0x%02x," % self.bytecode[i], end="") - print() - ip = self.ip - while ip < len(self.bytecode): - f, sz = mp_opcode_format(self.bytecode, ip, True) - if f == 1: - qst = self._unpack_qstr(ip + 1).qstr_id - extra = "" if sz == 3 else " 0x%02x," % self.bytecode[ip + 3] - print(" ", "0x%02x," % self.bytecode[ip], qst, "& 0xff,", qst, ">> 8,", extra) + def disassemble(self): + bc = self.fun_data + print("simple_name:", self.simple_name.str) + print(" raw bytecode:", len(bc), hexlify_to_str(bc)) + print(" prelude:", self.prelude) + print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) + print(" line info:", hexlify_to_str(bc[self.offset_line_info : self.offset_opcodes])) + ip = self.offset_opcodes + while ip < len(bc): + fmt, sz, arg = mp_opcode_decode(bc, ip) + if bc[ip] == Opcodes.MP_BC_LOAD_CONST_OBJ: + arg = "%r" % self.obj_table[arg] + if fmt == MP_BC_FORMAT_QSTR: + arg = self.qstr_table[arg].str + elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET): + pass else: - print(" ", "".join("0x%02x, " % self.bytecode[ip + i] for i in range(sz))) + arg = "" + print( + " %-11s %s %s" % (hexlify_to_str(bc[ip : ip + sz]), Opcodes.mapping[bc[ip]], arg) + ) ip += sz + self.disassemble_children() + + def freeze(self): + # generate bytecode data + bc = self.fun_data + print( + "// frozen bytecode for file %s, scope %s" + % (self.qstr_table[0].str, self.escaped_name) + ) + print("static const byte fun_data_%s[%u] = {" % (self.escaped_name, len(bc))) + + print(" ", end="") + for b in bc[: self.offset_names]: + print("0x%02x," % b, end="") + print(" // prelude") + + print(" ", end="") + for b in bc[self.offset_names : self.offset_line_info]: + print("0x%02x," % b, end="") + print(" // names: %s" % ", ".join(self.qstr_table[i].str for i in self.names)) + + print(" ", end="") + for b in bc[self.offset_line_info : self.offset_opcodes]: + print("0x%02x," % b, end="") + print(" // code info") + + ip = self.offset_opcodes + while ip < len(bc): + fmt, sz, arg = mp_opcode_decode(bc, ip) + opcode_name = Opcodes.mapping[bc[ip]] + if fmt == MP_BC_FORMAT_QSTR: + opcode_name += " " + self.qstr_table[arg].str + elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET): + opcode_name += " %u" % arg + print( + " %s, // %s" % (",".join("0x%02x" % b for b in bc[ip : ip + sz]), opcode_name) + ) + ip += sz + print("};") - self.freeze_constants() - self.freeze_module() + self.freeze_children() + self.freeze_raw_code() + + global bc_content + bc_content += len(bc) class RawCodeNative(RawCode): def __init__( self, - code_kind, + cm_escaped_name, + qstr_table, + kind, fun_data, prelude_offset, - prelude, qstr_links, - qstrs, - objs, - raw_codes, + scope_flags, + n_pos_args, type_sig, ): super(RawCodeNative, self).__init__( - code_kind, fun_data, prelude_offset, qstrs, objs, raw_codes + cm_escaped_name, qstr_table, fun_data, prelude_offset, kind ) - self.prelude = prelude + + if kind in (MP_CODE_NATIVE_VIPER, MP_CODE_NATIVE_ASM): + self.scope_flags = scope_flags + self.n_pos_args = n_pos_args + self.qstr_links = qstr_links self.type_sig = type_sig if config.native_arch in ( @@ -502,11 +925,32 @@ class RawCodeNative(RawCode): # ARMVxxM -- two byte align. self.fun_data_attributes += " __attribute__ ((aligned (2)))" + def disassemble(self): + fun_data = self.fun_data + print("simple_name:", self.simple_name.str) + print( + " raw data:", + len(fun_data), + hexlify_to_str(fun_data[:32]), + "..." if len(fun_data) > 32 else "", + ) + if self.code_kind != MP_CODE_NATIVE_PY: + return + print(" prelude:", self.prelude) + print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) + print(" line info:", fun_data[self.offset_line_info : self.offset_opcodes]) + ip = 0 + while ip < self.prelude_offset: + sz = 16 + print(" ", hexlify_to_str(fun_data[ip : min(ip + sz, self.prelude_offset)])) + ip += sz + self.disassemble_children() + def _asm_thumb_rewrite_mov(self, pc, val): - print(" (%u & 0xf0) | (%s >> 12)," % (self.bytecode[pc], val), end="") - print(" (%u & 0xfb) | (%s >> 9 & 0x04)," % (self.bytecode[pc + 1], val), end="") + print(" (%u & 0xf0) | (%s >> 12)," % (self.fun_data[pc], val), end="") + print(" (%u & 0xfb) | (%s >> 9 & 0x04)," % (self.fun_data[pc + 1], val), end="") print(" (%s & 0xff)," % (val,), end="") - print(" (%u & 0x07) | (%s >> 4 & 0x70)," % (self.bytecode[pc + 3], val)) + print(" (%u & 0x07) | (%s >> 4 & 0x70)," % (self.fun_data[pc + 3], val)) def _link_qstr(self, pc, kind, qst): if kind == 0: @@ -543,32 +987,22 @@ class RawCodeNative(RawCode): else: assert 0 - def freeze(self, parent_name): - if self.prelude[2] & ~0x0F: + def freeze(self): + if self.scope_flags & ~0x0F: raise FreezeError("unable to freeze code with relocations") - self.freeze_children(parent_name) - # generate native code data print() - if self.code_kind == MP_CODE_NATIVE_PY: - print( - "// frozen native code for file %s, scope %s%s" - % (self.source_file.str, parent_name, self.simple_name.str) - ) - elif self.code_kind == MP_CODE_NATIVE_VIPER: - print("// frozen viper code for scope %s" % (parent_name,)) - else: - print("// frozen assembler code for scope %s" % (parent_name,)) print( - "STATIC const byte fun_data_%s[%u] %s = {" - % (self.escaped_name, len(self.bytecode), self.fun_data_attributes) + "// frozen native code for file %s, scope %s" + % (self.qstr_table[0].str, self.escaped_name) + ) + print( + "static const byte fun_data_%s[%u] %s = {" + % (self.escaped_name, len(self.fun_data), self.fun_data_attributes) ) - if self.code_kind == MP_CODE_NATIVE_PY: - i_top = self.prelude_offset - else: - i_top = len(self.bytecode) + i_top = len(self.fun_data) i = 0 qi = 0 while i < i_top: @@ -585,229 +1019,253 @@ class RawCodeNative(RawCode): i16 = min(i16, self.qstr_links[qi][0]) print(" ", end="") for ii in range(i, i16): - print(" 0x%02x," % self.bytecode[ii], end="") + print(" 0x%02x," % self.fun_data[ii], end="") print() i = i16 - if self.code_kind == MP_CODE_NATIVE_PY: - print(" ", end="") - for i in range(self.prelude_offset, self.ip2): - print(" 0x%02x," % self.bytecode[i], end="") - print() - - print(" ", self.simple_name.qstr_id, "& 0xff,", self.simple_name.qstr_id, ">> 8,") - print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") - - print(" ", end="") - for i in range(self.ip2 + 4, self.ip): - print(" 0x%02x," % self.bytecode[i], end="") - print() - print("};") - self.freeze_constants() - self.freeze_module(self.qstr_links, self.type_sig) + self.freeze_children() + self.freeze_raw_code(self.qstr_links, self.type_sig) -class BytecodeBuffer: - def __init__(self, size): - self.buf = bytearray(size) - self.idx = 0 +class MPYSegment: + META = 0 + QSTR = 1 + OBJ = 2 + CODE = 3 - def is_full(self): - return self.idx == len(self.buf) - - def append(self, b): - self.buf[self.idx] = b - self.idx += 1 + def __init__(self, kind, name, start, end): + self.kind = kind + self.name = name + self.start = start + self.end = end -def read_byte(f, out=None): - b = bytes_cons(f.read(1))[0] - if out is not None: - out.append(b) - return b +class MPYReader: + def __init__(self, filename, fileobj): + self.filename = filename + self.fileobj = fileobj + + def tell(self): + return self.fileobj.tell() + + def read_byte(self): + return bytes_cons(self.fileobj.read(1))[0] + + def read_bytes(self, n): + return bytes_cons(self.fileobj.read(n)) + + def read_uint(self): + i = 0 + while True: + b = self.read_byte() + i = (i << 7) | (b & 0x7F) + if b & 0x80 == 0: + break + return i -def read_uint(f, out=None): - i = 0 - while True: - b = read_byte(f, out) - i = (i << 7) | (b & 0x7F) - if b & 0x80 == 0: - break - return i - - -def read_qstr(f, qstr_win): - ln = read_uint(f) - if ln == 0: - # static qstr - return bytes_cons(f.read(1))[0] +def read_qstr(reader, segments): + start_pos = reader.tell() + ln = reader.read_uint() if ln & 1: - # qstr in table - return qstr_win.access(ln >> 1) + # static qstr + segments.append( + MPYSegment(MPYSegment.META, global_qstrs[ln >> 1].str, start_pos, start_pos) + ) + return ln >> 1 ln >>= 1 - data = str_cons(f.read(ln), "utf8") + start_pos = reader.tell() + data = str_cons(reader.read_bytes(ln), "utf8") + reader.read_byte() # read and discard null terminator + segments.append(MPYSegment(MPYSegment.QSTR, data, start_pos, reader.tell())) global_qstrs.append(QStrType(data)) - qstr_win.push(len(global_qstrs) - 1) return len(global_qstrs) - 1 -def read_obj(f): - obj_type = f.read(1) - if obj_type == b"e": +def read_obj(reader, segments): + obj_type = reader.read_bytes(1) + if obj_type == b"t": + return MPFunTable() + elif obj_type == b"e": return Ellipsis else: - buf = f.read(read_uint(f)) + ln = reader.read_uint() + start_pos = reader.tell() + buf = reader.read_bytes(ln) + if obj_type in (b"s", b"b"): + reader.read_byte() # read and discard null terminator if obj_type == b"s": - return str_cons(buf, "utf8") + obj = str_cons(buf, "utf8") elif obj_type == b"b": - return bytes_cons(buf) + obj = buf elif obj_type == b"i": - return int(str_cons(buf, "ascii"), 10) + obj = int(str_cons(buf, "ascii"), 10) elif obj_type == b"f": - return float(str_cons(buf, "ascii")) + obj = float(str_cons(buf, "ascii")) elif obj_type == b"c": - return complex(str_cons(buf, "ascii")) + obj = complex(str_cons(buf, "ascii")) else: - assert 0 + raise MPYReadError(reader.filename, "corrupt .mpy file") + segments.append(MPYSegment(MPYSegment.OBJ, obj, start_pos, reader.tell())) + return obj -def read_prelude(f, bytecode, qstr_win): - ( - n_state, - n_exc_stack, - scope_flags, - n_pos_args, - n_kwonly_args, - n_def_pos_args, - ) = read_prelude_sig(lambda: read_byte(f, bytecode)) - n_info, n_cell = read_prelude_size(lambda: read_byte(f, bytecode)) - read_qstr_and_pack(f, bytecode, qstr_win) # simple_name - read_qstr_and_pack(f, bytecode, qstr_win) # source_file - for _ in range(n_info - 4 + n_cell): - read_byte(f, bytecode) - return n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args - - -def read_qstr_and_pack(f, bytecode, qstr_win): - qst = read_qstr(f, qstr_win) - bytecode.append(qst & 0xFF) - bytecode.append(qst >> 8) - - -def read_bytecode(file, bytecode, qstr_win): - while not bytecode.is_full(): - op = read_byte(file, bytecode) - f, sz = mp_opcode_format(bytecode.buf, bytecode.idx - 1, False) - sz -= 1 - if f == MP_BC_FORMAT_QSTR: - read_qstr_and_pack(file, bytecode, qstr_win) - sz -= 2 - elif f == MP_BC_FORMAT_VAR_UINT: - while read_byte(file, bytecode) & 0x80: - pass - for _ in range(sz): - read_byte(file, bytecode) - - -def read_raw_code(f, qstr_win): - kind_len = read_uint(f) +def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments): + # Read raw code header. + kind_len = reader.read_uint() kind = (kind_len & 3) + MP_CODE_BYTECODE - fun_data_len = kind_len >> 2 - fun_data = BytecodeBuffer(fun_data_len) + has_children = (kind_len >> 2) & 1 + fun_data_len = kind_len >> 3 + + # Read the body of the raw code. + file_offset = reader.tell() + fun_data = reader.read_bytes(fun_data_len) + segments_len = len(segments) if kind == MP_CODE_BYTECODE: - prelude = read_prelude(f, fun_data, qstr_win) - read_bytecode(f, fun_data, qstr_win) + # Create bytecode raw code. + rc = RawCodeBytecode(cm_escaped_name, qstr_table, obj_table, fun_data) else: - fun_data.buf[:] = f.read(fun_data_len) - + # Create native raw code. qstr_links = [] if kind in (MP_CODE_NATIVE_PY, MP_CODE_NATIVE_VIPER): - # load qstr link table - n_qstr_link = read_uint(f) + # Read qstr link table. + n_qstr_link = reader.read_uint() for _ in range(n_qstr_link): - off = read_uint(f) - qst = read_qstr(f, qstr_win) + off = reader.read_uint() + qst = read_qstr(reader, segments) qstr_links.append((off >> 2, off & 3, qst)) - type_sig = 0 + native_scope_flags = 0 + native_n_pos_args = 0 + native_type_sig = 0 if kind == MP_CODE_NATIVE_PY: - prelude_offset = read_uint(f) - _, name_idx, prelude = extract_prelude(fun_data.buf, prelude_offset) - fun_data.idx = name_idx # rewind to where qstrs are in prelude - read_qstr_and_pack(f, fun_data, qstr_win) # simple_name - read_qstr_and_pack(f, fun_data, qstr_win) # source_file + prelude_offset = reader.read_uint() else: - prelude_offset = None - scope_flags = read_uint(f) - n_pos_args = 0 - if kind == MP_CODE_NATIVE_ASM: - n_pos_args = read_uint(f) - type_sig = read_uint(f) - prelude = (None, None, scope_flags, n_pos_args, 0) + prelude_offset = 0 + native_scope_flags = reader.read_uint() + if kind == MP_CODE_NATIVE_VIPER: + # Read any additional sections for native viper. + if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: + rodata_size = reader.read_uint() + if native_scope_flags & MP_SCOPE_FLAG_VIPERBSS: + bss_size = reader.read_uint() + if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: + reader.read_bytes(rodata_size) + if native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC: + while True: + op = reader.read_byte() + if op == 0xFF: + break + if op & 1: + addr = reader.read_uint() + op >>= 1 + if op <= 5 and op & 1: + n = reader.read_uint() + else: + assert kind == MP_CODE_NATIVE_ASM + native_n_pos_args = reader.read_uint() + native_type_sig = reader.read_uint() - qstrs = [] - objs = [] - raw_codes = [] - if kind != MP_CODE_NATIVE_ASM: - # load constant table - n_obj = read_uint(f) - n_raw_code = read_uint(f) - qstrs = [read_qstr(f, qstr_win) for _ in range(prelude[3] + prelude[4])] - if kind != MP_CODE_BYTECODE: - objs.append(MPFunTable) - objs.extend([read_obj(f) for _ in range(n_obj)]) - raw_codes = [read_raw_code(f, qstr_win) for _ in range(n_raw_code)] - - if kind == MP_CODE_BYTECODE: - return RawCodeBytecode(fun_data.buf, qstrs, objs, raw_codes) - else: - return RawCodeNative( + rc = RawCodeNative( + cm_escaped_name, + qstr_table, kind, - fun_data.buf, + fun_data, prelude_offset, - prelude, qstr_links, - qstrs, - objs, - raw_codes, - type_sig, + native_scope_flags, + native_n_pos_args, + native_type_sig, ) + # Add a segment for the raw code data. + segments.insert( + segments_len, + MPYSegment(MPYSegment.CODE, rc.simple_name.str, file_offset, file_offset + fun_data_len), + ) + + # Read children, if there are any. + rc.children = [] + if has_children: + n_children = reader.read_uint() + for _ in range(n_children): + rc.children.append( + read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments) + ) + + return rc + def read_mpy(filename): - with open(filename, "rb") as f: - header = bytes_cons(f.read(4)) + with open(filename, "rb") as fileobj: + reader = MPYReader(filename, fileobj) + segments = [] + + # Read and verify the header. + header = reader.read_bytes(4) if header[0] != ord("M"): - raise Exception("not a valid .mpy file") + raise MPYReadError(filename, "not a valid .mpy file") if header[1] != config.MPY_VERSION: - raise Exception("incompatible .mpy version") + raise MPYReadError(filename, "incompatible .mpy version") feature_byte = header[2] - qw_size = read_uint(f) config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_byte & 2) != 0 mpy_native_arch = feature_byte >> 2 if mpy_native_arch != MP_NATIVE_ARCH_NONE: if config.native_arch == MP_NATIVE_ARCH_NONE: config.native_arch = mpy_native_arch elif config.native_arch != mpy_native_arch: - raise Exception("native architecture mismatch") + raise MPYReadError(filename, "native architecture mismatch") config.mp_small_int_bits = header[3] - qstr_win = QStrWindow(qw_size) - rc = read_raw_code(f, qstr_win) - rc.mpy_source_file = filename - rc.qstr_win_size = qw_size - return rc + + # Read number of qstrs, and number of objects. + n_qstr = reader.read_uint() + n_obj = reader.read_uint() + + # Read qstrs and construct qstr table. + qstr_table = [] + for i in range(n_qstr): + q = read_qstr(reader, segments) + qstr_table.append(global_qstrs[q]) + + # Read objects and construct object table. + obj_table = [] + for i in range(n_obj): + obj_table.append(read_obj(reader, segments)) + + # Compute the compiled-module escaped name. + cm_escaped_name = qstr_table[0].str.replace("/", "_")[:-3] + + # Read the outer raw code, which will in turn read all its children. + raw_code_file_offset = reader.tell() + raw_code = read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments) + + # Create the outer-level compiled module representing the whole .mpy file. + return CompiledModule( + filename, + segments, + header, + qstr_table, + obj_table, + raw_code, + raw_code_file_offset, + cm_escaped_name, + ) -def dump_mpy(raw_codes): - for rc in raw_codes: - rc.dump() +def hexdump_mpy(compiled_modules): + for cm in compiled_modules: + cm.hexdump() -def freeze_mpy(base_qstrs, raw_codes): +def disassemble_mpy(compiled_modules): + for cm in compiled_modules: + cm.disassemble() + + +def freeze_mpy(base_qstrs, compiled_modules): # add to qstrs new = {} for q in global_qstrs: @@ -864,6 +1322,17 @@ def freeze_mpy(base_qstrs, raw_codes): # As in qstr.c, set so that the first dynamically allocated pool is twice this size; must be <= the len qstr_pool_alloc = min(len(new), 10) + global bc_content, const_str_content, const_int_content, const_obj_content, const_table_qstr_content, const_table_ptr_content, raw_code_count, raw_code_content + qstr_content = 0 + bc_content = 0 + const_str_content = 0 + const_int_content = 0 + const_obj_content = 0 + const_table_qstr_content = 0 + const_table_ptr_content = 0 + raw_code_count = 0 + raw_code_content = 0 + print() print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {") qstr_size = {"metadata": 0, "data": 0} @@ -892,32 +1361,49 @@ def freeze_mpy(base_qstrs, raw_codes): print(" {") for _, _, qstr, qbytes in new: print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes)) + qstr_content += ( + config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + len(qbytes) + 1 + ) print(" },") print("};") - for rc in raw_codes: - rc.freeze(rc.source_file.str.replace("/", "_")[:-3] + "_") + # Freeze all modules. + for idx, cm in enumerate(compiled_modules): + cm.freeze(idx) + # Print separator, separating individual modules from global data structures. + print() + print("/" * 80) + print("// collection of all frozen modules") + + # Define the string of frozen module names. print() print("const char mp_frozen_names[] = {") - print("#ifdef MP_FROZEN_STR_NAMES") + print(" #ifdef MP_FROZEN_STR_NAMES") # makemanifest.py might also include some frozen string content. - print("MP_FROZEN_STR_NAMES") - print("#endif") - for rc in raw_codes: - module_name = rc.source_file.str - print('"%s\\0"' % module_name) - print('"\\0"};') - - print("const mp_raw_code_t *const mp_frozen_mpy_content[] = {") - for rc in raw_codes: - print(" &raw_code_%s," % rc.escaped_name) + print(" MP_FROZEN_STR_NAMES") + print(" #endif") + mp_frozen_mpy_names_content = 1 + for cm in compiled_modules: + module_name = cm.source_file.str + print(' "%s\\0"' % module_name) + mp_frozen_mpy_names_content += len(cm.source_file.str) + 1 + print(' "\\0"') print("};") + # Define the array of pointers to frozen module content. + print() + print("const mp_frozen_module_t *const mp_frozen_mpy_content[] = {") + for cm in compiled_modules: + print(" &frozen_module_%s," % cm.escaped_name) + print("};") + mp_frozen_mpy_content_size = len(compiled_modules * 4) + # If a port defines MICROPY_FROZEN_LIST_ITEM then list all modules wrapped in that macro. + print() print("#ifdef MICROPY_FROZEN_LIST_ITEM") - for rc in raw_codes: - module_name = rc.source_file.str + for cm in compiled_modules: + module_name = cm.source_file.str if module_name.endswith("/__init__.py"): short_name = module_name[: -len("/__init__.py")] else: @@ -925,45 +1411,94 @@ def freeze_mpy(base_qstrs, raw_codes): print('MICROPY_FROZEN_LIST_ITEM("%s", "%s")' % (short_name, module_name)) print("#endif") + print() + print("/*") + print("byte sizes:") + print("qstr content: %d unique, %d bytes" % (len(new), qstr_content)) + print("bc content: %d" % bc_content) + print("const str content: %d" % const_str_content) + print("const int content: %d" % const_int_content) + print("const obj content: %d" % const_obj_content) + print( + "const table qstr content: %d entries, %d bytes" + % (const_table_qstr_content, const_table_qstr_content * 4) + ) + print( + "const table ptr content: %d entries, %d bytes" + % (const_table_ptr_content, const_table_ptr_content * 4) + ) + print("raw code content: %d * 4 = %d" % (raw_code_count, raw_code_content)) + print("mp_frozen_mpy_names_content: %d" % mp_frozen_mpy_names_content) + print("mp_frozen_mpy_content_size: %d" % mp_frozen_mpy_content_size) + print( + "total: %d" + % ( + qstr_content + + bc_content + + const_str_content + + const_int_content + + const_obj_content + + const_table_qstr_content * 4 + + const_table_ptr_content * 4 + + raw_code_content + + mp_frozen_mpy_names_content + + mp_frozen_mpy_content_size + ) + ) + print("*/") + def merge_mpy(raw_codes, output_file): - assert len(raw_codes) <= 31 # so var-uints all fit in 1 byte + assert len(raw_codes) <= 2 # so var-uints all fit in 1 byte merged_mpy = bytearray() if len(raw_codes) == 1: with open(raw_codes[0].mpy_source_file, "rb") as f: merged_mpy.extend(f.read()) else: - header = bytearray(5) + main_rc = None + for rc in raw_codes: + if len(rc.qstr_table) > 1 or len(rc.obj_table) > 0: + # Must use qstr_table and obj_table from this raw_code + if main_rc is not None: + raise Exception( + "can't merge files when more than one has a populated qstr or obj table" + ) + main_rc = rc + if main_rc is None: + main_rc = raw_codes[0] + + header = bytearray(4) header[0] = ord("M") header[1] = config.MPY_VERSION header[2] = config.native_arch << 2 | config.MICROPY_PY_BUILTINS_STR_UNICODE << 1 header[3] = config.mp_small_int_bits - header[4] = 32 # qstr_win_size merged_mpy.extend(header) + # Copy n_qstr, n_obj, qstr_table, obj_table from main_rc. + with open(main_rc.mpy_source_file, "rb") as f: + data = f.read(main_rc.raw_code_file_offset) + merged_mpy.extend(data[4:]) + bytecode = bytearray() - bytecode_len = 6 + len(raw_codes) * 5 + 2 - bytecode.append(bytecode_len << 2) # kind and length + bytecode_len = 3 + len(raw_codes) * 5 + 2 + bytecode.append(bytecode_len << 3 | 1 << 2) # kind, has_children and length bytecode.append(0b00000000) # signature prelude - bytecode.append(0b00001000) # size prelude - bytecode.extend(b"\x00\x01") # MP_QSTR_ - bytecode.extend(b"\x00\x01") # MP_QSTR_ + bytecode.append(0b00000010) # size prelude; n_info=1 + bytecode.extend(b"\x00") # simple_name: qstr index 0 (will use source filename) for idx in range(len(raw_codes)): bytecode.append(0x32) # MP_BC_MAKE_FUNCTION bytecode.append(idx) # index raw code bytecode.extend(b"\x34\x00\x59") # MP_BC_CALL_FUNCTION, 0 args, MP_BC_POP_TOP bytecode.extend(b"\x51\x63") # MP_BC_LOAD_NONE, MP_BC_RETURN_VALUE - bytecode.append(0) # n_obj - bytecode.append(len(raw_codes)) # n_raw_code - merged_mpy.extend(bytecode) + merged_mpy.append(len(raw_codes)) # n_children + for rc in raw_codes: with open(rc.mpy_source_file, "rb") as f: - f.read(4) # skip header - read_uint(f) # skip qstr_win_size + f.seek(rc.raw_code_file_offset) data = f.read() # read rest of mpy file merged_mpy.extend(data) @@ -978,7 +1513,12 @@ def main(): import argparse cmd_parser = argparse.ArgumentParser(description="A tool to work with MicroPython .mpy files.") - cmd_parser.add_argument("-d", "--dump", action="store_true", help="dump contents of files") + cmd_parser.add_argument( + "-x", "--hexdump", action="store_true", help="output an annotated hex dump of files" + ) + cmd_parser.add_argument( + "-d", "--disassemble", action="store_true", help="output disassembled contents of files" + ) cmd_parser.add_argument("-f", "--freeze", action="store_true", help="freeze files") cmd_parser.add_argument( "--merge", action="store_true", help="merge multiple .mpy files into one" @@ -1018,20 +1558,32 @@ def main(): else: config.MICROPY_QSTR_BYTES_IN_LEN = 1 config.MICROPY_QSTR_BYTES_IN_HASH = 1 - base_qstrs = {} + base_qstrs = list(qstrutil.static_qstr_list) - raw_codes = [read_mpy(file) for file in args.files] + # Load all .mpy files. + try: + compiled_modules = [read_mpy(file) for file in args.files] + except MPYReadError as er: + print(er, file=sys.stderr) + sys.exit(1) - if args.dump: - dump_mpy(raw_codes) - elif args.freeze: + if args.hexdump: + hexdump_mpy(compiled_modules) + + if args.disassemble: + if args.hexdump: + print() + disassemble_mpy(compiled_modules) + + if args.freeze: try: - freeze_mpy(base_qstrs, raw_codes) + freeze_mpy(base_qstrs, compiled_modules) except FreezeError as er: print(er, file=sys.stderr) sys.exit(1) - elif args.merge: - merged_mpy = merge_mpy(raw_codes, args.output) + + if args.merge: + merge_mpy(compiled_modules, args.output) if __name__ == "__main__": diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 6bc1dbac05..0dac69e208 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -35,7 +35,7 @@ sys.path.append(os.path.dirname(__file__) + "/../py") import makeqstrdata as qstrutil # MicroPython constants -MPY_VERSION = 5 +MPY_VERSION = 6 MP_NATIVE_ARCH_X86 = 1 MP_NATIVE_ARCH_X64 = 2 MP_NATIVE_ARCH_ARMV7M = 5 @@ -860,11 +860,12 @@ class MPYOutput: def write_qstr(self, s): if s in qstrutil.static_qstr_list: - self.write_bytes(bytes([0, qstrutil.static_qstr_list.index(s) + 1])) + self.write_uint((qstrutil.static_qstr_list.index(s) + 1) << 1 | 1) else: s = bytes(s, "ascii") self.write_uint(len(s) << 1) self.write_bytes(s) + self.write_bytes(b"\x00") def write_reloc(self, base, offset, dest, n): need_offset = not (base == self.prev_base and offset == self.prev_offset + 1) @@ -905,14 +906,19 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.open(fmpy) # MPY: header - out.write_bytes( - bytearray( - [ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS, QSTR_WINDOW_SIZE] - ) - ) + out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) + + # MPY: n_qstr + out.write_uint(1) + + # MPY: n_obj + out.write_uint(0) + + # MPY: qstr table + out.write_qstr(fmpy) # filename # MPY: kind/len - out.write_uint(len(env.full_text) << 2 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) + out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) # MPY: machine code out.write_bytes(env.full_text) @@ -936,20 +942,15 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): scope_flags |= MP_SCOPE_FLAG_VIPERBSS out.write_uint(scope_flags) - # MPY: n_obj - out.write_uint(0) - - # MPY: n_raw_code - out.write_uint(0) - - # MPY: rodata and/or bss + # MPY: bss and/or rodata if len(env.full_rodata): rodata_const_table_idx = 1 out.write_uint(len(env.full_rodata)) - out.write_bytes(env.full_rodata) if len(env.full_bss): - bss_const_table_idx = bool(env.full_rodata) + 1 + bss_const_table_idx = 2 out.write_uint(len(env.full_bss)) + if len(env.full_rodata): + out.write_bytes(env.full_rodata) # MPY: relocation information prev_kind = None