From ea23520403777f9b026f49245d39f8be1ccdbdac Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 11 Feb 2016 22:30:53 +0000 Subject: [PATCH] py: Add MICROPY_DYNAMIC_COMPILER option to config compiler at runtime. This new compile-time option allows to make the bytecode compiler configurable at runtime by setting the fields in the mp_dynamic_compiler structure. By using this feature, the compiler can generate bytecode that targets any MicroPython runtime/VM, regardless of the host and target compile-time settings. Options so far that fall under this dynamic setting are: - maximum number of bits that a small int can hold; - whether caching of lookups is used in the bytecode; - whether to use unicode strings or not (lexer behaviour differs, and therefore generated string constants differ). --- py/compile.c | 16 ++++++++++++++++ py/emitbc.c | 8 ++++---- py/emitglue.c | 17 +++++++++++++++-- py/lexer.c | 35 +++++++++++++++++++---------------- py/mpconfig.h | 14 ++++++++++++++ py/mpstate.c | 4 ++++ py/mpstate.h | 10 ++++++++++ 7 files changed, 82 insertions(+), 22 deletions(-) diff --git a/py/compile.c b/py/compile.c index 7f96891027..0699692df4 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2486,7 +2486,23 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) { // pass } else if (MP_PARSE_NODE_IS_SMALL_INT(pn)) { mp_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn); + #if MICROPY_DYNAMIC_COMPILER + mp_uint_t sign_mask = -(1 << (mp_dynamic_compiler.small_int_bits - 1)); + if ((arg & sign_mask) == 0 || (arg & sign_mask) == sign_mask) { + // integer fits in target runtime's small-int + EMIT_ARG(load_const_small_int, arg); + } else { + // integer doesn't fit, so create a multi-precision int object + // (but only create the actual object on the last pass) + if (comp->pass != MP_PASS_EMIT) { + EMIT_ARG(load_const_obj, mp_const_none); + } else { + EMIT_ARG(load_const_obj, mp_obj_new_int_from_ll(arg)); + } + } + #else EMIT_ARG(load_const_small_int, arg); + #endif } else if (MP_PARSE_NODE_IS_LEAF(pn)) { uintptr_t arg = MP_PARSE_NODE_LEAF_ARG(pn); switch (MP_PARSE_NODE_LEAF_KIND(pn)) { diff --git a/py/emitbc.c b/py/emitbc.c index 2f5304deb1..14298bad49 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -579,7 +579,7 @@ void mp_emit_bc_load_name(emit_t *emit, qstr qst) { (void)qst; emit_bc_pre(emit, 1); emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_NAME, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { + if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { emit_write_bytecode_byte(emit, 0); } } @@ -588,7 +588,7 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst) { (void)qst; emit_bc_pre(emit, 1); emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_GLOBAL, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { + if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { emit_write_bytecode_byte(emit, 0); } } @@ -596,7 +596,7 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst) { void mp_emit_bc_load_attr(emit_t *emit, qstr qst) { emit_bc_pre(emit, 0); emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_ATTR, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { + if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { emit_write_bytecode_byte(emit, 0); } } @@ -646,7 +646,7 @@ void mp_emit_bc_store_global(emit_t *emit, qstr qst) { void mp_emit_bc_store_attr(emit_t *emit, qstr qst) { emit_bc_pre(emit, -2); emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_ATTR, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) { + if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { emit_write_bytecode_byte(emit, 0); } } diff --git a/py/emitglue.c b/py/emitglue.c index 4157593ba3..133ba13571 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -204,7 +204,13 @@ mp_obj_t mp_make_closure_from_raw_code(mp_raw_code_t *rc, mp_uint_t n_closed_ove ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) << 0) \ | ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \ ) +// This is a version of the flags that can be configured at runtime. +#define MPY_FEATURE_FLAGS_DYNAMIC ( \ + ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) << 0) \ + | ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \ + ) +#if MICROPY_PERSISTENT_CODE_LOAD || (MICROPY_PERSISTENT_CODE_SAVE && !MICROPY_DYNAMIC_COMPILER) // The bytecode will depend on the number of bits in a small-int, and // this function computes that (could make it a fixed constant, but it // would need to be defined in mpconfigport.h). @@ -217,6 +223,7 @@ STATIC int mp_small_int_bits(void) { } return n; } +#endif typedef struct _bytecode_prelude_t { uint n_state; @@ -366,7 +373,7 @@ mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "invalid .mpy file")); } - if (header[2] != MPY_FEATURE_FLAGS || header[3] != mp_small_int_bits()) { + if (header[2] != MPY_FEATURE_FLAGS || header[3] > mp_small_int_bits()) { nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "incompatible .mpy file")); } @@ -615,7 +622,13 @@ void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { // byte version // byte feature flags // byte number of bits in a small int - byte header[4] = {'M', 0, MPY_FEATURE_FLAGS, mp_small_int_bits()}; + byte header[4] = {'M', 0, MPY_FEATURE_FLAGS_DYNAMIC, + #if MICROPY_DYNAMIC_COMPILER + mp_dynamic_compiler.small_int_bits, + #else + mp_small_int_bits(), + #endif + }; mp_print_bytes(print, header, sizeof(header)); save_raw_code(print, rc); diff --git a/py/lexer.c b/py/lexer.c index 89ecc386e9..76abedd451 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -490,22 +490,25 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) { } } if (c != MP_LEXER_EOF) { - #if MICROPY_PY_BUILTINS_STR_UNICODE - if (c < 0x110000 && !is_bytes) { - vstr_add_char(&lex->vstr, c); - } else if (c < 0x100 && is_bytes) { - vstr_add_byte(&lex->vstr, c); - } - #else - // without unicode everything is just added as an 8-bit byte - if (c < 0x100) { - vstr_add_byte(&lex->vstr, c); - } - #endif - else { - // unicode character out of range - // this raises a generic SyntaxError; could provide more info - lex->tok_kind = MP_TOKEN_INVALID; + if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) { + if (c < 0x110000 && !is_bytes) { + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && is_bytes) { + vstr_add_byte(&lex->vstr, c); + } else { + // unicode character out of range + // this raises a generic SyntaxError; could provide more info + lex->tok_kind = MP_TOKEN_INVALID; + } + } else { + // without unicode everything is just added as an 8-bit byte + if (c < 0x100) { + vstr_add_byte(&lex->vstr, c); + } else { + // 8-bit character out of range + // this raises a generic SyntaxError; could provide more info + lex->tok_kind = MP_TOKEN_INVALID; + } } } } else { diff --git a/py/mpconfig.h b/py/mpconfig.h index cd9380aa6e..39070b712e 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -283,6 +283,20 @@ #define MICROPY_ENABLE_COMPILER (1) #endif +// Whether the compiler is dynamically configurable (ie at runtime) +#ifndef MICROPY_DYNAMIC_COMPILER +#define MICROPY_DYNAMIC_COMPILER (0) +#endif + +// Configure dynamic compiler macros +#if MICROPY_DYNAMIC_COMPILER +#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC (mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode) +#define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC (mp_dynamic_compiler.py_builtins_str_unicode) +#else +#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE +#define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC MICROPY_PY_BUILTINS_STR_UNICODE +#endif + // Whether to enable constant folding; eg 1+2 rewritten as 3 #ifndef MICROPY_COMP_CONST_FOLDING #define MICROPY_COMP_CONST_FOLDING (1) diff --git a/py/mpstate.c b/py/mpstate.c index 2ba3402ef1..4fc8bc506e 100644 --- a/py/mpstate.c +++ b/py/mpstate.c @@ -26,4 +26,8 @@ #include "py/mpstate.h" +#if MICROPY_DYNAMIC_COMPILER +mp_dynamic_compiler_t mp_dynamic_compiler = {0}; +#endif + mp_state_ctx_t mp_state_ctx; diff --git a/py/mpstate.h b/py/mpstate.h index 7f8325b9a8..0e77e65833 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -39,6 +39,16 @@ // memory system, runtime and virtual machine. The state is a global // variable, but in the future it is hoped that the state can become local. +// This structure contains dynamic configuration for the compiler. +#if MICROPY_DYNAMIC_COMPILER +typedef struct mp_dynamic_compiler_t { + uint8_t small_int_bits; // must be <= host small_int_bits + bool opt_cache_map_lookup_in_bytecode; + bool py_builtins_str_unicode; +} mp_dynamic_compiler_t; +extern mp_dynamic_compiler_t mp_dynamic_compiler; +#endif + // This structure hold information about the memory allocation system. typedef struct _mp_state_mem_t { #if MICROPY_MEM_STATS