py: Add MICROPY_DYNAMIC_COMPILER option to config compiler at runtime.

This new compile-time option allows to make the bytecode compiler
configurable at runtime by setting the fields in the mp_dynamic_compiler
structure.  By using this feature, the compiler can generate bytecode
that targets any MicroPython runtime/VM, regardless of the host and
target compile-time settings.

Options so far that fall under this dynamic setting are:
- maximum number of bits that a small int can hold;
- whether caching of lookups is used in the bytecode;
- whether to use unicode strings or not (lexer behaviour differs, and
  therefore generated string constants differ).

py/compile.c

@@ -2486,7 +2486,23 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) {
         // pass
     } else if (MP_PARSE_NODE_IS_SMALL_INT(pn)) {
         mp_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn);
+        #if MICROPY_DYNAMIC_COMPILER
+        mp_uint_t sign_mask = -(1 << (mp_dynamic_compiler.small_int_bits - 1));
+        if ((arg & sign_mask) == 0 || (arg & sign_mask) == sign_mask) {
+            // integer fits in target runtime's small-int
+            EMIT_ARG(load_const_small_int, arg);
+        } else {
+            // integer doesn't fit, so create a multi-precision int object
+            // (but only create the actual object on the last pass)
+            if (comp->pass != MP_PASS_EMIT) {
+                EMIT_ARG(load_const_obj, mp_const_none);
+            } else {
+                EMIT_ARG(load_const_obj, mp_obj_new_int_from_ll(arg));
+            }
+        }
+        #else
         EMIT_ARG(load_const_small_int, arg);
+        #endif
     } else if (MP_PARSE_NODE_IS_LEAF(pn)) {
         uintptr_t arg = MP_PARSE_NODE_LEAF_ARG(pn);
         switch (MP_PARSE_NODE_LEAF_KIND(pn)) {

py/emitbc.c

@@ -579,7 +579,7 @@ void mp_emit_bc_load_name(emit_t *emit, qstr qst) {
     (void)qst;
     emit_bc_pre(emit, 1);
     emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_NAME, qst);
-    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
         emit_write_bytecode_byte(emit, 0);
     }
 }
@@ -588,7 +588,7 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst) {
     (void)qst;
     emit_bc_pre(emit, 1);
     emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_GLOBAL, qst);
-    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
         emit_write_bytecode_byte(emit, 0);
     }
 }
@@ -596,7 +596,7 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst) {
 void mp_emit_bc_load_attr(emit_t *emit, qstr qst) {
     emit_bc_pre(emit, 0);
     emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_ATTR, qst);
-    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
         emit_write_bytecode_byte(emit, 0);
     }
 }
@@ -646,7 +646,7 @@ void mp_emit_bc_store_global(emit_t *emit, qstr qst) {
 void mp_emit_bc_store_attr(emit_t *emit, qstr qst) {
     emit_bc_pre(emit, -2);
     emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_ATTR, qst);
-    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+    if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
         emit_write_bytecode_byte(emit, 0);
     }
 }

py/emitglue.c

@@ -204,7 +204,13 @@ mp_obj_t mp_make_closure_from_raw_code(mp_raw_code_t *rc, mp_uint_t n_closed_ove
     ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) << 0) \
     | ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \
     )
+// This is a version of the flags that can be configured at runtime.
+#define MPY_FEATURE_FLAGS_DYNAMIC ( \
+    ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) << 0) \
+    | ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \
+    )
 
+#if MICROPY_PERSISTENT_CODE_LOAD || (MICROPY_PERSISTENT_CODE_SAVE && !MICROPY_DYNAMIC_COMPILER)
 // The bytecode will depend on the number of bits in a small-int, and
 // this function computes that (could make it a fixed constant, but it
 // would need to be defined in mpconfigport.h).
@@ -217,6 +223,7 @@ STATIC int mp_small_int_bits(void) {
     }
     return n;
 }
+#endif
 
 typedef struct _bytecode_prelude_t {
     uint n_state;
@@ -366,7 +373,7 @@ mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) {
         nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError,
             "invalid .mpy file"));
     }
-    if (header[2] != MPY_FEATURE_FLAGS || header[3] != mp_small_int_bits()) {
+    if (header[2] != MPY_FEATURE_FLAGS || header[3] > mp_small_int_bits()) {
         nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError,
             "incompatible .mpy file"));
     }
@@ -615,7 +622,13 @@ void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) {
     //  byte  version
     //  byte  feature flags
     //  byte  number of bits in a small int
-    byte header[4] = {'M', 0, MPY_FEATURE_FLAGS, mp_small_int_bits()};
+    byte header[4] = {'M', 0, MPY_FEATURE_FLAGS_DYNAMIC,
+        #if MICROPY_DYNAMIC_COMPILER
+        mp_dynamic_compiler.small_int_bits,
+        #else
+        mp_small_int_bits(),
+        #endif
+    };
     mp_print_bytes(print, header, sizeof(header));
 
     save_raw_code(print, rc);

py/lexer.c

@@ -490,22 +490,25 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
                         }
                     }
                     if (c != MP_LEXER_EOF) {
-                        #if MICROPY_PY_BUILTINS_STR_UNICODE
+                        if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) {
-                        if (c < 0x110000 && !is_bytes) {
+                            if (c < 0x110000 && !is_bytes) {
-                            vstr_add_char(&lex->vstr, c);
+                                vstr_add_char(&lex->vstr, c);
-                        } else if (c < 0x100 && is_bytes) {
+                            } else if (c < 0x100 && is_bytes) {
-                            vstr_add_byte(&lex->vstr, c);
+                                vstr_add_byte(&lex->vstr, c);
-                        }
+                            } else {
-                        #else
+                                // unicode character out of range
-                        // without unicode everything is just added as an 8-bit byte
+                                // this raises a generic SyntaxError; could provide more info
-                        if (c < 0x100) {
+                                lex->tok_kind = MP_TOKEN_INVALID;
-                            vstr_add_byte(&lex->vstr, c);
+                            }
-                        }
+                        } else {
-                        #endif
+                            // without unicode everything is just added as an 8-bit byte
-                        else {
+                            if (c < 0x100) {
-                            // unicode character out of range
+                                vstr_add_byte(&lex->vstr, c);
-                            // this raises a generic SyntaxError; could provide more info
+                            } else {
-                            lex->tok_kind = MP_TOKEN_INVALID;
+                                // 8-bit character out of range
+                                // this raises a generic SyntaxError; could provide more info
+                                lex->tok_kind = MP_TOKEN_INVALID;
+                            }
                         }
                     }
                 } else {

py/mpconfig.h

@@ -283,6 +283,20 @@
 #define MICROPY_ENABLE_COMPILER (1)
 #endif
 
+// Whether the compiler is dynamically configurable (ie at runtime)
+#ifndef MICROPY_DYNAMIC_COMPILER
+#define MICROPY_DYNAMIC_COMPILER (0)
+#endif
+
+// Configure dynamic compiler macros
+#if MICROPY_DYNAMIC_COMPILER
+#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC (mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode)
+#define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC (mp_dynamic_compiler.py_builtins_str_unicode)
+#else
+#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE
+#define MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC MICROPY_PY_BUILTINS_STR_UNICODE
+#endif
+
 // Whether to enable constant folding; eg 1+2 rewritten as 3
 #ifndef MICROPY_COMP_CONST_FOLDING
 #define MICROPY_COMP_CONST_FOLDING (1)

py/mpstate.c

@@ -26,4 +26,8 @@
 
 #include "py/mpstate.h"
 
+#if MICROPY_DYNAMIC_COMPILER
+mp_dynamic_compiler_t mp_dynamic_compiler = {0};
+#endif
+
 mp_state_ctx_t mp_state_ctx;

py/mpstate.h

@@ -39,6 +39,16 @@
 // memory system, runtime and virtual machine.  The state is a global
 // variable, but in the future it is hoped that the state can become local.
 
+// This structure contains dynamic configuration for the compiler.
+#if MICROPY_DYNAMIC_COMPILER
+typedef struct mp_dynamic_compiler_t {
+    uint8_t small_int_bits; // must be <= host small_int_bits
+    bool opt_cache_map_lookup_in_bytecode;
+    bool py_builtins_str_unicode;
+} mp_dynamic_compiler_t;
+extern mp_dynamic_compiler_t mp_dynamic_compiler;
+#endif
+
 // This structure hold information about the memory allocation system.
 typedef struct _mp_state_mem_t {
     #if MICROPY_MEM_STATS