From 1762990579ff8ddc285e24492626fe40b6f2621d Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 15 May 2022 13:37:06 +1000 Subject: [PATCH] py/bc: Provide separate code-state setup funcs for bytecode and native. mpy-cross will now generate native code based on the size of mp_code_state_native_t, and the runtime will use this struct to calculate the offset of the .state field. This makes native code generation and execution (which rely on this struct) independent to the settings MICROPY_STACKLESS and MICROPY_PY_SYS_SETTRACE, both of which change the size of the mp_code_state_t struct. Fixes issue #5059. Signed-off-by: Damien George --- py/bc.c | 79 ++++++++++++++++++++++++++++++----------------- py/bc.h | 12 +++++++ py/emitnative.c | 18 +++++------ py/nativeglue.c | 2 +- py/nativeglue.h | 2 +- py/objfun.c | 1 - py/objgenerator.c | 30 +++++++++++++++--- 7 files changed, 98 insertions(+), 46 deletions(-) diff --git a/py/bc.c b/py/bc.c index 2a21ffd4b7..2c26d74895 100644 --- a/py/bc.c +++ b/py/bc.c @@ -122,23 +122,15 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // contain the following valid entries: // - code_state->fun_bc should contain a pointer to the function object // - code_state->ip should contain a pointer to the beginning of the prelude +// - code_state->sp should be: &code_state->state[0] - 1 // - code_state->n_state should be the number of objects in the local state -void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { +STATIC void mp_setup_code_state_helper(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { // This function is pretty complicated. It's main aim is to be efficient in speed and RAM // usage for the common case of positional only args. // get the function object that we want to set up (could be bytecode or native code) mp_obj_fun_bc_t *self = code_state->fun_bc; - #if MICROPY_STACKLESS - code_state->prev = NULL; - #endif - - #if MICROPY_PY_SYS_SETTRACE - code_state->prev_state = NULL; - code_state->frame = NULL; - #endif - // Get cached n_state (rather than decode it again) size_t n_state = code_state->n_state; @@ -149,16 +141,16 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw (void)n_state_unused; (void)n_exc_stack_unused; - code_state->sp = &code_state->state[0] - 1; + mp_obj_t *code_state_state = code_state->sp + 1; code_state->exc_sp_idx = 0; // zero out the local stack to begin with - memset(code_state->state, 0, n_state * sizeof(*code_state->state)); + memset(code_state_state, 0, n_state * sizeof(*code_state->state)); const mp_obj_t *kwargs = args + n_args; // var_pos_kw_args points to the stack where the var-args tuple, and var-kw dict, should go (if they are needed) - mp_obj_t *var_pos_kw_args = &code_state->state[n_state - 1 - n_pos_args - n_kwonly_args]; + mp_obj_t *var_pos_kw_args = &code_state_state[n_state - 1 - n_pos_args - n_kwonly_args]; // check positional arguments @@ -181,7 +173,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw if (n_args >= (size_t)(n_pos_args - n_def_pos_args)) { // given enough arguments, but may need to use some default arguments for (size_t i = n_args; i < n_pos_args; i++) { - code_state->state[n_state - 1 - i] = self->extra_args[i - (n_pos_args - n_def_pos_args)]; + code_state_state[n_state - 1 - i] = self->extra_args[i - (n_pos_args - n_def_pos_args)]; } } else { fun_pos_args_mismatch(self, n_pos_args - n_def_pos_args, n_args); @@ -191,14 +183,14 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // copy positional args into state for (size_t i = 0; i < n_args; i++) { - code_state->state[n_state - 1 - i] = args[i]; + code_state_state[n_state - 1 - i] = args[i]; } // check keyword arguments if (n_kw != 0 || (scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) { DEBUG_printf("Initial args: "); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); mp_obj_t dict = MP_OBJ_NULL; if ((scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) { @@ -220,11 +212,11 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw arg_qstr = self->context->constants.qstr_table[arg_qstr]; #endif if (wanted_arg_name == MP_OBJ_NEW_QSTR(arg_qstr)) { - if (code_state->state[n_state - 1 - j] != MP_OBJ_NULL) { + if (code_state_state[n_state - 1 - j] != MP_OBJ_NULL) { mp_raise_msg_varg(&mp_type_TypeError, MP_ERROR_TEXT("function got multiple values for argument '%q'"), MP_OBJ_QSTR_VALUE(wanted_arg_name)); } - code_state->state[n_state - 1 - j] = kwargs[2 * i + 1]; + code_state_state[n_state - 1 - j] = kwargs[2 * i + 1]; goto continue2; } } @@ -242,10 +234,10 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw } DEBUG_printf("Args with kws flattened: "); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); // fill in defaults for positional args - mp_obj_t *d = &code_state->state[n_state - n_pos_args]; + mp_obj_t *d = &code_state_state[n_state - n_pos_args]; mp_obj_t *s = &self->extra_args[n_def_pos_args - 1]; for (size_t i = n_def_pos_args; i > 0; i--, d++, s--) { if (*d == MP_OBJ_NULL) { @@ -254,13 +246,13 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw } DEBUG_printf("Args after filling default positional: "); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); // Check that all mandatory positional args are specified - while (d < &code_state->state[n_state]) { + while (d < &code_state_state[n_state]) { if (*d++ == MP_OBJ_NULL) { mp_raise_msg_varg(&mp_type_TypeError, - MP_ERROR_TEXT("function missing required positional argument #%d"), &code_state->state[n_state] - d); + MP_ERROR_TEXT("function missing required positional argument #%d"), &code_state_state[n_state] - d); } } @@ -275,13 +267,13 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE arg_qstr = self->context->constants.qstr_table[arg_qstr]; #endif - if (code_state->state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { + if (code_state_state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { mp_map_elem_t *elem = NULL; if ((scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) { elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, MP_OBJ_NEW_QSTR(arg_qstr), MP_MAP_LOOKUP); } if (elem != NULL) { - code_state->state[n_state - 1 - n_pos_args - i] = elem->value; + code_state_state[n_state - 1 - n_pos_args - i] = elem->value; } else { mp_raise_msg_varg(&mp_type_TypeError, MP_ERROR_TEXT("function missing required keyword argument '%q'"), arg_qstr); @@ -305,18 +297,47 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // bytecode prelude: initialise closed over variables for (; n_cell; --n_cell) { size_t local_num = *ip++; - code_state->state[n_state - 1 - local_num] = - mp_obj_new_cell(code_state->state[n_state - 1 - local_num]); + code_state_state[n_state - 1 - local_num] = + mp_obj_new_cell(code_state_state[n_state - 1 - local_num]); } // now that we skipped over the prelude, set the ip for the VM code_state->ip = ip; DEBUG_printf("Calling: n_pos_args=%d, n_kwonly_args=%d\n", n_pos_args, n_kwonly_args); - dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); - dump_args(code_state->state, n_state); + dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args); + dump_args(code_state_state, n_state); } +// On entry code_state should be allocated somewhere (stack/heap) and +// contain the following valid entries: +// - code_state->fun_bc should contain a pointer to the function object +// - code_state->n_state should be the number of objects in the local state +void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { + code_state->ip = code_state->fun_bc->bytecode; + code_state->sp = &code_state->state[0] - 1; + #if MICROPY_STACKLESS + code_state->prev = NULL; + #endif + #if MICROPY_PY_SYS_SETTRACE + code_state->prev_state = NULL; + code_state->frame = NULL; + #endif + mp_setup_code_state_helper(code_state, n_args, n_kw, args); +} + +#if MICROPY_EMIT_NATIVE +// On entry code_state should be allocated somewhere (stack/heap) and +// contain the following valid entries: +// - code_state->fun_bc should contain a pointer to the function object +// - code_state->ip should contain a pointer to the beginning of the prelude +// - code_state->n_state should be the number of objects in the local state +void mp_setup_code_state_native(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { + code_state->sp = &code_state->state[0] - 1; + mp_setup_code_state_helper((mp_code_state_t *)code_state, n_args, n_kw, args); +} +#endif + #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE // The following table encodes the number of bytes that a specific opcode diff --git a/py/bc.h b/py/bc.h index 3b7b625fec..a888164f18 100644 --- a/py/bc.h +++ b/py/bc.h @@ -254,6 +254,17 @@ typedef struct _mp_code_state_t { // mp_exc_stack_t exc_state[0]; } mp_code_state_t; +// State for an executing native function (based on mp_code_state_t). +typedef struct _mp_code_state_native_t { + struct _mp_obj_fun_bc_t *fun_bc; + const byte *ip; + mp_obj_t *sp; + uint16_t n_state; + uint16_t exc_sp_idx; + mp_obj_dict_t *old_globals; + mp_obj_t state[0]; +} mp_code_state_native_t; + // Allocator may return NULL, in which case data is not stored (can be used to compute size). typedef uint8_t *(*mp_encode_uint_allocator_t)(void *env, size_t nbytes); @@ -269,6 +280,7 @@ mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); +void mp_setup_code_state_native(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_bytecode_print(const mp_print_t *print, const struct _mp_raw_code_t *rc, const mp_module_constants_t *cm); void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, struct _mp_raw_code_t *const *child_table, const mp_module_constants_t *cm); const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, const byte *ip, struct _mp_raw_code_t *const *child_table, const mp_module_constants_t *cm); diff --git a/py/emitnative.c b/py/emitnative.c index 056a1a8a61..bfb37ef4f7 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -63,7 +63,7 @@ // C stack layout for native functions: // 0: nlr_buf_t [optional] -// emit->code_state_start: mp_code_state_t +// emit->code_state_start: mp_code_state_native_t // emit->stack_start: Python object stack | emit->n_state // locals (reversed, L0 at end) | // @@ -71,7 +71,7 @@ // 0=emit->stack_start: nlr_buf_t // // Then REG_GENERATOR_STATE points to: -// 0=emit->code_state_start: mp_code_state_t +// 0=emit->code_state_start: mp_code_state_native_t // emit->stack_start: Python object stack | emit->n_state // locals (reversed, L0 at end) | // @@ -88,12 +88,12 @@ #else #define SIZEOF_NLR_BUF (sizeof(nlr_buf_t) / sizeof(uintptr_t)) #endif -#define SIZEOF_CODE_STATE (sizeof(mp_code_state_t) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_t, state) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t)) -#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t)) +#define SIZEOF_CODE_STATE (sizeof(mp_code_state_native_t) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_native_t, state) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_native_t, fun_bc) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_native_t, ip) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_native_t, sp) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_native_t, n_state) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_CONTEXT (offsetof(mp_obj_fun_bc_t, context) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_CHILD_TABLE (offsetof(mp_obj_fun_bc_t, child_table) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t)) @@ -412,7 +412,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // generate code for entry to function - // Work out start of code state (mp_code_state_t or reduced version for viper) + // Work out start of code state (mp_code_state_native_t or reduced version for viper) emit->code_state_start = 0; if (NEED_GLOBAL_EXC_HANDLER(emit)) { emit->code_state_start = SIZEOF_NLR_BUF; diff --git a/py/nativeglue.c b/py/nativeglue.c index aed6fecddf..743ff38ccb 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -302,7 +302,7 @@ const mp_fun_table_t mp_fun_table = { mp_delete_global, mp_obj_new_closure, mp_arg_check_num_sig, - mp_setup_code_state, + mp_setup_code_state_native, mp_small_int_floor_divide, mp_small_int_modulo, mp_native_yield_from, diff --git a/py/nativeglue.h b/py/nativeglue.h index 49ce665f2d..7b1ccd8d46 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -131,7 +131,7 @@ typedef struct _mp_fun_table_t { void (*delete_global)(qstr qst); mp_obj_t (*new_closure)(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed); void (*arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig); - void (*setup_code_state)(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); + void (*setup_code_state_native)(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); mp_int_t (*small_int_floor_divide)(mp_int_t num, mp_int_t denom); mp_int_t (*small_int_modulo)(mp_int_t dividend, mp_int_t divisor); bool (*yield_from)(mp_obj_t gen, mp_obj_t send_value, mp_obj_t *ret_value); diff --git a/py/objfun.c b/py/objfun.c index 0053db4493..8f0c3eb6d4 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -209,7 +209,6 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { #define INIT_CODESTATE(code_state, _fun_bc, _n_state, n_args, n_kw, args) \ code_state->fun_bc = _fun_bc; \ - code_state->ip = _fun_bc->bytecode; \ code_state->n_state = _n_state; \ mp_setup_code_state(code_state, n_args, n_kw, args); \ code_state->old_globals = mp_globals_get(); diff --git a/py/objgenerator.c b/py/objgenerator.c index c1747a88bc..12fd81a8bb 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -65,7 +65,6 @@ STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; - o->code_state.ip = self_fun->bytecode; o->code_state.n_state = n_state; mp_setup_code_state(&o->code_state, n_args, n_kw, args); return MP_OBJ_FROM_PTR(o); @@ -87,6 +86,13 @@ const mp_obj_type_t mp_type_gen_wrap = { #if MICROPY_EMIT_NATIVE +// Based on mp_obj_gen_instance_t. +typedef struct _mp_obj_gen_instance_native_t { + mp_obj_base_t base; + mp_obj_t pend_exc; + mp_code_state_native_t code_state; +} mp_obj_gen_instance_native_t; + STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { // The state for a native generating function is held in the same struct as a bytecode function mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in); @@ -106,14 +112,15 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k MP_BC_PRELUDE_SIG_DECODE(ip); // Allocate the generator object, with room for local stack (exception stack not needed). - mp_obj_gen_instance_t *o = mp_obj_malloc_var(mp_obj_gen_instance_t, byte, n_state * sizeof(mp_obj_t), &mp_type_gen_instance); + mp_obj_gen_instance_native_t *o = mp_obj_malloc_var(mp_obj_gen_instance_native_t, byte, n_state * sizeof(mp_obj_t), &mp_type_gen_instance); // Parse the input arguments and set up the code state o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; o->code_state.ip = prelude_ptr; o->code_state.n_state = n_state; - mp_setup_code_state(&o->code_state, n_args, n_kw, args); + o->code_state.sp = &o->code_state.state[0] - 1; + mp_setup_code_state_native(&o->code_state, n_args, n_kw, args); // Indicate we are a native function, which doesn't use this variable o->code_state.exc_sp_idx = MP_CODE_STATE_EXC_SP_IDX_SENTINEL; @@ -171,7 +178,13 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ #endif // If the generator is started, allow sending a value. - if (self->code_state.sp == self->code_state.state - 1) { + void *state_start = self->code_state.state - 1; + #if MICROPY_EMIT_NATIVE + if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { + state_start = ((mp_obj_gen_instance_native_t *)self)->code_state.state - 1; + } + #endif + if (self->code_state.sp == state_start) { if (send_value != mp_const_none) { mp_raise_TypeError(MP_ERROR_TEXT("can't send non-None value to a just-started generator")); } @@ -226,7 +239,14 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ case MP_VM_RETURN_EXCEPTION: { self->code_state.ip = 0; - *ret_val = self->code_state.state[0]; + #if MICROPY_EMIT_NATIVE + if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { + *ret_val = ((mp_obj_gen_instance_native_t *)self)->code_state.state[0]; + } else + #endif + { + *ret_val = self->code_state.state[0]; + } // PEP479: if StopIteration is raised inside a generator it is replaced with RuntimeError if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(*ret_val)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) { *ret_val = mp_obj_new_exception_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("generator raised StopIteration"));