From 5e50975a6dd9466afafbcd012c00078093fe1f57 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Mon, 5 Jun 2023 16:52:29 +1000 Subject: [PATCH] py/modsys: Allow sys.path to be assigned to. Previously sys.path could be modified by append/pop or slice assignment. This allows `sys.path = [...]`, which can be simpler in many cases, but also improves CPython compatibility. It also allows sys.path to be set to a tuple which means that you can clear sys.path (e.g. temporarily) with no allocations. This also makes sys.path (and sys.argv for consistency) able to be disabled via mpconfig. The unix port (and upytesthelper) require them, so they explicitly verify that they're enabled. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- ports/unix/main.c | 66 +++++++++++++++------------- ports/unix/mpconfigport.h | 2 +- py/builtinimport.c | 4 +- py/modsys.c | 18 ++++++-- py/mpconfig.h | 19 +++++++- py/mpstate.h | 3 ++ py/runtime.c | 6 ++- py/runtime.h | 7 ++- shared/upytesthelper/upytesthelper.c | 10 ++++- 9 files changed, 93 insertions(+), 42 deletions(-) diff --git a/ports/unix/main.c b/ports/unix/main.c index 16f663de19..b065706ba6 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -69,6 +69,14 @@ long heap_size = 1024 * 1024 * (sizeof(mp_uint_t) / 4); #define MICROPY_GC_SPLIT_HEAP_N_HEAPS (1) #endif +#if !MICROPY_PY_SYS_PATH +#error "The unix port requires MICROPY_PY_SYS_PATH=1" +#endif + +#if !MICROPY_PY_SYS_ARGV +#error "The unix port requires MICROPY_PY_SYS_ARGV=1" +#endif + STATIC void stderr_print_strn(void *env, const char *str, size_t len) { (void)env; ssize_t ret; @@ -538,44 +546,40 @@ MP_NOINLINE int main_(int argc, char **argv) { } #endif - char *home = getenv("HOME"); - char *path = getenv("MICROPYPATH"); - if (path == NULL) { - path = MICROPY_PY_SYS_PATH_DEFAULT; - } - size_t path_num = 1; // [0] is for current dir (or base dir of the script) - if (*path == PATHLIST_SEP_CHAR) { - path_num++; - } - for (char *p = path; p != NULL; p = strchr(p, PATHLIST_SEP_CHAR)) { - path_num++; - if (p != NULL) { - p++; - } - } - mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), path_num); - mp_obj_t *path_items; - mp_obj_list_get(mp_sys_path, &path_num, &path_items); - path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR_); { - char *p = path; - for (mp_uint_t i = 1; i < path_num; i++) { - char *p1 = strchr(p, PATHLIST_SEP_CHAR); - if (p1 == NULL) { - p1 = p + strlen(p); + // sys.path starts as [""] + mp_sys_path = mp_obj_new_list(0, NULL); + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); + + // Add colon-separated entries from MICROPYPATH. + char *home = getenv("HOME"); + char *path = getenv("MICROPYPATH"); + if (path == NULL) { + path = MICROPY_PY_SYS_PATH_DEFAULT; + } + if (*path == PATHLIST_SEP_CHAR) { + // First entry is empty. We've already added an empty entry to sys.path, so skip it. + ++path; + } + bool path_remaining = *path; + while (path_remaining) { + char *path_entry_end = strchr(path, PATHLIST_SEP_CHAR); + if (path_entry_end == NULL) { + path_entry_end = path + strlen(path); + path_remaining = false; } - if (p[0] == '~' && p[1] == '/' && home != NULL) { + if (path[0] == '~' && path[1] == '/' && home != NULL) { // Expand standalone ~ to $HOME int home_l = strlen(home); vstr_t vstr; - vstr_init(&vstr, home_l + (p1 - p - 1) + 1); + vstr_init(&vstr, home_l + (path_entry_end - path - 1) + 1); vstr_add_strn(&vstr, home, home_l); - vstr_add_strn(&vstr, p + 1, p1 - p - 1); - path_items[i] = mp_obj_new_str_from_vstr(&vstr); + vstr_add_strn(&vstr, path + 1, path_entry_end - path - 1); + mp_obj_list_append(mp_sys_path, mp_obj_new_str_from_vstr(&vstr)); } else { - path_items[i] = mp_obj_new_str_via_qstr(p, p1 - p); + mp_obj_list_append(mp_sys_path, mp_obj_new_str_via_qstr(path, path_entry_end - path)); } - p = p1 + 1; + path = path_entry_end + 1; } } @@ -710,7 +714,7 @@ MP_NOINLINE int main_(int argc, char **argv) { // Set base dir of the script as first entry in sys.path. char *p = strrchr(basedir, '/'); - path_items[0] = mp_obj_new_str_via_qstr(basedir, p - basedir); + mp_obj_list_store(mp_sys_path, MP_OBJ_NEW_SMALL_INT(0), mp_obj_new_str_via_qstr(basedir, p - basedir)); free(pathbuf); set_sys_argv(argv, argc, a); diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 58aba9700b..c20aff1683 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -154,7 +154,7 @@ typedef long mp_off_t; // Ensure builtinimport.c works with -m. #define MICROPY_MODULE_OVERRIDE_MAIN_IMPORT (1) -// Don't default sys.argv because we do that in main. +// Don't default sys.argv and sys.path because we do that in main. #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0) // Enable sys.executable. diff --git a/py/builtinimport.c b/py/builtinimport.c index 8a125fc538..4fee04b8f3 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -118,7 +118,7 @@ STATIC mp_import_stat_t stat_top_level(qstr mod_name, vstr_t *dest) { #if MICROPY_PY_SYS size_t path_num; mp_obj_t *path_items; - mp_obj_list_get(mp_sys_path, &path_num, &path_items); + mp_obj_get_array(mp_sys_path, &path_num, &path_items); // go through each sys.path entry, trying to import "/". for (size_t i = 0; i < path_num; i++) { @@ -365,7 +365,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, // which may have come from the filesystem. size_t path_num; mp_obj_t *path_items; - mp_obj_list_get(mp_sys_path, &path_num, &path_items); + mp_obj_get_array(mp_sys_path, &path_num, &path_items); if (path_num) #endif { diff --git a/py/modsys.c b/py/modsys.c index 72817ce009..9b3a2bc163 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -195,6 +195,9 @@ STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) { MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace); #endif // MICROPY_PY_SYS_SETTRACE +#if MICROPY_PY_SYS_PATH && !MICROPY_PY_SYS_ATTR_DELEGATION +#error "MICROPY_PY_SYS_PATH requires MICROPY_PY_SYS_ATTR_DELEGATION" +#endif #if MICROPY_PY_SYS_PS1_PS2 && !MICROPY_PY_SYS_ATTR_DELEGATION #error "MICROPY_PY_SYS_PS1_PS2 requires MICROPY_PY_SYS_ATTR_DELEGATION" @@ -211,6 +214,11 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace); #if MICROPY_PY_SYS_ATTR_DELEGATION // Must be kept in sync with the enum at the top of mpstate.h. STATIC const uint16_t sys_mutable_keys[] = { + #if MICROPY_PY_SYS_PATH + // Code should access this (as an mp_obj_t) for use with e.g. + // mp_obj_list_append by using the `mp_sys_path` macro defined in runtime.h. + MP_QSTR_path, + #endif #if MICROPY_PY_SYS_PS1_PS2 MP_QSTR_ps1, MP_QSTR_ps2, @@ -231,8 +239,9 @@ void mp_module_sys_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) }, - { MP_ROM_QSTR(MP_QSTR_path), MP_ROM_PTR(&MP_STATE_VM(mp_sys_path_obj)) }, + #if MICROPY_PY_SYS_ARGV { MP_ROM_QSTR(MP_QSTR_argv), MP_ROM_PTR(&MP_STATE_VM(mp_sys_argv_obj)) }, + #endif { MP_ROM_QSTR(MP_QSTR_version), MP_ROM_PTR(&mp_sys_version_obj) }, { MP_ROM_QSTR(MP_QSTR_version_info), MP_ROM_PTR(&mp_sys_version_info_obj) }, { MP_ROM_QSTR(MP_QSTR_implementation), MP_ROM_PTR(&mp_sys_implementation_obj) }, @@ -308,10 +317,11 @@ const mp_obj_module_t mp_module_sys = { // available. MP_REGISTER_MODULE(MP_QSTR_sys, mp_module_sys); -// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is not enabled then these two lists -// must be initialised after the call to mp_init. -MP_REGISTER_ROOT_POINTER(mp_obj_list_t mp_sys_path_obj); +#if MICROPY_PY_SYS_ARGV +// Code should access this (as an mp_obj_t) for use with e.g. +// mp_obj_list_append by using the `mp_sys_argv` macro defined in runtime.h. MP_REGISTER_ROOT_POINTER(mp_obj_list_t mp_sys_argv_obj); +#endif #if MICROPY_PY_SYS_EXC_INFO // current exception being handled, for sys.exc_info() diff --git a/py/mpconfig.h b/py/mpconfig.h index b6f8838662..eb23c5965c 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1421,6 +1421,23 @@ typedef double mp_float_t; #define MICROPY_PY_SYS_ATEXIT (0) #endif +// Whether to provide the "sys.path" attribute (which forces module delegation +// and mutable sys attributes to be enabled). +// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is enabled, this is initialised in +// mp_init to an empty list. Otherwise the port must initialise it using +// `mp_sys_path = mp_obj_new_list(...)`. +#ifndef MICROPY_PY_SYS_PATH +#define MICROPY_PY_SYS_PATH (1) +#endif + +// Whether to provide the "sys.argv" attribute. +// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is enabled, this is initialised in +// mp_init to an empty list. Otherwise the port must initialise it using +// `mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), ...);` +#ifndef MICROPY_PY_SYS_ARGV +#define MICROPY_PY_SYS_ARGV (1) +#endif + // Whether to provide sys.{ps1,ps2} mutable attributes, to control REPL prompts #ifndef MICROPY_PY_SYS_PS1_PS2 #define MICROPY_PY_SYS_PS1_PS2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1455,7 +1472,7 @@ typedef double mp_float_t; // Whether the sys module supports attribute delegation // This is enabled automatically when needed by other features #ifndef MICROPY_PY_SYS_ATTR_DELEGATION -#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT) +#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PATH || MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT) #endif // Whether to provide "errno" module diff --git a/py/mpstate.h b/py/mpstate.h index 3786131de6..080dc1380b 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -43,6 +43,9 @@ #if MICROPY_PY_SYS_ATTR_DELEGATION // Must be kept in sync with sys_mutable_keys in modsys.c. enum { + #if MICROPY_PY_SYS_PATH + MP_SYS_MUTABLE_PATH, + #endif #if MICROPY_PY_SYS_PS1_PS2 MP_SYS_MUTABLE_PS1, MP_SYS_MUTABLE_PS2, diff --git a/py/runtime.c b/py/runtime.c index 2326dfb3ca..f5d219728f 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -136,13 +136,17 @@ void mp_init(void) { #endif #if MICROPY_PY_SYS_PATH_ARGV_DEFAULTS - mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); + #if MICROPY_PY_SYS_PATH + mp_sys_path = mp_obj_new_list(0, NULL); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) #if MICROPY_MODULE_FROZEN mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); #endif + #endif + #if MICROPY_PY_SYS_ARGV mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); #endif + #endif // MICROPY_PY_SYS_PATH_ARGV_DEFAULTS #if MICROPY_PY_SYS_ATEXIT MP_STATE_VM(sys_exitfunc) = mp_const_none; diff --git a/py/runtime.h b/py/runtime.h index 78194973d6..c033c77b40 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -227,8 +227,13 @@ int mp_native_type_from_qstr(qstr qst); mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type); mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type); -#define mp_sys_path (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_path_obj))) +#if MICROPY_PY_SYS_PATH +#define mp_sys_path (MP_STATE_VM(sys_mutable[MP_SYS_MUTABLE_PATH])) +#endif + +#if MICROPY_PY_SYS_ARGV #define mp_sys_argv (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_argv_obj))) +#endif #if MICROPY_WARNINGS #ifndef mp_warning diff --git a/shared/upytesthelper/upytesthelper.c b/shared/upytesthelper/upytesthelper.c index 12fa8276b0..ba20037f7a 100644 --- a/shared/upytesthelper/upytesthelper.c +++ b/shared/upytesthelper/upytesthelper.c @@ -31,6 +31,14 @@ #include "py/compile.h" #include "upytesthelper.h" +#if !MICROPY_PY_SYS_PATH +#error "upytesthelper requires MICROPY_PY_SYS_PATH=1" +#endif + +#if !MICROPY_PY_SYS_ARGV +#error "upytesthelper requires MICROPY_PY_SYS_ARGV=1" +#endif + static const char *test_exp_output; static int test_exp_output_len, test_rem_output_len; static int test_failed; @@ -93,7 +101,7 @@ void upytest_execute_test(const char *src) { // reinitialized before running each. gc_init(heap_start, heap_end); mp_init(); - mp_obj_list_init(mp_sys_path, 0); + mp_sys_path = mp_obj_new_list(0, NULL); #if MICROPY_MODULE_FROZEN mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); #endif