From 65dc960e3b22a8426e369607e47c19b380ce30ea Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 14 Aug 2015 12:24:11 +0100 Subject: [PATCH] unix-cpy: Remove unix-cpy. It's no longer needed. unix-cpy was originally written to get semantic equivalent with CPython without writing functional tests. When writing the initial implementation of uPy it was a long way between lexer and functional tests, so the half-way test was to make sure that the bytecode was correct. The idea was that if the uPy bytecode matched CPython 1-1 then uPy would be proper Python if the bytecodes acted correctly. And having matching bytecode meant that it was less likely to miss some deep subtlety in the Python semantics that would require an architectural change later on. But that is all history and it no longer makes sense to retain the ability to output CPython bytecode, because: 1. It outputs CPython 3.3 compatible bytecode. CPython's bytecode changes from version to version, and seems to have changed quite a bit in 3.5. There's no point in changing the bytecode output to match CPython anymore. 2. uPy and CPy do different optimisations to the bytecode which makes it harder to match. 3. The bytecode tests are not run. They were never part of Travis and are not run locally anymore. 4. The EMIT_CPYTHON option needs a lot of extra source code which adds heaps of noise, especially in compile.c. 5. Now that there is an extensive test suite (which tests functionality) there is no need to match the bytecode. Some very subtle behaviour is tested with the test suite and passing these tests is a much better way to stay Python-language compliant, rather than trying to match CPy bytecode. --- .travis.yml | 1 - README.md | 1 - py/compile.c | 420 +------- py/emit.h | 8 - py/emitbc.c | 4 - py/emitcommon.c | 28 +- py/emitcpy.c | 901 ------------------ py/mkrules.mk | 2 +- py/mpconfig.h | 6 - py/parse.c | 4 +- py/py.mk | 1 - py/scope.c | 56 -- py/scope.h | 1 - tests/bytecode/.gitignore | 3 - tests/bytecode/README.md | 15 - tests/bytecode/check.py | 56 -- tests/bytecode/mp-tests/assert1.py | 2 - tests/bytecode/mp-tests/assign1.py | 10 - tests/bytecode/mp-tests/assign2.py | 21 - tests/bytecode/mp-tests/augassign1.py | 5 - tests/bytecode/mp-tests/call1.py | 1 - tests/bytecode/mp-tests/class1.py | 3 - tests/bytecode/mp-tests/class2.py | 4 - tests/bytecode/mp-tests/class3.py | 10 - tests/bytecode/mp-tests/class4.py | 9 - tests/bytecode/mp-tests/class5.py | 8 - tests/bytecode/mp-tests/class6.py | 7 - tests/bytecode/mp-tests/class7.py | 6 - tests/bytecode/mp-tests/closure1.py | 2 - tests/bytecode/mp-tests/closure2.py | 7 - tests/bytecode/mp-tests/closure3.py | 12 - tests/bytecode/mp-tests/closure4.py | 13 - tests/bytecode/mp-tests/compare1.py | 8 - tests/bytecode/mp-tests/const1.py | 9 - tests/bytecode/mp-tests/continue1.py | 44 - tests/bytecode/mp-tests/decorate1.py | 20 - tests/bytecode/mp-tests/del1.py | 16 - tests/bytecode/mp-tests/del2.py | 11 - tests/bytecode/mp-tests/dict1.py | 3 - tests/bytecode/mp-tests/dictcomp1.py | 2 - tests/bytecode/mp-tests/docstring1.py | 8 - tests/bytecode/mp-tests/docstring2.py | 3 - tests/bytecode/mp-tests/fun1.py | 2 - tests/bytecode/mp-tests/fun2.py | 23 - tests/bytecode/mp-tests/fun3.py | 3 - tests/bytecode/mp-tests/fun4.py | 5 - tests/bytecode/mp-tests/if1.py | 24 - tests/bytecode/mp-tests/if2.py | 26 - tests/bytecode/mp-tests/if3.py | 6 - tests/bytecode/mp-tests/if4.py | 8 - tests/bytecode/mp-tests/ifexpr1.py | 1 - tests/bytecode/mp-tests/import1.py | 5 - tests/bytecode/mp-tests/import2.py | 1 - tests/bytecode/mp-tests/import3.py | 8 - tests/bytecode/mp-tests/import4.py | 3 - tests/bytecode/mp-tests/import5.py | 4 - tests/bytecode/mp-tests/import6.py | 14 - tests/bytecode/mp-tests/lambda1.py | 2 - tests/bytecode/mp-tests/lambda2.py | 1 - tests/bytecode/mp-tests/list1.py | 8 - tests/bytecode/mp-tests/list2.py | 8 - tests/bytecode/mp-tests/listcomp1.py | 4 - tests/bytecode/mp-tests/listcomp2.py | 1 - tests/bytecode/mp-tests/listcomp3.py | 3 - tests/bytecode/mp-tests/listcomp4.py | 4 - tests/bytecode/mp-tests/listcomp5.py | 11 - tests/bytecode/mp-tests/locals1.py | 22 - tests/bytecode/mp-tests/ptex.py | 269 ------ tests/bytecode/mp-tests/raise1.py | 11 - tests/bytecode/mp-tests/scope0.py | 7 - tests/bytecode/mp-tests/scope1.py | 6 - tests/bytecode/mp-tests/scope2.py | 18 - tests/bytecode/mp-tests/scope3.py | 11 - tests/bytecode/mp-tests/scope4.py | 14 - tests/bytecode/mp-tests/scope5.py | 12 - tests/bytecode/mp-tests/scope6.py | 7 - tests/bytecode/mp-tests/scope7.py | 15 - tests/bytecode/mp-tests/set1.py | 6 - tests/bytecode/mp-tests/setcomp1.py | 2 - tests/bytecode/mp-tests/slice1.py | 16 - tests/bytecode/mp-tests/slice2.py | 3 - tests/bytecode/mp-tests/string1.py | 11 - tests/bytecode/mp-tests/string2.py | 14 - tests/bytecode/mp-tests/super1.py | 17 - tests/bytecode/mp-tests/try1.py | 13 - tests/bytecode/mp-tests/try2.py | 5 - tests/bytecode/mp-tests/try3.py | 14 - tests/bytecode/mp-tests/try4.py | 22 - tests/bytecode/mp-tests/try5.py | 8 - tests/bytecode/mp-tests/try6.py | 15 - tests/bytecode/mp-tests/tuple1.py | 17 - tests/bytecode/mp-tests/tuple2.py | 15 - tests/bytecode/mp-tests/tuple3.py | 4 - tests/bytecode/mp-tests/with1.py | 8 - tests/bytecode/mp-tests/yield1.py | 17 - tests/bytecode/mp-tests/yield2.py | 7 - tests/bytecode/pylib-tests/LICENSE.txt | 254 ----- tests/bytecode/pylib-tests/_compat_pickle.py | 81 -- .../bytecode/pylib-tests/_threading_local.py | 246 ----- tests/bytecode/pylib-tests/_weakrefset.py | 194 ---- tests/bytecode/pylib-tests/abc.py | 228 ----- tests/bytecode/pylib-tests/aifc.py | 895 ----------------- tests/bytecode/pylib-tests/antigravity.py | 17 - tests/bytecode/pylib-tests/base64.py | 410 -------- tests/bytecode/pylib-tests/bdb.py | 647 ------------- tests/bytecode/pylib-tests/binhex.py | 471 --------- tests/bytecode/pylib-tests/bisect.py | 92 -- tests/bytecode/pylib-tests/bz2.py | 504 ---------- tests/bytecode/pylib-tests/cProfile.py | 195 ---- tests/bytecode/pylib-tests/chunk.py | 167 ---- tests/bytecode/pylib-tests/code.py | 302 ------ tests/bytecode/pylib-tests/compileall.py | 240 ----- tests/bytecode/pylib-tests/contextlib.py | 255 ----- tests/bytecode/pylib-tests/crypt.py | 62 -- tests/bytecode/pylib-tests/dummy_threading.py | 78 -- tests/bytecode/pylib-tests/fnmatch.py | 109 --- tests/bytecode/pylib-tests/genericpath.py | 106 --- tests/bytecode/pylib-tests/getopt.py | 215 ----- tests/bytecode/pylib-tests/hashlib.py | 148 --- tests/bytecode/pylib-tests/heapq.py | 472 --------- tests/bytecode/pylib-tests/keyword.py | 93 -- tests/bytecode/pylib-tests/macurl2path.py | 97 -- tests/bytecode/pylib-tests/mimetypes.py | 589 ------------ tests/bytecode/pylib-tests/modulefinder.py | 663 ------------- tests/bytecode/pylib-tests/nturl2path.py | 66 -- tests/bytecode/pylib-tests/opcode.py | 185 ---- tests/bytecode/pylib-tests/pipes.py | 247 ----- tests/bytecode/pylib-tests/poplib.py | 374 -------- tests/bytecode/pylib-tests/pty.py | 180 ---- tests/bytecode/pylib-tests/reprlib.py | 157 --- tests/bytecode/pylib-tests/rlcompleter.py | 160 ---- tests/bytecode/pylib-tests/runpy.py | 262 ----- tests/bytecode/pylib-tests/sched.py | 168 ---- tests/bytecode/pylib-tests/shelve.py | 232 ----- tests/bytecode/pylib-tests/socket.py | 437 --------- tests/bytecode/pylib-tests/socketserver.py | 745 --------------- tests/bytecode/pylib-tests/sre_constants.py | 259 ----- tests/bytecode/pylib-tests/stat.py | 149 --- tests/bytecode/pylib-tests/struct.py | 14 - tests/bytecode/pylib-tests/sunau.py | 485 ---------- tests/bytecode/pylib-tests/symbol.py | 111 --- tests/bytecode/pylib-tests/tabnanny.py | 332 ------- tests/bytecode/pylib-tests/tempfile.py | 709 -------------- tests/bytecode/pylib-tests/this.py | 28 - tests/bytecode/pylib-tests/timeit.py | 334 ------- tests/bytecode/pylib-tests/tty.py | 36 - tests/bytecode/pylib-tests/types.py | 101 -- tests/bytecode/pylib-tests/uu.py | 199 ---- tests/bytecode/pylib-tests/wave.py | 504 ---------- tests/bytecode/pylib-tests/weakref.py | 385 -------- tests/bytecode/pylib-tests/xdrlib.py | 224 ----- tests/bytecode/run-tests | 30 - tests/bytecode/unpyc.py | 112 --- unix-cpy/.gitignore | 2 - unix-cpy/Makefile | 32 - unix-cpy/main.c | 88 -- unix-cpy/mpconfigport.h | 59 -- 157 files changed, 26 insertions(+), 17742 deletions(-) delete mode 100644 py/emitcpy.c delete mode 100644 tests/bytecode/.gitignore delete mode 100644 tests/bytecode/README.md delete mode 100644 tests/bytecode/check.py delete mode 100644 tests/bytecode/mp-tests/assert1.py delete mode 100644 tests/bytecode/mp-tests/assign1.py delete mode 100644 tests/bytecode/mp-tests/assign2.py delete mode 100644 tests/bytecode/mp-tests/augassign1.py delete mode 100644 tests/bytecode/mp-tests/call1.py delete mode 100644 tests/bytecode/mp-tests/class1.py delete mode 100644 tests/bytecode/mp-tests/class2.py delete mode 100644 tests/bytecode/mp-tests/class3.py delete mode 100644 tests/bytecode/mp-tests/class4.py delete mode 100644 tests/bytecode/mp-tests/class5.py delete mode 100644 tests/bytecode/mp-tests/class6.py delete mode 100644 tests/bytecode/mp-tests/class7.py delete mode 100644 tests/bytecode/mp-tests/closure1.py delete mode 100644 tests/bytecode/mp-tests/closure2.py delete mode 100644 tests/bytecode/mp-tests/closure3.py delete mode 100644 tests/bytecode/mp-tests/closure4.py delete mode 100644 tests/bytecode/mp-tests/compare1.py delete mode 100644 tests/bytecode/mp-tests/const1.py delete mode 100644 tests/bytecode/mp-tests/continue1.py delete mode 100644 tests/bytecode/mp-tests/decorate1.py delete mode 100644 tests/bytecode/mp-tests/del1.py delete mode 100644 tests/bytecode/mp-tests/del2.py delete mode 100644 tests/bytecode/mp-tests/dict1.py delete mode 100644 tests/bytecode/mp-tests/dictcomp1.py delete mode 100644 tests/bytecode/mp-tests/docstring1.py delete mode 100644 tests/bytecode/mp-tests/docstring2.py delete mode 100644 tests/bytecode/mp-tests/fun1.py delete mode 100644 tests/bytecode/mp-tests/fun2.py delete mode 100644 tests/bytecode/mp-tests/fun3.py delete mode 100644 tests/bytecode/mp-tests/fun4.py delete mode 100644 tests/bytecode/mp-tests/if1.py delete mode 100644 tests/bytecode/mp-tests/if2.py delete mode 100644 tests/bytecode/mp-tests/if3.py delete mode 100644 tests/bytecode/mp-tests/if4.py delete mode 100644 tests/bytecode/mp-tests/ifexpr1.py delete mode 100644 tests/bytecode/mp-tests/import1.py delete mode 100644 tests/bytecode/mp-tests/import2.py delete mode 100644 tests/bytecode/mp-tests/import3.py delete mode 100644 tests/bytecode/mp-tests/import4.py delete mode 100644 tests/bytecode/mp-tests/import5.py delete mode 100644 tests/bytecode/mp-tests/import6.py delete mode 100644 tests/bytecode/mp-tests/lambda1.py delete mode 100644 tests/bytecode/mp-tests/lambda2.py delete mode 100644 tests/bytecode/mp-tests/list1.py delete mode 100644 tests/bytecode/mp-tests/list2.py delete mode 100644 tests/bytecode/mp-tests/listcomp1.py delete mode 100644 tests/bytecode/mp-tests/listcomp2.py delete mode 100644 tests/bytecode/mp-tests/listcomp3.py delete mode 100644 tests/bytecode/mp-tests/listcomp4.py delete mode 100644 tests/bytecode/mp-tests/listcomp5.py delete mode 100644 tests/bytecode/mp-tests/locals1.py delete mode 100644 tests/bytecode/mp-tests/ptex.py delete mode 100644 tests/bytecode/mp-tests/raise1.py delete mode 100644 tests/bytecode/mp-tests/scope0.py delete mode 100644 tests/bytecode/mp-tests/scope1.py delete mode 100644 tests/bytecode/mp-tests/scope2.py delete mode 100644 tests/bytecode/mp-tests/scope3.py delete mode 100644 tests/bytecode/mp-tests/scope4.py delete mode 100644 tests/bytecode/mp-tests/scope5.py delete mode 100644 tests/bytecode/mp-tests/scope6.py delete mode 100644 tests/bytecode/mp-tests/scope7.py delete mode 100644 tests/bytecode/mp-tests/set1.py delete mode 100644 tests/bytecode/mp-tests/setcomp1.py delete mode 100644 tests/bytecode/mp-tests/slice1.py delete mode 100644 tests/bytecode/mp-tests/slice2.py delete mode 100644 tests/bytecode/mp-tests/string1.py delete mode 100644 tests/bytecode/mp-tests/string2.py delete mode 100644 tests/bytecode/mp-tests/super1.py delete mode 100644 tests/bytecode/mp-tests/try1.py delete mode 100644 tests/bytecode/mp-tests/try2.py delete mode 100644 tests/bytecode/mp-tests/try3.py delete mode 100644 tests/bytecode/mp-tests/try4.py delete mode 100644 tests/bytecode/mp-tests/try5.py delete mode 100644 tests/bytecode/mp-tests/try6.py delete mode 100644 tests/bytecode/mp-tests/tuple1.py delete mode 100644 tests/bytecode/mp-tests/tuple2.py delete mode 100644 tests/bytecode/mp-tests/tuple3.py delete mode 100644 tests/bytecode/mp-tests/with1.py delete mode 100644 tests/bytecode/mp-tests/yield1.py delete mode 100644 tests/bytecode/mp-tests/yield2.py delete mode 100644 tests/bytecode/pylib-tests/LICENSE.txt delete mode 100644 tests/bytecode/pylib-tests/_compat_pickle.py delete mode 100644 tests/bytecode/pylib-tests/_threading_local.py delete mode 100644 tests/bytecode/pylib-tests/_weakrefset.py delete mode 100644 tests/bytecode/pylib-tests/abc.py delete mode 100644 tests/bytecode/pylib-tests/aifc.py delete mode 100644 tests/bytecode/pylib-tests/antigravity.py delete mode 100644 tests/bytecode/pylib-tests/base64.py delete mode 100644 tests/bytecode/pylib-tests/bdb.py delete mode 100644 tests/bytecode/pylib-tests/binhex.py delete mode 100644 tests/bytecode/pylib-tests/bisect.py delete mode 100644 tests/bytecode/pylib-tests/bz2.py delete mode 100644 tests/bytecode/pylib-tests/cProfile.py delete mode 100644 tests/bytecode/pylib-tests/chunk.py delete mode 100644 tests/bytecode/pylib-tests/code.py delete mode 100644 tests/bytecode/pylib-tests/compileall.py delete mode 100644 tests/bytecode/pylib-tests/contextlib.py delete mode 100644 tests/bytecode/pylib-tests/crypt.py delete mode 100644 tests/bytecode/pylib-tests/dummy_threading.py delete mode 100644 tests/bytecode/pylib-tests/fnmatch.py delete mode 100644 tests/bytecode/pylib-tests/genericpath.py delete mode 100644 tests/bytecode/pylib-tests/getopt.py delete mode 100644 tests/bytecode/pylib-tests/hashlib.py delete mode 100644 tests/bytecode/pylib-tests/heapq.py delete mode 100644 tests/bytecode/pylib-tests/keyword.py delete mode 100644 tests/bytecode/pylib-tests/macurl2path.py delete mode 100644 tests/bytecode/pylib-tests/mimetypes.py delete mode 100644 tests/bytecode/pylib-tests/modulefinder.py delete mode 100644 tests/bytecode/pylib-tests/nturl2path.py delete mode 100644 tests/bytecode/pylib-tests/opcode.py delete mode 100644 tests/bytecode/pylib-tests/pipes.py delete mode 100644 tests/bytecode/pylib-tests/poplib.py delete mode 100644 tests/bytecode/pylib-tests/pty.py delete mode 100644 tests/bytecode/pylib-tests/reprlib.py delete mode 100644 tests/bytecode/pylib-tests/rlcompleter.py delete mode 100644 tests/bytecode/pylib-tests/runpy.py delete mode 100644 tests/bytecode/pylib-tests/sched.py delete mode 100644 tests/bytecode/pylib-tests/shelve.py delete mode 100644 tests/bytecode/pylib-tests/socket.py delete mode 100644 tests/bytecode/pylib-tests/socketserver.py delete mode 100644 tests/bytecode/pylib-tests/sre_constants.py delete mode 100644 tests/bytecode/pylib-tests/stat.py delete mode 100644 tests/bytecode/pylib-tests/struct.py delete mode 100644 tests/bytecode/pylib-tests/sunau.py delete mode 100644 tests/bytecode/pylib-tests/symbol.py delete mode 100644 tests/bytecode/pylib-tests/tabnanny.py delete mode 100644 tests/bytecode/pylib-tests/tempfile.py delete mode 100644 tests/bytecode/pylib-tests/this.py delete mode 100644 tests/bytecode/pylib-tests/timeit.py delete mode 100644 tests/bytecode/pylib-tests/tty.py delete mode 100644 tests/bytecode/pylib-tests/types.py delete mode 100644 tests/bytecode/pylib-tests/uu.py delete mode 100644 tests/bytecode/pylib-tests/wave.py delete mode 100644 tests/bytecode/pylib-tests/weakref.py delete mode 100644 tests/bytecode/pylib-tests/xdrlib.py delete mode 100755 tests/bytecode/run-tests delete mode 100644 tests/bytecode/unpyc.py delete mode 100644 unix-cpy/.gitignore delete mode 100644 unix-cpy/Makefile delete mode 100644 unix-cpy/main.c delete mode 100644 unix-cpy/mpconfigport.h diff --git a/.travis.yml b/.travis.yml index c598783f44..dc6c3de05c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ before_script: script: - make -C minimal test - make -C unix CC=gcc-4.7 - - make -C unix-cpy CC=gcc-4.7 - make -C bare-arm - make -C qemu-arm test - make -C stmhal diff --git a/README.md b/README.md index 59bf5940fb..ba4a0322fa 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ Additional components: - pic16bit/ -- a version of Micro Python for 16-bit PIC microcontrollers. - cc3200/ -- a version of Micro Python that runs on the CC3200 from TI. - esp8266/ -- an experimental port for ESP8266 WiFi modules. -- unix-cpy/ -- a version of Micro Python that outputs bytecode (for testing). - tests/ -- test framework and test scripts. - tools/ -- various tools, including the pyboard.py module. - examples/ -- a few example Python scripts. diff --git a/py/compile.c b/py/compile.c index a124203e4b..b7ec070299 100644 --- a/py/compile.c +++ b/py/compile.c @@ -49,7 +49,7 @@ typedef enum { PN_const_object, // special node for a constant, generic Python object } pn_kind_t; -#define NEED_METHOD_TABLE (MICROPY_EMIT_CPYTHON || MICROPY_EMIT_NATIVE) +#define NEED_METHOD_TABLE MICROPY_EMIT_NATIVE #if NEED_METHOD_TABLE @@ -323,23 +323,6 @@ STATIC mp_parse_node_t fold_constants(compiler_t *comp, mp_parse_node_t pn, mp_m case PN_power: if (0) { -#if MICROPY_EMIT_CPYTHON - } else if (MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[0]) && MP_PARSE_NODE_IS_NULL(pns->nodes[1]) && !MP_PARSE_NODE_IS_NULL(pns->nodes[2])) { - // int ** x - // can overflow; enabled only to compare with CPython - mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t*)pns->nodes[2]; - if (MP_PARSE_NODE_IS_SMALL_INT(pns2->nodes[0])) { - int power = MP_PARSE_NODE_LEAF_SMALL_INT(pns2->nodes[0]); - if (power >= 0) { - int ans = 1; - int base = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[0]); - for (; power > 0; power--) { - ans *= base; - } - pn = mp_parse_node_new_leaf(MP_PARSE_NODE_SMALL_INT, ans); - } - } -#endif #if MICROPY_COMP_MODULE_CONST } else if (MP_PARSE_NODE_IS_ID(pns->nodes[0]) && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_trailer_period) && MP_PARSE_NODE_IS_NULL(pns->nodes[2])) { // id.id @@ -464,163 +447,7 @@ STATIC void compile_delete_id(compiler_t *comp, qstr qst) { } } -#if MICROPY_EMIT_CPYTHON -STATIC bool cpython_c_tuple_is_const(mp_parse_node_t pn) { - if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_string)) { - return true; - } - if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_bytes)) { - return true; - } - if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_const_object)) { - return true; - } - if (!MP_PARSE_NODE_IS_LEAF(pn)) { - return false; - } - if (MP_PARSE_NODE_IS_ID(pn)) { - return false; - } - return true; -} - -STATIC void cpython_c_print_quoted_str(vstr_t *vstr, const char *str, uint len, bool bytes) { - bool has_single_quote = false; - bool has_double_quote = false; - for (int i = 0; i < len; i++) { - if (str[i] == '\'') { - has_single_quote = true; - } else if (str[i] == '"') { - has_double_quote = true; - } - } - if (bytes) { - vstr_printf(vstr, "b"); - } - bool quote_single = false; - if (has_single_quote && !has_double_quote) { - vstr_printf(vstr, "\""); - } else { - quote_single = true; - vstr_printf(vstr, "'"); - } - for (int i = 0; i < len; i++) { - if (str[i] == '\n') { - vstr_printf(vstr, "\\n"); - } else if (str[i] == '\\') { - vstr_printf(vstr, "\\\\"); - } else if (str[i] == '\'' && quote_single) { - vstr_printf(vstr, "\\'"); - } else { - vstr_printf(vstr, "%c", str[i]); - } - } - if (has_single_quote && !has_double_quote) { - vstr_printf(vstr, "\""); - } else { - vstr_printf(vstr, "'"); - } -} - -STATIC void cpython_c_tuple_emit_const(compiler_t *comp, mp_parse_node_t pn, vstr_t *vstr) { - if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_string) || MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_bytes)) { - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; - cpython_c_print_quoted_str(vstr, (const char*)pns->nodes[0], (mp_uint_t)pns->nodes[1], MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_bytes)); - return; - } - - if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_const_object)) { - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; - mp_obj_print((mp_obj_t)pns->nodes[0], PRINT_REPR); - return; - } - - assert(MP_PARSE_NODE_IS_LEAF(pn)); - if (MP_PARSE_NODE_IS_SMALL_INT(pn)) { - vstr_printf(vstr, INT_FMT, MP_PARSE_NODE_LEAF_SMALL_INT(pn)); - return; - } - - mp_uint_t arg = MP_PARSE_NODE_LEAF_ARG(pn); - switch (MP_PARSE_NODE_LEAF_KIND(pn)) { - case MP_PARSE_NODE_ID: assert(0); - case MP_PARSE_NODE_STRING: - case MP_PARSE_NODE_BYTES: { - mp_uint_t len; - const byte *str = qstr_data(arg, &len); - cpython_c_print_quoted_str(vstr, (const char*)str, len, MP_PARSE_NODE_LEAF_KIND(pn) == MP_PARSE_NODE_BYTES); - break; - } - case MP_PARSE_NODE_TOKEN: - switch (arg) { - case MP_TOKEN_KW_FALSE: vstr_printf(vstr, "False"); break; - case MP_TOKEN_KW_NONE: vstr_printf(vstr, "None"); break; - case MP_TOKEN_KW_TRUE: vstr_printf(vstr, "True"); break; - default: assert(0); // shouldn't happen - } - break; - default: assert(0); - } -} - -STATIC void cpython_c_tuple(compiler_t *comp, mp_parse_node_t pn, mp_parse_node_struct_t *pns_list) { - int n = 0; - if (pns_list != NULL) { - n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns_list); - } - int total = n; - bool is_const = true; - if (!MP_PARSE_NODE_IS_NULL(pn)) { - total += 1; - if (!cpython_c_tuple_is_const(pn)) { - is_const = false; - } - } - for (int i = 0; i < n; i++) { - if (!cpython_c_tuple_is_const(pns_list->nodes[i])) { - is_const = false; - break; - } - } - if (total > 0 && is_const) { - bool need_comma = false; - vstr_t *vstr = vstr_new(); - vstr_printf(vstr, "("); - if (!MP_PARSE_NODE_IS_NULL(pn)) { - cpython_c_tuple_emit_const(comp, pn, vstr); - need_comma = true; - } - for (int i = 0; i < n; i++) { - if (need_comma) { - vstr_printf(vstr, ", "); - } - cpython_c_tuple_emit_const(comp, pns_list->nodes[i], vstr); - need_comma = true; - } - if (total == 1) { - vstr_printf(vstr, ",)"); - } else { - vstr_printf(vstr, ")"); - } - EMIT_ARG(load_const_verbatim_strn, vstr_str(vstr), vstr_len(vstr)); - vstr_free(vstr); - } else { - if (!MP_PARSE_NODE_IS_NULL(pn)) { - compile_node(comp, pn); - } - for (int i = 0; i < n; i++) { - compile_node(comp, pns_list->nodes[i]); - } - EMIT_ARG(build_tuple, total); - } -} -#endif - -// funnelling all tuple creations through this function is purely so we can optionally agree with CPython STATIC void c_tuple(compiler_t *comp, mp_parse_node_t pn, mp_parse_node_struct_t *pns_list) { -#if MICROPY_EMIT_CPYTHON - cpython_c_tuple(comp, pn, pns_list); -#else int total = 0; if (!MP_PARSE_NODE_IS_NULL(pn)) { compile_node(comp, pn); @@ -634,7 +461,6 @@ STATIC void c_tuple(compiler_t *comp, mp_parse_node_t pn, mp_parse_node_struct_t total += n; } EMIT_ARG(build_tuple, total); -#endif } STATIC void compile_generic_tuple(compiler_t *comp, mp_parse_node_struct_t *pns) { @@ -652,66 +478,7 @@ STATIC bool node_is_const_true(mp_parse_node_t pn) { || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_SMALL_INT(pn) != 0); } -#if MICROPY_EMIT_CPYTHON -// the is_nested variable is purely to match with CPython, which doesn't fully optimise not's -STATIC void cpython_c_if_cond(compiler_t *comp, mp_parse_node_t pn, bool jump_if, int label, bool is_nested) { - if (node_is_const_false(pn)) { - if (jump_if == false) { - EMIT_ARG(jump, label); - } - return; - } else if (node_is_const_true(pn)) { - if (jump_if == true) { - EMIT_ARG(jump, label); - } - return; - } else if (MP_PARSE_NODE_IS_STRUCT(pn)) { - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; - int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); - if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_or_test) { - if (jump_if == false) { - uint label2 = comp_next_label(comp); - for (int i = 0; i < n - 1; i++) { - cpython_c_if_cond(comp, pns->nodes[i], true, label2, true); - } - cpython_c_if_cond(comp, pns->nodes[n - 1], false, label, true); - EMIT_ARG(label_assign, label2); - } else { - for (int i = 0; i < n; i++) { - cpython_c_if_cond(comp, pns->nodes[i], true, label, true); - } - } - return; - } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_and_test) { - if (jump_if == false) { - for (int i = 0; i < n; i++) { - cpython_c_if_cond(comp, pns->nodes[i], false, label, true); - } - } else { - uint label2 = comp_next_label(comp); - for (int i = 0; i < n - 1; i++) { - cpython_c_if_cond(comp, pns->nodes[i], false, label2, true); - } - cpython_c_if_cond(comp, pns->nodes[n - 1], true, label, true); - EMIT_ARG(label_assign, label2); - } - return; - } else if (!is_nested && MP_PARSE_NODE_STRUCT_KIND(pns) == PN_not_test_2) { - cpython_c_if_cond(comp, pns->nodes[0], !jump_if, label, true); - return; - } - } - - // nothing special, fall back to default compiling for node and jump - compile_node(comp, pn); - EMIT_ARG(pop_jump_if, jump_if, label); -} -#endif - STATIC void c_if_cond(compiler_t *comp, mp_parse_node_t pn, bool jump_if, int label) { -#if MICROPY_EMIT_CPYTHON - cpython_c_if_cond(comp, pn, jump_if, label, false); -#else if (node_is_const_false(pn)) { if (jump_if == false) { EMIT_ARG(jump, label); @@ -773,7 +540,6 @@ STATIC void c_if_cond(compiler_t *comp, mp_parse_node_t pn, bool jump_if, int la // nothing special, fall back to default compiling for node and jump compile_node(comp, pn); EMIT_ARG(pop_jump_if, jump_if, label); -#endif } typedef enum { ASSIGN_STORE, ASSIGN_AUG_LOAD, ASSIGN_AUG_STORE } assign_kind_t; @@ -992,8 +758,7 @@ STATIC void c_assign(compiler_t *comp, mp_parse_node_t pn, assign_kind_t assign_ compile_syntax_error(comp, pn, "illegal expression for augmented assignment"); } -// stuff for lambda and comprehensions and generators -// if we are not in CPython compatibility mode then: +// stuff for lambda and comprehensions and generators: // if n_pos_defaults > 0 then there is a tuple on the stack with the positional defaults // if n_kw_defaults > 0 then there is a dictionary on the stack with the keyword defaults // if both exist, the tuple is above the dictionary (ie the first pop gets the tuple) @@ -1011,12 +776,8 @@ STATIC void close_over_variables_etc(compiler_t *comp, scope_t *this_scope, int for (int j = 0; j < this_scope->id_info_len; j++) { id_info_t *id2 = &this_scope->id_info[j]; if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) { -#if MICROPY_EMIT_CPYTHON - EMIT_ARG(load_closure, id->qst, id->local_num); -#else // in Micro Python we load closures using LOAD_FAST EMIT_LOAD_FAST(id->qst, id->local_num); -#endif nfree += 1; } } @@ -1085,10 +846,6 @@ STATIC void compile_funcdef_param(compiler_t *comp, mp_parse_node_t pn) { if (comp->have_star) { comp->num_dict_params += 1; -#if MICROPY_EMIT_CPYTHON - EMIT_ARG(load_const_str, MP_PARSE_NODE_LEAF_ARG(pn_id)); - compile_node(comp, pn_equal); -#else // in Micro Python we put the default dict parameters into a dictionary using the bytecode if (comp->num_dict_params == 1) { // in Micro Python we put the default positional parameters into a tuple using the bytecode @@ -1106,7 +863,6 @@ STATIC void compile_funcdef_param(compiler_t *comp, mp_parse_node_t pn) { compile_node(comp, pn_equal); EMIT_ARG(load_const_str, MP_PARSE_NODE_LEAF_ARG(pn_id)); EMIT(store_map); -#endif } else { comp->num_default_params += 1; compile_node(comp, pn_equal); @@ -1143,14 +899,12 @@ STATIC qstr compile_funcdef_helper(compiler_t *comp, mp_parse_node_struct_t *pns return MP_QSTR_NULL; } -#if !MICROPY_EMIT_CPYTHON // in Micro Python we put the default positional parameters into a tuple using the bytecode // the default keywords args may have already made the tuple; if not, do it now if (comp->num_default_params > 0 && comp->num_dict_params == 0) { EMIT_ARG(build_tuple, comp->num_default_params); EMIT(load_null); // sentinel indicating empty default keyword args } -#endif // get the scope for this function scope_t *fscope = (scope_t*)pns->nodes[4]; @@ -1558,12 +1312,8 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(load_const_small_int, import_level); // build the "fromlist" tuple -#if MICROPY_EMIT_CPYTHON - EMIT_ARG(load_const_verbatim_strn, "('*',)", 6); -#else EMIT_ARG(load_const_str, MP_QSTR__star_); EMIT_ARG(build_tuple, 1); -#endif // do the import qstr dummy_q; @@ -1576,31 +1326,6 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { // build the "fromlist" tuple mp_parse_node_t *pn_nodes; int n = mp_parse_node_extract_list(&pns->nodes[1], PN_import_as_names, &pn_nodes); -#if MICROPY_EMIT_CPYTHON - { - vstr_t *vstr = vstr_new(); - vstr_printf(vstr, "("); - for (int i = 0; i < n; i++) { - assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn_nodes[i], PN_import_as_name)); - mp_parse_node_struct_t *pns3 = (mp_parse_node_struct_t*)pn_nodes[i]; - qstr id2 = MP_PARSE_NODE_LEAF_ARG(pns3->nodes[0]); // should be id - if (i > 0) { - vstr_printf(vstr, ", "); - } - vstr_printf(vstr, "'"); - mp_uint_t len; - const byte *str = qstr_data(id2, &len); - vstr_add_strn(vstr, (const char*)str, len); - vstr_printf(vstr, "'"); - } - if (n == 1) { - vstr_printf(vstr, ","); - } - vstr_printf(vstr, ")"); - EMIT_ARG(load_const_verbatim_strn, vstr_str(vstr), vstr_len(vstr)); - vstr_free(vstr); - } -#else for (int i = 0; i < n; i++) { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn_nodes[i], PN_import_as_name)); mp_parse_node_struct_t *pns3 = (mp_parse_node_struct_t*)pn_nodes[i]; @@ -1608,7 +1333,6 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(load_const_str, id2); } EMIT_ARG(build_tuple, n); -#endif // do the import qstr dummy_q; @@ -1704,21 +1428,21 @@ STATIC void compile_if_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { uint l_end = comp_next_label(comp); - // optimisation: don't emit anything when "if False" (not in CPython) - if (MICROPY_EMIT_CPYTHON || !node_is_const_false(pns->nodes[0])) { + // optimisation: don't emit anything when "if False" + if (!node_is_const_false(pns->nodes[0])) { uint l_fail = comp_next_label(comp); c_if_cond(comp, pns->nodes[0], false, l_fail); // if condition compile_node(comp, pns->nodes[1]); // if block - // optimisation: skip everything else when "if True" (not in CPython) - if (!MICROPY_EMIT_CPYTHON && node_is_const_true(pns->nodes[0])) { + // optimisation: skip everything else when "if True" + if (node_is_const_true(pns->nodes[0])) { goto done; } if ( - // optimisation: don't jump over non-existent elif/else blocks (not in CPython) - (MICROPY_EMIT_CPYTHON || !(MP_PARSE_NODE_IS_NULL(pns->nodes[2]) && MP_PARSE_NODE_IS_NULL(pns->nodes[3]))) + // optimisation: don't jump over non-existent elif/else blocks + !(MP_PARSE_NODE_IS_NULL(pns->nodes[2]) && MP_PARSE_NODE_IS_NULL(pns->nodes[3])) // optimisation: don't jump if last instruction was return && !EMIT(last_emit_was_return_value) ) { @@ -1736,15 +1460,15 @@ STATIC void compile_if_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn_elif[i], PN_if_stmt_elif)); // should be mp_parse_node_struct_t *pns_elif = (mp_parse_node_struct_t*)pn_elif[i]; - // optimisation: don't emit anything when "if False" (not in CPython) - if (MICROPY_EMIT_CPYTHON || !node_is_const_false(pns_elif->nodes[0])) { + // optimisation: don't emit anything when "if False" + if (!node_is_const_false(pns_elif->nodes[0])) { uint l_fail = comp_next_label(comp); c_if_cond(comp, pns_elif->nodes[0], false, l_fail); // elif condition compile_node(comp, pns_elif->nodes[1]); // elif block - // optimisation: skip everything else when "elif True" (not in CPython) - if (!MICROPY_EMIT_CPYTHON && node_is_const_true(pns_elif->nodes[0])) { + // optimisation: skip everything else when "elif True" + if (node_is_const_true(pns_elif->nodes[0])) { goto done; } @@ -1781,23 +1505,6 @@ done: STATIC void compile_while_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { START_BREAK_CONTINUE_BLOCK - // compared to CPython, we have an optimised version of while loops -#if MICROPY_EMIT_CPYTHON - uint done_label = comp_next_label(comp); - EMIT_ARG(setup_loop, break_label); - EMIT_ARG(label_assign, continue_label); - c_if_cond(comp, pns->nodes[0], false, done_label); // condition - compile_node(comp, pns->nodes[1]); // body - if (!EMIT(last_emit_was_return_value)) { - EMIT_ARG(jump, continue_label); - } - EMIT_ARG(label_assign, done_label); - // CPython does not emit POP_BLOCK if the condition was a constant; don't undertand why - // this is a small hack to agree with CPython - if (!node_is_const_true(pns->nodes[0])) { - EMIT(pop_block); - } -#else if (!node_is_const_false(pns->nodes[0])) { // optimisation: don't emit anything for "while False" uint top_label = comp_next_label(comp); if (!node_is_const_true(pns->nodes[0])) { // optimisation: don't jump to cond for "while True" @@ -1808,7 +1515,6 @@ STATIC void compile_while_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(label_assign, continue_label); c_if_cond(comp, pns->nodes[0], true, top_label); // condition } -#endif // break/continue apply to outer loop (if any) in the else block END_BREAK_CONTINUE_BLOCK @@ -1818,7 +1524,6 @@ STATIC void compile_while_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(label_assign, break_label); } -#if !MICROPY_EMIT_CPYTHON // This function compiles an optimised for-loop of the form: // for in range(, , ): // @@ -1899,10 +1604,8 @@ STATIC void compile_for_stmt_optimised_range(compiler_t *comp, mp_parse_node_t p EMIT(pop_top); } } -#endif STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { -#if !MICROPY_EMIT_CPYTHON // this bit optimises: for in range(...), turning it into an explicitly incremented variable // this is actually slower, but uses no heap memory // for viper it will be much, much faster @@ -1945,7 +1648,6 @@ STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { } } } -#endif START_BREAK_CONTINUE_BLOCK comp->break_label |= MP_EMIT_BREAK_FROM_FOR; @@ -1953,11 +1655,6 @@ STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { uint pop_label = comp_next_label(comp); uint end_label = comp_next_label(comp); - // I don't think our implementation needs SETUP_LOOP/POP_BLOCK for for-statements -#if MICROPY_EMIT_CPYTHON - EMIT_ARG(setup_loop, end_label); -#endif - compile_node(comp, pns->nodes[1]); // iterator EMIT(get_iter); EMIT_ARG(label_assign, continue_label); @@ -1973,10 +1670,6 @@ STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { // break/continue apply to outer loop (if any) in the else block END_BREAK_CONTINUE_BLOCK -#if MICROPY_EMIT_CPYTHON - EMIT(pop_block); -#endif - compile_node(comp, pns->nodes[3]); // else (not tested) EMIT_ARG(label_assign, break_label); @@ -2233,7 +1926,7 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns1->nodes[0]) == 2 && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 2) { - // optimisation for a, b = c, d; to match CPython's optimisation + // optimisation for a, b = c, d mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns1->nodes[0]; mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) @@ -2251,7 +1944,7 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns1->nodes[0]) == 3 && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 3) { - // optimisation for a, b, c = d, e, f; to match CPython's optimisation + // optimisation for a, b, c = d, e, f mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns1->nodes[0]; mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) @@ -2485,7 +2178,6 @@ STATIC void compile_power(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_arglist, bool is_method_call, int n_positional_extra) { // function to call is on top of stack -#if !MICROPY_EMIT_CPYTHON // this is to handle special super() call if (MP_PARSE_NODE_IS_NULL(pn_arglist) && comp->func_arg_is_super && comp->scope_cur->kind == SCOPE_FUNCTION) { compile_load_id(comp, MP_QSTR___class__); @@ -2501,7 +2193,6 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar compile_syntax_error(comp, MP_PARSE_NODE_NULL, "super() call cannot find self"); // really a TypeError return; } -#endif // get the list of arguments mp_parse_node_t *args; @@ -3184,7 +2875,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_t pn_iter, m } STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { -#if MICROPY_EMIT_CPYTHON || MICROPY_ENABLE_DOC_STRING +#if MICROPY_ENABLE_DOC_STRING // see http://www.python.org/dev/peps/pep-0257/ // look for the first statement @@ -3240,12 +2931,6 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { scope->exc_stack_size = 0; } -#if MICROPY_EMIT_CPYTHON - if (comp->pass == MP_PASS_EMIT) { - scope_print_info(scope); - } -#endif - // compile if (MP_PARSE_NODE_IS_STRUCT_KIND(scope->pn, PN_eval_input)) { assert(scope->kind == SCOPE_MODULE); @@ -3331,11 +3016,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { // CPython uses .0, but we should be able to use anything that won't // clash with a user defined variable. Best to use an existing qstr, // so we use the blank qstr. -#if MICROPY_EMIT_CPYTHON - qstr qstr_arg = QSTR_FROM_STR_STATIC(".0"); -#else qstr qstr_arg = MP_QSTR_; -#endif if (comp->pass == MP_PASS_SCOPE) { bool added; id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qstr_arg, &added); @@ -3395,11 +3076,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { if (id->kind == ID_INFO_KIND_LOCAL) { EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); } else { -#if MICROPY_EMIT_CPYTHON - EMIT_ARG(load_closure, MP_QSTR___class__, 0); // XXX check this is the correct local num -#else EMIT_LOAD_FAST(MP_QSTR___class__, id->local_num); -#endif } EMIT(return_value); } @@ -3547,7 +3224,6 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind #endif STATIC void scope_compute_things(scope_t *scope) { -#if !MICROPY_EMIT_CPYTHON // in Micro Python we put the *x parameter after all other parameters (except **y) if (scope->scope_flags & MP_SCOPE_FLAG_VARARGS) { id_info_t *id_param = NULL; @@ -3564,7 +3240,6 @@ STATIC void scope_compute_things(scope_t *scope) { } } } -#endif // in functions, turn implicit globals into explicit globals // compute the index of each local @@ -3584,19 +3259,9 @@ STATIC void scope_compute_things(scope_t *scope) { } } - // compute the index of cell vars (freevars[idx] in CPython) -#if MICROPY_EMIT_CPYTHON - int num_cell = 0; -#endif + // compute the index of cell vars for (int i = 0; i < scope->id_info_len; i++) { id_info_t *id = &scope->id_info[i]; -#if MICROPY_EMIT_CPYTHON - // in CPython the cells are numbered starting from 0 - if (id->kind == ID_INFO_KIND_CELL) { - id->local_num = num_cell; - num_cell += 1; - } -#else // in Micro Python the cells come right after the fast locals // parameters are not counted here, since they remain at the start // of the locals, even if they are cell vars @@ -3604,10 +3269,9 @@ STATIC void scope_compute_things(scope_t *scope) { id->local_num = scope->num_locals; scope->num_locals += 1; } -#endif } - // compute the index of free vars (freevars[idx] in CPython) + // compute the index of free vars // make sure they are in the order of the parent scope if (scope->parent != NULL) { int num_free = 0; @@ -3618,19 +3282,13 @@ STATIC void scope_compute_things(scope_t *scope) { id_info_t *id2 = &scope->id_info[j]; if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) { assert(!(id2->flags & ID_FLAG_IS_PARAM)); // free vars should not be params -#if MICROPY_EMIT_CPYTHON - // in CPython the frees are numbered after the cells - id2->local_num = num_cell + num_free; -#else // in Micro Python the frees come first, before the params id2->local_num = num_free; -#endif num_free += 1; } } } } -#if !MICROPY_EMIT_CPYTHON // in Micro Python shift all other locals after the free locals if (num_free > 0) { for (int i = 0; i < scope->id_info_len; i++) { @@ -3642,23 +3300,9 @@ STATIC void scope_compute_things(scope_t *scope) { scope->num_pos_args += num_free; // free vars are counted as params for passing them into the function scope->num_locals += num_free; } -#endif } // compute scope_flags - -#if MICROPY_EMIT_CPYTHON - // these flags computed here are for CPython compatibility only - if (scope->kind == SCOPE_FUNCTION || scope->kind == SCOPE_LAMBDA || scope->kind == SCOPE_LIST_COMP || scope->kind == SCOPE_DICT_COMP || scope->kind == SCOPE_SET_COMP || scope->kind == SCOPE_GEN_EXPR) { - assert(scope->parent != NULL); - scope->scope_flags |= MP_SCOPE_FLAG_NEWLOCALS; - scope->scope_flags |= MP_SCOPE_FLAG_OPTIMISED; - if ((SCOPE_FUNCTION <= scope->parent->kind && scope->parent->kind <= SCOPE_SET_COMP) || (scope->parent->kind == SCOPE_CLASS && scope->parent->parent->kind == SCOPE_FUNCTION)) { - scope->scope_flags |= MP_SCOPE_FLAG_NESTED; - } - } -#endif - int num_free = 0; for (int i = 0; i < scope->id_info_len; i++) { id_info_t *id = &scope->id_info[i]; @@ -3688,22 +3332,13 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is mp_map_deinit(&consts); // create standard emitter; it's used at least for MP_PASS_SCOPE - #if MICROPY_EMIT_CPYTHON - emit_t *emit_cpython = emit_cpython_new(); - #else emit_t *emit_bc = emit_bc_new(); - #endif // compile pass 1 - #if MICROPY_EMIT_CPYTHON - comp->emit = emit_cpython; - comp->emit_method_table = &emit_cpython_method_table; - #else comp->emit = emit_bc; #if MICROPY_EMIT_NATIVE comp->emit_method_table = &emit_bc_method_table; #endif - #endif #if MICROPY_EMIT_INLINE_THUMB comp->emit_inline_asm = NULL; comp->emit_inline_asm_method_table = NULL; @@ -3731,21 +3366,15 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is } // set max number of labels now that it's calculated - #if MICROPY_EMIT_CPYTHON - emit_cpython_set_max_num_labels(emit_cpython, max_num_labels); - #else emit_bc_set_max_num_labels(emit_bc, max_num_labels); - #endif // compile pass 2 and 3 -#if !MICROPY_EMIT_CPYTHON #if MICROPY_EMIT_NATIVE emit_t *emit_native = NULL; #endif #if MICROPY_EMIT_INLINE_THUMB emit_inline_asm_t *emit_inline_thumb = NULL; #endif -#endif // !MICROPY_EMIT_CPYTHON for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) { if (false) { // dummy @@ -3770,10 +3399,6 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is // choose the emit type -#if MICROPY_EMIT_CPYTHON - comp->emit = emit_cpython; - comp->emit_method_table = &emit_cpython_method_table; -#else switch (s->emit_options) { #if MICROPY_EMIT_NATIVE @@ -3812,7 +3437,6 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is #endif break; } -#endif // !MICROPY_EMIT_CPYTHON // need a pass to compute stack size compile_scope(comp, s, MP_PASS_STACK_SIZE); @@ -3840,9 +3464,6 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is // free the emitters -#if MICROPY_EMIT_CPYTHON - emit_cpython_free(emit_cpython); -#else emit_bc_free(emit_bc); #if MICROPY_EMIT_NATIVE if (emit_native != NULL) { @@ -3862,7 +3483,6 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is emit_inline_thumb_free(emit_inline_thumb); } #endif -#endif // MICROPY_EMIT_CPYTHON // free the parse tree mp_parse_node_free(module_scope->pn); @@ -3882,13 +3502,7 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is if (compile_error != MP_OBJ_NULL) { nlr_raise(compile_error); } else { -#if MICROPY_EMIT_CPYTHON - // can't create code, so just return true - (void)outer_raw_code; // to suppress warning that outer_raw_code is unused - return mp_const_true; -#else // return function that executes the outer module return mp_make_function_from_raw_code(outer_raw_code, MP_OBJ_NULL, MP_OBJ_NULL); -#endif } } diff --git a/py/emit.h b/py/emit.h index 8b980c1023..4f60a6fff8 100644 --- a/py/emit.h +++ b/py/emit.h @@ -145,14 +145,6 @@ typedef struct _emit_method_table_t { // they may or may not emit code void (*start_except_handler)(emit_t *emit); void (*end_except_handler)(emit_t *emit); - -#if MICROPY_EMIT_CPYTHON - // these methods are only needed for emitcpy - void (*load_const_verbatim_strn)(emit_t *emit, const char *str, mp_uint_t len); - void (*load_closure)(emit_t *emit, qstr qst, mp_uint_t local_num); - void (*setup_loop)(emit_t *emit, mp_uint_t label); -#endif - } emit_method_table_t; void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst); diff --git a/py/emitbc.c b/py/emitbc.c index fef9054c75..11e65f62e3 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -34,8 +34,6 @@ #include "py/emit.h" #include "py/bc0.h" -#if !MICROPY_EMIT_CPYTHON - #define BYTES_FOR_INT ((BYTES_PER_WORD * 8 + 6) / 7) #define DUMMY_DATA_SIZE (BYTES_FOR_INT) @@ -1030,5 +1028,3 @@ const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_delete_id_ops = { mp_emit_bc_delete_global, }; #endif - -#endif // !MICROPY_EMIT_CPYTHON diff --git a/py/emitcommon.c b/py/emitcommon.c index 7eda5e05fd..1fd2bd1dc1 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -33,28 +33,12 @@ void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst) { bool added; id_info_t *id = scope_find_or_add_id(scope, qst, &added); if (added) { -#if MICROPY_EMIT_CPYTHON - if (qst == MP_QSTR_super && scope->kind == SCOPE_FUNCTION) { - // special case, super is a global, and also counts as use of __class__ - id->kind = ID_INFO_KIND_GLOBAL_EXPLICIT; - id_info_t *id2 = scope_find_local_in_parent(scope, MP_QSTR___class__); - if (id2 != NULL) { - id2 = scope_find_or_add_id(scope, MP_QSTR___class__, &added); - if (added) { - id2->kind = ID_INFO_KIND_FREE; - scope_close_over_in_parents(scope, MP_QSTR___class__); - } - } - } else -#endif - { - id_info_t *id2 = scope_find_local_in_parent(scope, qst); - if (id2 != NULL && (id2->kind == ID_INFO_KIND_LOCAL || id2->kind == ID_INFO_KIND_CELL || id2->kind == ID_INFO_KIND_FREE)) { - id->kind = ID_INFO_KIND_FREE; - scope_close_over_in_parents(scope, qst); - } else { - id->kind = ID_INFO_KIND_GLOBAL_IMPLICIT; - } + id_info_t *id2 = scope_find_local_in_parent(scope, qst); + if (id2 != NULL && (id2->kind == ID_INFO_KIND_LOCAL || id2->kind == ID_INFO_KIND_CELL || id2->kind == ID_INFO_KIND_FREE)) { + id->kind = ID_INFO_KIND_FREE; + scope_close_over_in_parents(scope, qst); + } else { + id->kind = ID_INFO_KIND_GLOBAL_IMPLICIT; } } } diff --git a/py/emitcpy.c b/py/emitcpy.c deleted file mode 100644 index 3bdec1716d..0000000000 --- a/py/emitcpy.c +++ /dev/null @@ -1,901 +0,0 @@ -/* - * This file is part of the Micro Python project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2013, 2014 Damien P. George - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include "py/emit.h" - -// wrapper around everything in this file -#if MICROPY_EMIT_CPYTHON - -struct _emit_t { - int pass; - int bytecode_offset; - int stack_size; - bool last_emit_was_return_value; - - scope_t *scope; - - mp_uint_t max_num_labels; - mp_uint_t *label_offsets; -}; - -emit_t *emit_cpython_new(void) { - emit_t *emit = m_new0(emit_t, 1); - return emit; -} - -void emit_cpython_set_max_num_labels(emit_t* emit, mp_uint_t max_num_labels) { - emit->max_num_labels = max_num_labels; - emit->label_offsets = m_new(mp_uint_t, max_num_labels); -} - -void emit_cpython_free(emit_t *emit) { - m_del(mp_uint_t, emit->label_offsets, emit->max_num_labels); - m_del_obj(emit_t, emit); -} - -STATIC void emit_cpy_set_native_type(emit_t *emit, mp_uint_t op, mp_uint_t arg1, qstr arg2) { -} - -STATIC void emit_cpy_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { - emit->pass = pass; - emit->bytecode_offset = 0; - emit->stack_size = 0; - emit->last_emit_was_return_value = false; - emit->scope = scope; - if (pass < MP_PASS_EMIT) { - memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(mp_uint_t)); - } -} - -STATIC void emit_cpy_end_pass(emit_t *emit) { - if (emit->pass == MP_PASS_SCOPE) { - return; - } - // check stack is back to zero size - if (emit->stack_size != 0) { - printf("ERROR: stack size not back to zero; got %d\n", emit->stack_size); - } -} - -STATIC bool emit_cpy_last_emit_was_return_value(emit_t *emit) { - return emit->last_emit_was_return_value; -} - -STATIC void emit_cpy_adjust_stack_size(emit_t *emit, mp_int_t delta) { - emit->stack_size += delta; -} - -STATIC void emit_cpy_set_source_line(emit_t *emit, mp_uint_t source_line) { -} - -// TODO: module-polymorphic function (read: name clash if made global) -static void emit_pre(emit_t *emit, int stack_size_delta, int bytecode_size) { - if (emit->pass == MP_PASS_SCOPE) { - return; - } - emit->stack_size += stack_size_delta; - if (emit->stack_size > emit->scope->stack_size) { - emit->scope->stack_size = emit->stack_size; - } - emit->last_emit_was_return_value = false; - if (emit->pass == MP_PASS_EMIT && bytecode_size > 0) { - if (emit->bytecode_offset >= 1000) { - printf("%d ", emit->bytecode_offset); - } else { - printf("% 4d ", emit->bytecode_offset); - } - } - emit->bytecode_offset += bytecode_size; -} - -STATIC void emit_cpy_label_assign(emit_t *emit, mp_uint_t l) { - emit_pre(emit, 0, 0); - if (emit->pass == MP_PASS_SCOPE) { - return; - } - assert(l < emit->max_num_labels); - if (emit->pass < MP_PASS_EMIT) { - // assign label offset - assert(emit->label_offsets[l] == -1); - emit->label_offsets[l] = emit->bytecode_offset; - } else { - // ensure label offset has not changed from MP_PASS_CODE_SIZE to MP_PASS_EMIT - assert(emit->label_offsets[l] == emit->bytecode_offset); - //printf("l%d: (at %d)\n", l, emit->bytecode_offset); - } -} - -STATIC void emit_cpy_import_name(emit_t *emit, qstr qst) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("IMPORT_NAME %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_import_from(emit_t *emit, qstr qst) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("IMPORT_FROM %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_import_star(emit_t *emit) { - emit_pre(emit, -1, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("IMPORT_STAR\n"); - } -} - -STATIC void emit_cpy_load_const_tok(emit_t *emit, mp_token_kind_t tok) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST "); - switch (tok) { - case MP_TOKEN_KW_FALSE: printf("False"); break; - case MP_TOKEN_KW_NONE: printf("None"); break; - case MP_TOKEN_KW_TRUE: printf("True"); break; - default: printf("?=%d\n", tok); return; assert(0); - } - printf("\n"); - } -} - -STATIC void emit_cpy_load_const_small_int(emit_t *emit, mp_int_t arg) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST " INT_FMT "\n", arg); - } -} - -STATIC void print_quoted_str(qstr qst) { - const char *str = qstr_str(qst); - int len = strlen(str); - bool has_single_quote = false; - bool has_double_quote = false; - for (int i = 0; i < len; i++) { - if (str[i] == '\'') { - has_single_quote = true; - } else if (str[i] == '"') { - has_double_quote = true; - } - } - int quote_char = '\''; - if (has_single_quote && !has_double_quote) { - quote_char = '"'; - } - printf("%c", quote_char); - for (const char *s = str, *top = str + len; s < top; s++) { - if (*s == quote_char) { - printf("\\%c", quote_char); - } else if (*s == '\\') { - printf("\\\\"); - } else if (32 <= *s && *s <= 126) { - printf("%c", *s); - } else if (*s == '\n') { - printf("\\n"); - // TODO add more escape codes here - } else { - printf("\\x%02x", (*s) & 0xff); - } - } - printf("%c", quote_char); -} - -STATIC void emit_cpy_load_const_str(emit_t *emit, qstr qst) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST "); - print_quoted_str(qst); - printf("\n"); - } -} - -STATIC void emit_cpy_load_const_obj(emit_t *emit, void *obj) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST "); - mp_obj_print(obj, PRINT_REPR); - printf("\n"); - } -} - -STATIC void emit_cpy_load_null(emit_t *emit) { - // unused for cpy - assert(0); -} - -STATIC void emit_cpy_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_FAST " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_load_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_DEREF " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_load_name(emit_t *emit, qstr qst) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_NAME %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_load_global(emit_t *emit, qstr qst) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_GLOBAL %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_load_attr(emit_t *emit, qstr qst) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_ATTR %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_load_method(emit_t *emit, qstr qst) { - emit_cpy_load_attr(emit, qst); -} - -STATIC void emit_cpy_load_build_class(emit_t *emit) { - emit_pre(emit, 1, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_BUILD_CLASS\n"); - } -} - -STATIC void emit_cpy_load_subscr(emit_t *emit) { - emit_pre(emit, -1, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("BINARY_SUBSCR\n"); - } -} - -STATIC void emit_cpy_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_FAST " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_store_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_DEREF " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_store_name(emit_t *emit, qstr qst) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_NAME %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_store_global(emit_t *emit, qstr qst) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_GLOBAL %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_store_attr(emit_t *emit, qstr qst) { - emit_pre(emit, -2, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_ATTR %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_store_subscr(emit_t *emit) { - emit_pre(emit, -3, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_SUBSCR\n"); - } -} - -STATIC void emit_cpy_delete_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("DELETE_FAST " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_delete_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("DELETE_DEREF " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_delete_name(emit_t *emit, qstr qst) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("DELETE_NAME %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_delete_global(emit_t *emit, qstr qst) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("DELETE_GLOBAL %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_delete_attr(emit_t *emit, qstr qst) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("DELETE_ATTR %s\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_delete_subscr(emit_t *emit) { - emit_pre(emit, -2, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("DELETE_SUBSCR\n"); - } -} - -STATIC void emit_cpy_dup_top(emit_t *emit) { - emit_pre(emit, 1, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("DUP_TOP\n"); - } -} - -STATIC void emit_cpy_dup_top_two(emit_t *emit) { - emit_pre(emit, 2, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("DUP_TOP_TWO\n"); - } -} - -STATIC void emit_cpy_pop_top(emit_t *emit) { - emit_pre(emit, -1, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("POP_TOP\n"); - } -} - -STATIC void emit_cpy_rot_two(emit_t *emit) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("ROT_TWO\n"); - } -} - -STATIC void emit_cpy_rot_three(emit_t *emit) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("ROT_THREE\n"); - } -} - -STATIC void emit_cpy_jump(emit_t *emit, mp_uint_t label) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - int dest = emit->label_offsets[label]; - if (dest < emit->bytecode_offset) { - printf("JUMP_ABSOLUTE " UINT_FMT "\n", emit->label_offsets[label]); - } else { - printf("JUMP_FORWARD " UINT_FMT "\n", emit->label_offsets[label]); - } - } -} - -STATIC void emit_cpy_pop_jump_if(emit_t *emit, bool cond, mp_uint_t label) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - if (cond) { - printf("POP_JUMP_IF_TRUE " UINT_FMT "\n", emit->label_offsets[label]); - } else { - printf("POP_JUMP_IF_FALSE " UINT_FMT "\n", emit->label_offsets[label]); - } - } -} - -STATIC void emit_cpy_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - if (cond) { - printf("JUMP_IF_TRUE_OR_POP " UINT_FMT "\n", emit->label_offsets[label]); - } else { - printf("JUMP_IF_FALSE_OR_POP " UINT_FMT "\n", emit->label_offsets[label]); - } - } -} - -STATIC void emit_cpy_break_loop(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("BREAK_LOOP\n"); - } -} - -STATIC void emit_cpy_continue_loop(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) { - if (except_depth == 0) { - emit_cpy_jump(emit, label); - } else { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("CONTINUE_LOOP " UINT_FMT "\n", emit->label_offsets[label]); - } - } -} - -STATIC void emit_cpy_setup_with(emit_t *emit, mp_uint_t label) { - emit_pre(emit, 7, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("SETUP_WITH " UINT_FMT "\n", emit->label_offsets[label]); - } -} - -STATIC void emit_cpy_with_cleanup(emit_t *emit) { - emit_pre(emit, -7, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("WITH_CLEANUP\n"); - } -} - -STATIC void emit_cpy_setup_except(emit_t *emit, mp_uint_t label) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("SETUP_EXCEPT " UINT_FMT "\n", emit->label_offsets[label]); - } -} - -STATIC void emit_cpy_setup_finally(emit_t *emit, mp_uint_t label) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("SETUP_FINALLY " UINT_FMT "\n", emit->label_offsets[label]); - } -} - -STATIC void emit_cpy_end_finally(emit_t *emit) { - emit_pre(emit, -1, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("END_FINALLY\n"); - } -} - -STATIC void emit_cpy_get_iter(emit_t *emit) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("GET_ITER\n"); - } -} - -STATIC void emit_cpy_for_iter(emit_t *emit, mp_uint_t label) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("FOR_ITER " UINT_FMT "\n", emit->label_offsets[label]); - } -} - -STATIC void emit_cpy_for_iter_end(emit_t *emit) { - emit_pre(emit, -1, 0); -} - -STATIC void emit_cpy_pop_block(emit_t *emit) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("POP_BLOCK\n"); - } -} - -STATIC void emit_cpy_pop_except(emit_t *emit) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("POP_EXCEPT\n"); - } -} - -STATIC void emit_cpy_unary_op(emit_t *emit, mp_unary_op_t op) { - emit_pre(emit, 0, 1); - if (emit->pass == MP_PASS_EMIT) { - switch (op) { - case MP_UNARY_OP_POSITIVE: printf("UNARY_POSITIVE\n"); break; - case MP_UNARY_OP_NEGATIVE: printf("UNARY_NEGATIVE\n"); break; - case MP_UNARY_OP_INVERT: printf("UNARY_INVERT\n"); break; - case MP_UNARY_OP_NOT: printf("UNARY_NOT\n"); break; - default: assert(0); - } - } -} - -STATIC void emit_cpy_binary_op(emit_t *emit, mp_binary_op_t op) { - if (op <= MP_BINARY_OP_INPLACE_POWER) { - // CPython uses a byte code for each binary op - emit_pre(emit, -1, 1); - } else { - // CPython uses a byte code plus an argument for compare ops - emit_pre(emit, -1, 3); - } - if (emit->pass == MP_PASS_EMIT) { - switch (op) { - case MP_BINARY_OP_OR: printf("BINARY_OR\n"); break; - case MP_BINARY_OP_XOR: printf("BINARY_XOR\n"); break; - case MP_BINARY_OP_AND: printf("BINARY_AND\n"); break; - case MP_BINARY_OP_LSHIFT: printf("BINARY_LSHIFT\n"); break; - case MP_BINARY_OP_RSHIFT: printf("BINARY_RSHIFT\n"); break; - case MP_BINARY_OP_ADD: printf("BINARY_ADD\n"); break; - case MP_BINARY_OP_SUBTRACT: printf("BINARY_SUBTRACT\n"); break; - case MP_BINARY_OP_MULTIPLY: printf("BINARY_MULTIPLY\n"); break; - case MP_BINARY_OP_FLOOR_DIVIDE: printf("BINARY_FLOOR_DIVIDE\n"); break; - case MP_BINARY_OP_TRUE_DIVIDE: printf("BINARY_TRUE_DIVIDE\n"); break; - case MP_BINARY_OP_MODULO: printf("BINARY_MODULO\n"); break; - case MP_BINARY_OP_POWER: printf("BINARY_POWER\n"); break; - case MP_BINARY_OP_INPLACE_OR: printf("INPLACE_OR\n"); break; - case MP_BINARY_OP_INPLACE_XOR: printf("INPLACE_XOR\n"); break; - case MP_BINARY_OP_INPLACE_AND: printf("INPLACE_AND\n"); break; - case MP_BINARY_OP_INPLACE_LSHIFT: printf("INPLACE_LSHIFT\n"); break; - case MP_BINARY_OP_INPLACE_RSHIFT: printf("INPLACE_RSHIFT\n"); break; - case MP_BINARY_OP_INPLACE_ADD: printf("INPLACE_ADD\n"); break; - case MP_BINARY_OP_INPLACE_SUBTRACT: printf("INPLACE_SUBTRACT\n"); break; - case MP_BINARY_OP_INPLACE_MULTIPLY: printf("INPLACE_MULTIPLY\n"); break; - case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: printf("INPLACE_FLOOR_DIVIDE\n"); break; - case MP_BINARY_OP_INPLACE_TRUE_DIVIDE: printf("INPLACE_TRUE_DIVIDE\n"); break; - case MP_BINARY_OP_INPLACE_MODULO: printf("INPLACE_MODULO\n"); break; - case MP_BINARY_OP_INPLACE_POWER: printf("INPLACE_POWER\n"); break; - case MP_BINARY_OP_LESS: printf("COMPARE_OP <\n"); break; - case MP_BINARY_OP_MORE: printf("COMPARE_OP >\n"); break; - case MP_BINARY_OP_EQUAL: printf("COMPARE_OP ==\n"); break; - case MP_BINARY_OP_LESS_EQUAL: printf("COMPARE_OP <=\n"); break; - case MP_BINARY_OP_MORE_EQUAL: printf("COMPARE_OP >=\n"); break; - case MP_BINARY_OP_NOT_EQUAL: printf("COMPARE_OP !=\n"); break; - case MP_BINARY_OP_IN: printf("COMPARE_OP in\n"); break; - case MP_BINARY_OP_IS: printf("COMPARE_OP is\n"); break; - case MP_BINARY_OP_EXCEPTION_MATCH: printf("COMPARE_OP exception match\n"); break; - case MP_BINARY_OP_NOT_IN: printf("COMPARE_OP not in\n"); break; - case MP_BINARY_OP_IS_NOT: printf("COMPARE_OP is not\n"); break; - default: assert(0); - } - } -} - -STATIC void emit_cpy_build_tuple(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, 1 - n_args, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("BUILD_TUPLE " UINT_FMT "\n", n_args); - } -} - -STATIC void emit_cpy_build_list(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, 1 - n_args, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("BUILD_LIST " UINT_FMT "\n", n_args); - } -} - -STATIC void emit_cpy_list_append(emit_t *emit, mp_uint_t list_index) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LIST_APPEND " UINT_FMT "\n", list_index); - } -} - -STATIC void emit_cpy_build_map(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("BUILD_MAP " UINT_FMT "\n", n_args); - } -} - -STATIC void emit_cpy_store_map(emit_t *emit) { - emit_pre(emit, -2, 1); - if (emit->pass == MP_PASS_EMIT) { - printf("STORE_MAP\n"); - } -} - -STATIC void emit_cpy_map_add(emit_t *emit, mp_uint_t map_index) { - emit_pre(emit, -2, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("MAP_ADD " UINT_FMT "\n", map_index); - } -} - -STATIC void emit_cpy_build_set(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, 1 - n_args, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("BUILD_SET " UINT_FMT "\n", n_args); - } -} - -STATIC void emit_cpy_set_add(emit_t *emit, mp_uint_t set_index) { - emit_pre(emit, -1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("SET_ADD " UINT_FMT "\n", set_index); - } -} - -STATIC void emit_cpy_build_slice(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, 1 - n_args, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("BUILD_SLICE " UINT_FMT "\n", n_args); - } -} - -STATIC void emit_cpy_unpack_sequence(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, -1 + n_args, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("UNPACK_SEQUENCE " UINT_FMT "\n", n_args); - } -} - -STATIC void emit_cpy_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right) { - emit_pre(emit, -1 + n_left + n_right + 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("UNPACK_EX " UINT_FMT "\n", n_left | (n_right << 8)); - } -} - -STATIC void emit_cpy_call_function(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) { - mp_int_t s = 0; - if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) { - s += 1; - } - if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { - s += 1; - } - emit_pre(emit, -(mp_int_t)n_positional - 2 * (mp_int_t)n_keyword - s, 3); - if (emit->pass == MP_PASS_EMIT) { - if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) { - if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { - printf("CALL_FUNCTION_VAR_KW"); - } else { - printf("CALL_FUNCTION_VAR"); - } - } else { - if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { - printf("CALL_FUNCTION_KW"); - } else { - printf("CALL_FUNCTION"); - } - } - printf(" " UINT_FMT ", " UINT_FMT "\n", n_positional, n_keyword); - } -} - -STATIC void emit_cpy_call_method(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) { - emit_cpy_call_function(emit, n_positional, n_keyword, star_flags); -} - -STATIC void emit_cpy_return_value(emit_t *emit) { - emit_pre(emit, -1, 1); - emit->last_emit_was_return_value = true; - if (emit->pass == MP_PASS_EMIT) { - printf("RETURN_VALUE\n"); - } -} - -STATIC void emit_cpy_raise_varargs(emit_t *emit, mp_uint_t n_args) { - emit_pre(emit, -n_args, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("RAISE_VARARGS " UINT_FMT "\n", n_args); - } -} - -STATIC void load_cpy_const_code_and_name(emit_t *emit, qstr qst) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST code %s\n", qstr_str(qst)); - } - // load qualified name - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST '"); - // code just to work out the qualname (or whatever it is) - { - int depth = 0; - for (scope_t *s = emit->scope; s->parent != NULL; s = s->parent) { - depth += 1; - } - for (int wanted_depth = depth; wanted_depth >= 0; wanted_depth--) { - scope_t *s = emit->scope; - for (int i = 0; i < wanted_depth; i++) { - s = s->parent; - } - if (s->kind == SCOPE_FUNCTION) { - printf("%s..", qstr_str(s->simple_name)); - } else if (s->kind == SCOPE_CLASS) { - printf("%s.", qstr_str(s->simple_name)); - } - } - } - printf("%s'\n", qstr_str(qst)); - } -} - -STATIC void emit_cpy_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { - load_cpy_const_code_and_name(emit, scope->simple_name); - emit_pre(emit, -1 - n_pos_defaults - 2 * n_kw_defaults, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("MAKE_FUNCTION " UINT_FMT "\n", (n_kw_defaults << 8) | n_pos_defaults); - } -} - -STATIC void emit_cpy_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { - emit_cpy_build_tuple(emit, n_closed_over); - load_cpy_const_code_and_name(emit, scope->simple_name); - emit_pre(emit, -2 - n_pos_defaults - 2 * n_kw_defaults, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("MAKE_CLOSURE " UINT_FMT "\n", (n_kw_defaults << 8) | n_pos_defaults); - } -} - -STATIC void emit_cpy_yield_value(emit_t *emit) { - emit_pre(emit, 0, 1); - emit->scope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; - if (emit->pass == MP_PASS_EMIT) { - printf("YIELD_VALUE\n"); - } -} - -STATIC void emit_cpy_yield_from(emit_t *emit) { - emit_pre(emit, -1, 1); - emit->scope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; - if (emit->pass == MP_PASS_EMIT) { - printf("YIELD_FROM\n"); - } -} - -STATIC void emit_cpy_start_except_handler(emit_t *emit) { - emit_cpy_adjust_stack_size(emit, 3); // stack adjust for the 3 exception items -} - -STATIC void emit_cpy_end_except_handler(emit_t *emit) { - emit_cpy_adjust_stack_size(emit, -2); // stack adjust -} - -STATIC void emit_cpy_load_const_verbatim_strn(emit_t *emit, const char *str, mp_uint_t len) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CONST %.*s\n", (int)len, str); - } -} - -STATIC void emit_cpy_load_closure(emit_t *emit, qstr qst, mp_uint_t local_num) { - emit_pre(emit, 1, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("LOAD_CLOSURE " UINT_FMT " %s\n", local_num, qstr_str(qst)); - } -} - -STATIC void emit_cpy_setup_loop(emit_t *emit, mp_uint_t label) { - emit_pre(emit, 0, 3); - if (emit->pass == MP_PASS_EMIT) { - printf("SETUP_LOOP " UINT_FMT "\n", emit->label_offsets[label]); - } -} - -const emit_method_table_t emit_cpython_method_table = { - emit_cpy_set_native_type, - emit_cpy_start_pass, - emit_cpy_end_pass, - emit_cpy_last_emit_was_return_value, - emit_cpy_adjust_stack_size, - emit_cpy_set_source_line, - - { - emit_cpy_load_fast, - emit_cpy_load_deref, - emit_cpy_load_name, - emit_cpy_load_global, - }, - { - emit_cpy_store_fast, - emit_cpy_store_deref, - emit_cpy_store_name, - emit_cpy_store_global, - }, - { - emit_cpy_delete_fast, - emit_cpy_delete_deref, - emit_cpy_delete_name, - emit_cpy_delete_global, - }, - - emit_cpy_label_assign, - emit_cpy_import_name, - emit_cpy_import_from, - emit_cpy_import_star, - emit_cpy_load_const_tok, - emit_cpy_load_const_small_int, - emit_cpy_load_const_str, - emit_cpy_load_const_obj, - emit_cpy_load_null, - emit_cpy_load_attr, - emit_cpy_load_method, - emit_cpy_load_build_class, - emit_cpy_load_subscr, - emit_cpy_store_attr, - emit_cpy_store_subscr, - emit_cpy_delete_attr, - emit_cpy_delete_subscr, - emit_cpy_dup_top, - emit_cpy_dup_top_two, - emit_cpy_pop_top, - emit_cpy_rot_two, - emit_cpy_rot_three, - emit_cpy_jump, - emit_cpy_pop_jump_if, - emit_cpy_jump_if_or_pop, - emit_cpy_break_loop, - emit_cpy_continue_loop, - emit_cpy_setup_with, - emit_cpy_with_cleanup, - emit_cpy_setup_except, - emit_cpy_setup_finally, - emit_cpy_end_finally, - emit_cpy_get_iter, - emit_cpy_for_iter, - emit_cpy_for_iter_end, - emit_cpy_pop_block, - emit_cpy_pop_except, - emit_cpy_unary_op, - emit_cpy_binary_op, - emit_cpy_build_tuple, - emit_cpy_build_list, - emit_cpy_list_append, - emit_cpy_build_map, - emit_cpy_store_map, - emit_cpy_map_add, - emit_cpy_build_set, - emit_cpy_set_add, - emit_cpy_build_slice, - emit_cpy_unpack_sequence, - emit_cpy_unpack_ex, - emit_cpy_make_function, - emit_cpy_make_closure, - emit_cpy_call_function, - emit_cpy_call_method, - emit_cpy_return_value, - emit_cpy_raise_varargs, - emit_cpy_yield_value, - emit_cpy_yield_from, - - emit_cpy_start_except_handler, - emit_cpy_end_except_handler, - - // emitcpy specific functions - emit_cpy_load_const_verbatim_strn, - emit_cpy_load_closure, - emit_cpy_setup_loop, -}; - -#endif // MICROPY_EMIT_CPYTHON diff --git a/py/mkrules.mk b/py/mkrules.mk index a994016148..fe70a0ee37 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -75,7 +75,7 @@ $(HEADER_BUILD): $(MKDIR) -p $@ ifneq ($(PROG),) -# Build a standalone executable (unix and unix-cpy do this) +# Build a standalone executable (unix does this) all: $(PROG) diff --git a/py/mpconfig.h b/py/mpconfig.h index 35a14ba88e..bfa7a1f68d 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -173,12 +173,6 @@ /*****************************************************************************/ /* Micro Python emitters */ -// Whether to emit CPython byte codes (for debugging/testing) -// Enabling this overrides all other emitters -#ifndef MICROPY_EMIT_CPYTHON -#define MICROPY_EMIT_CPYTHON (0) -#endif - // Whether to emit x64 native code #ifndef MICROPY_EMIT_X64 #define MICROPY_EMIT_X64 (0) diff --git a/py/parse.c b/py/parse.c index b20fdaacd2..cd3acb2593 100644 --- a/py/parse.c +++ b/py/parse.c @@ -548,7 +548,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } -#if !MICROPY_EMIT_CPYTHON && !MICROPY_ENABLE_DOC_STRING + #if !MICROPY_ENABLE_DOC_STRING // this code discards lonely statements, such as doc strings if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { mp_parse_node_t p = peek_result(&parser, 1); @@ -559,7 +559,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { break; } } -#endif + #endif // always emit these rules, even if they have only 1 argument if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) { diff --git a/py/py.mk b/py/py.mk index 84f3295e14..5ba0f55085 100644 --- a/py/py.mk +++ b/py/py.mk @@ -32,7 +32,6 @@ PY_O_BASENAME = \ scope.o \ compile.o \ emitcommon.o \ - emitcpy.o \ emitbc.o \ asmx64.o \ emitnx64.o \ diff --git a/py/scope.c b/py/scope.c index 4739ac3236..f9b1fb122b 100644 --- a/py/scope.c +++ b/py/scope.c @@ -149,59 +149,3 @@ void scope_close_over_in_parents(scope_t *scope, qstr qst) { } assert(0); // we should have found the variable in one of the parents } - -#if MICROPY_EMIT_CPYTHON -#include - -void scope_print_info(scope_t *s) { - if (s->kind == SCOPE_MODULE) { - printf("code \n"); - } else if (s->kind == SCOPE_LAMBDA) { - printf("code \n"); - } else if (s->kind == SCOPE_LIST_COMP) { - printf("code \n"); - } else if (s->kind == SCOPE_DICT_COMP) { - printf("code \n"); - } else if (s->kind == SCOPE_SET_COMP) { - printf("code \n"); - } else if (s->kind == SCOPE_GEN_EXPR) { - printf("code \n"); - } else { - printf("code %s\n", qstr_str(s->simple_name)); - } - /* - printf("var global:"); - for (int i = 0; i < s->id_info_len; i++) { - if (s->id_info[i].kind == ID_INFO_KIND_GLOBAL_EXPLICIT) { - printf(" %s", qstr_str(s->id_info[i].qst)); - } - } - printf("\n"); - printf("var name:"); - for (int i = 0; i < s->id_info_len; i++) { - if (s->id_info[i].kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { - printf(" %s", qstr_str(s->id_info[i].qst)); - } - } - printf("\n"); - printf("var local:"); - for (int i = 0; i < s->id_info_len; i++) { - if (s->id_info[i].kind == ID_INFO_KIND_LOCAL) { - printf(" %s", qstr_str(s->id_info[i].qst)); - } - } - printf("\n"); - printf("var free:"); - for (int i = 0; i < s->id_info_len; i++) { - if (s->id_info[i].kind == ID_INFO_KIND_FREE) { - printf(" %s", qstr_str(s->id_info[i].qst)); - } - } - printf("\n"); - */ - printf(" flags %04x\n", s->scope_flags); - printf(" argcount %d\n", s->num_pos_args); - printf(" nlocals %d\n", s->num_locals); - printf(" stacksize %d\n", s->stack_size); -} -#endif diff --git a/py/scope.h b/py/scope.h index 6d74b17d28..0ea003516a 100644 --- a/py/scope.h +++ b/py/scope.h @@ -81,6 +81,5 @@ id_info_t *scope_find(scope_t *scope, qstr qstr); id_info_t *scope_find_global(scope_t *scope, qstr qstr); id_info_t *scope_find_local_in_parent(scope_t *scope, qstr qstr); void scope_close_over_in_parents(scope_t *scope, qstr qstr); -void scope_print_info(scope_t *s); #endif // __MICROPY_INCLUDED_PY_SCOPE_H__ diff --git a/tests/bytecode/.gitignore b/tests/bytecode/.gitignore deleted file mode 100644 index cc6d8816aa..0000000000 --- a/tests/bytecode/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -output -mp-tests/__pycache__ -pylib-tests/__pycache__ diff --git a/tests/bytecode/README.md b/tests/bytecode/README.md deleted file mode 100644 index 0d5245cb7f..0000000000 --- a/tests/bytecode/README.md +++ /dev/null @@ -1,15 +0,0 @@ -This directory contains the framework and test files for testing the byte code -output of the Micro Python compiler. - -You need to first build the 'cpy' executable in the directory micropython/unix-cpy/. -This executable is a minimal version of Micro Python which compiles a single source -file and outputs the corresponding byte code. - -The output of Micro Python is checked against CPython 3.4. - -To run the tests use: - - ./run-tests - -Note that the tests in pylib-test/ are from the Python 3.3 library, and are licensed -under the relevant license, as per pylib-test/LICENSE.txt. diff --git a/tests/bytecode/check.py b/tests/bytecode/check.py deleted file mode 100644 index b5211b0fe8..0000000000 --- a/tests/bytecode/check.py +++ /dev/null @@ -1,56 +0,0 @@ -import sys -name = sys.argv[1].split('/')[-1].split('.')[0] -with open(sys.argv[1]) as f: - lines_correct = [l.strip('\n') for l in f.readlines()] -lines_me = [l.strip('\n') for l in sys.stdin.readlines()] -if len(lines_me) != len(lines_correct): - if len(lines_me) == 0: - print('{:<20}: no output'.format(name)) - elif lines_me[0].find('syntax error') >= 0: - print('{:<20}: syntax error'.format(name)) - elif lines_me[0].find(' cannot be compiled') >= 0: - print('{:<20}: compile error: {}'.format(name, lines_me[0])) - else: - print('{:<20}: mismatch in number of lines'.format(name)) -else: - total = len(lines_me) - same = 0 - bad_num_fields = 0 - bad_2 = 0 - bad_3 = 0 - jump_op = ['JUMP_FORWARD', 'JUMP_ABSOLUTE', 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE', 'SETUP_LOOP'] - jump_abs_op = ['JUMP_FORWARD', 'JUMP_ABSOLUTE'] - for i in range(total): - if lines_me[i] == lines_correct[i]: - same += 1 - else: - # line is different - line_me = lines_me[i].strip().split(' ', 2) - line_correct = lines_correct[i].strip().split(' ', 2) - allow = False - if len(line_me) != len(line_correct): - bad_num_fields += 1 - elif len(line_me) == 2: - if line_me[0] == line_correct[0] == 'stacksize': - allow = True - else: - bad_2 += 1 - else: - assert(len(line_me) == 3) - if line_me[0] == line_correct[0] and line_me[1] in jump_abs_op and line_correct[1] in jump_abs_op: - allow = True - elif line_me[0] == line_correct[0] and line_me[1] == line_correct[1] in jump_op: - allow = True - else: - bad_3 += 1 - #if not allow: - # print(line_me, 'vs', line_correct) - - bad_str = '' - if bad_num_fields > 0: - bad_str += ', {} bad num fields'.format(bad_num_fields) - if bad_2 > 0: - bad_str += ', {} bad 2-field'.format(bad_2) - if bad_3 > 0: - bad_str += ', {} bad 3-field'.format(bad_3) - print('{:<20}: {:>6} lines, {:>5.1f}% correct{}'.format(name, total, 100 * same / total, bad_str)) diff --git a/tests/bytecode/mp-tests/assert1.py b/tests/bytecode/mp-tests/assert1.py deleted file mode 100644 index 077defc970..0000000000 --- a/tests/bytecode/mp-tests/assert1.py +++ /dev/null @@ -1,2 +0,0 @@ -assert x -assert x, 'test' diff --git a/tests/bytecode/mp-tests/assign1.py b/tests/bytecode/mp-tests/assign1.py deleted file mode 100644 index 64ae4a0c59..0000000000 --- a/tests/bytecode/mp-tests/assign1.py +++ /dev/null @@ -1,10 +0,0 @@ -[] = () -[] = [] -a = b -(a) = b -a, b = c, d -a, b, c = d, e, f -a, b, c, d = e, f, g, h -#(a, b) = c, d -#a, b = (c, d) -#(a, b) = (c, d) diff --git a/tests/bytecode/mp-tests/assign2.py b/tests/bytecode/mp-tests/assign2.py deleted file mode 100644 index cb03593d2e..0000000000 --- a/tests/bytecode/mp-tests/assign2.py +++ /dev/null @@ -1,21 +0,0 @@ -*a, = b -a, *b = c -a, *b, = c -a, *b, c = d - -[*a] = b -[*a,] = b -[a, *b] = c -[a, *b,] = c -[a, *b, c] = d - -(*a,) = x -(*a, b) = x -(a, *b) = x -(*a, b, c) = x -(a, *b, c) = x -(a, b, *c) = x -(*a, b, c, d) = x -(a, *b, c, d) = x -(a, b, *c, d) = x -(a, b, c, *d) = x diff --git a/tests/bytecode/mp-tests/augassign1.py b/tests/bytecode/mp-tests/augassign1.py deleted file mode 100644 index 38a376af46..0000000000 --- a/tests/bytecode/mp-tests/augassign1.py +++ /dev/null @@ -1,5 +0,0 @@ -[] = () -x += 1 -x.y += 1 -x.f().y += 1 -x[1] += 2 diff --git a/tests/bytecode/mp-tests/call1.py b/tests/bytecode/mp-tests/call1.py deleted file mode 100644 index eb8a8bf5f1..0000000000 --- a/tests/bytecode/mp-tests/call1.py +++ /dev/null @@ -1 +0,0 @@ -f(a, b=c) diff --git a/tests/bytecode/mp-tests/class1.py b/tests/bytecode/mp-tests/class1.py deleted file mode 100644 index bc87666806..0000000000 --- a/tests/bytecode/mp-tests/class1.py +++ /dev/null @@ -1,3 +0,0 @@ -class C: - pass -C() diff --git a/tests/bytecode/mp-tests/class2.py b/tests/bytecode/mp-tests/class2.py deleted file mode 100644 index 1a3e89849d..0000000000 --- a/tests/bytecode/mp-tests/class2.py +++ /dev/null @@ -1,4 +0,0 @@ -class A: - x = 1 - y = x + z -A() diff --git a/tests/bytecode/mp-tests/class3.py b/tests/bytecode/mp-tests/class3.py deleted file mode 100644 index f49e2e8114..0000000000 --- a/tests/bytecode/mp-tests/class3.py +++ /dev/null @@ -1,10 +0,0 @@ -class A: - def f(x): - return x - def g(y): - def h(z): - return x + y + z - h(y) -A() -A.f(1) -A.g(2)(3) diff --git a/tests/bytecode/mp-tests/class4.py b/tests/bytecode/mp-tests/class4.py deleted file mode 100644 index 4cb6258093..0000000000 --- a/tests/bytecode/mp-tests/class4.py +++ /dev/null @@ -1,9 +0,0 @@ -class A: - def __init__(self, x): - self.x = x - self.y = 0 - - def get(self): - return self.x + self.y -A(1) -A(2).get() diff --git a/tests/bytecode/mp-tests/class5.py b/tests/bytecode/mp-tests/class5.py deleted file mode 100644 index 4bf96c8e2f..0000000000 --- a/tests/bytecode/mp-tests/class5.py +++ /dev/null @@ -1,8 +0,0 @@ -class A(B): - pass -class A(object): - pass -class A(x.y()): - pass -class A(B, C): - pass diff --git a/tests/bytecode/mp-tests/class6.py b/tests/bytecode/mp-tests/class6.py deleted file mode 100644 index 05a2454f50..0000000000 --- a/tests/bytecode/mp-tests/class6.py +++ /dev/null @@ -1,7 +0,0 @@ -class A: - def f(self): - pass - -class B(A): - def f(self): - super().f() diff --git a/tests/bytecode/mp-tests/class7.py b/tests/bytecode/mp-tests/class7.py deleted file mode 100644 index 3de41dbb52..0000000000 --- a/tests/bytecode/mp-tests/class7.py +++ /dev/null @@ -1,6 +0,0 @@ -# accessing super, but not as a function call - -class A: - def f(): - #x = super - print(super) diff --git a/tests/bytecode/mp-tests/closure1.py b/tests/bytecode/mp-tests/closure1.py deleted file mode 100644 index fdfb4eaf27..0000000000 --- a/tests/bytecode/mp-tests/closure1.py +++ /dev/null @@ -1,2 +0,0 @@ -# basic closure -# to write! diff --git a/tests/bytecode/mp-tests/closure2.py b/tests/bytecode/mp-tests/closure2.py deleted file mode 100644 index 08b4205810..0000000000 --- a/tests/bytecode/mp-tests/closure2.py +++ /dev/null @@ -1,7 +0,0 @@ -# test closing over an argument - -def f(x): - y = 2 * x - def g(z): - return x + y + z - return g diff --git a/tests/bytecode/mp-tests/closure3.py b/tests/bytecode/mp-tests/closure3.py deleted file mode 100644 index 905211317a..0000000000 --- a/tests/bytecode/mp-tests/closure3.py +++ /dev/null @@ -1,12 +0,0 @@ -# test when different variables are closed over by different functions - -def f(): - l1 = 1 - l2 = 2 - l3 = 3 - - def g(): - return l1 + l2 - - def h(): - return l2 + l3 diff --git a/tests/bytecode/mp-tests/closure4.py b/tests/bytecode/mp-tests/closure4.py deleted file mode 100644 index 6828f89008..0000000000 --- a/tests/bytecode/mp-tests/closure4.py +++ /dev/null @@ -1,13 +0,0 @@ -# test when a function has cell and free vars - -def f(): - f_local = 1 - f_cell = 2 - - def g(): - g_local = 3 - g_cell = f_cell + 4 - - def h(): - h1_local = 4 - h2_local = f_cell + g_cell diff --git a/tests/bytecode/mp-tests/compare1.py b/tests/bytecode/mp-tests/compare1.py deleted file mode 100644 index 32ba43e3bf..0000000000 --- a/tests/bytecode/mp-tests/compare1.py +++ /dev/null @@ -1,8 +0,0 @@ -if 1 <= x <= 5: - f() - -if 1 <= x <= y <= 7: - f() - -if a < b > c in l != c is not d: - f() diff --git a/tests/bytecode/mp-tests/const1.py b/tests/bytecode/mp-tests/const1.py deleted file mode 100644 index 545b334344..0000000000 --- a/tests/bytecode/mp-tests/const1.py +++ /dev/null @@ -1,9 +0,0 @@ -x = 1 -#x = 1.2 -#x = 1.2e5 -#x = 1.2e+5 -#x = 1.2e-5 -x = () -x = (1,) -x = (1,2) -x = ('a',None,3) diff --git a/tests/bytecode/mp-tests/continue1.py b/tests/bytecode/mp-tests/continue1.py deleted file mode 100644 index 3600691b1f..0000000000 --- a/tests/bytecode/mp-tests/continue1.py +++ /dev/null @@ -1,44 +0,0 @@ -for a in b: - continue - -for a in b: - try: - f() - except: - continue - g() - -for a in b: - try: - f() - continue - except: - g() - -for a in b: - try: - f() - except: - try: - g() - except: - continue - -for a in b: - try: - f() - except: - try: - g() - continue - except: - h() - -for a in b: - try: - f() - except: - pass - else: - continue - g() diff --git a/tests/bytecode/mp-tests/decorate1.py b/tests/bytecode/mp-tests/decorate1.py deleted file mode 100644 index 208aebc5bf..0000000000 --- a/tests/bytecode/mp-tests/decorate1.py +++ /dev/null @@ -1,20 +0,0 @@ -@d -def f(): - pass - -@d -@e -def g(): - pass - -@d.e.f -def h(): - pass - -@d(a + 1) -def i(): - pass - -@d(a + 1, b + 2) -def i(): - pass diff --git a/tests/bytecode/mp-tests/del1.py b/tests/bytecode/mp-tests/del1.py deleted file mode 100644 index 0a259fac78..0000000000 --- a/tests/bytecode/mp-tests/del1.py +++ /dev/null @@ -1,16 +0,0 @@ -del x -del x.y -del x().y -del g -del x[a] -def f(): - global g - del x - del g - local = 1 - local2 = 2 - local3 = 3 - del local, local2, local3 - def f2(): - nonlocal local3 - del local2, local3 diff --git a/tests/bytecode/mp-tests/del2.py b/tests/bytecode/mp-tests/del2.py deleted file mode 100644 index 1c63d15fcb..0000000000 --- a/tests/bytecode/mp-tests/del2.py +++ /dev/null @@ -1,11 +0,0 @@ -del x -del x, -del x, y -del x, y, -del x, y, z -del (x) -del (x,) -del (x, y) -del (x, y,) -del (x, y, z) -del a, (b, c) diff --git a/tests/bytecode/mp-tests/dict1.py b/tests/bytecode/mp-tests/dict1.py deleted file mode 100644 index 3243faa632..0000000000 --- a/tests/bytecode/mp-tests/dict1.py +++ /dev/null @@ -1,3 +0,0 @@ -x = {} -x = {'a':1} -x = {'a':1, 'b':2} diff --git a/tests/bytecode/mp-tests/dictcomp1.py b/tests/bytecode/mp-tests/dictcomp1.py deleted file mode 100644 index 9dca499c57..0000000000 --- a/tests/bytecode/mp-tests/dictcomp1.py +++ /dev/null @@ -1,2 +0,0 @@ -x = {a:None for a in l} -x = {b:c for c, b in l if c} diff --git a/tests/bytecode/mp-tests/docstring1.py b/tests/bytecode/mp-tests/docstring1.py deleted file mode 100644 index d1e0184547..0000000000 --- a/tests/bytecode/mp-tests/docstring1.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Module""" - -class A: - """Class""" - pass - -class B: - """Class B""" diff --git a/tests/bytecode/mp-tests/docstring2.py b/tests/bytecode/mp-tests/docstring2.py deleted file mode 100644 index 5a2183aef9..0000000000 --- a/tests/bytecode/mp-tests/docstring2.py +++ /dev/null @@ -1,3 +0,0 @@ -# comment before doc string - -"""Doc string""" diff --git a/tests/bytecode/mp-tests/fun1.py b/tests/bytecode/mp-tests/fun1.py deleted file mode 100644 index 36e079c01e..0000000000 --- a/tests/bytecode/mp-tests/fun1.py +++ /dev/null @@ -1,2 +0,0 @@ -def f(*args): - g(*args) diff --git a/tests/bytecode/mp-tests/fun2.py b/tests/bytecode/mp-tests/fun2.py deleted file mode 100644 index a6cba92aad..0000000000 --- a/tests/bytecode/mp-tests/fun2.py +++ /dev/null @@ -1,23 +0,0 @@ -def f(*, b): - return b - -def f(a, *, b): - return a + b - -def f(a, *, b, c): - return a + b + c - -def f(a, *, b=c): - return a + b - -def f(a, *, b=c, c): - return a + b + c - -def f(a, *, b=c, c=d): - return a + b + c - -def f(a, *, b=c, c, d=e): - return a + b + c + d - -def f(a=None, *, b=None): - return a + b diff --git a/tests/bytecode/mp-tests/fun3.py b/tests/bytecode/mp-tests/fun3.py deleted file mode 100644 index 5336a70797..0000000000 --- a/tests/bytecode/mp-tests/fun3.py +++ /dev/null @@ -1,3 +0,0 @@ -def f(a, b): - def g(c, d=None, *, e=True): - return a + b + c + d + e diff --git a/tests/bytecode/mp-tests/fun4.py b/tests/bytecode/mp-tests/fun4.py deleted file mode 100644 index b8d2ac159e..0000000000 --- a/tests/bytecode/mp-tests/fun4.py +++ /dev/null @@ -1,5 +0,0 @@ -def f(a, b=1, *c, d): - pass - #print(a,b,c,d) # bug in uPy! -f = lambda a, b, *c, d: None # default arg -#f = lambda a, b=1, *c, d: None # default arg for lambda not implemented diff --git a/tests/bytecode/mp-tests/if1.py b/tests/bytecode/mp-tests/if1.py deleted file mode 100644 index 8c8a08ccdd..0000000000 --- a/tests/bytecode/mp-tests/if1.py +++ /dev/null @@ -1,24 +0,0 @@ -if x: - x() -if x: - x() -elif y: - y() -if x: - x() -else: - zz() -if x: - x() -elif y: - y() -else: - zz() -if x: - x() -elif y: - y() -elif z: - z() -else: - zz() diff --git a/tests/bytecode/mp-tests/if2.py b/tests/bytecode/mp-tests/if2.py deleted file mode 100644 index deb0cd5811..0000000000 --- a/tests/bytecode/mp-tests/if2.py +++ /dev/null @@ -1,26 +0,0 @@ -def f(x): - if x: - return - if x: - return - elif y: - return - if x: - return - else: - return - if x: - return - elif y: - return - else: - return - if x: - return - elif y: - return - elif z: - return - else: - return - return None diff --git a/tests/bytecode/mp-tests/if3.py b/tests/bytecode/mp-tests/if3.py deleted file mode 100644 index bd01514d63..0000000000 --- a/tests/bytecode/mp-tests/if3.py +++ /dev/null @@ -1,6 +0,0 @@ -if a and b: - f() -if a or b: - f() -if a and (b or c): - f() diff --git a/tests/bytecode/mp-tests/if4.py b/tests/bytecode/mp-tests/if4.py deleted file mode 100644 index 4d5a86cd8b..0000000000 --- a/tests/bytecode/mp-tests/if4.py +++ /dev/null @@ -1,8 +0,0 @@ -if not a: - f() -if not a and b: - f() -if not a and not b: - f() -while not a: - f() diff --git a/tests/bytecode/mp-tests/ifexpr1.py b/tests/bytecode/mp-tests/ifexpr1.py deleted file mode 100644 index bdb2efc0a1..0000000000 --- a/tests/bytecode/mp-tests/ifexpr1.py +++ /dev/null @@ -1 +0,0 @@ -x = 1 if a else 2 diff --git a/tests/bytecode/mp-tests/import1.py b/tests/bytecode/mp-tests/import1.py deleted file mode 100644 index 696f3a2708..0000000000 --- a/tests/bytecode/mp-tests/import1.py +++ /dev/null @@ -1,5 +0,0 @@ -a = 1 -def f(): - global a -import a -import b, c diff --git a/tests/bytecode/mp-tests/import2.py b/tests/bytecode/mp-tests/import2.py deleted file mode 100644 index 2a89703d9d..0000000000 --- a/tests/bytecode/mp-tests/import2.py +++ /dev/null @@ -1 +0,0 @@ -from a import b diff --git a/tests/bytecode/mp-tests/import3.py b/tests/bytecode/mp-tests/import3.py deleted file mode 100644 index 7f365a51eb..0000000000 --- a/tests/bytecode/mp-tests/import3.py +++ /dev/null @@ -1,8 +0,0 @@ -import a.b -import a.b.c -from a.b import d -from a.b.c import d - -from a import * -from a import d, e -from a import (d, e) diff --git a/tests/bytecode/mp-tests/import4.py b/tests/bytecode/mp-tests/import4.py deleted file mode 100644 index ecc3786755..0000000000 --- a/tests/bytecode/mp-tests/import4.py +++ /dev/null @@ -1,3 +0,0 @@ -import a as y -import a.b as y -import a.b.c as y diff --git a/tests/bytecode/mp-tests/import5.py b/tests/bytecode/mp-tests/import5.py deleted file mode 100644 index fb93862d19..0000000000 --- a/tests/bytecode/mp-tests/import5.py +++ /dev/null @@ -1,4 +0,0 @@ -from a import b as c -from a.b import c as d -from a.b.c import d as e -from a.b.c import d as e, f as h diff --git a/tests/bytecode/mp-tests/import6.py b/tests/bytecode/mp-tests/import6.py deleted file mode 100644 index 7cbb3c6d73..0000000000 --- a/tests/bytecode/mp-tests/import6.py +++ /dev/null @@ -1,14 +0,0 @@ -from . import bar -from .. import bar -from ... import bar -from .... import bar -from ..... import bar -from ...... import bar -from . import bar as abc -from .foo import bar -from ..foo import bar -from ...foo import bar -from .foo.bar import baz -from ..foo.bar import baz -from ...foo.bar import baz -from .foo.bar import baz as abc diff --git a/tests/bytecode/mp-tests/lambda1.py b/tests/bytecode/mp-tests/lambda1.py deleted file mode 100644 index 559c7c20f5..0000000000 --- a/tests/bytecode/mp-tests/lambda1.py +++ /dev/null @@ -1,2 +0,0 @@ -f = lambda: 0 -f = lambda x: 1 + x diff --git a/tests/bytecode/mp-tests/lambda2.py b/tests/bytecode/mp-tests/lambda2.py deleted file mode 100644 index 1b4500c08f..0000000000 --- a/tests/bytecode/mp-tests/lambda2.py +++ /dev/null @@ -1 +0,0 @@ -f = lambda *args: args diff --git a/tests/bytecode/mp-tests/list1.py b/tests/bytecode/mp-tests/list1.py deleted file mode 100644 index e2a1a3e9fa..0000000000 --- a/tests/bytecode/mp-tests/list1.py +++ /dev/null @@ -1,8 +0,0 @@ -x = [] -x = [1] -x = [1,] # not implemented -x = [1, 2] -x = [1, 2,] -x = [1, 2, 3] -x = [1, 2, 3, 4] -x = [1, 2, 3, 4, 5] diff --git a/tests/bytecode/mp-tests/list2.py b/tests/bytecode/mp-tests/list2.py deleted file mode 100644 index 90b21184da..0000000000 --- a/tests/bytecode/mp-tests/list2.py +++ /dev/null @@ -1,8 +0,0 @@ -x = [()] -x = [(a)] -x = [(a,)] -x = [(a)] -x = [(a,)] -x = [a, b] -x = [(a, b)] -x = [(a, b, c)] diff --git a/tests/bytecode/mp-tests/listcomp1.py b/tests/bytecode/mp-tests/listcomp1.py deleted file mode 100644 index 3a0ef49791..0000000000 --- a/tests/bytecode/mp-tests/listcomp1.py +++ /dev/null @@ -1,4 +0,0 @@ -x = (a for a in l) - -f(a for a in l) -f(a + b for a, b in f()) diff --git a/tests/bytecode/mp-tests/listcomp2.py b/tests/bytecode/mp-tests/listcomp2.py deleted file mode 100644 index 5f52a5e6b0..0000000000 --- a/tests/bytecode/mp-tests/listcomp2.py +++ /dev/null @@ -1 +0,0 @@ -[x.y for x in k.l] diff --git a/tests/bytecode/mp-tests/listcomp3.py b/tests/bytecode/mp-tests/listcomp3.py deleted file mode 100644 index 77a8f2be20..0000000000 --- a/tests/bytecode/mp-tests/listcomp3.py +++ /dev/null @@ -1,3 +0,0 @@ -x = (a + 1 for a in l if a.f()) - -x = [a + 1 for a in l if a.f()] diff --git a/tests/bytecode/mp-tests/listcomp4.py b/tests/bytecode/mp-tests/listcomp4.py deleted file mode 100644 index 6b29993097..0000000000 --- a/tests/bytecode/mp-tests/listcomp4.py +++ /dev/null @@ -1,4 +0,0 @@ -# closing over a local variable in a list comprehension -def f(): - a = 1 - x = [a + b for b in l] diff --git a/tests/bytecode/mp-tests/listcomp5.py b/tests/bytecode/mp-tests/listcomp5.py deleted file mode 100644 index a42d811b75..0000000000 --- a/tests/bytecode/mp-tests/listcomp5.py +++ /dev/null @@ -1,11 +0,0 @@ -# nested ifs -x = [a for a in l if a if a + 1] -x = [a for a in l if a if a + 1 if a + 2] - -# nested for loops -x = [a for a in l for l in ls] -x = [a for ls in lss for l in ls for a in l] -x = [a for a in l for l in ls for ls in lss] - -# nested ifs and for loops -x = [a for a in l if a for l in ls if l if a for ls in lss if ls] diff --git a/tests/bytecode/mp-tests/locals1.py b/tests/bytecode/mp-tests/locals1.py deleted file mode 100644 index 49c34da1ad..0000000000 --- a/tests/bytecode/mp-tests/locals1.py +++ /dev/null @@ -1,22 +0,0 @@ -# to test the order of locals and arguments (LOAD_FAST, STORE_FAST) - -def f1(): - b = 1 - a = 2 - return a + b - -def f2(b): - a = 2 - return a + b - -def f3(): - def f3f(): - return True - a = 1 - return f3f(a) - -def f4(): - x = 1 - def f3f(): - return True - return f3f(x) diff --git a/tests/bytecode/mp-tests/ptex.py b/tests/bytecode/mp-tests/ptex.py deleted file mode 100644 index 8f23d78009..0000000000 --- a/tests/bytecode/mp-tests/ptex.py +++ /dev/null @@ -1,269 +0,0 @@ -import sys -import os -import os.path -import datetime -import argparse -from xml.etree.ElementTree import Element, SubElement, tostring - -from log import Log -from texparser import TexParser -from latexparser import LatexParser -from gettexfile import file_has_suffix -from gettexfile import get_tex_file - -from xiwi.common.misc import buildFileList -from xiwi.common import arxivid -from xiwi.common.stats import Statistics - -def str_contains(s1, s2): - return s1.find(s2) != -1 - -def str_contains_one_of(st, st_list): - for st2 in st_list: - if str_contains(st, st2): - return True - return False - -def detect_file_kind(file_obj): - """Simple detection of kind of source file.""" - kind = 'unknown' - firstline = file_obj.readline() - while firstline.isspace(): - firstline = file_obj.readline() - if firstline.startswith('%!PS'): - kind = 'PS' - elif firstline.startswith('%auto-ignore'): - kind = 'auto-ignore' - else: - file_obj.seek(0) - for line in file_obj: - if str_contains(line, '\\def'): - # might be tex, if we don't find anything else - kind = 'tex' - if str_contains(line, '\\input'): - # might be tex, if we don't find anything else - kind = 'tex' - if str_contains(line, 'amstex') or str_contains(line, 'harvmac'): - # definitely tex - kind = 'tex' - break - if str_contains(line, '\\documentclass'): - # definitely latex - kind = 'latex' - break - if str_contains(line, '\\documentstyle'): - # could be tex or latex - if str_contains(line, 'amsppt'): - kind = 'tex' - break - else: - kind = 'latex' - break - file_obj.seek(0) - return kind - -class WithdrawnPaper(object): - def __init__(self): - pass - - def __getitem__(self, item): - if item == 'refs': - return [] - elif item == 'success': - return True - - def parse(self): - pass - -def process_article(filename): - """Returns TexParserBase derived object on success, None on failure.""" - - # get the tex file - filename, file_obj, tarfile_obj = get_tex_file(filename) - if file_obj is None: - return None - - # detect the type of file - kind = detect_file_kind(file_obj) - - # act on the type of file - parser = None - if kind == 'PS': - print('skipping postscript file') - elif kind == 'auto-ignore': - print('asked to ignore file, most likely it was withdrawn') - parser = WithdrawnPaper() - if kind == 'tex': - print('parsing as TeX') - parser = TexParser(filename, file_obj, tarfile_obj) - elif kind == 'latex': - print('parsing as LaTeX') - parser = LatexParser(filename, file_obj, tarfile_obj) - else: - print('cannot determine kind of file') - - # attempt to parse the document - try: - if parser is not None: - parser.parse() - except Exception as e: - print('exception while trying to parse file:') - print(str(e)) - parser = None - - # close the files - file_obj.close() - if tarfile_obj is not None: - tarfile_obj.close() - - # return the parsed document - return parser - -arxiv_classes = [ - 'acc-phys', 'adap-org', 'alg-geom', 'ao-sci', 'astro-ph', 'atom-ph', - 'bayes-an', 'chao-dyn', 'chem-ph', 'cmp-lg', 'comp-gas', 'cond-mat', - 'cs', 'dg-ga', 'funct-an', 'gr-qc', 'hep-ex', 'hep-lat', - 'hep-ph', 'hep-th', 'math', 'math-ph', 'mtrl-th', 'nlin', - 'nucl-ex', 'nucl-th', 'patt-sol', 'physics', 'plasm-ph', 'q-alg', - 'q-bio', 'quant-ph', 'solv-int', 'supr-con' -] - -def do_single_file(file_name, print_xml, write_xml_dir): - arxiv_id, arxiv_version = arxivid.filenameToArxivAndVersion(file_name) - if arxiv_id is None: - print('WARN: could not determine arXiv identifier for', file_name) - arxiv_id = '' - arxiv_version = 0 - - Log.reset() - Statistics.begin_item(arxiv_id) - - if file_has_suffix(file_name, '.pdf'): - Statistics.count('1) pdf') - succ = True - else: - Statistics.count('2) processed') - - parser = process_article(file_name) - - if parser is not None : - succ = parser['success'] - bib_refs = parser['refs'] - else : - succ = False - bib_refs = [] - - if str_contains_one_of(arxiv_id, ['gr-qc', 'hep-']): - Statistics.count('hep-processed') - if succ: - Statistics.count('hep-success') - if succ: - print('-success--------') - Statistics.count('3) success') - else: - print('-fail-----------') - Statistics.count('4) fail') - - show_ref = False - - if succ and show_ref: - for bib_ref in bib_refs: - print(bib_ref.key, 'with', bib_ref.cite_count, 'citations in paper') - if len(bib_ref.bib_info) == 0: - print('no reference') - else: - print(bib_ref.bib_info_as_str(keep_comments=True)) - - if succ and (print_xml or write_xml_dir): - xml = Element('article') - SubElement(xml, 'id').text = arxiv_id - if arxiv_version > 0: - SubElement(xml, 'version').text = str(arxiv_version) - refs = SubElement(xml, 'refs') - for bib_ref in bib_refs: - bib_text = bib_ref.bib_info_as_str(keep_comments=True) - if len(bib_text) != 0: - ncites = bib_ref.cite_count - if ncites < 1: - ncites = 1 - ref = SubElement(refs, 'ref', order=str(bib_ref.ref_order_num), freq=str(ncites)) - ref.text = bib_text - if print_xml: - print(tostring(xml)) - if isinstance(write_xml_dir, str): - if arxiv_id != '': - xml_file_name = os.path.join(write_xml_dir, arxiv_id.replace('/', '') + '.xml') - else: - fname = os.path.split(file_name)[1] - if fname.rfind('.') > 0: - fname = fname[:fname.rfind('.')] - xml_file_name = write_xml_dir + '/' + fname + '.xml' - file_obj = open(xml_file_name, 'wb') - file_obj.write(tostring(xml, encoding='utf-8')) - file_obj.close() - - Statistics.end_item() - - return succ - -summaryStrs = [] - -if __name__ == "__main__": - cmd_parser = argparse.ArgumentParser(description='Parse TeX/LaTeX to find references.') - cmd_parser.add_argument('--filelist', action='store_true', help='file names on the command line each contain a list of files to process') - cmd_parser.add_argument('--print-xml', action='store_true', help='print XML output to stdout') - cmd_parser.add_argument('--write-xml', metavar='', help='destination directory to write XML output files') - cmd_parser.add_argument('--failed', metavar='', help='output file to write list of failed files') - cmd_parser.add_argument('files', nargs='+', help='input files') - args = cmd_parser.parse_args() - - # print date stamp - timeStart = datetime.datetime.now() - print('[ptex] started processing at', str(timeStart)) - - print('given', len(args.files), 'files, first file:', args.files[0]) - print('================') - - Statistics.clear('article') - - # build list of files to process - file_list = buildFileList(args.filelist, args.files) - - # ensure the destination directory exists - if args.write_xml is not None and os.path.exists(args.write_xml): - try: - os.makedirs(args.write_xml) - except: - pass - - # process the files - failed_files = [] - for file_name in file_list: - success = do_single_file(file_name, args.print_xml, args.write_xml) - if not success: - failed_files.append(file_name) - - # write the failed files to an output file, if requested - if args.failed is not None: - file_obj = open(args.failed, 'w') - file_obj.writelines(f + '\n' for f in failed_files) - file_obj.close() - - print('================') - Statistics.show() - Statistics.show_detail('fail') - #Statistics.show_detail('cite-range') - #Statistics.show_detail('bad-ascii') - #Statistics.show_detail('non-ascii') - - print('================') - - # print date stamp - timeEnd = datetime.datetime.now() - print('[ptex] finished processing at', str(timeEnd)) - - # print summary for email - summaryStrs.extend(Statistics.get_summary()) - summaryStrs.insert(0, 'started processing at %s, took %.1f minutes' % (timeStart.strftime('%H:%M'), (timeEnd - timeStart).total_seconds() / 60)) - for s in summaryStrs: - print('**SUMMARY** [ptex]', s) diff --git a/tests/bytecode/mp-tests/raise1.py b/tests/bytecode/mp-tests/raise1.py deleted file mode 100644 index 9cceed4944..0000000000 --- a/tests/bytecode/mp-tests/raise1.py +++ /dev/null @@ -1,11 +0,0 @@ -def f(): - raise -def g(): - raise 1 -def h(): - raise 1 from 2 -def i(): - try: - f() - except: - raise diff --git a/tests/bytecode/mp-tests/scope0.py b/tests/bytecode/mp-tests/scope0.py deleted file mode 100644 index 5d81345ea4..0000000000 --- a/tests/bytecode/mp-tests/scope0.py +++ /dev/null @@ -1,7 +0,0 @@ -x = 1 -print(x) - -# local store after load -def f(): - print(x) - x = 1 diff --git a/tests/bytecode/mp-tests/scope1.py b/tests/bytecode/mp-tests/scope1.py deleted file mode 100644 index 92a0f9fa8c..0000000000 --- a/tests/bytecode/mp-tests/scope1.py +++ /dev/null @@ -1,6 +0,0 @@ -x = 1 -print(x) -def f1(): - print(x) -def f2(x): - print(x) diff --git a/tests/bytecode/mp-tests/scope2.py b/tests/bytecode/mp-tests/scope2.py deleted file mode 100644 index af9e372318..0000000000 --- a/tests/bytecode/mp-tests/scope2.py +++ /dev/null @@ -1,18 +0,0 @@ -# scope - -gl = 1 - -def f(x): - global gl - gl += 2 - lo1 = 3 - lo2 = 4 - lo3 = 5 - - def f2(x, y): - global gl - nonlocal lo3 - lo3 = 5 - lo4 = gl + lo2 + lo3 - - return f2 diff --git a/tests/bytecode/mp-tests/scope3.py b/tests/bytecode/mp-tests/scope3.py deleted file mode 100644 index a5fc8d09fe..0000000000 --- a/tests/bytecode/mp-tests/scope3.py +++ /dev/null @@ -1,11 +0,0 @@ -# test nested functions and scope - -def f(x): - def f2(y): - return y + x - print(f2(x)) - return f2 -x=f(2) -print(x, x(5)) -f=123 -print(f(f)) diff --git a/tests/bytecode/mp-tests/scope4.py b/tests/bytecode/mp-tests/scope4.py deleted file mode 100644 index 70968cdf30..0000000000 --- a/tests/bytecode/mp-tests/scope4.py +++ /dev/null @@ -1,14 +0,0 @@ -# test scope - -def f(x): - global x42 - print(x, x42) - x42 = x - -x42 = 123 -f(1) -print(x42) - -x42 = 456 -f(2) -print(x42) diff --git a/tests/bytecode/mp-tests/scope5.py b/tests/bytecode/mp-tests/scope5.py deleted file mode 100644 index a14de350ed..0000000000 --- a/tests/bytecode/mp-tests/scope5.py +++ /dev/null @@ -1,12 +0,0 @@ -# test scope - -def f(x): - def f2(y): - print(y, x42, y42) - x42 = x = y42 = 123 - myf2 = f2 - x42 = 456 - return myf2 - -myf = f(1) -myf(1) diff --git a/tests/bytecode/mp-tests/scope6.py b/tests/bytecode/mp-tests/scope6.py deleted file mode 100644 index 4848378887..0000000000 --- a/tests/bytecode/mp-tests/scope6.py +++ /dev/null @@ -1,7 +0,0 @@ -# closed over variable 2 deep - -def f(): - x = 1 - def g(): - def h(): - return 1 + x diff --git a/tests/bytecode/mp-tests/scope7.py b/tests/bytecode/mp-tests/scope7.py deleted file mode 100644 index 699d12510c..0000000000 --- a/tests/bytecode/mp-tests/scope7.py +++ /dev/null @@ -1,15 +0,0 @@ -# test order of closed over locals -# not that CPython seems to sort closed over variables (but not fast locals) - -def f(): - l1 = 1 - l2 = 4 - l3 = 3 - l4 = 2 - l5 = 5 - - def g(): - return l1 + l4 + l3 + l2 + l5 - - def h(): - return l1 + l2 + l3 + l4 + l5 diff --git a/tests/bytecode/mp-tests/set1.py b/tests/bytecode/mp-tests/set1.py deleted file mode 100644 index f6de75606c..0000000000 --- a/tests/bytecode/mp-tests/set1.py +++ /dev/null @@ -1,6 +0,0 @@ -x = set() -x = {1} -x = {1,} -x = {1, 2} -x = {1, 2,} -x = {1, 2, 3} diff --git a/tests/bytecode/mp-tests/setcomp1.py b/tests/bytecode/mp-tests/setcomp1.py deleted file mode 100644 index 82927f5d19..0000000000 --- a/tests/bytecode/mp-tests/setcomp1.py +++ /dev/null @@ -1,2 +0,0 @@ -x = {a for a in l} -x = {a + b for a, b in l if b} diff --git a/tests/bytecode/mp-tests/slice1.py b/tests/bytecode/mp-tests/slice1.py deleted file mode 100644 index 008e57c182..0000000000 --- a/tests/bytecode/mp-tests/slice1.py +++ /dev/null @@ -1,16 +0,0 @@ -x = x[:] -x = x[::] -x = x[::c] -x = x[:b] -x = x[:b:] -x = x[:b:c] -x = x[a] -x = x[a:] -x = x[a::] -x = x[a::c] -x = x[a:b] -x = x[a:b:] -x = x[a:b:c] - -x[0] = 1 -x[x] = x diff --git a/tests/bytecode/mp-tests/slice2.py b/tests/bytecode/mp-tests/slice2.py deleted file mode 100644 index e329156c3c..0000000000 --- a/tests/bytecode/mp-tests/slice2.py +++ /dev/null @@ -1,3 +0,0 @@ -x = x[a, b] - -x[a, b] = x diff --git a/tests/bytecode/mp-tests/string1.py b/tests/bytecode/mp-tests/string1.py deleted file mode 100644 index d6ddc7ae43..0000000000 --- a/tests/bytecode/mp-tests/string1.py +++ /dev/null @@ -1,11 +0,0 @@ -x = 'abc' -x = "abc" -x = r'abc' -x = 'abc' \ - 'def' -x = ('abc' - 'def') - -x = 'ab"c' -x = "ab'c" -x = '''ab'c''' diff --git a/tests/bytecode/mp-tests/string2.py b/tests/bytecode/mp-tests/string2.py deleted file mode 100644 index 70dc9924b0..0000000000 --- a/tests/bytecode/mp-tests/string2.py +++ /dev/null @@ -1,14 +0,0 @@ -'abc' -class f: - u"123" - pass -x = 'abc' -x = u"abc" -x = u"ab\\c" -x = r"ab\\c" -x = b"abc" -x = rb"abc" -x = b"ab\\c" -x = rb"ab\\c" -x = """abc""" -x = b"""abc""" diff --git a/tests/bytecode/mp-tests/super1.py b/tests/bytecode/mp-tests/super1.py deleted file mode 100644 index 1512429939..0000000000 --- a/tests/bytecode/mp-tests/super1.py +++ /dev/null @@ -1,17 +0,0 @@ -class A(B): - def f(): - super.a() - -class B(C): - def g(): - def h(): - super.a() - -super.a() - -def i(): - super.a() - -def j(): - def k(): - super.a() diff --git a/tests/bytecode/mp-tests/try1.py b/tests/bytecode/mp-tests/try1.py deleted file mode 100644 index 10344c8ae3..0000000000 --- a/tests/bytecode/mp-tests/try1.py +++ /dev/null @@ -1,13 +0,0 @@ -def f(x): - try: - f(x) - except: - f(x) - try: - f(x) - except Exception: - f(x) - try: - f(x) - except Exception as e: - f(x, e) diff --git a/tests/bytecode/mp-tests/try2.py b/tests/bytecode/mp-tests/try2.py deleted file mode 100644 index efdac04756..0000000000 --- a/tests/bytecode/mp-tests/try2.py +++ /dev/null @@ -1,5 +0,0 @@ -def f(): - try: - f() - finally: - g() diff --git a/tests/bytecode/mp-tests/try3.py b/tests/bytecode/mp-tests/try3.py deleted file mode 100644 index 9741aaf681..0000000000 --- a/tests/bytecode/mp-tests/try3.py +++ /dev/null @@ -1,14 +0,0 @@ -def f(): - try: - f() - except: - g() - finally: - f() - - try: - f() - except Exception: - g() - finally: - f() diff --git a/tests/bytecode/mp-tests/try4.py b/tests/bytecode/mp-tests/try4.py deleted file mode 100644 index 412cb74ee5..0000000000 --- a/tests/bytecode/mp-tests/try4.py +++ /dev/null @@ -1,22 +0,0 @@ -try: - f() -except A: - g() -except: - h() - -try: - f() -except A: - g() -except B as c: - h() - -try: - f() -except A: - g() -except B as c: - h() -except: - i() diff --git a/tests/bytecode/mp-tests/try5.py b/tests/bytecode/mp-tests/try5.py deleted file mode 100644 index 7ba7949125..0000000000 --- a/tests/bytecode/mp-tests/try5.py +++ /dev/null @@ -1,8 +0,0 @@ -try: - f() -except A: - g() -except B as b: - h() -finally: - i() diff --git a/tests/bytecode/mp-tests/try6.py b/tests/bytecode/mp-tests/try6.py deleted file mode 100644 index d5b68722e9..0000000000 --- a/tests/bytecode/mp-tests/try6.py +++ /dev/null @@ -1,15 +0,0 @@ -try: - f() -except: - g() -else: - h() - -try: - f() -except: - g() -else: - h() -finally: - i() diff --git a/tests/bytecode/mp-tests/tuple1.py b/tests/bytecode/mp-tests/tuple1.py deleted file mode 100644 index d70e4cf569..0000000000 --- a/tests/bytecode/mp-tests/tuple1.py +++ /dev/null @@ -1,17 +0,0 @@ -x = () -x = a -x = a, -x = a, 2 -x = a, 2, -x = a, 2, 3 -x = a, 2, 3, 4 -x = a, 2, 3, 4, 5 - -x = () -x = (a) -x = (a,) -x = (a, 2) -x = (a, 2,) -x = (a, 2, 3) -x = (a, 2, 3, 4) -x = (a, 2, 3, 4, 5) diff --git a/tests/bytecode/mp-tests/tuple2.py b/tests/bytecode/mp-tests/tuple2.py deleted file mode 100644 index df11e74ce0..0000000000 --- a/tests/bytecode/mp-tests/tuple2.py +++ /dev/null @@ -1,15 +0,0 @@ -x = t -x, = t -x, y = t -x, y, = t -x, y, z = t -x, y, z, = t -x, y, z, z = a, b, c, d - -(x) = t -(x,) = t -(x, y) = t -(x, y,) = t -(x, y, z) = t -(x, y, z,) = t -(x, y, z, z) = a, b, c, d diff --git a/tests/bytecode/mp-tests/tuple3.py b/tests/bytecode/mp-tests/tuple3.py deleted file mode 100644 index 29ddd86d08..0000000000 --- a/tests/bytecode/mp-tests/tuple3.py +++ /dev/null @@ -1,4 +0,0 @@ -def f(x): - return x, x + 1 -for a in b, c: - f(a) diff --git a/tests/bytecode/mp-tests/with1.py b/tests/bytecode/mp-tests/with1.py deleted file mode 100644 index 897ec530fa..0000000000 --- a/tests/bytecode/mp-tests/with1.py +++ /dev/null @@ -1,8 +0,0 @@ -with x: - f() -with x(): - f() -with f() as x: - f(x) -with f() as x, g() as y: - f(x, y) diff --git a/tests/bytecode/mp-tests/yield1.py b/tests/bytecode/mp-tests/yield1.py deleted file mode 100644 index 114151e718..0000000000 --- a/tests/bytecode/mp-tests/yield1.py +++ /dev/null @@ -1,17 +0,0 @@ -# generators and yield - -def main(): - def f(): - print(123) - yield - print(456) - yield 2 - print(789) - - a = f() - print(a) - print(a.__next__()) - print(a.__next__()) - #print(a.__next__()) - -main() diff --git a/tests/bytecode/mp-tests/yield2.py b/tests/bytecode/mp-tests/yield2.py deleted file mode 100644 index acc0ec8e99..0000000000 --- a/tests/bytecode/mp-tests/yield2.py +++ /dev/null @@ -1,7 +0,0 @@ -def f(): - yield from a - yield from (a, b) - yield from f(a) - -lambda:(yield) -lambda:(yield 1) + 2 diff --git a/tests/bytecode/pylib-tests/LICENSE.txt b/tests/bytecode/pylib-tests/LICENSE.txt deleted file mode 100644 index 56a5d5cc7f..0000000000 --- a/tests/bytecode/pylib-tests/LICENSE.txt +++ /dev/null @@ -1,254 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013 Python Software Foundation; All Rights Reserved" are retained -in Python alone or in any derivative version prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/tests/bytecode/pylib-tests/_compat_pickle.py b/tests/bytecode/pylib-tests/_compat_pickle.py deleted file mode 100644 index 700c80cd57..0000000000 --- a/tests/bytecode/pylib-tests/_compat_pickle.py +++ /dev/null @@ -1,81 +0,0 @@ -# This module is used to map the old Python 2 names to the new names used in -# Python 3 for the pickle module. This needed to make pickle streams -# generated with Python 2 loadable by Python 3. - -# This is a copy of lib2to3.fixes.fix_imports.MAPPING. We cannot import -# lib2to3 and use the mapping defined there, because lib2to3 uses pickle. -# Thus, this could cause the module to be imported recursively. -IMPORT_MAPPING = { - 'StringIO': 'io', - 'cStringIO': 'io', - 'cPickle': 'pickle', - '__builtin__' : 'builtins', - 'copy_reg': 'copyreg', - 'Queue': 'queue', - 'SocketServer': 'socketserver', - 'ConfigParser': 'configparser', - 'repr': 'reprlib', - 'FileDialog': 'tkinter.filedialog', - 'tkFileDialog': 'tkinter.filedialog', - 'SimpleDialog': 'tkinter.simpledialog', - 'tkSimpleDialog': 'tkinter.simpledialog', - 'tkColorChooser': 'tkinter.colorchooser', - 'tkCommonDialog': 'tkinter.commondialog', - 'Dialog': 'tkinter.dialog', - 'Tkdnd': 'tkinter.dnd', - 'tkFont': 'tkinter.font', - 'tkMessageBox': 'tkinter.messagebox', - 'ScrolledText': 'tkinter.scrolledtext', - 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', - 'ttk': 'tkinter.ttk', - 'Tkinter': 'tkinter', - 'markupbase': '_markupbase', - '_winreg': 'winreg', - 'thread': '_thread', - 'dummy_thread': '_dummy_thread', - 'dbhash': 'dbm.bsd', - 'dumbdbm': 'dbm.dumb', - 'dbm': 'dbm.ndbm', - 'gdbm': 'dbm.gnu', - 'xmlrpclib': 'xmlrpc.client', - 'DocXMLRPCServer': 'xmlrpc.server', - 'SimpleXMLRPCServer': 'xmlrpc.server', - 'httplib': 'http.client', - 'htmlentitydefs' : 'html.entities', - 'HTMLParser' : 'html.parser', - 'Cookie': 'http.cookies', - 'cookielib': 'http.cookiejar', - 'BaseHTTPServer': 'http.server', - 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', - 'test.test_support': 'test.support', - 'commands': 'subprocess', - 'UserString' : 'collections', - 'UserList' : 'collections', - 'urlparse' : 'urllib.parse', - 'robotparser' : 'urllib.robotparser', - 'whichdb': 'dbm', - 'anydbm': 'dbm' -} - - -# This contains rename rules that are easy to handle. We ignore the more -# complex stuff (e.g. mapping the names in the urllib and types modules). -# These rules should be run before import names are fixed. -NAME_MAPPING = { - ('__builtin__', 'xrange'): ('builtins', 'range'), - ('__builtin__', 'reduce'): ('functools', 'reduce'), - ('__builtin__', 'intern'): ('sys', 'intern'), - ('__builtin__', 'unichr'): ('builtins', 'chr'), - ('__builtin__', 'basestring'): ('builtins', 'str'), - ('__builtin__', 'long'): ('builtins', 'int'), - ('itertools', 'izip'): ('builtins', 'zip'), - ('itertools', 'imap'): ('builtins', 'map'), - ('itertools', 'ifilter'): ('builtins', 'filter'), - ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'), -} - -# Same, but for 3.x to 2.x -REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items()) -REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items()) diff --git a/tests/bytecode/pylib-tests/_threading_local.py b/tests/bytecode/pylib-tests/_threading_local.py deleted file mode 100644 index 4ec4828144..0000000000 --- a/tests/bytecode/pylib-tests/_threading_local.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Thread-local objects. - -(Note that this module provides a Python version of the threading.local - class. Depending on the version of Python you're using, there may be a - faster one available. You should always import the `local` class from - `threading`.) - -Thread-local objects support the management of thread-local data. -If you have data that you want to be local to a thread, simply create -a thread-local object and use its attributes: - - >>> mydata = local() - >>> mydata.number = 42 - >>> mydata.number - 42 - -You can also access the local-object's dictionary: - - >>> mydata.__dict__ - {'number': 42} - >>> mydata.__dict__.setdefault('widgets', []) - [] - >>> mydata.widgets - [] - -What's important about thread-local objects is that their data are -local to a thread. If we access the data in a different thread: - - >>> log = [] - >>> def f(): - ... items = sorted(mydata.__dict__.items()) - ... log.append(items) - ... mydata.number = 11 - ... log.append(mydata.number) - - >>> import threading - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[], 11] - -we get different data. Furthermore, changes made in the other thread -don't affect data seen in this thread: - - >>> mydata.number - 42 - -Of course, values you get from a local object, including a __dict__ -attribute, are for whatever thread was current at the time the -attribute was read. For that reason, you generally don't want to save -these values across threads, as they apply only to the thread they -came from. - -You can create custom local objects by subclassing the local class: - - >>> class MyLocal(local): - ... number = 2 - ... initialized = False - ... def __init__(self, **kw): - ... if self.initialized: - ... raise SystemError('__init__ called too many times') - ... self.initialized = True - ... self.__dict__.update(kw) - ... def squared(self): - ... return self.number ** 2 - -This can be useful to support default values, methods and -initialization. Note that if you define an __init__ method, it will be -called each time the local object is used in a separate thread. This -is necessary to initialize each thread's dictionary. - -Now if we create a local object: - - >>> mydata = MyLocal(color='red') - -Now we have a default number: - - >>> mydata.number - 2 - -an initial color: - - >>> mydata.color - 'red' - >>> del mydata.color - -And a method that operates on the data: - - >>> mydata.squared() - 4 - -As before, we can access the data in a separate thread: - - >>> log = [] - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[('color', 'red'), ('initialized', True)], 11] - -without affecting this thread's data: - - >>> mydata.number - 2 - >>> mydata.color - Traceback (most recent call last): - ... - AttributeError: 'MyLocal' object has no attribute 'color' - -Note that subclasses can define slots, but they are not thread -local. They are shared across threads: - - >>> class MyLocal(local): - ... __slots__ = 'number' - - >>> mydata = MyLocal() - >>> mydata.number = 42 - >>> mydata.color = 'red' - -So, the separate thread: - - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - -affects what we see: - - >>> mydata.number - 11 - ->>> del mydata -""" - -from weakref import ref -from contextlib import contextmanager - -__all__ = ["local"] - -# We need to use objects from the threading module, but the threading -# module may also want to use our `local` class, if support for locals -# isn't compiled in to the `thread` module. This creates potential problems -# with circular imports. For that reason, we don't import `threading` -# until the bottom of this file (a hack sufficient to worm around the -# potential problems). Note that all platforms on CPython do have support -# for locals in the `thread` module, and there is no circular import problem -# then, so problems introduced by fiddling the order of imports here won't -# manifest. - -class _localimpl: - """A class managing thread-local dicts""" - __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__' - - def __init__(self): - # The key used in the Thread objects' attribute dicts. - # We keep it a string for speed but make it unlikely to clash with - # a "real" attribute. - self.key = '_threading_local._localimpl.' + str(id(self)) - # { id(Thread) -> (ref(Thread), thread-local dict) } - self.dicts = {} - - def get_dict(self): - """Return the dict for the current thread. Raises KeyError if none - defined.""" - thread = current_thread() - return self.dicts[id(thread)][1] - - def create_dict(self): - """Create a new dict for the current thread, and return it.""" - localdict = {} - key = self.key - thread = current_thread() - idt = id(thread) - def local_deleted(_, key=key): - # When the localimpl is deleted, remove the thread attribute. - thread = wrthread() - if thread is not None: - del thread.__dict__[key] - def thread_deleted(_, idt=idt): - # When the thread is deleted, remove the local dict. - # Note that this is suboptimal if the thread object gets - # caught in a reference loop. We would like to be called - # as soon as the OS-level thread ends instead. - local = wrlocal() - if local is not None: - dct = local.dicts.pop(idt) - wrlocal = ref(self, local_deleted) - wrthread = ref(thread, thread_deleted) - thread.__dict__[key] = wrlocal - self.dicts[idt] = wrthread, localdict - return localdict - - -@contextmanager -def _patch(self): - impl = object.__getattribute__(self, '_local__impl') - try: - dct = impl.get_dict() - except KeyError: - dct = impl.create_dict() - args, kw = impl.localargs - self.__init__(*args, **kw) - with impl.locallock: - object.__setattr__(self, '__dict__', dct) - yield - - -class local: - __slots__ = '_local__impl', '__dict__' - - def __new__(cls, *args, **kw): - if (args or kw) and (cls.__init__ is object.__init__): - raise TypeError("Initialization arguments are not supported") - self = object.__new__(cls) - impl = _localimpl() - impl.localargs = (args, kw) - impl.locallock = RLock() - object.__setattr__(self, '_local__impl', impl) - # We need to create the thread dict in anticipation of - # __init__ being called, to make sure we don't call it - # again ourselves. - impl.create_dict() - return self - - def __getattribute__(self, name): - with _patch(self): - return object.__getattribute__(self, name) - - def __setattr__(self, name, value): - if name == '__dict__': - raise AttributeError( - "%r object attribute '__dict__' is read-only" - % self.__class__.__name__) - with _patch(self): - return object.__setattr__(self, name, value) - - def __delattr__(self, name): - if name == '__dict__': - raise AttributeError( - "%r object attribute '__dict__' is read-only" - % self.__class__.__name__) - with _patch(self): - return object.__delattr__(self, name) - - -from threading import current_thread, RLock diff --git a/tests/bytecode/pylib-tests/_weakrefset.py b/tests/bytecode/pylib-tests/_weakrefset.py deleted file mode 100644 index 6a98b88e33..0000000000 --- a/tests/bytecode/pylib-tests/_weakrefset.py +++ /dev/null @@ -1,194 +0,0 @@ -# Access WeakSet through the weakref module. -# This code is separated-out because it is needed -# by abc.py to load everything else at startup. - -from _weakref import ref - -__all__ = ['WeakSet'] - - -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - -class WeakSet: - def __init__(self, data=None): - self.data = set() - def _remove(item, selfref=ref(self)): - self = selfref() - if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) - self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() - if data is not None: - self.update(data) - - def _commit_removals(self): - l = self._pending_removals - discard = self.data.discard - while l: - discard(l.pop()) - - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - yield item - - def __len__(self): - return len(self.data) - len(self._pending_removals) - - def __contains__(self, item): - try: - wr = ref(item) - except TypeError: - return False - return wr in self.data - - def __reduce__(self): - return (self.__class__, (list(self),), - getattr(self, '__dict__', None)) - - def add(self, item): - if self._pending_removals: - self._commit_removals() - self.data.add(ref(item, self._remove)) - - def clear(self): - if self._pending_removals: - self._commit_removals() - self.data.clear() - - def copy(self): - return self.__class__(self) - - def pop(self): - if self._pending_removals: - self._commit_removals() - while True: - try: - itemref = self.data.pop() - except KeyError: - raise KeyError('pop from empty WeakSet') - item = itemref() - if item is not None: - return item - - def remove(self, item): - if self._pending_removals: - self._commit_removals() - self.data.remove(ref(item)) - - def discard(self, item): - if self._pending_removals: - self._commit_removals() - self.data.discard(ref(item)) - - def update(self, other): - if self._pending_removals: - self._commit_removals() - for element in other: - self.add(element) - - def __ior__(self, other): - self.update(other) - return self - - def difference(self, other): - newset = self.copy() - newset.difference_update(other) - return newset - __sub__ = difference - - def difference_update(self, other): - self.__isub__(other) - def __isub__(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.difference_update(ref(item) for item in other) - return self - - def intersection(self, other): - return self.__class__(item for item in other if item in self) - __and__ = intersection - - def intersection_update(self, other): - self.__iand__(other) - def __iand__(self, other): - if self._pending_removals: - self._commit_removals() - self.data.intersection_update(ref(item) for item in other) - return self - - def issubset(self, other): - return self.data.issubset(ref(item) for item in other) - __le__ = issubset - - def __lt__(self, other): - return self.data < set(ref(item) for item in other) - - def issuperset(self, other): - return self.data.issuperset(ref(item) for item in other) - __ge__ = issuperset - - def __gt__(self, other): - return self.data > set(ref(item) for item in other) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return NotImplemented - return self.data == set(ref(item) for item in other) - - def symmetric_difference(self, other): - newset = self.copy() - newset.symmetric_difference_update(other) - return newset - __xor__ = symmetric_difference - - def symmetric_difference_update(self, other): - self.__ixor__(other) - def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.symmetric_difference_update(ref(item, self._remove) for item in other) - return self - - def union(self, other): - return self.__class__(e for s in (self, other) for e in s) - __or__ = union - - def isdisjoint(self, other): - return len(self.intersection(other)) == 0 diff --git a/tests/bytecode/pylib-tests/abc.py b/tests/bytecode/pylib-tests/abc.py deleted file mode 100644 index 09778e8609..0000000000 --- a/tests/bytecode/pylib-tests/abc.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Abstract Base Classes (ABCs) according to PEP 3119.""" - -from _weakrefset import WeakSet - -def abstractmethod(funcobj): - """A decorator indicating abstract methods. - - Requires that the metaclass is ABCMeta or derived from it. A - class that has a metaclass derived from ABCMeta cannot be - instantiated unless all of its abstract methods are overridden. - The abstract methods can be called using any of the normal - 'super' call mechanisms. - - Usage: - - class C(metaclass=ABCMeta): - @abstractmethod - def my_abstract_method(self, ...): - ... - """ - funcobj.__isabstractmethod__ = True - return funcobj - - -class abstractclassmethod(classmethod): - """ - A decorator indicating abstract classmethods. - - Similar to abstractmethod. - - Usage: - - class C(metaclass=ABCMeta): - @abstractclassmethod - def my_abstract_classmethod(cls, ...): - ... - - 'abstractclassmethod' is deprecated. Use 'classmethod' with - 'abstractmethod' instead. - """ - - __isabstractmethod__ = True - - def __init__(self, callable): - callable.__isabstractmethod__ = True - super().__init__(callable) - - -class abstractstaticmethod(staticmethod): - """ - A decorator indicating abstract staticmethods. - - Similar to abstractmethod. - - Usage: - - class C(metaclass=ABCMeta): - @abstractstaticmethod - def my_abstract_staticmethod(...): - ... - - 'abstractstaticmethod' is deprecated. Use 'staticmethod' with - 'abstractmethod' instead. - """ - - __isabstractmethod__ = True - - def __init__(self, callable): - callable.__isabstractmethod__ = True - super().__init__(callable) - - -class abstractproperty(property): - """ - A decorator indicating abstract properties. - - Requires that the metaclass is ABCMeta or derived from it. A - class that has a metaclass derived from ABCMeta cannot be - instantiated unless all of its abstract properties are overridden. - The abstract properties can be called using any of the normal - 'super' call mechanisms. - - Usage: - - class C(metaclass=ABCMeta): - @abstractproperty - def my_abstract_property(self): - ... - - This defines a read-only property; you can also define a read-write - abstract property using the 'long' form of property declaration: - - class C(metaclass=ABCMeta): - def getx(self): ... - def setx(self, value): ... - x = abstractproperty(getx, setx) - - 'abstractproperty' is deprecated. Use 'property' with 'abstractmethod' - instead. - """ - - __isabstractmethod__ = True - - -class ABCMeta(type): - - """Metaclass for defining Abstract Base Classes (ABCs). - - Use this metaclass to create an ABC. An ABC can be subclassed - directly, and then acts as a mix-in class. You can also register - unrelated concrete classes (even built-in classes) and unrelated - ABCs as 'virtual subclasses' -- these and their descendants will - be considered subclasses of the registering ABC by the built-in - issubclass() function, but the registering ABC won't show up in - their MRO (Method Resolution Order) nor will method - implementations defined by the registering ABC be callable (not - even via super()). - - """ - - # A global counter that is incremented each time a class is - # registered as a virtual subclass of anything. It forces the - # negative cache to be cleared before its next use. - _abc_invalidation_counter = 0 - - def __new__(mcls, name, bases, namespace): - cls = super().__new__(mcls, name, bases, namespace) - # Compute set of abstract method names - abstracts = {name - for name, value in namespace.items() - if getattr(value, "__isabstractmethod__", False)} - for base in bases: - for name in getattr(base, "__abstractmethods__", set()): - value = getattr(cls, name, None) - if getattr(value, "__isabstractmethod__", False): - abstracts.add(name) - cls.__abstractmethods__ = frozenset(abstracts) - # Set up inheritance registry - cls._abc_registry = WeakSet() - cls._abc_cache = WeakSet() - cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter - return cls - - def register(cls, subclass): - """Register a virtual subclass of an ABC. - - Returns the subclass, to allow usage as a class decorator. - """ - if not isinstance(subclass, type): - raise TypeError("Can only register classes") - if issubclass(subclass, cls): - return subclass # Already a subclass - # Subtle: test for cycles *after* testing for "already a subclass"; - # this means we allow X.register(X) and interpret it as a no-op. - if issubclass(cls, subclass): - # This would create a cycle, which is bad for the algorithm below - raise RuntimeError("Refusing to create an inheritance cycle") - cls._abc_registry.add(subclass) - ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache - return subclass - - def _dump_registry(cls, file=None): - """Debug helper to print the ABC registry.""" - print("Class: %s.%s" % (cls.__module__, cls.__name__), file=file) - print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file) - for name in sorted(cls.__dict__.keys()): - if name.startswith("_abc_"): - value = getattr(cls, name) - print("%s: %r" % (name, value), file=file) - - def __instancecheck__(cls, instance): - """Override for isinstance(instance, cls).""" - # Inline the cache checking - subclass = instance.__class__ - if subclass in cls._abc_cache: - return True - subtype = type(instance) - if subtype is subclass: - if (cls._abc_negative_cache_version == - ABCMeta._abc_invalidation_counter and - subclass in cls._abc_negative_cache): - return False - # Fall back to the subclass check. - return cls.__subclasscheck__(subclass) - return any(cls.__subclasscheck__(c) for c in {subclass, subtype}) - - def __subclasscheck__(cls, subclass): - """Override for issubclass(subclass, cls).""" - # Check cache - if subclass in cls._abc_cache: - return True - # Check negative cache; may have to invalidate - if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: - # Invalidate the negative cache - cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter - elif subclass in cls._abc_negative_cache: - return False - # Check the subclass hook - ok = cls.__subclasshook__(subclass) - if ok is not NotImplemented: - assert isinstance(ok, bool) - if ok: - cls._abc_cache.add(subclass) - else: - cls._abc_negative_cache.add(subclass) - return ok - # Check if it's a direct subclass - if cls in getattr(subclass, '__mro__', ()): - cls._abc_cache.add(subclass) - return True - # Check if it's a subclass of a registered class (recursive) - for rcls in cls._abc_registry: - if issubclass(subclass, rcls): - cls._abc_cache.add(subclass) - return True - # Check if it's a subclass of a subclass (recursive) - for scls in cls.__subclasses__(): - if issubclass(subclass, scls): - cls._abc_cache.add(subclass) - return True - # No dice; update negative cache - cls._abc_negative_cache.add(subclass) - return False diff --git a/tests/bytecode/pylib-tests/aifc.py b/tests/bytecode/pylib-tests/aifc.py deleted file mode 100644 index dd17d1dc27..0000000000 --- a/tests/bytecode/pylib-tests/aifc.py +++ /dev/null @@ -1,895 +0,0 @@ -"""Stuff to parse AIFF-C and AIFF files. - -Unless explicitly stated otherwise, the description below is true -both for AIFF-C files and AIFF files. - -An AIFF-C file has the following structure. - - +-----------------+ - | FORM | - +-----------------+ - | | - +----+------------+ - | | AIFC | - | +------------+ - | | | - | | . | - | | . | - | | . | - +----+------------+ - -An AIFF file has the string "AIFF" instead of "AIFC". - -A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, -big endian order), followed by the data. The size field does not include -the size of the 8 byte header. - -The following chunk types are recognized. - - FVER - (AIFF-C only). - MARK - <# of markers> (2 bytes) - list of markers: - (2 bytes, must be > 0) - (4 bytes) - ("pstring") - COMM - <# of channels> (2 bytes) - <# of sound frames> (4 bytes) - (2 bytes) - (10 bytes, IEEE 80-bit extended - floating point) - in AIFF-C files only: - (4 bytes) - ("pstring") - SSND - (4 bytes, not used by this program) - (4 bytes, not used by this program) - - -A pstring consists of 1 byte length, a string of characters, and 0 or 1 -byte pad to make the total length even. - -Usage. - -Reading AIFF files: - f = aifc.open(file, 'r') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods read(), seek(), and close(). -In some types of audio files, if the setpos() method is not used, -the seek() method is not necessary. - -This returns an instance of a class with the following public methods: - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' for AIFF files) - getcompname() -- returns human-readable version of - compression type ('not compressed' for AIFF files) - getparams() -- returns a tuple consisting of all of the - above in the above order - getmarkers() -- get the list of marks in the audio file or None - if there are no marks - getmark(id) -- get mark with the specified id (raises an error - if the mark does not exist) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) -The position returned by tell(), the position given to setpos() and -the position of marks are all compatible and have nothing to do with -the actual position in the file. -The close() method is called automatically when the class instance -is destroyed. - -Writing AIFF files: - f = aifc.open(file, 'w') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods write(), tell(), seek(), and -close(). - -This returns an instance of a class with the following public methods: - aiff() -- create an AIFF file (AIFF-C default) - aifc() -- create an AIFF-C file - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple) - -- set all parameters at once - setmark(id, pos, name) - -- add specified mark to the list of marks - tell() -- return current position in output file (useful - in combination with setmark()) - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file -You should set the parameters before the first writeframesraw or -writeframes. The total number of frames does not need to be set, -but when it is set to the correct value, the header does not have to -be patched up. -It is best to first set all parameters, perhaps possibly the -compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes('') or -close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, ypu must call -close() after all frames have been written. -The close() method is called automatically when the class instance -is destroyed. - -When a file is opened with the extension '.aiff', an AIFF file is -written, otherwise an AIFF-C file is written. This default can be -changed by calling aiff() or aifc() before the first writeframes or -writeframesraw. -""" - -import struct -import builtins -import warnings - -__all__ = ["Error", "open", "openfp"] - -class Error(Exception): - pass - -_AIFC_version = 0xA2805140 # Version 1 of AIFF-C - -def _read_long(file): - try: - return struct.unpack('>l', file.read(4))[0] - except struct.error: - raise EOFError - -def _read_ulong(file): - try: - return struct.unpack('>L', file.read(4))[0] - except struct.error: - raise EOFError - -def _read_short(file): - try: - return struct.unpack('>h', file.read(2))[0] - except struct.error: - raise EOFError - -def _read_ushort(file): - try: - return struct.unpack('>H', file.read(2))[0] - except struct.error: - raise EOFError - -def _read_string(file): - length = ord(file.read(1)) - if length == 0: - data = b'' - else: - data = file.read(length) - if length & 1 == 0: - dummy = file.read(1) - return data - -_HUGE_VAL = 1.79769313486231e+308 # See - -def _read_float(f): # 10 bytes - expon = _read_short(f) # 2 bytes - sign = 1 - if expon < 0: - sign = -1 - expon = expon + 0x8000 - himant = _read_ulong(f) # 4 bytes - lomant = _read_ulong(f) # 4 bytes - if expon == himant == lomant == 0: - f = 0.0 - elif expon == 0x7FFF: - f = _HUGE_VAL - else: - expon = expon - 16383 - f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63) - return sign * f - -def _write_short(f, x): - f.write(struct.pack('>h', x)) - -def _write_ushort(f, x): - f.write(struct.pack('>H', x)) - -def _write_long(f, x): - f.write(struct.pack('>l', x)) - -def _write_ulong(f, x): - f.write(struct.pack('>L', x)) - -def _write_string(f, s): - if len(s) > 255: - raise ValueError("string exceeds maximum pstring length") - f.write(struct.pack('B', len(s))) - f.write(s) - if len(s) & 1 == 0: - f.write(b'\x00') - -def _write_float(f, x): - import math - if x < 0: - sign = 0x8000 - x = x * -1 - else: - sign = 0 - if x == 0: - expon = 0 - himant = 0 - lomant = 0 - else: - fmant, expon = math.frexp(x) - if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN - expon = sign|0x7FFF - himant = 0 - lomant = 0 - else: # Finite - expon = expon + 16382 - if expon < 0: # denormalized - fmant = math.ldexp(fmant, expon) - expon = 0 - expon = expon | sign - fmant = math.ldexp(fmant, 32) - fsmant = math.floor(fmant) - himant = int(fsmant) - fmant = math.ldexp(fmant - fsmant, 32) - fsmant = math.floor(fmant) - lomant = int(fsmant) - _write_ushort(f, expon) - _write_ulong(f, himant) - _write_ulong(f, lomant) - -from chunk import Chunk - -class Aifc_read: - # Variables used in this class: - # - # These variables are available to the user though appropriate - # methods of this class: - # _file -- the open file with methods read(), close(), and seek() - # set through the __init__() method - # _nchannels -- the number of audio channels - # available through the getnchannels() method - # _nframes -- the number of audio frames - # available through the getnframes() method - # _sampwidth -- the number of bytes per audio sample - # available through the getsampwidth() method - # _framerate -- the sampling frequency - # available through the getframerate() method - # _comptype -- the AIFF-C compression type ('NONE' if AIFF) - # available through the getcomptype() method - # _compname -- the human-readable AIFF-C compression type - # available through the getcomptype() method - # _markers -- the marks in the audio file - # available through the getmarkers() and getmark() - # methods - # _soundpos -- the position in the audio stream - # available through the tell() method, set through the - # setpos() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _decomp -- the decompressor from builtin module cl - # _comm_chunk_read -- 1 iff the COMM chunk has been read - # _aifc -- 1 iff reading an AIFF-C file - # _ssnd_seek_needed -- 1 iff positioned correctly in audio - # file for readframes() - # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk - # _framesize -- size of one frame in the file - - def initfp(self, file): - self._version = 0 - self._convert = None - self._markers = [] - self._soundpos = 0 - self._file = file - chunk = Chunk(file) - if chunk.getname() != b'FORM': - raise Error('file does not start with FORM id') - formdata = chunk.read(4) - if formdata == b'AIFF': - self._aifc = 0 - elif formdata == b'AIFC': - self._aifc = 1 - else: - raise Error('not an AIFF or AIFF-C file') - self._comm_chunk_read = 0 - while 1: - self._ssnd_seek_needed = 1 - try: - chunk = Chunk(self._file) - except EOFError: - break - chunkname = chunk.getname() - if chunkname == b'COMM': - self._read_comm_chunk(chunk) - self._comm_chunk_read = 1 - elif chunkname == b'SSND': - self._ssnd_chunk = chunk - dummy = chunk.read(8) - self._ssnd_seek_needed = 0 - elif chunkname == b'FVER': - self._version = _read_ulong(chunk) - elif chunkname == b'MARK': - self._readmark(chunk) - chunk.skip() - if self._comm_chunk_read or self._ssnd_chunk: - raise Error('COMM chunk and/or SSND chunk missing') - - def __init__(self, f): - if isinstance(f, str): - f = builtins.open(f, 'rb') - # else, assume it is an open file object already - self.initfp(f) - - # - # User visible methods. - # - def getfp(self): - return self._file - - def rewind(self): - self._ssnd_seek_needed = 1 - self._soundpos = 0 - - def close(self): - self._file.close() - - def tell(self): - return self._soundpos - - def getnchannels(self): - return self._nchannels - - def getnframes(self): - return self._nframes - - def getsampwidth(self): - return self._sampwidth - - def getframerate(self): - return self._framerate - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def getversion(self): -## return self._version - - def getparams(self): - return self.getnchannels(), self.getsampwidth(), \ - self.getframerate(), self.getnframes(), \ - self.getcomptype(), self.getcompname() - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def setpos(self, pos): - if pos < 0 or pos > self._nframes: - raise Error('position not in range') - self._soundpos = pos - self._ssnd_seek_needed = 1 - - def readframes(self, nframes): - if self._ssnd_seek_needed: - self._ssnd_chunk.seek(0) - dummy = self._ssnd_chunk.read(8) - pos = self._soundpos * self._framesize - if pos: - self._ssnd_chunk.seek(pos + 8) - self._ssnd_seek_needed = 0 - if nframes == 0: - return b'' - data = self._ssnd_chunk.read(nframes * self._framesize) - if self._convert and data: - data = self._convert(data) - self._soundpos = self._soundpos + len(data) // (self._nchannels - * self._sampwidth) - return data - - # - # Internal methods. - # - - def _alaw2lin(self, data): - import audioop - return audioop.alaw2lin(data, 2) - - def _ulaw2lin(self, data): - import audioop - return audioop.ulaw2lin(data, 2) - - def _adpcm2lin(self, data): - import audioop - if not hasattr(self, '_adpcmstate'): - # first time - self._adpcmstate = None - data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate) - return data - - def _read_comm_chunk(self, chunk): - self._nchannels = _read_short(chunk) - self._nframes = _read_long(chunk) - self._sampwidth = (_read_short(chunk) + 7) // 8 - self._framerate = int(_read_float(chunk)) - self._framesize = self._nchannels * self._sampwidth - if self._aifc: - #DEBUG: SGI's soundeditor produces a bad size :-( - kludge = 0 - if chunk.chunksize == 18: - kludge = 1 - warnings.warn('Warning: bad COMM chunk size') - chunk.chunksize = 23 - #DEBUG end - self._comptype = chunk.read(4) - #DEBUG start - if kludge: - length = ord(chunk.file.read(1)) - if length & 1 == 0: - length = length + 1 - chunk.chunksize = chunk.chunksize + length - chunk.file.seek(-1, 1) - #DEBUG end - self._compname = _read_string(chunk) - if self._comptype != b'NONE': - if self._comptype == b'G722': - self._convert = self._adpcm2lin - self._framesize = self._framesize // 4 - elif self._comptype in (0+b'ulaw', b'ULAW'): - self._convert = self._ulaw2lin - self._framesize = self._framesize // 2 - elif self._comptype in (0+b'alaw', b'ALAW'): - self._convert = self._alaw2lin - self._framesize = self._framesize // 2 - else: - raise Error('unsupported compression type') - else: - self._comptype = b'NONE' - self._compname = b'not compressed' - - def _readmark(self, chunk): - nmarkers = _read_short(chunk) - # Some files appear to contain invalid counts. - # Cope with this by testing for EOF. - try: - for i in range(nmarkers): - id = _read_short(chunk) - pos = _read_long(chunk) - name = _read_string(chunk) - if pos or name: - # some files appear to have - # dummy markers consisting of - # a position 0 and name '' - self._markers.append((id, pos, name)) - except EOFError: - w = ('Warning: MARK chunk contains only %s marker%s instead of %s' % - (len(self._markers), '' if len(self._markers) == 1 else 's', - nmarkers)) - warnings.warn(w) - -class Aifc_write: - # Variables used in this class: - # - # These variables are user settable through appropriate methods - # of this class: - # _file -- the open file with methods write(), close(), tell(), seek() - # set through the __init__() method - # _comptype -- the AIFF-C compression type ('NONE' in AIFF) - # set through the setcomptype() or setparams() method - # _compname -- the human-readable AIFF-C compression type - # set through the setcomptype() or setparams() method - # _nchannels -- the number of audio channels - # set through the setnchannels() or setparams() method - # _sampwidth -- the number of bytes per audio sample - # set through the setsampwidth() or setparams() method - # _framerate -- the sampling frequency - # set through the setframerate() or setparams() method - # _nframes -- the number of audio frames written to the header - # set through the setnframes() or setparams() method - # _aifc -- whether we're writing an AIFF-C file or an AIFF file - # set through the aifc() method, reset through the - # aiff() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _comp -- the compressor from builtin module cl - # _nframeswritten -- the number of audio frames actually written - # _datalength -- the size of the audio samples written to the header - # _datawritten -- the size of the audio samples actually written - - def __init__(self, f): - if isinstance(f, str): - filename = f - f = builtins.open(f, 'wb') - else: - # else, assume it is an open file object already - filename = '???' - self.initfp(f) - if filename[-5:] == '.aiff': - self._aifc = 0 - else: - self._aifc = 1 - - def initfp(self, file): - self._file = file - self._version = _AIFC_version - self._comptype = b'NONE' - self._compname = b'not compressed' - self._convert = None - self._nchannels = 0 - self._sampwidth = 0 - self._framerate = 0 - self._nframes = 0 - self._nframeswritten = 0 - self._datawritten = 0 - self._datalength = 0 - self._markers = [] - self._marklength = 0 - self._aifc = 1 # AIFF-C is default - - def __del__(self): - self.close() - - # - # User visible methods. - # - def aiff(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 0 - - def aifc(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 1 - - def setnchannels(self, nchannels): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nchannels < 1: - raise Error('bad # of channels') - self._nchannels = nchannels - - def getnchannels(self): - if not self._nchannels: - raise Error('number of channels not set') - return self._nchannels - - def setsampwidth(self, sampwidth): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if sampwidth < 1 or sampwidth > 4: - raise Error('bad sample width') - self._sampwidth = sampwidth - - def getsampwidth(self): - if not self._sampwidth: - raise Error('sample width not set') - return self._sampwidth - - def setframerate(self, framerate): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if framerate <= 0: - raise Error('bad frame rate') - self._framerate = framerate - - def getframerate(self): - if not self._framerate: - raise Error('frame rate not set') - return self._framerate - - def setnframes(self, nframes): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._nframes = nframes - - def getnframes(self): - return self._nframeswritten - - def setcomptype(self, comptype, compname): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (0+b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722'): - raise Error('unsupported compression type') - self._comptype = comptype - self._compname = compname - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def setversion(self, version): -## if self._nframeswritten: -## raise Error, 'cannot change parameters after starting to write' -## self._version = version - - def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (0+b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722'): - raise Error('unsupported compression type') - self.setnchannels(nchannels) - self.setsampwidth(sampwidth) - self.setframerate(framerate) - self.setnframes(nframes) - self.setcomptype(comptype, compname) - - def getparams(self): - if self._nchannels or self._sampwidth or self._framerate: - raise Error('not all parameters set') - return self._nchannels, self._sampwidth, self._framerate, \ - self._nframes, self._comptype, self._compname - - def setmark(self, id, pos, name): - if id <= 0: - raise Error('marker ID must be > 0') - if pos < 0: - raise Error('marker position must be >= 0') - if not isinstance(name, bytes): - raise Error('marker name must be bytes') - for i in range(len(self._markers)): - if id == self._markers[i][0]: - self._markers[i] = id, pos, name - return - self._markers.append((id, pos, name)) - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def tell(self): - return self._nframeswritten - - def writeframesraw(self, data): - self._ensure_header_written(len(data)) - nframes = len(data) // (self._sampwidth * self._nchannels) - if self._convert: - data = self._convert(data) - self._file.write(data) - self._nframeswritten = self._nframeswritten + nframes - self._datawritten = self._datawritten + len(data) - - def writeframes(self, data): - self.writeframesraw(data) - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - - def close(self): - if self._file is None: - return - try: - self._ensure_header_written(0) - if self._datawritten & 1: - # quick pad to even size - self._file.write(b'\x00') - self._datawritten = self._datawritten + 1 - self._writemarkers() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten or \ - self._marklength: - self._patchheader() - finally: - # Prevent ref cycles - self._convert = None - f = self._file - self._file = None - f.close() - - # - # Internal methods. - # - - def _lin2alaw(self, data): - import audioop - return audioop.lin2alaw(data, 2) - - def _lin2ulaw(self, data): - import audioop - return audioop.lin2ulaw(data, 2) - - def _lin2adpcm(self, data): - import audioop - if not hasattr(self, '_adpcmstate'): - self._adpcmstate = None - data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate) - return data - - def _ensure_header_written(self, datasize): - if not self._nframeswritten: - if self._comptype in (0+b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): - if not self._sampwidth: - self._sampwidth = 2 - if self._sampwidth != 2: - raise Error('sample width must be 2 when compressing ' - 'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)') - if not self._nchannels: - raise Error('# channels not specified') - if not self._sampwidth: - raise Error('sample width not specified') - if not self._framerate: - raise Error('sampling rate not specified') - self._write_header(datasize) - - def _init_compression(self): - if self._comptype == b'G722': - self._convert = self._lin2adpcm - elif self._comptype in (0+b'ulaw', b'ULAW'): - self._convert = self._lin2ulaw - elif self._comptype in (0+b'alaw', b'ALAW'): - self._convert = self._lin2alaw - - def _write_header(self, initlength): - if self._aifc and self._comptype != b'NONE': - self._init_compression() - self._file.write(b'FORM') - if not self._nframes: - self._nframes = initlength // (self._nchannels * self._sampwidth) - self._datalength = self._nframes * self._nchannels * self._sampwidth - if self._datalength & 1: - self._datalength = self._datalength + 1 - if self._aifc: - if self._comptype in (0+b'ulaw', b'ULAW', b'alaw', b'ALAW'): - self._datalength = self._datalength // 2 - if self._datalength & 1: - self._datalength = self._datalength + 1 - elif self._comptype == b'G722': - self._datalength = (self._datalength + 3) // 4 - if self._datalength & 1: - self._datalength = self._datalength + 1 - self._form_length_pos = self._file.tell() - commlength = self._write_form_length(self._datalength) - if self._aifc: - self._file.write(b'AIFC') - self._file.write(b'FVER') - _write_ulong(self._file, 4) - _write_ulong(self._file, self._version) - else: - self._file.write(b'AIFF') - self._file.write(b'COMM') - _write_ulong(self._file, commlength) - _write_short(self._file, self._nchannels) - self._nframes_pos = self._file.tell() - _write_ulong(self._file, self._nframes) - _write_short(self._file, self._sampwidth * 8) - _write_float(self._file, self._framerate) - if self._aifc: - self._file.write(self._comptype) - _write_string(self._file, self._compname) - self._file.write(b'SSND') - self._ssnd_length_pos = self._file.tell() - _write_ulong(self._file, self._datalength + 8) - _write_ulong(self._file, 0) - _write_ulong(self._file, 0) - - def _write_form_length(self, datalength): - if self._aifc: - commlength = 23 + len(self._compname) - if commlength & 1: - commlength = commlength + 1 - verslength = 12 - else: - commlength = 18 - verslength = 0 - _write_ulong(self._file, 4 + verslength + self._marklength + \ - 8 + commlength + 16 + datalength) - return commlength - - def _patchheader(self): - curpos = self._file.tell() - if self._datawritten & 1: - datalength = self._datawritten + 1 - self._file.write(b'\x00') - else: - datalength = self._datawritten - if datalength == self._datalength and \ - self._nframes == self._nframeswritten and \ - self._marklength == 0: - self._file.seek(curpos, 0) - return - self._file.seek(self._form_length_pos, 0) - dummy = self._write_form_length(datalength) - self._file.seek(self._nframes_pos, 0) - _write_ulong(self._file, self._nframeswritten) - self._file.seek(self._ssnd_length_pos, 0) - _write_ulong(self._file, datalength + 8) - self._file.seek(curpos, 0) - self._nframes = self._nframeswritten - self._datalength = datalength - - def _writemarkers(self): - if len(self._markers) == 0: - return - self._file.write(b'MARK') - length = 2 - for marker in self._markers: - id, pos, name = marker - length = length + len(name) + 1 + 6 - if len(name) & 1 == 0: - length = length + 1 - _write_ulong(self._file, length) - self._marklength = length + 8 - _write_short(self._file, len(self._markers)) - for marker in self._markers: - id, pos, name = marker - _write_short(self._file, id) - _write_ulong(self._file, pos) - _write_string(self._file, name) - -def open(f, mode=None): - if mode is None: - if hasattr(f, 'mode'): - mode = f.mode - else: - mode = 'rb' - if mode in (0+'r', 'rb'): - return Aifc_read(f) - elif mode in (0+'w', 'wb'): - return Aifc_write(f) - else: - raise Error("mode must be 'r', 'rb', 'w', or 'wb'") - -openfp = open # B/W compatibility - -if __name__ == '__main__': - import sys - if sys.argv[1:]: - sys.argv.append('/usr/demos/data/audio/bach.aiff') - fn = sys.argv[1] - f = open(fn, 'r') - print("Reading", fn) - print("nchannels =", f.getnchannels()) - print("nframes =", f.getnframes()) - print("sampwidth =", f.getsampwidth()) - print("framerate =", f.getframerate()) - print("comptype =", f.getcomptype()) - print("compname =", f.getcompname()) - if sys.argv[2:]: - gn = sys.argv[2] - print("Writing", gn) - g = open(gn, 'w') - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if data: - break - g.writeframes(data) - g.close() - f.close() - print("Done.") diff --git a/tests/bytecode/pylib-tests/antigravity.py b/tests/bytecode/pylib-tests/antigravity.py deleted file mode 100644 index 7670187f83..0000000000 --- a/tests/bytecode/pylib-tests/antigravity.py +++ /dev/null @@ -1,17 +0,0 @@ - -import webbrowser -import hashlib - -webbrowser.open("http://xkcd.com/353/") - -def geohash(latitude, longitude, datedow): - '''Compute geohash() using the Munroe algorithm. - - >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68') - 37.857713 -122.544543 - - ''' - # http://xkcd.com/426/ - h = hashlib.md5(datedow).hexdigest() - p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])] - print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:])) diff --git a/tests/bytecode/pylib-tests/base64.py b/tests/bytecode/pylib-tests/base64.py deleted file mode 100644 index 17c6d1f3ee..0000000000 --- a/tests/bytecode/pylib-tests/base64.py +++ /dev/null @@ -1,410 +0,0 @@ -#! /usr/bin/env python3 - -"""RFC 3548: Base16, Base32, Base64 Data Encodings""" - -# Modified 04-Oct-1995 by Jack Jansen to use binascii module -# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support -# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere - -import re -import struct -import binascii - - -__all__ = [ - # Legacy interface exports traditional RFC 1521 Base64 encodings - 'encode', 'decode', 'encodebytes', 'decodebytes', - # Generalized interface for other encodings - 'b64encode', 'b64decode', 'b32encode', 'b32decode', - 'b16encode', 'b16decode', - # Standard Base64 encoding - 'standard_b64encode', 'standard_b64decode', - # Some common Base64 alternatives. As referenced by RFC 3458, see thread - # starting at: - # - # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html - 'urlsafe_b64encode', 'urlsafe_b64decode', - ] - - -bytes_types = (bytes, bytearray) # Types acceptable as binary data - -def _bytes_from_decode_data(s): - if isinstance(s, str): - try: - return s.encode('ascii') - except UnicodeEncodeError: - raise ValueError('string argument should contain only ASCII characters') - elif isinstance(s, bytes_types): - return s - else: - raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__) - - - -# Base64 encoding/decoding uses binascii - -def b64encode(s, altchars=None): - """Encode a byte string using Base64. - - s is the byte string to encode. Optional altchars must be a byte - string of length 2 which specifies an alternative alphabet for the - '+' and '/' characters. This allows an application to - e.g. generate url or filesystem safe Base64 strings. - - The encoded byte string is returned. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - # Strip off the trailing newline - encoded = binascii.b2a_base64(s)[:-1] - if altchars is not None: - if not isinstance(altchars, bytes_types): - raise TypeError("expected bytes, not %s" - % altchars.__class__.__name__) - assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b'+/', altchars)) - return encoded - - -def b64decode(s, altchars=None, validate=False): - """Decode a Base64 encoded byte string. - - s is the byte string to decode. Optional altchars must be a - string of length 2 which specifies the alternative alphabet used - instead of the '+' and '/' characters. - - The decoded string is returned. A binascii.Error is raised if s is - incorrectly padded. - - If validate is False (the default), non-base64-alphabet characters are - discarded prior to the padding check. If validate is True, - non-base64-alphabet characters in the input result in a binascii.Error. - """ - s = _bytes_from_decode_data(s) - if altchars is not None: - altchars = _bytes_from_decode_data(altchars) - assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b'+/')) - if validate and re.match(b'^[A-Za-z0-9+/]*={0,2}$', s): - raise binascii.Error('Non-base64 digit found') - return binascii.a2b_base64(s) - - -def standard_b64encode(s): - """Encode a byte string using the standard Base64 alphabet. - - s is the byte string to encode. The encoded byte string is returned. - """ - return b64encode(s) - -def standard_b64decode(s): - """Decode a byte string encoded with the standard Base64 alphabet. - - s is the byte string to decode. The decoded byte string is - returned. binascii.Error is raised if the input is incorrectly - padded or if there are non-alphabet characters present in the - input. - """ - return b64decode(s) - - -_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') -_urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') - -def urlsafe_b64encode(s): - """Encode a byte string using a url-safe Base64 alphabet. - - s is the byte string to encode. The encoded byte string is - returned. The alphabet uses '-' instead of '+' and '_' instead of - '/'. - """ - return b64encode(s).translate(_urlsafe_encode_translation) - -def urlsafe_b64decode(s): - """Decode a byte string encoded with the standard Base64 alphabet. - - s is the byte string to decode. The decoded byte string is - returned. binascii.Error is raised if the input is incorrectly - padded or if there are non-alphabet characters present in the - input. - - The alphabet uses '-' instead of '+' and '_' instead of '/'. - """ - s = _bytes_from_decode_data(s) - s = s.translate(_urlsafe_decode_translation) - return b64decode(s) - - - -# Base32 encoding/decoding must be done in Python -_b32alphabet = { - 0: b'A', 9: b'J', 18: b'S', 27: b'3', - 1: b'B', 10: b'K', 19: b'T', 28: b'4', - 2: b'C', 11: b'L', 20: b'U', 29: b'5', - 3: b'D', 12: b'M', 21: b'V', 30: b'6', - 4: b'E', 13: b'N', 22: b'W', 31: b'7', - 5: b'F', 14: b'O', 23: b'X', - 6: b'G', 15: b'P', 24: b'Y', - 7: b'H', 16: b'Q', 25: b'Z', - 8: b'I', 17: b'R', 26: b'2', - } - -_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())] -_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()]) - - -def b32encode(s): - """Encode a byte string using Base32. - - s is the byte string to encode. The encoded byte string is returned. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - quanta, leftover = divmod(len(s), 5) - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + bytes(5 - leftover) # Don't use += ! - quanta += 1 - encoded = bytes() - for i in range(quanta): - # c1 and c2 are 16 bits wide, c3 is 8 bits wide. The intent of this - # code is to process the 40 bits in units of 5 bits. So we take the 1 - # leftover bit of c1 and tack it onto c2. Then we take the 2 leftover - # bits of c2 and tack them onto c3. The shifts and masks are intended - # to give us values of exactly 5 bits in width. - c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5]) - c2 += (c1 & 1) << 16 # 17 bits wide - c3 += (c2 & 3) << 8 # 10 bits wide - encoded += bytes([_b32tab[c1 >> 11], # bits 1 - 5 - _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10 - _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15 - _b32tab[c2 >> 12], # bits 16 - 20 (1 - 5) - _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10) - _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15) - _b32tab[c3 >> 5], # bits 31 - 35 (1 - 5) - _b32tab[c3 & 0x1f], # bits 36 - 40 (1 - 5) - ]) - # Adjust for any leftover partial quanta - if leftover == 1: - return encoded[:-6] + b'======' - elif leftover == 2: - return encoded[:-4] + b'====' - elif leftover == 3: - return encoded[:-3] + b'===' - elif leftover == 4: - return encoded[:-1] + b'=' - return encoded - - -def b32decode(s, casefold=False, map01=None): - """Decode a Base32 encoded byte string. - - s is the byte string to decode. Optional casefold is a flag - specifying whether a lowercase alphabet is acceptable as input. - For security purposes, the default is False. - - RFC 3548 allows for optional mapping of the digit 0 (zero) to the - letter O (oh), and for optional mapping of the digit 1 (one) to - either the letter I (eye) or letter L (el). The optional argument - map01 when not None, specifies which letter the digit 1 should be - mapped to (when map01 is not None, the digit 0 is always mapped to - the letter O). For security purposes the default is None, so that - 0 and 1 are not allowed in the input. - - The decoded byte string is returned. binascii.Error is raised if - the input is incorrectly padded or if there are non-alphabet - characters present in the input. - """ - s = _bytes_from_decode_data(s) - quanta, leftover = divmod(len(s), 8) - if leftover: - raise binascii.Error('Incorrect padding') - # Handle section 2.4 zero and one mapping. The flag map01 will be either - # False, or the character to map the digit 1 (one) to. It should be - # either L (el) or I (eye). - if map01 is not None: - map01 = _bytes_from_decode_data(map01) - assert len(map01) == 1, repr(map01) - s = s.translate(bytes.maketrans(b'01', b'O' + map01)) - if casefold: - s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - padchars = 0 - mo = re.search(b'(?P[=]*)$', s) - if mo: - padchars = len(mo.group('pad')) - if padchars > 0: - s = s[:-padchars] - # Now decode the full quanta - parts = [] - acc = 0 - shift = 35 - for c in s: - val = _b32rev.get(c) - if val is None: - raise TypeError('Non-base32 digit found') - acc += _b32rev[c] << shift - shift -= 5 - if shift < 0: - parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii"))) - acc = 0 - shift = 35 - # Process the last, partial quanta - last = binascii.unhexlify(bytes('%010x' % acc, "ascii")) - if padchars == 0: - last = b'' # No characters - elif padchars == 1: - last = last[:-1] - elif padchars == 3: - last = last[:-2] - elif padchars == 4: - last = last[:-3] - elif padchars == 6: - last = last[:-4] - else: - raise binascii.Error('Incorrect padding') - parts.append(last) - return b''.join(parts) - - - -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns -# lowercase. The RFC also recommends against accepting input case -# insensitively. -def b16encode(s): - """Encode a byte string using Base16. - - s is the byte string to encode. The encoded byte string is returned. - """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - return binascii.hexlify(s).upper() - - -def b16decode(s, casefold=False): - """Decode a Base16 encoded byte string. - - s is the byte string to decode. Optional casefold is a flag - specifying whether a lowercase alphabet is acceptable as input. - For security purposes, the default is False. - - The decoded byte string is returned. binascii.Error is raised if - s were incorrectly padded or if there are non-alphabet characters - present in the string. - """ - s = _bytes_from_decode_data(s) - if casefold: - s = s.upper() - if re.search(b'[^0-9A-F]', s): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) - - - -# Legacy interface. This code could be cleaned up since I don't believe -# binascii has any line length limitations. It just doesn't seem worth it -# though. The files should be opened in binary mode. - -MAXLINESIZE = 76 # Excluding the CRLF -MAXBINSIZE = (MAXLINESIZE//4)*3 - -def encode(input, output): - """Encode a file; input and output are binary files.""" - while True: - s = input.read(MAXBINSIZE) - if not s: - break - while len(s) < MAXBINSIZE: - ns = input.read(MAXBINSIZE-len(s)) - if not ns: - break - s += ns - line = binascii.b2a_base64(s) - output.write(line) - - -def decode(input, output): - """Decode a file; input and output are binary files.""" - while True: - line = input.readline() - if not line: - break - s = binascii.a2b_base64(line) - output.write(s) - - -def encodebytes(s): - """Encode a bytestring into a bytestring containing multiple lines - of base-64 data.""" - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - pieces = [] - for i in range(0, len(s), MAXBINSIZE): - chunk = s[i : i + MAXBINSIZE] - pieces.append(binascii.b2a_base64(chunk)) - return b"".join(pieces) - -def encodestring(s): - """Legacy alias of encodebytes().""" - import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", - DeprecationWarning, 2) - return encodebytes(s) - - -def decodebytes(s): - """Decode a bytestring of base-64 data into a bytestring.""" - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) - return binascii.a2b_base64(s) - -def decodestring(s): - """Legacy alias of decodebytes().""" - import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", - DeprecationWarning, 2) - return decodebytes(s) - - -# Usable as a script... -def main(): - """Small main program""" - import sys, getopt - try: - opts, args = getopt.getopt(sys.argv[1:], 'deut') - except getopt.error as msg: - sys.stdout = sys.stderr - print(msg) - print("""usage: %s [-d|-e|-u|-t] [file|-] - -d, -u: decode - -e: encode (default) - -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]) - sys.exit(2) - func = encode - for o, a in opts: - if o == '-e': func = encode - if o == '-d': func = decode - if o == '-u': func = decode - if o == '-t': test(); return - if args and args[0] != '-': - with open(args[0], 'rb') as f: - func(f, sys.stdout.buffer) - else: - func(sys.stdin.buffer, sys.stdout.buffer) - - -def test(): - s0 = b"Aladdin:open sesame" - print(repr(s0)) - s1 = encodebytes(s0) - print(repr(s1)) - s2 = decodebytes(s1) - print(repr(s2)) - assert s0 == s2 - - -if __name__ == '__main__': - main() diff --git a/tests/bytecode/pylib-tests/bdb.py b/tests/bytecode/pylib-tests/bdb.py deleted file mode 100644 index 0579296de8..0000000000 --- a/tests/bytecode/pylib-tests/bdb.py +++ /dev/null @@ -1,647 +0,0 @@ -"""Debugger basics""" - -import fnmatch -import sys -import os - -__all__ = ["BdbQuit", "Bdb", "Breakpoint"] - -class BdbQuit(Exception): - """Exception to give up completely.""" - - -class Bdb: - """Generic Python debugger base class. - - This class takes care of details of the trace facility; - a derived class should implement user interaction. - The standard debugger class (pdb.Pdb) is an example. - """ - - def __init__(self, skip=None): - self.skip = set(skip) if skip else None - self.breaks = {} - self.fncache = {} - self.frame_returning = None - - def canonic(self, filename): - if filename == "<" + filename[1:-1] + ">": - return filename - canonic = self.fncache.get(filename) - if not canonic: - canonic = os.path.abspath(filename) - canonic = os.path.normcase(canonic) - self.fncache[filename] = canonic - return canonic - - def reset(self): - import linecache - linecache.checkcache() - self.botframe = None - self._set_stopinfo(None, None) - - def trace_dispatch(self, frame, event, arg): - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': - return self.trace_dispatch - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch - - def dispatch_line(self, frame): - if self.stop_here(frame) or self.break_here(frame): - self.user_line(frame) - if self.quitting: raise BdbQuit - return self.trace_dispatch - - def dispatch_call(self, frame, arg): - # XXX 'arg' is no longer used - if self.botframe is None: - # First call of dispatch since reset() - self.botframe = frame.f_back # (CT) Note that this may also be None! - return self.trace_dispatch - if (self.stop_here(frame) or self.break_anywhere(frame)): - # No need to trace this function - return # None - self.user_call(frame, arg) - if self.quitting: raise BdbQuit - return self.trace_dispatch - - def dispatch_return(self, frame, arg): - if self.stop_here(frame) or frame == self.returnframe: - try: - self.frame_returning = frame - self.user_return(frame, arg) - finally: - self.frame_returning = None - if self.quitting: raise BdbQuit - return self.trace_dispatch - - def dispatch_exception(self, frame, arg): - if self.stop_here(frame): - self.user_exception(frame, arg) - if self.quitting: raise BdbQuit - return self.trace_dispatch - - # Normally derived classes don't override the following - # methods, but they may if they want to redefine the - # definition of stopping and breakpoints. - - def is_skipped_module(self, module_name): - for pattern in self.skip: - if fnmatch.fnmatch(module_name, pattern): - return True - return False - - def stop_here(self, frame): - # (CT) stopframe may now also be None, see dispatch_call. - # (CT) the former test for None is therefore removed from here. - if self.skip and \ - self.is_skipped_module(frame.f_globals.get('__name__')): - return False - if frame is self.stopframe: - if self.stoplineno == -1: - return False - return frame.f_lineno >= self.stoplineno - while frame is not None and frame is not self.stopframe: - if frame is self.botframe: - return True - frame = frame.f_back - return False - - def break_here(self, frame): - filename = self.canonic(frame.f_code.co_filename) - if filename not in self.breaks: - return False - lineno = frame.f_lineno - if lineno not in self.breaks[filename]: - # The line itself has no breakpoint, but maybe the line is the - # first line of a function with breakpoint set by function name. - lineno = frame.f_code.co_firstlineno - if lineno not in self.breaks[filename]: - return False - - # flag says ok to delete temp. bp - (bp, flag) = effective(filename, lineno, frame) - if bp: - self.currentbp = bp.number - if (flag and bp.temporary): - self.do_clear(str(bp.number)) - return True - else: - return False - - def do_clear(self, arg): - raise NotImplementedError("subclass of bdb must implement do_clear()") - - def break_anywhere(self, frame): - return self.canonic(frame.f_code.co_filename) in self.breaks - - # Derived classes should override the user_* methods - # to gain control. - - def user_call(self, frame, argument_list): - """This method is called when there is the remote possibility - that we ever need to stop in this function.""" - pass - - def user_line(self, frame): - """This method is called when we stop or break at this line.""" - pass - - def user_return(self, frame, return_value): - """This method is called when a return trap is set here.""" - pass - - def user_exception(self, frame, exc_info): - """This method is called if an exception occurs, - but only if we are to stop at or just below this level.""" - pass - - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): - self.stopframe = stopframe - self.returnframe = returnframe - self.quitting = False - # stoplineno >= 0 means: stop at line >= the stoplineno - # stoplineno -1 means: don't stop at all - self.stoplineno = stoplineno - - # Derived classes and clients can call the following methods - # to affect the stepping state. - - def set_until(self, frame, lineno=None): - """Stop when the line with the line no greater than the current one is - reached or when returning from current frame""" - # the name "until" is borrowed from gdb - if lineno is None: - lineno = frame.f_lineno + 1 - self._set_stopinfo(frame, frame, lineno) - - def set_step(self): - """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch - self._set_stopinfo(None, None) - - def set_next(self, frame): - """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) - - def set_return(self, frame): - """Stop when returning from the given frame.""" - self._set_stopinfo(frame.f_back, frame) - - def set_trace(self, frame=None): - """Start debugging from `frame`. - - If frame is not specified, debugging starts from caller's frame. - """ - if frame is None: - frame = sys._getframe().f_back - self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - frame = frame.f_back - self.set_step() - sys.settrace(self.trace_dispatch) - - def set_continue(self): - # Don't stop except at breakpoints or when finished - self._set_stopinfo(self.botframe, None, -1) - if not self.breaks: - # no breakpoints; run without debugger overhead - sys.settrace(None) - frame = sys._getframe().f_back - while frame and frame is not self.botframe: - del frame.f_trace - frame = frame.f_back - - def set_quit(self): - self.stopframe = self.botframe - self.returnframe = None - self.quitting = True - sys.settrace(None) - - # Derived classes and clients can call the following methods - # to manipulate breakpoints. These methods return an - # error message is something went wrong, None if all is well. - # Set_break prints out the breakpoint line and file:lineno. - # Call self.get_*break*() to see the breakpoints or better - # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint(). - - def set_break(self, filename, lineno, temporary=False, cond=None, - funcname=None): - filename = self.canonic(filename) - import linecache # Import as late as possible - line = linecache.getline(filename, lineno) - if not line: - return 'Line %s:%d does not exist' % (filename, lineno) - list = self.breaks.setdefault(filename, []) - if lineno not in list: - list.append(lineno) - bp = Breakpoint(filename, lineno, temporary, cond, funcname) - - def _prune_breaks(self, filename, lineno): - if (filename, lineno) not in Breakpoint.bplist: - self.breaks[filename].remove(lineno) - if not self.breaks[filename]: - del self.breaks[filename] - - def clear_break(self, filename, lineno): - filename = self.canonic(filename) - if filename not in self.breaks: - return 'There are no breakpoints in %s' % filename - if lineno not in self.breaks[filename]: - return 'There is no breakpoint at %s:%d' % (filename, lineno) - # If there's only one bp in the list for that file,line - # pair, then remove the breaks entry - for bp in Breakpoint.bplist[filename, lineno][:]: - bp.deleteMe() - self._prune_breaks(filename, lineno) - - def clear_bpbynumber(self, arg): - try: - bp = self.get_bpbynumber(arg) - except ValueError as err: - return str(err) - bp.deleteMe() - self._prune_breaks(bp.file, bp.line) - - def clear_all_file_breaks(self, filename): - filename = self.canonic(filename) - if filename not in self.breaks: - return 'There are no breakpoints in %s' % filename - for line in self.breaks[filename]: - blist = Breakpoint.bplist[filename, line] - for bp in blist: - bp.deleteMe() - del self.breaks[filename] - - def clear_all_breaks(self): - if not self.breaks: - return 'There are no breakpoints' - for bp in Breakpoint.bpbynumber: - if bp: - bp.deleteMe() - self.breaks = {} - - def get_bpbynumber(self, arg): - if not arg: - raise ValueError('Breakpoint number expected') - try: - number = int(arg) - except ValueError: - raise ValueError('Non-numeric breakpoint number %s' % arg) - try: - bp = Breakpoint.bpbynumber[number] - except IndexError: - raise ValueError('Breakpoint number %d out of range' % number) - if bp is None: - raise ValueError('Breakpoint %d already deleted' % number) - return bp - - def get_break(self, filename, lineno): - filename = self.canonic(filename) - return filename in self.breaks and \ - lineno in self.breaks[filename] - - def get_breaks(self, filename, lineno): - filename = self.canonic(filename) - return filename in self.breaks and \ - lineno in self.breaks[filename] and \ - Breakpoint.bplist[filename, lineno] or [] - - def get_file_breaks(self, filename): - filename = self.canonic(filename) - if filename in self.breaks: - return self.breaks[filename] - else: - return [] - - def get_all_breaks(self): - return self.breaks - - # Derived classes and clients can call the following method - # to get a data structure representing a stack trace. - - def get_stack(self, f, t): - stack = [] - if t and t.tb_frame is f: - t = t.tb_next - while f is not None: - stack.append((f, f.f_lineno)) - if f is self.botframe: - break - f = f.f_back - stack.reverse() - i = max(0, len(stack) - 1) - while t is not None: - stack.append((t.tb_frame, t.tb_lineno)) - t = t.tb_next - if f is None: - i = max(0, len(stack) - 1) - return stack, i - - def format_stack_entry(self, frame_lineno, lprefix=': '): - import linecache, reprlib - frame, lineno = frame_lineno - filename = self.canonic(frame.f_code.co_filename) - s = '%s(%r)' % (filename, lineno) - if frame.f_code.co_name: - s += frame.f_code.co_name - else: - s += "" - if '__args__' in frame.f_locals: - args = frame.f_locals['__args__'] - else: - args = None - if args: - s += reprlib.repr(args) - else: - s += '()' - if '__return__' in frame.f_locals: - rv = frame.f_locals['__return__'] - s += '->' - s += reprlib.repr(rv) - line = linecache.getline(filename, lineno, frame.f_globals) - if line: - s += lprefix + line.strip() - return s - - # The following methods can be called by clients to use - # a debugger to debug a statement or an expression. - # Both can be given as a string, or a code object. - - def run(self, cmd, globals=None, locals=None): - if globals is None: - import __main__ - globals = __main__.__dict__ - if locals is None: - locals = globals - self.reset() - if isinstance(cmd, str): - cmd = compile(cmd, "", "exec") - sys.settrace(self.trace_dispatch) - try: - exec(cmd, globals, locals) - except BdbQuit: - pass - finally: - self.quitting = True - sys.settrace(None) - - def runeval(self, expr, globals=None, locals=None): - if globals is None: - import __main__ - globals = __main__.__dict__ - if locals is None: - locals = globals - self.reset() - sys.settrace(self.trace_dispatch) - try: - return eval(expr, globals, locals) - except BdbQuit: - pass - finally: - self.quitting = True - sys.settrace(None) - - def runctx(self, cmd, globals, locals): - # B/W compatibility - self.run(cmd, globals, locals) - - # This method is more useful to debug a single function call. - - def runcall(self, func, *args, **kwds): - self.reset() - sys.settrace(self.trace_dispatch) - res = None - try: - res = func(*args, **kwds) - except BdbQuit: - pass - finally: - self.quitting = True - sys.settrace(None) - return res - - -def set_trace(): - Bdb().set_trace() - - -class Breakpoint: - """Breakpoint class. - - Implements temporary breakpoints, ignore counts, disabling and - (re)-enabling, and conditionals. - - Breakpoints are indexed by number through bpbynumber and by - the file,line tuple using bplist. The former points to a - single instance of class Breakpoint. The latter points to a - list of such instances since there may be more than one - breakpoint per line. - - """ - - # XXX Keeping state in the class is a mistake -- this means - # you cannot have more than one active Bdb instance. - - next = 1 # Next bp to be assigned - bplist = {} # indexed by (file, lineno) tuple - bpbynumber = [None] # Each entry is None or an instance of Bpt - # index 0 is unused, except for marking an - # effective break .... see effective() - - def __init__(self, file, line, temporary=False, cond=None, funcname=None): - self.funcname = funcname - # Needed if funcname is not None. - self.func_first_executable_line = None - self.file = file # This better be in canonical form! - self.line = line - self.temporary = temporary - self.cond = cond - self.enabled = True - self.ignore = 0 - self.hits = 0 - self.number = Breakpoint.next - Breakpoint.next += 1 - # Build the two lists - self.bpbynumber.append(self) - if (file, line) in self.bplist: - self.bplist[file, line].append(self) - else: - self.bplist[file, line] = [self] - - def deleteMe(self): - index = (self.file, self.line) - self.bpbynumber[self.number] = None # No longer in list - self.bplist[index].remove(self) - if not self.bplist[index]: - # No more bp for this f:l combo - del self.bplist[index] - - def enable(self): - self.enabled = True - - def disable(self): - self.enabled = False - - def bpprint(self, out=None): - if out is None: - out = sys.stdout - print(self.bpformat(), file=out) - - def bpformat(self): - if self.temporary: - disp = 'del ' - else: - disp = 'keep ' - if self.enabled: - disp = disp + 'yes ' - else: - disp = disp + 'no ' - ret = '%-4dbreakpoint %s at %s:%d' % (self.number, disp, - self.file, self.line) - if self.cond: - ret += '\n\tstop only if %s' % (self.cond,) - if self.ignore: - ret += '\n\tignore next %d hits' % (self.ignore,) - if self.hits: - if self.hits > 1: - ss = 's' - else: - ss = '' - ret += '\n\tbreakpoint already hit %d time%s' % (self.hits, ss) - return ret - - def __str__(self): - return 'breakpoint %s at %s:%s' % (self.number, self.file, self.line) - -# -----------end of Breakpoint class---------- - -def checkfuncname(b, frame): - """Check whether we should break here because of `b.funcname`.""" - if not b.funcname: - # Breakpoint was set via line number. - if b.line != frame.f_lineno: - # Breakpoint was set at a line with a def statement and the function - # defined is called: don't break. - return False - return True - - # Breakpoint set via function name. - - if frame.f_code.co_name != b.funcname: - # It's not a function call, but rather execution of def statement. - return False - - # We are in the right frame. - if not b.func_first_executable_line: - # The function is entered for the 1st time. - b.func_first_executable_line = frame.f_lineno - - if b.func_first_executable_line != frame.f_lineno: - # But we are not at the first line number: don't break. - return False - return True - -# Determines if there is an effective (active) breakpoint at this -# line of code. Returns breakpoint number or 0 if none -def effective(file, line, frame): - """Determine which breakpoint for this file:line is to be acted upon. - - Called only if we know there is a bpt at this - location. Returns breakpoint that was triggered and a flag - that indicates if it is ok to delete a temporary bp. - - """ - possibles = Breakpoint.bplist[file, line] - for b in possibles: - if not b.enabled: - continue - if not checkfuncname(b, frame): - continue - # Count every hit when bp is enabled - b.hits += 1 - if not b.cond: - # If unconditional, and ignoring go on to next, else break - if b.ignore > 0: - b.ignore -= 1 - continue - else: - # breakpoint and marker that it's ok to delete if temporary - return (b, True) - else: - # Conditional bp. - # Ignore count applies only to those bpt hits where the - # condition evaluates to true. - try: - val = eval(b.cond, frame.f_globals, frame.f_locals) - if val: - if b.ignore > 0: - b.ignore -= 1 - # continue - else: - return (b, True) - # else: - # continue - except: - # if eval fails, most conservative thing is to stop on - # breakpoint regardless of ignore count. Don't delete - # temporary, as another hint to user. - return (b, False) - return (0+None, None) - - -# -------------------- testing -------------------- - -class Tdb(Bdb): - def user_call(self, frame, args): - name = frame.f_code.co_name - if not name: name = '???' - print('+++ call', name, args) - def user_line(self, frame): - import linecache - name = frame.f_code.co_name - if not name: name = '???' - fn = self.canonic(frame.f_code.co_filename) - line = linecache.getline(fn, frame.f_lineno, frame.f_globals) - print('+++', fn, frame.f_lineno, name, ':', line.strip()) - def user_return(self, frame, retval): - print('+++ return', retval) - def user_exception(self, frame, exc_stuff): - print('+++ exception', exc_stuff) - self.set_continue() - -def foo(n): - print('foo(', n, ')') - x = bar(n*10) - print('bar returned', x) - -def bar(a): - print('bar(', a, ')') - return a/2 - -def test(): - t = Tdb() - t.run('import bdb; bdb.foo(10)') diff --git a/tests/bytecode/pylib-tests/binhex.py b/tests/bytecode/pylib-tests/binhex.py deleted file mode 100644 index ec5624f9e1..0000000000 --- a/tests/bytecode/pylib-tests/binhex.py +++ /dev/null @@ -1,471 +0,0 @@ -"""Macintosh binhex compression/decompression. - -easy interface: -binhex(inputfilename, outputfilename) -hexbin(inputfilename, outputfilename) -""" - -# -# Jack Jansen, CWI, August 1995. -# -# The module is supposed to be as compatible as possible. Especially the -# easy interface should work "as expected" on any platform. -# XXXX Note: currently, textfiles appear in mac-form on all platforms. -# We seem to lack a simple character-translate in python. -# (we should probably use ISO-Latin-1 on all but the mac platform). -# XXXX The simple routines are too simple: they expect to hold the complete -# files in-core. Should be fixed. -# XXXX It would be nice to handle AppleDouble format on unix -# (for servers serving macs). -# XXXX I don't understand what happens when you get 0x90 times the same byte on -# input. The resulting code (xx 90 90) would appear to be interpreted as an -# escaped *value* of 0x90. All coders I've seen appear to ignore this nicety... -# -import io -import os -import struct -import binascii - -__all__ = ["binhex","hexbin","Error"] - -class Error(Exception): - pass - -# States (what have we written) -[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3) - -# Various constants -REASONABLY_LARGE = 32768 # Minimal amount we pass the rle-coder -LINELEN = 64 -RUNCHAR = b"\x90" - -# -# This code is no longer byte-order dependent - - -class FInfo: - def __init__(self): - self.Type = '????' - self.Creator = '????' - self.Flags = 0 - -def getfileinfo(name): - finfo = FInfo() - with io.open(name, 'rb') as fp: - # Quick check for textfile - data = fp.read(512) - if 0 not in data: - finfo.Type = 'TEXT' - fp.seek(0, 2) - dsize = fp.tell() - dir, file = os.path.split(name) - file = file.replace(':', '-', 1) - return file, finfo, dsize, 0 - -class openrsrc: - def __init__(self, *args): - pass - - def read(self, *args): - return b'' - - def write(self, *args): - pass - - def close(self): - pass - -class _Hqxcoderengine: - """Write data to the coder in 3-byte chunks""" - - def __init__(self, ofp): - self.ofp = ofp - self.data = b'' - self.hqxdata = b'' - self.linelen = LINELEN - 1 - - def write(self, data): - self.data = self.data + data - datalen = len(self.data) - todo = (datalen // 3) * 3 - data = self.data[:todo] - self.data = self.data[todo:] - if not data: - return - self.hqxdata = self.hqxdata + binascii.b2a_hqx(data) - self._flush(0) - - def _flush(self, force): - first = 0 - while first <= len(self.hqxdata) - self.linelen: - last = first + self.linelen - self.ofp.write(self.hqxdata[first:last] + b'\n') - self.linelen = LINELEN - first = last - self.hqxdata = self.hqxdata[first:] - if force: - self.ofp.write(self.hqxdata + b':\n') - - def close(self): - if self.data: - self.hqxdata = self.hqxdata + binascii.b2a_hqx(self.data) - self._flush(1) - self.ofp.close() - del self.ofp - -class _Rlecoderengine: - """Write data to the RLE-coder in suitably large chunks""" - - def __init__(self, ofp): - self.ofp = ofp - self.data = b'' - - def write(self, data): - self.data = self.data + data - if len(self.data) < REASONABLY_LARGE: - return - rledata = binascii.rlecode_hqx(self.data) - self.ofp.write(rledata) - self.data = b'' - - def close(self): - if self.data: - rledata = binascii.rlecode_hqx(self.data) - self.ofp.write(rledata) - self.ofp.close() - del self.ofp - -class BinHex: - def __init__(self, name_finfo_dlen_rlen, ofp): - name, finfo, dlen, rlen = name_finfo_dlen_rlen - close_on_error = False - if isinstance(ofp, str): - ofname = ofp - ofp = io.open(ofname, 'wb') - close_on_error = True - try: - ofp.write(b'(This file must be converted with BinHex 4.0)\r\r:') - hqxer = _Hqxcoderengine(ofp) - self.ofp = _Rlecoderengine(hqxer) - self.crc = 0 - if finfo is None: - finfo = FInfo() - self.dlen = dlen - self.rlen = rlen - self._writeinfo(name, finfo) - self.state = _DID_HEADER - except: - if close_on_error: - ofp.close() - raise - - def _writeinfo(self, name, finfo): - nl = len(name) - if nl > 63: - raise Error('Filename too long') - d = bytes([nl]) + name.encode("latin-1") + b'\0' - tp, cr = finfo.Type, finfo.Creator - if isinstance(tp, str): - tp = tp.encode("latin-1") - if isinstance(cr, str): - cr = cr.encode("latin-1") - d2 = tp + cr - - # Force all structs to be packed with big-endian - d3 = struct.pack('>h', finfo.Flags) - d4 = struct.pack('>ii', self.dlen, self.rlen) - info = d + d2 + d3 + d4 - self._write(info) - self._writecrc() - - def _write(self, data): - self.crc = binascii.crc_hqx(data, self.crc) - self.ofp.write(data) - - def _writecrc(self): - # XXXX Should this be here?? - # self.crc = binascii.crc_hqx('\0\0', self.crc) - if self.crc < 0: - fmt = '>h' - else: - fmt = '>H' - self.ofp.write(struct.pack(fmt, self.crc)) - self.crc = 0 - - def write(self, data): - if self.state != _DID_HEADER: - raise Error('Writing data at the wrong time') - self.dlen = self.dlen - len(data) - self._write(data) - - def close_data(self): - if self.dlen != 0: - raise Error('Incorrect data size, diff=%r' % (self.rlen,)) - self._writecrc() - self.state = _DID_DATA - - def write_rsrc(self, data): - if self.state < _DID_DATA: - self.close_data() - if self.state != _DID_DATA: - raise Error('Writing resource data at the wrong time') - self.rlen = self.rlen - len(data) - self._write(data) - - def close(self): - if self.state < _DID_DATA: - self.close_data() - if self.state != _DID_DATA: - raise Error('Close at the wrong time') - if self.rlen != 0: - raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,)) - self._writecrc() - self.ofp.close() - self.state = None - del self.ofp - -def binhex(inp, out): - """binhex(infilename, outfilename): create binhex-encoded copy of a file""" - finfo = getfileinfo(inp) - ofp = BinHex(finfo, out) - - ifp = io.open(inp, 'rb') - # XXXX Do textfile translation on non-mac systems - while True: - d = ifp.read(128000) - if not d: break - ofp.write(d) - ofp.close_data() - ifp.close() - - ifp = openrsrc(inp, 'rb') - while True: - d = ifp.read(128000) - if not d: break - ofp.write_rsrc(d) - ofp.close() - ifp.close() - -class _Hqxdecoderengine: - """Read data via the decoder in 4-byte chunks""" - - def __init__(self, ifp): - self.ifp = ifp - self.eof = 0 - - def read(self, totalwtd): - """Read at least wtd bytes (or until EOF)""" - decdata = b'' - wtd = totalwtd - # - # The loop here is convoluted, since we don't really now how - # much to decode: there may be newlines in the incoming data. - while wtd > 0: - if self.eof: return decdata - wtd = ((wtd + 2) // 3) * 4 - data = self.ifp.read(wtd) - # - # Next problem: there may not be a complete number of - # bytes in what we pass to a2b. Solve by yet another - # loop. - # - while True: - try: - decdatacur, self.eof = binascii.a2b_hqx(data) - break - except binascii.Incomplete: - pass - newdata = self.ifp.read(1) - if newdata: - raise Error('Premature EOF on binhex file') - data = data + newdata - decdata = decdata + decdatacur - wtd = totalwtd - len(decdata) - if decdata and self.eof: - raise Error('Premature EOF on binhex file') - return decdata - - def close(self): - self.ifp.close() - -class _Rledecoderengine: - """Read data via the RLE-coder""" - - def __init__(self, ifp): - self.ifp = ifp - self.pre_buffer = b'' - self.post_buffer = b'' - self.eof = 0 - - def read(self, wtd): - if wtd > len(self.post_buffer): - self._fill(wtd - len(self.post_buffer)) - rv = self.post_buffer[:wtd] - self.post_buffer = self.post_buffer[wtd:] - return rv - - def _fill(self, wtd): - self.pre_buffer = self.pre_buffer + self.ifp.read(wtd + 4) - if self.ifp.eof: - self.post_buffer = self.post_buffer + \ - binascii.rledecode_hqx(self.pre_buffer) - self.pre_buffer = b'' - return - - # - # Obfuscated code ahead. We have to take care that we don't - # end up with an orphaned RUNCHAR later on. So, we keep a couple - # of bytes in the buffer, depending on what the end of - # the buffer looks like: - # '\220\0\220' - Keep 3 bytes: repeated \220 (escaped as \220\0) - # '?\220' - Keep 2 bytes: repeated something-else - # '\220\0' - Escaped \220: Keep 2 bytes. - # '?\220?' - Complete repeat sequence: decode all - # otherwise: keep 1 byte. - # - mark = len(self.pre_buffer) - if self.pre_buffer[-3:] == RUNCHAR + b'\0' + RUNCHAR: - mark = mark - 3 - elif self.pre_buffer[-1:] == RUNCHAR: - mark = mark - 2 - elif self.pre_buffer[-2:] == RUNCHAR + b'\0': - mark = mark - 2 - elif self.pre_buffer[-2:-1] == RUNCHAR: - pass # Decode all - else: - mark = mark - 1 - - self.post_buffer = self.post_buffer + \ - binascii.rledecode_hqx(self.pre_buffer[:mark]) - self.pre_buffer = self.pre_buffer[mark:] - - def close(self): - self.ifp.close() - -class HexBin: - def __init__(self, ifp): - if isinstance(ifp, str): - ifp = io.open(ifp, 'rb') - # - # Find initial colon. - # - while True: - ch = ifp.read(1) - if not ch: - raise Error("No binhex data found") - # Cater for \r\n terminated lines (which show up as \n\r, hence - # all lines start with \r) - if ch == b'\r': - continue - if ch == b':': - break - - hqxifp = _Hqxdecoderengine(ifp) - self.ifp = _Rledecoderengine(hqxifp) - self.crc = 0 - self._readheader() - - def _read(self, len): - data = self.ifp.read(len) - self.crc = binascii.crc_hqx(data, self.crc) - return data - - def _checkcrc(self): - filecrc = struct.unpack('>h', self.ifp.read(2))[0] & 0xffff - #self.crc = binascii.crc_hqx('\0\0', self.crc) - # XXXX Is this needed?? - self.crc = self.crc & 0xffff - if filecrc != self.crc: - raise Error('CRC error, computed %x, read %x' - % (self.crc, filecrc)) - self.crc = 0 - - def _readheader(self): - len = self._read(1) - fname = self._read(ord(len)) - rest = self._read(19) - self._checkcrc() - - type = rest[1:5] - creator = rest[5:9] - flags = struct.unpack('>h', rest[9:11])[0] - self.dlen = struct.unpack('>l', rest[11:15])[0] - self.rlen = struct.unpack('>l', rest[15:19])[0] - - self.FName = fname - self.FInfo = FInfo() - self.FInfo.Creator = creator - self.FInfo.Type = type - self.FInfo.Flags = flags - - self.state = _DID_HEADER - - def read(self, *n): - if self.state != _DID_HEADER: - raise Error('Read data at wrong time') - if n: - n = n[0] - n = min(n, self.dlen) - else: - n = self.dlen - rv = b'' - while len(rv) < n: - rv = rv + self._read(n-len(rv)) - self.dlen = self.dlen - n - return rv - - def close_data(self): - if self.state != _DID_HEADER: - raise Error('close_data at wrong time') - if self.dlen: - dummy = self._read(self.dlen) - self._checkcrc() - self.state = _DID_DATA - - def read_rsrc(self, *n): - if self.state == _DID_HEADER: - self.close_data() - if self.state != _DID_DATA: - raise Error('Read resource data at wrong time') - if n: - n = n[0] - n = min(n, self.rlen) - else: - n = self.rlen - self.rlen = self.rlen - n - return self._read(n) - - def close(self): - if self.rlen: - dummy = self.read_rsrc(self.rlen) - self._checkcrc() - self.state = _DID_RSRC - self.ifp.close() - -def hexbin(inp, out): - """hexbin(infilename, outfilename) - Decode binhexed file""" - ifp = HexBin(inp) - finfo = ifp.FInfo - if not out: - out = ifp.FName - - ofp = io.open(out, 'wb') - # XXXX Do translation on non-mac systems - while True: - d = ifp.read(128000) - if not d: break - ofp.write(d) - ofp.close() - ifp.close_data() - - d = ifp.read_rsrc(128000) - if d: - ofp = openrsrc(out, 'wb') - ofp.write(d) - while True: - d = ifp.read_rsrc(128000) - if not d: break - ofp.write(d) - ofp.close() - - ifp.close() diff --git a/tests/bytecode/pylib-tests/bisect.py b/tests/bytecode/pylib-tests/bisect.py deleted file mode 100644 index 4a4d05255e..0000000000 --- a/tests/bytecode/pylib-tests/bisect.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Bisection algorithms.""" - -def insort_right(a, x, lo=0, hi=None): - """Insert item x in list a, and keep it sorted assuming a is sorted. - - If x is already in a, insert it to the right of the rightmost x. - - Optional args lo (default 0) and hi (default len(a)) bound the - slice of a to be searched. - """ - - if lo < 0: - raise ValueError('lo must be non-negative') - if hi is None: - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - if x < a[mid]: hi = mid - else: lo = mid+1 - a.insert(lo, x) - -insort = insort_right # backward compatibility - -def bisect_right(a, x, lo=0, hi=None): - """Return the index where to insert item x in list a, assuming a is sorted. - - The return value i is such that all e in a[:i] have e <= x, and all e in - a[i:] have e > x. So if x already appears in the list, a.insert(x) will - insert just after the rightmost x already there. - - Optional args lo (default 0) and hi (default len(a)) bound the - slice of a to be searched. - """ - - if lo < 0: - raise ValueError('lo must be non-negative') - if hi is None: - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - if x < a[mid]: hi = mid - else: lo = mid+1 - return lo - -bisect = bisect_right # backward compatibility - -def insort_left(a, x, lo=0, hi=None): - """Insert item x in list a, and keep it sorted assuming a is sorted. - - If x is already in a, insert it to the left of the leftmost x. - - Optional args lo (default 0) and hi (default len(a)) bound the - slice of a to be searched. - """ - - if lo < 0: - raise ValueError('lo must be non-negative') - if hi is None: - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - if a[mid] < x: lo = mid+1 - else: hi = mid - a.insert(lo, x) - - -def bisect_left(a, x, lo=0, hi=None): - """Return the index where to insert item x in list a, assuming a is sorted. - - The return value i is such that all e in a[:i] have e < x, and all e in - a[i:] have e >= x. So if x already appears in the list, a.insert(x) will - insert just before the leftmost x already there. - - Optional args lo (default 0) and hi (default len(a)) bound the - slice of a to be searched. - """ - - if lo < 0: - raise ValueError('lo must be non-negative') - if hi is None: - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - if a[mid] < x: lo = mid+1 - else: hi = mid - return lo - -# Overwrite above definitions with a fast C implementation -try: - from _bisect import * -except ImportError: - pass diff --git a/tests/bytecode/pylib-tests/bz2.py b/tests/bytecode/pylib-tests/bz2.py deleted file mode 100644 index 6a4fd505b0..0000000000 --- a/tests/bytecode/pylib-tests/bz2.py +++ /dev/null @@ -1,504 +0,0 @@ -"""Interface to the libbzip2 compression library. - -This module provides a file interface, classes for incremental -(de)compression, and functions for one-shot (de)compression. -""" - -__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", - "open", "compress", "decompress"] - -__author__ = "Nadeem Vawda " - -import builtins -import io -import warnings - -try: - from threading import RLock -except ImportError: - from dummy_threading import RLock - -from _bz2 import BZ2Compressor, BZ2Decompressor - - -_MODE_CLOSED = 0 -_MODE_READ = 1 -_MODE_READ_EOF = 2 -_MODE_WRITE = 3 - -_BUFFER_SIZE = 8192 - - -class BZ2File(io.BufferedIOBase): - - """A file object providing transparent bzip2 (de)compression. - - A BZ2File can act as a wrapper for an existing file object, or refer - directly to a named file on disk. - - Note that BZ2File provides a *binary* file interface - data read is - returned as bytes, and data to be written should be given as bytes. - """ - - def __init__(self, filename, mode="r", buffering=None, compresslevel=9): - """Open a bzip2-compressed file. - - If filename is a str or bytes object, is gives the name of the file to - be opened. Otherwise, it should be a file object, which will be used to - read or write the compressed data. - - mode can be 'r' for reading (default), 'w' for (over)writing, or 'a' for - appending. These can equivalently be given as 'rb', 'wb', and 'ab'. - - buffering is ignored. Its use is deprecated. - - If mode is 'w' or 'a', compresslevel can be a number between 1 - and 9 specifying the level of compression: 1 produces the least - compression, and 9 (default) produces the most compression. - - If mode is 'r', the input file may be the concatenation of - multiple compressed streams. - """ - # This lock must be recursive, so that BufferedIOBase's - # readline(), readlines() and writelines() don't deadlock. - self._lock = RLock() - self._fp = None - self._closefp = False - self._mode = _MODE_CLOSED - self._pos = 0 - self._size = -1 - - if buffering is not None: - warnings.warn("Use of 'buffering' argument is deprecated", - DeprecationWarning) - - if not (1 <= compresslevel <= 9): - raise ValueError("compresslevel must be between 1 and 9") - - if mode in (0+"", "r", "rb"): - mode = "rb" - mode_code = _MODE_READ - self._decompressor = BZ2Decompressor() - self._buffer = b"" - self._buffer_offset = 0 - elif mode in (0+"w", "wb"): - mode = "wb" - mode_code = _MODE_WRITE - self._compressor = BZ2Compressor(compresslevel) - elif mode in (0+"a", "ab"): - mode = "ab" - mode_code = _MODE_WRITE - self._compressor = BZ2Compressor(compresslevel) - else: - raise ValueError("Invalid mode: {!r}".format(mode)) - - if isinstance(filename, (str, bytes)): - self._fp = builtins.open(filename, mode) - self._closefp = True - self._mode = mode_code - elif hasattr(filename, "read") or hasattr(filename, "write"): - self._fp = filename - self._mode = mode_code - else: - raise TypeError("filename must be a str or bytes object, or a file") - - def close(self): - """Flush and close the file. - - May be called more than once without error. Once the file is - closed, any other operation on it will raise a ValueError. - """ - with self._lock: - if self._mode == _MODE_CLOSED: - return - try: - if self._mode in (_MODE_READ, _MODE_READ_EOF): - self._decompressor = None - elif self._mode == _MODE_WRITE: - self._fp.write(self._compressor.flush()) - self._compressor = None - finally: - try: - if self._closefp: - self._fp.close() - finally: - self._fp = None - self._closefp = False - self._mode = _MODE_CLOSED - self._buffer = b"" - self._buffer_offset = 0 - - @property - def closed(self): - """True if this file is closed.""" - return self._mode == _MODE_CLOSED - - def fileno(self): - """Return the file descriptor for the underlying file.""" - self._check_not_closed() - return self._fp.fileno() - - def seekable(self): - """Return whether the file supports seeking.""" - return self.readable() and self._fp.seekable() - - def readable(self): - """Return whether the file was opened for reading.""" - self._check_not_closed() - return self._mode in (_MODE_READ, _MODE_READ_EOF) - - def writable(self): - """Return whether the file was opened for writing.""" - self._check_not_closed() - return self._mode == _MODE_WRITE - - # Mode-checking helper functions. - - def _check_not_closed(self): - if self.closed: - raise ValueError("I/O operation on closed file") - - def _check_can_read(self): - if self._mode not in (_MODE_READ, _MODE_READ_EOF): - self._check_not_closed() - raise io.UnsupportedOperation("File not open for reading") - - def _check_can_write(self): - if self._mode != _MODE_WRITE: - self._check_not_closed() - raise io.UnsupportedOperation("File not open for writing") - - def _check_can_seek(self): - if self._mode not in (_MODE_READ, _MODE_READ_EOF): - self._check_not_closed() - raise io.UnsupportedOperation("Seeking is only supported " - "on files open for reading") - if not self._fp.seekable(): - raise io.UnsupportedOperation("The underlying file object " - "does not support seeking") - - # Fill the readahead buffer if it is empty. Returns False on EOF. - def _fill_buffer(self): - if self._mode == _MODE_READ_EOF: - return False - # Depending on the input data, our call to the decompressor may not - # return any data. In this case, try again after reading another block. - while self._buffer_offset == len(self._buffer): - rawblock = (self._decompressor.unused_data or - self._fp.read(_BUFFER_SIZE)) - - if not rawblock: - if self._decompressor.eof: - self._mode = _MODE_READ_EOF - self._size = self._pos - return False - else: - raise EOFError("Compressed file ended before the " - "end-of-stream marker was reached") - - # Continue to next stream. - if self._decompressor.eof: - self._decompressor = BZ2Decompressor() - - self._buffer = self._decompressor.decompress(rawblock) - self._buffer_offset = 0 - return True - - # Read data until EOF. - # If return_data is false, consume the data without returning it. - def _read_all(self, return_data=True): - # The loop assumes that _buffer_offset is 0. Ensure that this is true. - self._buffer = self._buffer[self._buffer_offset:] - self._buffer_offset = 0 - - blocks = [] - while self._fill_buffer(): - if return_data: - blocks.append(self._buffer) - self._pos += len(self._buffer) - self._buffer = b"" - if return_data: - return b"".join(blocks) - - # Read a block of up to n bytes. - # If return_data is false, consume the data without returning it. - def _read_block(self, n, return_data=True): - # If we have enough data buffered, return immediately. - end = self._buffer_offset + n - if end <= len(self._buffer): - data = self._buffer[self._buffer_offset : end] - self._buffer_offset = end - self._pos += len(data) - return data if return_data else None - - # The loop assumes that _buffer_offset is 0. Ensure that this is true. - self._buffer = self._buffer[self._buffer_offset:] - self._buffer_offset = 0 - - blocks = [] - while n > 0 and self._fill_buffer(): - if n < len(self._buffer): - data = self._buffer[:n] - self._buffer_offset = n - else: - data = self._buffer - self._buffer = b"" - if return_data: - blocks.append(data) - self._pos += len(data) - n -= len(data) - if return_data: - return b"".join(blocks) - - def peek(self, n=0): - """Return buffered data without advancing the file position. - - Always returns at least one byte of data, unless at EOF. - The exact number of bytes returned is unspecified. - """ - with self._lock: - self._check_can_read() - if not self._fill_buffer(): - return b"" - return self._buffer[self._buffer_offset:] - - def read(self, size=-1): - """Read up to size uncompressed bytes from the file. - - If size is negative or omitted, read until EOF is reached. - Returns b'' if the file is already at EOF. - """ - with self._lock: - self._check_can_read() - if size == 0: - return b"" - elif size < 0: - return self._read_all() - else: - return self._read_block(size) - - def read1(self, size=-1): - """Read up to size uncompressed bytes, while trying to avoid - making multiple reads from the underlying stream. - - Returns b'' if the file is at EOF. - """ - # Usually, read1() calls _fp.read() at most once. However, sometimes - # this does not give enough data for the decompressor to make progress. - # In this case we make multiple reads, to avoid returning b"". - with self._lock: - self._check_can_read() - if (size == 0 or - # Only call _fill_buffer() if the buffer is actually empty. - # This gives a significant speedup if *size* is small. - (self._buffer_offset == len(self._buffer) and self._fill_buffer())): - return b"" - if size > 0: - data = self._buffer[self._buffer_offset : - self._buffer_offset + size] - self._buffer_offset += len(data) - else: - data = self._buffer[self._buffer_offset:] - self._buffer = b"" - self._buffer_offset = 0 - self._pos += len(data) - return data - - def readinto(self, b): - """Read up to len(b) bytes into b. - - Returns the number of bytes read (0 for EOF). - """ - with self._lock: - return io.BufferedIOBase.readinto(self, b) - - def readline(self, size=-1): - """Read a line of uncompressed bytes from the file. - - The terminating newline (if present) is retained. If size is - non-negative, no more than size bytes will be read (in which - case the line may be incomplete). Returns b'' if already at EOF. - """ - if not isinstance(size, int): - if not hasattr(size, "__index__"): - raise TypeError("Integer argument expected") - size = size.__index__() - with self._lock: - self._check_can_read() - # Shortcut for the common case - the whole line is in the buffer. - if size < 0: - end = self._buffer.find(b"\n", self._buffer_offset) + 1 - if end > 0: - line = self._buffer[self._buffer_offset : end] - self._buffer_offset = end - self._pos += len(line) - return line - return io.BufferedIOBase.readline(self, size) - - def readlines(self, size=-1): - """Read a list of lines of uncompressed bytes from the file. - - size can be specified to control the number of lines read: no - further lines will be read once the total size of the lines read - so far equals or exceeds size. - """ - if not isinstance(size, int): - if not hasattr(size, "__index__"): - raise TypeError("Integer argument expected") - size = size.__index__() - with self._lock: - return io.BufferedIOBase.readlines(self, size) - - def write(self, data): - """Write a byte string to the file. - - Returns the number of uncompressed bytes written, which is - always len(data). Note that due to buffering, the file on disk - may not reflect the data written until close() is called. - """ - with self._lock: - self._check_can_write() - compressed = self._compressor.compress(data) - self._fp.write(compressed) - self._pos += len(data) - return len(data) - - def writelines(self, seq): - """Write a sequence of byte strings to the file. - - Returns the number of uncompressed bytes written. - seq can be any iterable yielding byte strings. - - Line separators are not added between the written byte strings. - """ - with self._lock: - return io.BufferedIOBase.writelines(self, seq) - - # Rewind the file to the beginning of the data stream. - def _rewind(self): - self._fp.seek(0, 0) - self._mode = _MODE_READ - self._pos = 0 - self._decompressor = BZ2Decompressor() - self._buffer = b"" - self._buffer_offset = 0 - - def seek(self, offset, whence=0): - """Change the file position. - - The new position is specified by offset, relative to the - position indicated by whence. Values for whence are: - - 0: start of stream (default); offset must not be negative - 1: current stream position - 2: end of stream; offset must not be positive - - Returns the new file position. - - Note that seeking is emulated, so depending on the parameters, - this operation may be extremely slow. - """ - with self._lock: - self._check_can_seek() - - # Recalculate offset as an absolute file position. - if whence == 0: - pass - elif whence == 1: - offset = self._pos + offset - elif whence == 2: - # Seeking relative to EOF - we need to know the file's size. - if self._size < 0: - self._read_all(return_data=False) - offset = self._size + offset - else: - raise ValueError("Invalid value for whence: {}".format(whence)) - - # Make it so that offset is the number of bytes to skip forward. - if offset < self._pos: - self._rewind() - else: - offset -= self._pos - - # Read and discard data until we reach the desired position. - self._read_block(offset, return_data=False) - - return self._pos - - def tell(self): - """Return the current file position.""" - with self._lock: - self._check_not_closed() - return self._pos - - -def open(filename, mode="rb", compresslevel=9, - encoding=None, errors=None, newline=None): - """Open a bzip2-compressed file in binary or text mode. - - The filename argument can be an actual filename (a str or bytes object), or - an existing file object to read from or write to. - - The mode argument can be "r", "rb", "w", "wb", "a" or "ab" for binary mode, - or "rt", "wt" or "at" for text mode. The default mode is "rb", and the - default compresslevel is 9. - - For binary mode, this function is equivalent to the BZ2File constructor: - BZ2File(filename, mode, compresslevel). In this case, the encoding, errors - and newline arguments must not be provided. - - For text mode, a BZ2File object is created, and wrapped in an - io.TextIOWrapper instance with the specified encoding, error handling - behavior, and line ending(s). - - """ - if "t" in mode: - if "b" in mode: - raise ValueError("Invalid mode: %r" % (mode,)) - else: - if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") - if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") - if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") - - bz_mode = mode.replace("t", "") - binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel) - - if "t" in mode: - return io.TextIOWrapper(binary_file, encoding, errors, newline) - else: - return binary_file - - -def compress(data, compresslevel=9): - """Compress a block of data. - - compresslevel, if given, must be a number between 1 and 9. - - For incremental compression, use a BZ2Compressor object instead. - """ - comp = BZ2Compressor(compresslevel) - return comp.compress(data) + comp.flush() - - -def decompress(data): - """Decompress a block of data. - - For incremental decompression, use a BZ2Decompressor object instead. - """ - if len(data) == 0: - return b"" - - results = [] - while True: - decomp = BZ2Decompressor() - results.append(decomp.decompress(data)) - if not decomp.eof: - raise ValueError("Compressed data ended before the " - "end-of-stream marker was reached") - if not decomp.unused_data: - return b"".join(results) - # There is unused data left over. Proceed to next stream. - data = decomp.unused_data diff --git a/tests/bytecode/pylib-tests/cProfile.py b/tests/bytecode/pylib-tests/cProfile.py deleted file mode 100644 index c24d45bab4..0000000000 --- a/tests/bytecode/pylib-tests/cProfile.py +++ /dev/null @@ -1,195 +0,0 @@ -#! /usr/bin/env python3 - -"""Python interface for the 'lsprof' profiler. - Compatible with the 'profile' module. -""" - -__all__ = ["run", "runctx", "Profile"] - -import _lsprof - -# ____________________________________________________________ -# Simple interface - -def run(statement, filename=None, sort=-1): - """Run statement under profiler optionally saving results in filename - - This function takes a single argument that can be passed to the - "exec" statement, and an optional file name. In all cases this - routine attempts to "exec" its first argument and gather profiling - statistics from the execution. If no file name is present, then this - function automatically prints a simple profiling report, sorted by the - standard name string (file/line/function-name) that is presented in - each line. - """ - prof = Profile() - result = None - try: - try: - prof = prof.run(statement) - except SystemExit: - pass - finally: - if filename is not None: - prof.dump_stats(filename) - else: - result = prof.print_stats(sort) - return result - -def runctx(statement, globals, locals, filename=None, sort=-1): - """Run statement under profiler, supplying your own globals and locals, - optionally saving results in filename. - - statement and filename have the same semantics as profile.run - """ - prof = Profile() - result = None - try: - try: - prof = prof.runctx(statement, globals, locals) - except SystemExit: - pass - finally: - if filename is not None: - prof.dump_stats(filename) - else: - result = prof.print_stats(sort) - return result - -# ____________________________________________________________ - -class Profile(_lsprof.Profiler): - """Profile(custom_timer=None, time_unit=None, subcalls=True, builtins=True) - - Builds a profiler object using the specified timer function. - The default timer is a fast built-in one based on real time. - For custom timer functions returning integers, time_unit can - be a float specifying a scale (i.e. how long each integer unit - is, in seconds). - """ - - # Most of the functionality is in the base class. - # This subclass only adds convenient and backward-compatible methods. - - def print_stats(self, sort=-1): - import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats() - - def dump_stats(self, file): - import marshal - f = open(file, 'wb') - self.create_stats() - marshal.dump(self.stats, f) - f.close() - - def create_stats(self): - self.disable() - self.snapshot_stats() - - def snapshot_stats(self): - entries = self.getstats() - self.stats = {} - callersdicts = {} - # call information - for entry in entries: - func = label(entry.code) - nc = entry.callcount # ncalls column of pstats (before '/') - cc = nc - entry.reccallcount # ncalls column of pstats (after '/') - tt = entry.inlinetime # tottime column of pstats - ct = entry.totaltime # cumtime column of pstats - callers = {} - callersdicts[id(entry.code)] = callers - self.stats[func] = cc, nc, tt, ct, callers - # subcall information - for entry in entries: - if entry.calls: - func = label(entry.code) - for subentry in entry.calls: - try: - callers = callersdicts[id(subentry.code)] - except KeyError: - continue - nc = subentry.callcount - cc = nc - subentry.reccallcount - tt = subentry.inlinetime - ct = subentry.totaltime - if func in callers: - prev = callers[func] - nc += prev[0] - cc += prev[1] - tt += prev[2] - ct += prev[3] - callers[func] = nc, cc, tt, ct - - # The following two methods can be called by clients to use - # a profiler to profile a statement, given as a string. - - def run(self, cmd): - import __main__ - dict = __main__.__dict__ - return self.runctx(cmd, dict, dict) - - def runctx(self, cmd, globals, locals): - self.enable() - try: - exec(cmd, globals, locals) - finally: - self.disable() - return self - - # This method is more useful to profile a single function call. - def runcall(self, func, *args, **kw): - self.enable() - try: - return func(*args, **kw) - finally: - self.disable() - -# ____________________________________________________________ - -def label(code): - if isinstance(code, str): - return ('~', 0, code) # built-in functions ('~' sorts at the end) - else: - return (code.co_filename, code.co_firstlineno, code.co_name) - -# ____________________________________________________________ - -def main(): - import os, sys - from optparse import OptionParser - usage = "cProfile.py [-o output_file_path] [-s sort] scriptfile [arg] ..." - parser = OptionParser(usage=usage) - parser.allow_interspersed_args = False - parser.add_option('-o', '--outfile', dest="outfile", - help="Save stats to ", default=None) - parser.add_option('-s', '--sort', dest="sort", - help="Sort order when printing to stdout, based on pstats.Stats class", - default=-1) - - if not sys.argv[1:]: - parser.print_usage() - sys.exit(2) - - (options, args) = parser.parse_args() - sys.argv[:] = args - - if len(args) > 0: - progname = args[0] - sys.path.insert(0, os.path.dirname(progname)) - with open(progname, 'rb') as fp: - code = compile(fp.read(), progname, 'exec') - globs = { - '__file__': progname, - '__name__': '__main__', - '__package__': None, - '__cached__': None, - } - runctx(code, globs, None, options.outfile, options.sort) - else: - parser.print_usage() - return parser - -# When invoked as main program, invoke the profiler on a script -if __name__ == '__main__': - main() diff --git a/tests/bytecode/pylib-tests/chunk.py b/tests/bytecode/pylib-tests/chunk.py deleted file mode 100644 index 5863ed0846..0000000000 --- a/tests/bytecode/pylib-tests/chunk.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Simple class to read IFF chunks. - -An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File -Format)) has the following structure: - -+----------------+ -| ID (4 bytes) | -+----------------+ -| size (4 bytes) | -+----------------+ -| data | -| ... | -+----------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) -gives the size of the whole chunk, including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed -usage of the Chunk class defined here is to instantiate an instance at -the start of each chunk and read from the instance until it reaches -the end, after which a new instance can be instantiated. At the end -of the file, creating a new instance will fail with a EOFError -exception. - -Usage: -while True: - try: - chunk = Chunk(file) - except EOFError: - break - chunktype = chunk.getname() - while True: - data = chunk.read(nbytes) - if not data: - pass - # do something with data - -The interface is file-like. The implemented methods are: -read, close, seek, tell, isatty. -Extra methods are: skip() (called by close, skips to the end of the chunk), -getname() (returns the name (ID) of the chunk) - -The __init__ method has one required argument, a file-like object -(including a chunk instance), and one optional argument, a flag which -specifies whether or not chunks are aligned on 2-byte boundaries. The -default is 1, i.e. aligned. -""" - -class Chunk: - def __init__(self, file, align=True, bigendian=True, inclheader=False): - import struct - self.closed = False - self.align = align # whether to align to word (2-byte) boundaries - if bigendian: - strflag = '>' - else: - strflag = '<' - self.file = file - self.chunkname = file.read(4) - if len(self.chunkname) < 4: - raise EOFError - try: - self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0] - except struct.error: - raise EOFError - if inclheader: - self.chunksize = self.chunksize - 8 # subtract header - self.size_read = 0 - try: - self.offset = self.file.tell() - except (AttributeError, IOError): - self.seekable = False - else: - self.seekable = True - - def getname(self): - """Return the name (ID) of the current chunk.""" - return self.chunkname - - def getsize(self): - """Return the size of the current chunk.""" - return self.chunksize - - def close(self): - if not self.closed: - self.skip() - self.closed = True - - def isatty(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return False - - def seek(self, pos, whence=0): - """Seek to specified position into the chunk. - Default position is 0 (start of chunk). - If the file is not seekable, this will result in an error. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if not self.seekable: - raise IOError("cannot seek") - if whence == 1: - pos = pos + self.size_read - elif whence == 2: - pos = pos + self.chunksize - if pos < 0 or pos > self.chunksize: - raise RuntimeError - self.file.seek(self.offset + pos, 0) - self.size_read = pos - - def tell(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return self.size_read - - def read(self, size=-1): - """Read at most size bytes from the chunk. - If size is omitted or negative, read until the end - of the chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.size_read >= self.chunksize: - return '' - if size < 0: - size = self.chunksize - self.size_read - if size > self.chunksize - self.size_read: - size = self.chunksize - self.size_read - data = self.file.read(size) - self.size_read = self.size_read + len(data) - if self.size_read == self.chunksize and \ - self.align and \ - (self.chunksize & 1): - dummy = self.file.read(1) - self.size_read = self.size_read + len(dummy) - return data - - def skip(self): - """Skip the rest of the chunk. - If you are not interested in the contents of the chunk, - this method should be called so that the file points to - the start of the next chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.seekable: - try: - n = self.chunksize - self.size_read - # maybe fix alignment - if self.align and (self.chunksize & 1): - n = n + 1 - self.file.seek(n, 1) - self.size_read = self.size_read + n - return - except IOError: - pass - while self.size_read < self.chunksize: - n = min(8192, self.chunksize - self.size_read) - dummy = self.read(n) - if not dummy: - raise EOFError diff --git a/tests/bytecode/pylib-tests/code.py b/tests/bytecode/pylib-tests/code.py deleted file mode 100644 index 9020aab701..0000000000 --- a/tests/bytecode/pylib-tests/code.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Utilities needed to emulate Python's interactive interpreter. - -""" - -# Inspired by similar code by Jeff Epler and Fredrik Lundh. - - -import sys -import traceback -from codeop import CommandCompiler, compile_command - -__all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", - "compile_command"] - -class InteractiveInterpreter: - """Base class for InteractiveConsole. - - This class deals with parsing and interpreter state (the user's - namespace); it doesn't deal with input buffering or prompting or - input file naming (the filename is always passed in explicitly). - - """ - - def __init__(self, locals=None): - """Constructor. - - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. - - """ - if locals is None: - locals = {"__name__": "__console__", "__doc__": None} - self.locals = locals - self.compile = CommandCompiler() - - def runsource(self, source, filename="", symbol="single"): - """Compile and run some source in the interpreter. - - Arguments are as for compile_command(). - - One several things can happen: - - 1) The input is incorrect; compile_command() raised an - exception (SyntaxError or OverflowError). A syntax traceback - will be printed by calling the showsyntaxerror() method. - - 2) The input is incomplete, and more input is required; - compile_command() returned None. Nothing happens. - - 3) The input is complete; compile_command() returned a code - object. The code is executed by calling self.runcode() (which - also handles run-time exceptions, except for SystemExit). - - The return value is True in case 2, False in the other cases (unless - an exception is raised). The return value can be used to - decide whether to use sys.ps1 or sys.ps2 to prompt the next - line. - - """ - try: - code = self.compile(source, filename, symbol) - except (OverflowError, SyntaxError, ValueError): - # Case 1 - self.showsyntaxerror(filename) - return False - - if code is None: - # Case 2 - return True - - # Case 3 - self.runcode(code) - return False - - def runcode(self, code): - """Execute a code object. - - When an exception occurs, self.showtraceback() is called to - display a traceback. All exceptions are caught except - SystemExit, which is reraised. - - A note about KeyboardInterrupt: this exception may occur - elsewhere in this code, and may not always be caught. The - caller should be prepared to deal with it. - - """ - try: - exec(code, self.locals) - except SystemExit: - raise - except: - self.showtraceback() - - def showsyntaxerror(self, filename=None): - """Display the syntax error that just occurred. - - This doesn't display a stack trace because there isn't one. - - If a filename is given, it is stuffed in the exception instead - of what was there before (because Python's parser always uses - "" when reading from a string). - - The output is written by self.write(), below. - - """ - type, value, tb = sys.exc_info() - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) - - def showtraceback(self): - """Display the exception that just occurred. - - We remove the first stack item because it is our own code. - - The output is written by self.write(), below. - - """ - try: - type, value, tb = sys.exc_info() - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - tblist = traceback.extract_tb(tb) - del tblist[:1] - lines = traceback.format_list(tblist) - if lines: - lines.insert(0, "Traceback (most recent call last):\n") - lines.extend(traceback.format_exception_only(type, value)) - finally: - tblist = tb = None - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) - - def write(self, data): - """Write a string. - - The base implementation writes to sys.stderr; a subclass may - replace this with a different implementation. - - """ - sys.stderr.write(data) - - -class InteractiveConsole(InteractiveInterpreter): - """Closely emulate the behavior of the interactive Python interpreter. - - This class builds on InteractiveInterpreter and adds prompting - using the familiar sys.ps1 and sys.ps2, and input buffering. - - """ - - def __init__(self, locals=None, filename=""): - """Constructor. - - The optional locals argument will be passed to the - InteractiveInterpreter base class. - - The optional filename argument should specify the (file)name - of the input stream; it will show up in tracebacks. - - """ - InteractiveInterpreter.__init__(self, locals) - self.filename = filename - self.resetbuffer() - - def resetbuffer(self): - """Reset the input buffer.""" - self.buffer = [] - - def interact(self, banner=None): - """Closely emulate the interactive Python console. - - The optional banner argument specifies the banner to print - before the first interaction; by default it prints a banner - similar to the one printed by the real Python interpreter, - followed by the current class name in parentheses (so as not - to confuse this with the real interpreter -- since it's so - close!). - - """ - try: - sys.ps1 - except AttributeError: - sys.ps1 = ">>> " - try: - sys.ps2 - except AttributeError: - sys.ps2 = "... " - cprt = 'Type "help", "copyright", "credits" or "license" for more information.' - if banner is None: - self.write("Python %s on %s\n%s\n(%s)\n" % - (sys.version, sys.platform, cprt, - self.__class__.__name__)) - else: - self.write("%s\n" % str(banner)) - more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 - try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - - def push(self, line): - """Push a line to the interpreter. - - The line should not have a trailing newline; it may have - internal newlines. The line is appended to a buffer and the - interpreter's runsource() method is called with the - concatenated contents of the buffer as source. If this - indicates that the command was executed or invalid, the buffer - is reset; otherwise, the command is incomplete, and the buffer - is left as it was after the line was appended. The return - value is 1 if more input is required, 0 if the line was dealt - with in some way (this is the same as runsource()). - - """ - self.buffer.append(line) - source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) - if not more: - self.resetbuffer() - return more - - def raw_input(self, prompt=""): - """Write a prompt and read a line. - - The returned line does not include the trailing newline. - When the user enters the EOF key sequence, EOFError is raised. - - The base implementation uses the built-in function - input(); a subclass may replace this with a different - implementation. - - """ - return input(prompt) - - - -def interact(banner=None, readfunc=None, local=None): - """Closely emulate the interactive Python interpreter. - - This is a backwards compatible interface to the InteractiveConsole - class. When readfunc is not specified, it attempts to import the - readline module to enable GNU readline if it is available. - - Arguments (all optional, all default to None): - - banner -- passed to InteractiveConsole.interact() - readfunc -- if not None, replaces InteractiveConsole.raw_input() - local -- passed to InteractiveInterpreter.__init__() - - """ - console = InteractiveConsole(local) - if readfunc is not None: - console.raw_input = readfunc - else: - try: - import readline - except ImportError: - pass - console.interact(banner) - - -if __name__ == "__main__": - interact() diff --git a/tests/bytecode/pylib-tests/compileall.py b/tests/bytecode/pylib-tests/compileall.py deleted file mode 100644 index d92a322cf6..0000000000 --- a/tests/bytecode/pylib-tests/compileall.py +++ /dev/null @@ -1,240 +0,0 @@ -"""Module/script to byte-compile all .py files to .pyc (or .pyo) files. - -When called as a script with arguments, this compiles the directories -given as arguments recursively; the -l option prevents it from -recursing into directories. - -Without arguments, if compiles all modules on sys.path, without -recursing into subdirectories. (Even though it should do so for -packages -- for now, you'll have to deal with packages separately.) - -See module py_compile for details of the actual byte-compilation. -""" -import os -import sys -import errno -import imp -import py_compile -import struct - -__all__ = ["compile_dir","compile_file","compile_path"] - -def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=False, legacy=False, optimize=-1): - """Byte-compile all modules in the given directory tree. - - Arguments (only dir is required): - - dir: the directory to byte-compile - maxlevels: maximum recursion level (default 10) - ddir: the directory that will be prepended to the path to the - file as it is compiled into each byte-code file. - force: if True, force compilation, even if timestamps are up-to-date - quiet: if True, be quiet during compilation - legacy: if True, produce legacy pyc paths instead of PEP 3147 paths - optimize: optimization level or -1 for level of the interpreter - """ - if not quiet: - print('Listing {!r}...'.format(dir)) - try: - names = os.listdir(dir) - except os.error: - print("Can't list {!r}".format(dir)) - names = [] - names.sort() - success = 1 - for name in names: - if name == '__pycache__': - continue - fullname = os.path.join(dir, name) - if ddir is not None: - dfile = os.path.join(ddir, name) - else: - dfile = None - if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet, - legacy, optimize): - success = 0 - elif (maxlevels > 0 and name != os.curdir and name != os.pardir and - os.path.isdir(fullname) and not os.path.islink(fullname)): - if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, - quiet, legacy, optimize): - success = 0 - return success - -def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, - legacy=False, optimize=-1): - """Byte-compile one file. - - Arguments (only fullname is required): - - fullname: the file to byte-compile - ddir: if given, the directory name compiled in to the - byte-code file. - force: if True, force compilation, even if timestamps are up-to-date - quiet: if True, be quiet during compilation - legacy: if True, produce legacy pyc paths instead of PEP 3147 paths - optimize: optimization level or -1 for level of the interpreter - """ - success = 1 - name = os.path.basename(fullname) - if ddir is not None: - dfile = os.path.join(ddir, name) - else: - dfile = None - if rx is not None: - mo = rx.search(fullname) - if mo: - return success - if os.path.isfile(fullname): - if legacy: - cfile = fullname + 'c' - else: - if optimize >= 0: - cfile = imp.cache_from_source(fullname, - debug_override=not optimize) - else: - cfile = imp.cache_from_source(fullname) - cache_dir = os.path.dirname(cfile) - head, tail = name[:-3], name[-3:] - if tail == '.py': - if not force: - try: - mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack('<4sl', imp.get_magic(), mtime) - with open(cfile, 'rb') as chandle: - actual = chandle.read(8) - if expect == actual: - return success - except IOError: - pass - if not quiet: - print('Compiling {!r}...'.format(fullname)) - try: - ok = py_compile.compile(fullname, cfile, dfile, True, - optimize=optimize) - except py_compile.PyCompileError as err: - if quiet: - print('*** Error compiling {!r}...'.format(fullname)) - else: - print('*** ', end='') - # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, - errors='backslashreplace') - msg = msg.decode(sys.stdout.encoding) - print(msg) - success = 0 - except (SyntaxError, UnicodeError, IOError) as e: - if quiet: - print('*** Error compiling {!r}...'.format(fullname)) - else: - print('*** ', end='') - print(e.__class__.__name__ + ':', e) - success = 0 - else: - if ok == 0: - success = 0 - return success - -def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, - legacy=False, optimize=-1): - """Byte-compile all module on sys.path. - - Arguments (all optional): - - skip_curdir: if true, skip current directory (default True) - maxlevels: max recursion level (default 0) - force: as for compile_dir() (default False) - quiet: as for compile_dir() (default False) - legacy: as for compile_dir() (default False) - optimize: as for compile_dir() (default -1) - """ - success = 1 - for dir in sys.path: - if (not dir or dir == os.curdir) and skip_curdir: - print('Skipping current directory') - else: - success = success and compile_dir(dir, maxlevels, None, - force, quiet=quiet, - legacy=legacy, optimize=optimize) - return success - - -def main(): - """Script main program.""" - import argparse - - parser = argparse.ArgumentParser( - description='Utilities to support installing Python libraries.') - parser.add_argument('-l', action='store_const', const=0, - default=10, dest='maxlevels', - help="don't recurse into subdirectories") - parser.add_argument('-f', action='store_true', dest='force', - help='force rebuild even if timestamps are up to date') - parser.add_argument('-q', action='store_true', dest='quiet', - help='output only error messages') - parser.add_argument('-b', action='store_true', dest='legacy', - help='use legacy (pre-PEP3147) compiled file locations') - parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, - help=('directory to prepend to file paths for use in ' - 'compile-time tracebacks and in runtime ' - 'tracebacks in cases where the source file is ' - 'unavailable')) - parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, - help=('skip files matching the regular expression; ' - 'the regexp is searched for in the full path ' - 'of each file considered for compilation')) - parser.add_argument('-i', metavar='FILE', dest='flist', - help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation; ' - 'if "-", names are read from stdin')) - parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', - help=('zero or more file and directory names ' - 'to compile; if no arguments given, defaults ' - 'to the equivalent of -l sys.path')) - args = parser.parse_args() - - compile_dests = args.compile_dest - - if (args.ddir and (len(compile_dests) != 1 - or not os.path.isdir(compile_dests[0]))): - parser.exit('-d destdir requires exactly one directory argument') - if args.rx: - import re - args.rx = re.compile(args.rx) - - # if flist is provided then load it - if args.flist: - try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: - for line in f: - compile_dests.append(line.strip()) - except EnvironmentError: - print("Error reading file list {}".format(args.flist)) - return False - - success = True - try: - if compile_dests: - for dest in compile_dests: - if os.path.isfile(dest): - if not compile_file(dest, args.ddir, args.force, args.rx, - args.quiet, args.legacy): - success = False - else: - if not compile_dir(dest, args.maxlevels, args.ddir, - args.force, args.rx, args.quiet, - args.legacy): - success = False - return success - else: - return compile_path(legacy=args.legacy) - except KeyboardInterrupt: - print("\n[interrupted]") - return False - return True - - -if __name__ == '__main__': - exit_status = int(not main()) - sys.exit(exit_status) diff --git a/tests/bytecode/pylib-tests/contextlib.py b/tests/bytecode/pylib-tests/contextlib.py deleted file mode 100644 index 0b6bf71b08..0000000000 --- a/tests/bytecode/pylib-tests/contextlib.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Utilities for with-statement contexts. See PEP 343.""" - -import sys -from collections import deque -from functools import wraps - -__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack"] - - -class ContextDecorator(object): - "A base class or mixin that enables context managers to work as decorators." - - def _recreate_cm(self): - """Return a recreated instance of self. - - Allows an otherwise one-shot context manager like - _GeneratorContextManager to support use as - a decorator via implicit recreation. - - This is a private interface just for _GeneratorContextManager. - See issue #11647 for details. - """ - return self - - def __call__(self, func): - @wraps(func) - def inner(*args, **kwds): - with self._recreate_cm(): - return func(*args, **kwds) - return inner - - -class _GeneratorContextManager(ContextDecorator): - """Helper for @contextmanager decorator.""" - - def __init__(self, func, *args, **kwds): - self.gen = func(*args, **kwds) - self.func, self.args, self.kwds = func, args, kwds - - def _recreate_cm(self): - # _GCM instances are one-shot context managers, so the - # CM must be recreated each time a decorated function is - # called - return self.__class__(self.func, *self.args, **self.kwds) - - def __enter__(self): - try: - return next(self.gen) - except StopIteration: - raise RuntimeError("generator didn't yield") - - def __exit__(self, type, value, traceback): - if type is None: - try: - next(self.gen) - except StopIteration: - return - else: - raise RuntimeError("generator didn't stop") - else: - if value is None: - # Need to force instantiation so we can reliably - # tell if we get the same exception back - value = type() - try: - self.gen.throw(type, value, traceback) - raise RuntimeError("generator didn't stop after throw()") - except StopIteration as exc: - # Suppress the exception *unless* it's the same exception that - # was passed to throw(). This prevents a StopIteration - # raised inside the "with" statement from being suppressed - return exc is not value - except: - # only re-raise if it's *not* the exception that was - # passed to throw(), because __exit__() must not raise - # an exception unless __exit__() itself failed. But throw() - # has to raise the exception to signal propagation, so this - # fixes the impedance mismatch between the throw() protocol - # and the __exit__() protocol. - # - if sys.exc_info()[1] is not value: - raise - - -def contextmanager(func): - """@contextmanager decorator. - - Typical usage: - - @contextmanager - def some_generator(): - - try: - yield - finally: - - - This makes this: - - with some_generator() as : - - - equivalent to this: - - - try: - = - - finally: - - - """ - @wraps(func) - def helper(*args, **kwds): - return _GeneratorContextManager(func, *args, **kwds) - return helper - - -class closing(object): - """Context to automatically close something at the end of a block. - - Code like this: - - with closing(.open()) as f: - - - is equivalent to this: - - f = .open() - try: - - finally: - f.close() - - """ - def __init__(self, thing): - self.thing = thing - def __enter__(self): - return self.thing - def __exit__(self, *exc_info): - self.thing.close() - - -# Inspired by discussions on http://bugs.python.org/issue13585 -class ExitStack(object): - """Context manager for dynamic management of a stack of exit callbacks - - For example: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list raise an exception - - """ - def __init__(self): - self._exit_callbacks = deque() - - def pop_all(self): - """Preserve the context stack by transferring it to a new instance""" - new_stack = type(self)() - new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() - return new_stack - - def _push_cm_exit(self, cm, cm_exit): - """Helper to correctly register callbacks to __exit__ methods""" - def _exit_wrapper(*exc_details): - return cm_exit(cm, *exc_details) - _exit_wrapper.__self__ = cm - self.push(_exit_wrapper) - - def push(self, exit): - """Registers a callback with the standard __exit__ method signature - - Can suppress exceptions the same way __exit__ methods can. - - Also accepts any object with an __exit__ method (registering a call - to the method instead of the object itself) - """ - # We use an unbound method rather than a bound method to follow - # the standard lookup behaviour for special methods - _cb_type = type(exit) - try: - exit_method = _cb_type.__exit__ - except AttributeError: - # Not a context manager, so assume its a callable - self._exit_callbacks.append(exit) - else: - self._push_cm_exit(exit, exit_method) - return exit # Allow use as a decorator - - def callback(self, callback, *args, **kwds): - """Registers an arbitrary callback and arguments. - - Cannot suppress exceptions. - """ - def _exit_wrapper(exc_type, exc, tb): - callback(*args, **kwds) - # We changed the signature, so using @wraps is not appropriate, but - # setting __wrapped__ may still help with introspection - _exit_wrapper.__wrapped__ = callback - self.push(_exit_wrapper) - return callback # Allow use as a decorator - - def enter_context(self, cm): - """Enters the supplied context manager - - If successful, also pushes its __exit__ method as a callback and - returns the result of the __enter__ method. - """ - # We look up the special methods on the type to match the with statement - _cm_type = type(cm) - _exit = _cm_type.__exit__ - result = _cm_type.__enter__(cm) - self._push_cm_exit(cm, _exit) - return result - - def close(self): - """Immediately unwind the context stack""" - self.__exit__(None, None, None) - - def __enter__(self): - return self - - def __exit__(self, *exc_details): - # We manipulate the exception state so it behaves as though - # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] - def _fix_exception_context(new_exc, old_exc): - while 1: - exc_context = new_exc.__context__ - if exc_context in (None, frame_exc): - break - new_exc = exc_context - new_exc.__context__ = old_exc - - # Callbacks are invoked in LIFO order to match the behaviour of - # nested context managers - suppressed_exc = False - while self._exit_callbacks: - cb = self._exit_callbacks.pop() - try: - if cb(*exc_details): - suppressed_exc = True - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() - # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) - if not self._exit_callbacks: - raise - exc_details = new_exc_details - return suppressed_exc diff --git a/tests/bytecode/pylib-tests/crypt.py b/tests/bytecode/pylib-tests/crypt.py deleted file mode 100644 index b90c81cc40..0000000000 --- a/tests/bytecode/pylib-tests/crypt.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Wrapper to the POSIX crypt library call and associated functionality.""" - -import _crypt -import string as _string -from random import SystemRandom as _SystemRandom -from collections import namedtuple as _namedtuple - - -_saltchars = _string.ascii_letters + _string.digits + './' -_sr = _SystemRandom() - - -class _Method(_namedtuple('_Method', 'name ident salt_chars total_size')): - - """Class representing a salt method per the Modular Crypt Format or the - legacy 2-character crypt method.""" - - def __repr__(self): - return ''.format(self.name) - - -def mksalt(method=None): - """Generate a salt for the specified method. - - If not specified, the strongest available method will be used. - - """ - if method is None: - method = methods[0] - s = '${}$'.format(method.ident) if method.ident else '' - s += ''.join(_sr.sample(_saltchars, method.salt_chars)) - return s - - -def crypt(word, salt=None): - """Return a string representing the one-way hash of a password, with a salt - prepended. - - If ``salt`` is not specified or is ``None``, the strongest - available method will be selected and a salt generated. Otherwise, - ``salt`` may be one of the ``crypt.METHOD_*`` values, or a string as - returned by ``crypt.mksalt()``. - - """ - if salt is None or isinstance(salt, _Method): - salt = mksalt(salt) - return _crypt.crypt(word, salt) - - -# available salting/crypto methods -METHOD_CRYPT = _Method('CRYPT', None, 2, 13) -METHOD_MD5 = _Method('MD5', '1', 8, 34) -METHOD_SHA256 = _Method('SHA256', '5', 16, 63) -METHOD_SHA512 = _Method('SHA512', '6', 16, 106) - -methods = [] -for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): - _result = crypt('', _method) - if _result and len(_result) == _method.total_size: - methods.append(_method) -methods.append(METHOD_CRYPT) -del _result, _method diff --git a/tests/bytecode/pylib-tests/dummy_threading.py b/tests/bytecode/pylib-tests/dummy_threading.py deleted file mode 100644 index 1bb7eee338..0000000000 --- a/tests/bytecode/pylib-tests/dummy_threading.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Faux ``threading`` version using ``dummy_thread`` instead of ``thread``. - -The module ``_dummy_threading`` is added to ``sys.modules`` in order -to not have ``threading`` considered imported. Had ``threading`` been -directly imported it would have made all subsequent imports succeed -regardless of whether ``_thread`` was available which is not desired. - -""" -from sys import modules as sys_modules - -import _dummy_thread - -# Declaring now so as to not have to nest ``try``s to get proper clean-up. -holding_thread = False -holding_threading = False -holding__threading_local = False - -try: - # Could have checked if ``_thread`` was not in sys.modules and gone - # a different route, but decided to mirror technique used with - # ``threading`` below. - if '_thread' in sys_modules: - held_thread = sys_modules['_thread'] - holding_thread = True - # Must have some module named ``_thread`` that implements its API - # in order to initially import ``threading``. - sys_modules['_thread'] = sys_modules['_dummy_thread'] - - if 'threading' in sys_modules: - # If ``threading`` is already imported, might as well prevent - # trying to import it more than needed by saving it if it is - # already imported before deleting it. - held_threading = sys_modules['threading'] - holding_threading = True - del sys_modules['threading'] - - if '_threading_local' in sys_modules: - # If ``_threading_local`` is already imported, might as well prevent - # trying to import it more than needed by saving it if it is - # already imported before deleting it. - held__threading_local = sys_modules['_threading_local'] - holding__threading_local = True - del sys_modules['_threading_local'] - - import threading - # Need a copy of the code kept somewhere... - sys_modules['_dummy_threading'] = sys_modules['threading'] - del sys_modules['threading'] - sys_modules['_dummy__threading_local'] = sys_modules['_threading_local'] - del sys_modules['_threading_local'] - from _dummy_threading import * - from _dummy_threading import __all__ - -finally: - # Put back ``threading`` if we overwrote earlier - - if holding_threading: - sys_modules['threading'] = held_threading - del held_threading - del holding_threading - - # Put back ``_threading_local`` if we overwrote earlier - - if holding__threading_local: - sys_modules['_threading_local'] = held__threading_local - del held__threading_local - del holding__threading_local - - # Put back ``thread`` if we overwrote, else del the entry we made - if holding_thread: - sys_modules['_thread'] = held_thread - del held_thread - else: - del sys_modules['_thread'] - del holding_thread - - del _dummy_thread - del sys_modules diff --git a/tests/bytecode/pylib-tests/fnmatch.py b/tests/bytecode/pylib-tests/fnmatch.py deleted file mode 100644 index 6330b0cfda..0000000000 --- a/tests/bytecode/pylib-tests/fnmatch.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Filename matching with shell patterns. - -fnmatch(FILENAME, PATTERN) matches according to the local convention. -fnmatchcase(FILENAME, PATTERN) always takes case in account. - -The functions operate by translating the pattern into a regular -expression. They cache the compiled regular expressions for speed. - -The function translate(PATTERN) returns a regular expression -corresponding to PATTERN. (It does not compile it.) -""" -import os -import posixpath -import re -import functools - -__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] - -def fnmatch(name, pat): - """Test whether FILENAME matches PATTERN. - - Patterns are Unix shell style: - - * matches everything - ? matches any single character - [seq] matches any character in seq - [!seq] matches any char not in seq - - An initial period in FILENAME is not special. - Both FILENAME and PATTERN are first case-normalized - if the operating system requires it. - If you don't want this, use fnmatchcase(FILENAME, PATTERN). - """ - name = os.path.normcase(name) - pat = os.path.normcase(pat) - return fnmatchcase(name, pat) - -@functools.lru_cache(maxsize=256, typed=True) -def _compile_pattern(pat): - if isinstance(pat, bytes): - pat_str = str(pat, 'ISO-8859-1') - res_str = translate(pat_str) - res = bytes(res_str, 'ISO-8859-1') - else: - res = translate(pat) - return re.compile(res).match - -def filter(names, pat): - """Return the subset of the list NAMES that match PAT.""" - result = [] - pat = os.path.normcase(pat) - match = _compile_pattern(pat) - if os.path is posixpath: - # normcase on posix is NOP. Optimize it away from the loop. - for name in names: - if match(name): - result.append(name) - else: - for name in names: - if match(os.path.normcase(name)): - result.append(name) - return result - -def fnmatchcase(name, pat): - """Test whether FILENAME matches PATTERN, including case. - - This is a version of fnmatch() which doesn't case-normalize - its arguments. - """ - match = _compile_pattern(pat) - return match(name) is not None - - -def translate(pat): - """Translate a shell PATTERN to a regular expression. - - There is no way to quote meta-characters. - """ - - i, n = 0, len(pat) - res = '' - while i < n: - c = pat[i] - i = i+1 - if c == '*': - res = res + '.*' - elif c == '?': - res = res + '.' - elif c == '[': - j = i - if j < n and pat[j] == '!': - j = j+1 - if j < n and pat[j] == ']': - j = j+1 - while j < n and pat[j] != ']': - j = j+1 - if j >= n: - res = res + '\\[' - else: - stuff = pat[i:j].replace('\\','\\\\') - i = j+1 - if stuff[0] == '!': - stuff = '^' + stuff[1:] - elif stuff[0] == '^': - stuff = '\\' + stuff - res = '%s[%s]' % (res, stuff) - else: - res = res + re.escape(c) - return res + '\Z(?ms)' diff --git a/tests/bytecode/pylib-tests/genericpath.py b/tests/bytecode/pylib-tests/genericpath.py deleted file mode 100644 index 2174187a03..0000000000 --- a/tests/bytecode/pylib-tests/genericpath.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Path operations common to more than one OS -Do not use directly. The OS specific modules import the appropriate -functions from this module themselves. -""" -import os -import stat - -__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile'] - - -# Does a path exist? -# This is false for dangling symbolic links on systems that support them. -def exists(path): - """Test whether a path exists. Returns False for broken symbolic links""" - try: - os.stat(path) - except os.error: - return False - return True - - -# This follows symbolic links, so both islink() and isdir() can be true -# for the same path ono systems that support symlinks -def isfile(path): - """Test whether a path is a regular file""" - try: - st = os.stat(path) - except os.error: - return False - return stat.S_ISREG(st.st_mode) - - -# Is a path a directory? -# This follows symbolic links, so both islink() and isdir() -# can be true for the same path on systems that support symlinks -def isdir(s): - """Return true if the pathname refers to an existing directory.""" - try: - st = os.stat(s) - except os.error: - return False - return stat.S_ISDIR(st.st_mode) - - -def getsize(filename): - """Return the size of a file, reported by os.stat().""" - return os.stat(filename).st_size - - -def getmtime(filename): - """Return the last modification time of a file, reported by os.stat().""" - return os.stat(filename).st_mtime - - -def getatime(filename): - """Return the last access time of a file, reported by os.stat().""" - return os.stat(filename).st_atime - - -def getctime(filename): - """Return the metadata change time of a file, reported by os.stat().""" - return os.stat(filename).st_ctime - - -# Return the longest prefix of all list elements. -def commonprefix(m): - "Given a list of pathnames, returns the longest common leading component" - if not m: return '' - s1 = min(m) - s2 = max(m) - for i, c in enumerate(s1): - if c != s2[i]: - return s1[:i] - return s1 - -# Split a path in root and extension. -# The extension is everything starting at the last dot in the last -# pathname component; the root is everything before that. -# It is always true that root + ext == p. - -# Generic implementation of splitext, to be parametrized with -# the separators -def _splitext(p, sep, altsep, extsep): - """Split the extension from a pathname. - - Extension is everything from the last dot to the end, ignoring - leading dots. Returns "(root, ext)"; ext may be empty.""" - # NOTE: This code must work for text and bytes strings. - - sepIndex = p.rfind(sep) - if altsep: - altsepIndex = p.rfind(altsep) - sepIndex = max(sepIndex, altsepIndex) - - dotIndex = p.rfind(extsep) - if dotIndex > sepIndex: - # skip all leading dots - filenameIndex = sepIndex + 1 - while filenameIndex < dotIndex: - if p[filenameIndex:filenameIndex+1] != extsep: - return p[:dotIndex], p[dotIndex:] - filenameIndex += 1 - - return p, p[:0] diff --git a/tests/bytecode/pylib-tests/getopt.py b/tests/bytecode/pylib-tests/getopt.py deleted file mode 100644 index 3d6ecbddb9..0000000000 --- a/tests/bytecode/pylib-tests/getopt.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Parser for command line options. - -This module helps scripts to parse the command line arguments in -sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software -may be used as well via an optional third argument. This module -provides two functions and an exception: - -getopt() -- Parse command line options -gnu_getopt() -- Like getopt(), but allow option and non-option arguments -to be intermixed. -GetoptError -- exception (class) raised with 'opt' attribute, which is the -option involved with the exception. -""" - -# Long option support added by Lars Wirzenius . -# -# Gerrit Holl moved the string-based exceptions -# to class-based exceptions. -# -# Peter Åstrand added gnu_getopt(). -# -# TODO for gnu_getopt(): -# -# - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons -# - a option string with a W followed by semicolon should -# treat "-W foo" as "--foo" - -__all__ = ["GetoptError","error","getopt","gnu_getopt"] - -import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s - -class GetoptError(Exception): - opt = '' - msg = '' - def __init__(self, msg, opt=''): - self.msg = msg - self.opt = opt - Exception.__init__(self, msg, opt) - - def __str__(self): - return self.msg - -error = GetoptError # backward compatibility - -def getopt(args, shortopts, longopts = []): - """getopt(args, options[, long_options]) -> opts, args - - Parses command line options and parameter list. args is the - argument list to be parsed, without the leading reference to the - running program. Typically, this means "sys.argv[1:]". shortopts - is the string of option letters that the script wants to - recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If - specified, longopts is a list of strings with the names of the - long options which should be supported. The leading '--' - characters should not be included in the option name. Options - which require an argument should be followed by an equal sign - ('='). - - The return value consists of two elements: the first is a list of - (option, value) pairs; the second is the list of program arguments - left after the option list was stripped (this is a trailing slice - of the first argument). Each option-and-value pair returned has - the option as its first element, prefixed with a hyphen (e.g., - '-x'), and the option argument as its second element, or an empty - string if the option has no argument. The options occur in the - list in the same order in which they were found, thus allowing - multiple occurrences. Long and short options may be mixed. - - """ - - opts = [] - if type(longopts) == type(""): - longopts = [longopts] - else: - longopts = list(longopts) - while args and args[0].startswith('-') and args[0] != '-': - if args[0] == '--': - args = args[1:] - break - if args[0].startswith('--'): - opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) - else: - opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) - - return opts, args - -def gnu_getopt(args, shortopts, longopts = []): - """getopt(args, options[, long_options]) -> opts, args - - This function works like getopt(), except that GNU style scanning - mode is used by default. This means that option and non-option - arguments may be intermixed. The getopt() function stops - processing options as soon as a non-option argument is - encountered. - - If the first character of the option string is `+', or if the - environment variable POSIXLY_CORRECT is set, then option - processing stops as soon as a non-option argument is encountered. - - """ - - opts = [] - prog_args = [] - if isinstance(longopts, str): - longopts = [longopts] - else: - longopts = list(longopts) - - # Allow options after non-option arguments? - if shortopts.startswith('+'): - shortopts = shortopts[1:] - all_options_first = True - elif os.environ.get("POSIXLY_CORRECT"): - all_options_first = True - else: - all_options_first = False - - while args: - if args[0] == '--': - prog_args += args[1:] - break - - if args[0][:2] == '--': - opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) - elif args[0][:1] == '-' and args[0] != '-': - opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) - else: - if all_options_first: - prog_args += args - break - else: - prog_args.append(args[0]) - args = args[1:] - - return opts, prog_args - -def do_longs(opts, opt, longopts, args): - try: - i = opt.index('=') - except ValueError: - optarg = None - else: - opt, optarg = opt[:i], opt[i+1:] - - has_arg, opt = long_has_args(opt, longopts) - if has_arg: - if optarg is None: - if not args: - raise GetoptError(_('option --%s requires argument') % opt, opt) - optarg, args = args[0], args[1:] - elif optarg is not None: - raise GetoptError(_('option --%s must not have an argument') % opt, opt) - opts.append(('--' + opt, optarg or '')) - return opts, args - -# Return: -# has_arg? -# full option name -def long_has_args(opt, longopts): - possibilities = [o for o in longopts if o.startswith(opt)] - if not possibilities: - raise GetoptError(_('option --%s not recognized') % opt, opt) - # Is there an exact match? - if opt in possibilities: - return False, opt - elif opt + '=' in possibilities: - return True, opt - # No exact match, so better be unique. - if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError(_('option --%s not a unique prefix') % opt, opt) - assert len(possibilities) == 1 - unique_match = possibilities[0] - has_arg = unique_match.endswith('=') - if has_arg: - unique_match = unique_match[:-1] - return has_arg, unique_match - -def do_shorts(opts, optstring, shortopts, args): - while optstring != '': - opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': - if not args: - raise GetoptError(_('option -%s requires argument') % opt, - opt) - optstring, args = args[0], args[1:] - optarg, optstring = optstring, '' - else: - optarg = '' - opts.append(('-' + opt, optarg)) - return opts, args - -def short_has_arg(opt, shortopts): - for i in range(len(shortopts)): - if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) - raise GetoptError(_('option -%s not recognized') % opt, opt) - -if __name__ == '__main__': - import sys - print(getopt(sys.argv[1:], "a:b", ["alpha=", "beta"])) diff --git a/tests/bytecode/pylib-tests/hashlib.py b/tests/bytecode/pylib-tests/hashlib.py deleted file mode 100644 index 21454c7d30..0000000000 --- a/tests/bytecode/pylib-tests/hashlib.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) -# Licensed to PSF under a Contributor Agreement. -# - -__doc__ = """hashlib module - A common interface to many hash functions. - -new(name, data=b'') - returns a new hash object implementing the - given hash function; initializing the hash - using the given binary data. - -Named constructor functions are also available, these are faster -than using new(name): - -md5(), sha1(), sha224(), sha256(), sha384(), and sha512() - -More algorithms may be available on your platform but the above are guaranteed -to exist. See the algorithms_guaranteed and algorithms_available attributes -to find out what algorithm names can be passed to new(). - -NOTE: If you want the adler32 or crc32 hash functions they are available in -the zlib module. - -Choose your hash function wisely. Some have known collision weaknesses. -sha384 and sha512 will be slow on 32 bit platforms. - -Hash objects have these methods: - - update(arg): Update the hash object with the bytes in arg. Repeated calls - are equivalent to a single call with the concatenation of all - the arguments. - - digest(): Return the digest of the bytes passed to the update() method - so far. - - hexdigest(): Like digest() except the digest is returned as a unicode - object of double length, containing only hexadecimal digits. - - copy(): Return a copy (clone) of the hash object. This can be used to - efficiently compute the digests of strings that share a common - initial substring. - -For example, to obtain the digest of the string 'Nobody inspects the -spammish repetition': - - >>> import hashlib - >>> m = hashlib.md5() - >>> m.update(b"Nobody inspects") - >>> m.update(b" the spammish repetition") - >>> m.digest() - b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9' - -More condensed: - - >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() - 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' - -""" - -# This tuple and __get_builtin_constructor() must be modified if a new -# always available algorithm is added. -__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') - -algorithms_guaranteed = set(__always_supported) -algorithms_available = set(__always_supported) - -__all__ = __always_supported + ('new', 'algorithms_guaranteed', - 'algorithms_available') - - -def __get_builtin_constructor(name): - try: - if name in ('SHA1', 'sha1'): - import _sha1 - return _sha1.sha1 - elif name in ('MD5', 'md5'): - import _md5 - return _md5.md5 - elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): - import _sha256 - bs = name[3:] - if bs == '256': - return _sha256.sha256 - elif bs == '224': - return _sha256.sha224 - elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): - import _sha512 - bs = name[3:] - if bs == '512': - return _sha512.sha512 - elif bs == '384': - return _sha512.sha384 - except ImportError: - pass # no extension module, this hash is unsupported. - - raise ValueError('unsupported hash type ' + name) - - -def __get_openssl_constructor(name): - try: - f = getattr(_hashlib, 'openssl_' + name) - # Allow the C module to raise ValueError. The function will be - # defined but the hash not actually available thanks to OpenSSL. - f() - # Use the C function directly (very fast) - return f - except (AttributeError, ValueError): - return __get_builtin_constructor(name) - - -def __py_new(name, data=b''): - """new(name, data=b'') - Return a new hashing object using the named algorithm; - optionally initialized with data (which must be bytes). - """ - return __get_builtin_constructor(name)(data) - - -def __hash_new(name, data=b''): - """new(name, data=b'') - Return a new hashing object using the named algorithm; - optionally initialized with data (which must be bytes). - """ - try: - return _hashlib.new(name, data) - except ValueError: - # If the _hashlib module (OpenSSL) doesn't support the named - # hash, try using our builtin implementations. - # This allows for SHA224/256 and SHA384/512 support even though - # the OpenSSL library prior to 0.9.8 doesn't provide them. - return __get_builtin_constructor(name)(data) - - -try: - import _hashlib - new = __hash_new - __get_hash = __get_openssl_constructor - algorithms_available = algorithms_available.union( - _hashlib.openssl_md_meth_names) -except ImportError: - new = __py_new - __get_hash = __get_builtin_constructor - -for __func_name in __always_supported: - # try them all, some may not work due to the OpenSSL - # version not supporting that algorithm. - try: - globals()[__func_name] = __get_hash(__func_name) - except ValueError: - import logging - logging.exception('code for hash %s was not found.', __func_name) - -# Cleanup locals() -del __always_supported, __func_name, __get_hash -del __py_new, __hash_new, __get_openssl_constructor diff --git a/tests/bytecode/pylib-tests/heapq.py b/tests/bytecode/pylib-tests/heapq.py deleted file mode 100644 index 7834d06d57..0000000000 --- a/tests/bytecode/pylib-tests/heapq.py +++ /dev/null @@ -1,472 +0,0 @@ -"""Heap queue algorithm (a.k.a. priority queue). - -Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for -all k, counting elements from 0. For the sake of comparison, -non-existing elements are considered to be infinite. The interesting -property of a heap is that a[0] is always its smallest element. - -Usage: - -heap = [] # creates an empty heap -heappush(heap, item) # pushes a new item on the heap -item = heappop(heap) # pops the smallest item from the heap -item = heap[0] # smallest item on the heap without popping it -heapify(x) # transforms list into a heap, in-place, in linear time -item = heapreplace(heap, item) # pops and returns smallest item, and adds - # new item; the heap size is unchanged - -Our API differs from textbook heap algorithms as follows: - -- We use 0-based indexing. This makes the relationship between the - index for a node and the indexes for its children slightly less - obvious, but is more suitable since Python uses 0-based indexing. - -- Our heappop() method returns the smallest item, not the largest. - -These two make it possible to view the heap as a regular Python list -without surprises: heap[0] is the smallest item, and heap.sort() -maintains the heap invariant! -""" - -# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger - -__about__ = """Heap queues - -[explanation by Francois Pinard] - -Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for -all k, counting elements from 0. For the sake of comparison, -non-existing elements are considered to be infinite. The interesting -property of a heap is that a[0] is always its smallest element. - -The strange invariant above is meant to be an efficient memory -representation for a tournament. The numbers below are `k', not a[k]: - - 0 - - 1 2 - - 3 4 5 6 - - 7 8 9 10 11 12 13 14 - - 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 - - -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In -an usual binary tournament we see in sports, each cell is the winner -over the two cells it tops, and we can trace the winner down the tree -to see all opponents s/he had. However, in many computer applications -of such tournaments, we do not need to trace the history of a winner. -To be more memory efficient, when a winner is promoted, we try to -replace it by something else at a lower level, and the rule becomes -that a cell and the two cells it tops contain three different items, -but the top cell "wins" over the two topped cells. - -If this heap invariant is protected at all time, index 0 is clearly -the overall winner. The simplest algorithmic way to remove it and -find the "next" winner is to move some loser (let's say cell 30 in the -diagram above) into the 0 position, and then percolate this new 0 down -the tree, exchanging values, until the invariant is re-established. -This is clearly logarithmic on the total number of items in the tree. -By iterating over all items, you get an O(n ln n) sort. - -A nice feature of this sort is that you can efficiently insert new -items while the sort is going on, provided that the inserted items are -not "better" than the last 0'th element you extracted. This is -especially useful in simulation contexts, where the tree holds all -incoming events, and the "win" condition means the smallest scheduled -time. When an event schedule other events for execution, they are -scheduled into the future, so they can easily go into the heap. So, a -heap is a good structure for implementing schedulers (this is what I -used for my MIDI sequencer :-). - -Various structures for implementing schedulers have been extensively -studied, and heaps are good for this, as they are reasonably speedy, -the speed is almost constant, and the worst case is not much different -than the average case. However, there are other representations which -are more efficient overall, yet the worst cases might be terrible. - -Heaps are also very useful in big disk sorts. You most probably all -know that a big sort implies producing "runs" (which are pre-sorted -sequences, which size is usually related to the amount of CPU memory), -followed by a merging passes for these runs, which merging is often -very cleverly organised[1]. It is very important that the initial -sort produces the longest runs possible. Tournaments are a good way -to that. If, using all the memory available to hold a tournament, you -replace and percolate items that happen to fit the current run, you'll -produce runs which are twice the size of the memory for random input, -and much better for input fuzzily ordered. - -Moreover, if you output the 0'th item on disk and get an input which -may not fit in the current tournament (because the value "wins" over -the last output value), it cannot fit in the heap, so the size of the -heap decreases. The freed memory could be cleverly reused immediately -for progressively building a second heap, which grows at exactly the -same rate the first heap is melting. When the first heap completely -vanishes, you switch heaps and start a new run. Clever and quite -effective! - -In a word, heaps are useful memory structures to know. I use them in -a few applications, and I think it is good to keep a `heap' module -around. :-) - --------------------- -[1] The disk balancing algorithms which are current, nowadays, are -more annoying than clever, and this is a consequence of the seeking -capabilities of the disks. On devices which cannot seek, like big -tape drives, the story was quite different, and one had to be very -clever to ensure (far in advance) that each tape movement will be the -most effective possible (that is, will best participate at -"progressing" the merge). Some tapes were even able to read -backwards, and this was also used to avoid the rewinding time. -Believe me, real good tape sorts were quite spectacular to watch! -From all times, sorting has always been a Great Art! :-) -""" - -__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', - 'nlargest', 'nsmallest', 'heappushpop'] - -from itertools import islice, count, tee, chain - -def heappush(heap, item): - """Push item onto heap, maintaining the heap invariant.""" - heap.append(item) - _siftdown(heap, 0, len(heap)-1) - -def heappop(heap): - """Pop the smallest item off the heap, maintaining the heap invariant.""" - lastelt = heap.pop() # raises appropriate IndexError if heap is empty - if heap: - returnitem = heap[0] - heap[0] = lastelt - _siftup(heap, 0) - else: - returnitem = lastelt - return returnitem - -def heapreplace(heap, item): - """Pop and return the current smallest value, and add the new item. - - This is more efficient than heappop() followed by heappush(), and can be - more appropriate when using a fixed-size heap. Note that the value - returned may be larger than item! That constrains reasonable uses of - this routine unless written as part of a conditional replacement: - - if item > heap[0]: - item = heapreplace(heap, item) - """ - returnitem = heap[0] # raises appropriate IndexError if heap is empty - heap[0] = item - _siftup(heap, 0) - return returnitem - -def heappushpop(heap, item): - """Fast version of a heappush followed by a heappop.""" - if heap and heap[0] < item: - item, heap[0] = heap[0], item - _siftup(heap, 0) - return item - -def heapify(x): - """Transform list into a heap, in-place, in O(len(x)) time.""" - n = len(x) - # Transform bottom-up. The largest index there's any point to looking at - # is the largest with a child index in-range, so must have 2*i + 1 < n, - # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so - # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is - # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. - for i in reversed(range(n//2)): - _siftup(x, i) - -def _heappushpop_max(heap, item): - """Maxheap version of a heappush followed by a heappop.""" - if heap and item < heap[0]: - item, heap[0] = heap[0], item - _siftup_max(heap, 0) - return item - -def _heapify_max(x): - """Transform list into a maxheap, in-place, in O(len(x)) time.""" - n = len(x) - for i in reversed(range(n//2)): - _siftup_max(x, i) - -def nlargest(n, iterable): - """Find the n largest elements in a dataset. - - Equivalent to: sorted(iterable, reverse=True)[:n] - """ - if n < 0: - return [] - it = iter(iterable) - result = list(islice(it, n)) - if not result: - return result - heapify(result) - _heappushpop = heappushpop - for elem in it: - _heappushpop(result, elem) - result.sort(reverse=True) - return result - -def nsmallest(n, iterable): - """Find the n smallest elements in a dataset. - - Equivalent to: sorted(iterable)[:n] - """ - if n < 0: - return [] - it = iter(iterable) - result = list(islice(it, n)) - if not result: - return result - _heapify_max(result) - _heappushpop = _heappushpop_max - for elem in it: - _heappushpop(result, elem) - result.sort() - return result - -# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos -# is the index of a leaf with a possibly out-of-order value. Restore the -# heap invariant. -def _siftdown(heap, startpos, pos): - newitem = heap[pos] - # Follow the path to the root, moving parents down until finding a place - # newitem fits. - while pos > startpos: - parentpos = (pos - 1) >> 1 - parent = heap[parentpos] - if newitem < parent: - heap[pos] = parent - pos = parentpos - continue - break - heap[pos] = newitem - -# The child indices of heap index pos are already heaps, and we want to make -# a heap at index pos too. We do this by bubbling the smaller child of -# pos up (and so on with that child's children, etc) until hitting a leaf, -# then using _siftdown to move the oddball originally at index pos into place. -# -# We *could* break out of the loop as soon as we find a pos where newitem <= -# both its children, but turns out that's not a good idea, and despite that -# many books write the algorithm that way. During a heap pop, the last array -# element is sifted in, and that tends to be large, so that comparing it -# against values starting from the root usually doesn't pay (= usually doesn't -# get us out of the loop early). See Knuth, Volume 3, where this is -# explained and quantified in an exercise. -# -# Cutting the # of comparisons is important, since these routines have no -# way to extract "the priority" from an array element, so that intelligence -# is likely to be hiding in custom comparison methods, or in array elements -# storing (priority, record) tuples. Comparisons are thus potentially -# expensive. -# -# On random arrays of length 1000, making this change cut the number of -# comparisons made by heapify() a little, and those made by exhaustive -# heappop() a lot, in accord with theory. Here are typical results from 3 -# runs (3 just to demonstrate how small the variance is): -# -# Compares needed by heapify Compares needed by 1000 heappops -# -------------------------- -------------------------------- -# 1837 cut to 1663 14996 cut to 8680 -# 1855 cut to 1659 14966 cut to 8678 -# 1847 cut to 1660 15024 cut to 8703 -# -# Building the heap by using heappush() 1000 times instead required -# 2198, 2148, and 2219 compares: heapify() is more efficient, when -# you can use it. -# -# The total compares needed by list.sort() on the same lists were 8627, -# 8627, and 8632 (this should be compared to the sum of heapify() and -# heappop() compares): list.sort() is (unsurprisingly!) more efficient -# for sorting. - -def _siftup(heap, pos): - endpos = len(heap) - startpos = pos - newitem = heap[pos] - # Bubble up the smaller child until hitting a leaf. - childpos = 2*pos + 1 # leftmost child position - while childpos < endpos: - # Set childpos to index of smaller child. - rightpos = childpos + 1 - if rightpos < endpos and not heap[childpos] < heap[rightpos]: - childpos = rightpos - # Move the smaller child up. - heap[pos] = heap[childpos] - pos = childpos - childpos = 2*pos + 1 - # The leaf at pos is empty now. Put newitem there, and bubble it up - # to its final resting place (by sifting its parents down). - heap[pos] = newitem - _siftdown(heap, startpos, pos) - -def _siftdown_max(heap, startpos, pos): - 'Maxheap variant of _siftdown' - newitem = heap[pos] - # Follow the path to the root, moving parents down until finding a place - # newitem fits. - while pos > startpos: - parentpos = (pos - 1) >> 1 - parent = heap[parentpos] - if parent < newitem: - heap[pos] = parent - pos = parentpos - continue - break - heap[pos] = newitem - -def _siftup_max(heap, pos): - 'Maxheap variant of _siftup' - endpos = len(heap) - startpos = pos - newitem = heap[pos] - # Bubble up the larger child until hitting a leaf. - childpos = 2*pos + 1 # leftmost child position - while childpos < endpos: - # Set childpos to index of larger child. - rightpos = childpos + 1 - if rightpos < endpos and not heap[rightpos] < heap[childpos]: - childpos = rightpos - # Move the larger child up. - heap[pos] = heap[childpos] - pos = childpos - childpos = 2*pos + 1 - # The leaf at pos is empty now. Put newitem there, and bubble it up - # to its final resting place (by sifting its parents down). - heap[pos] = newitem - _siftdown_max(heap, startpos, pos) - -# If available, use C implementation -try: - from _heapq import * -except ImportError: - pass - -def merge(*iterables): - '''Merge multiple sorted inputs into a single sorted output. - - Similar to sorted(itertools.chain(*iterables)) but returns a generator, - does not pull the data into memory all at once, and assumes that each of - the input streams is already sorted (smallest to largest). - - >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) - [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] - - ''' - _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration - - h = [] - h_append = h.append - for itnum, it in enumerate(map(iter, iterables)): - try: - next = it.__next__ - h_append([next(), itnum, next]) - except _StopIteration: - pass - heapify(h) - - while 1: - try: - while 1: - v, itnum, next = s = h[0] # raises IndexError when h is empty - yield v - s[0] = next() # raises StopIteration when exhausted - _heapreplace(h, s) # restore heap condition - except _StopIteration: - _heappop(h) # remove empty iterator - except IndexError: - return - -# Extend the implementations of nsmallest and nlargest to use a key= argument -_nsmallest = nsmallest -def nsmallest(n, iterable, key=None): - """Find the n smallest elements in a dataset. - - Equivalent to: sorted(iterable, key=key)[:n] - """ - # Short-cut for n==1 is to use min() when len(iterable)>0 - if n == 1: - it = iter(iterable) - head = list(islice(it, 1)) - if not head: - return [] - if key is None: - return [min(chain(head, it))] - return [min(chain(head, it), key=key)] - - # When n>=size, it's faster to use sorted() - try: - size = len(iterable) - except (TypeError, AttributeError): - pass - else: - if n >= size: - return sorted(iterable, key=key)[:n] - - # When key is none, use simpler decoration - if key is None: - it = zip(iterable, count()) # decorate - result = _nsmallest(n, it) - return [r[0] for r in result] # undecorate - - # General case, slowest method - in1, in2 = tee(iterable) - it = zip(map(key, in1), count(), in2) # decorate - result = _nsmallest(n, it) - return [r[2] for r in result] # undecorate - -_nlargest = nlargest -def nlargest(n, iterable, key=None): - """Find the n largest elements in a dataset. - - Equivalent to: sorted(iterable, key=key, reverse=True)[:n] - """ - - # Short-cut for n==1 is to use max() when len(iterable)>0 - if n == 1: - it = iter(iterable) - head = list(islice(it, 1)) - if not head: - return [] - if key is None: - return [max(chain(head, it))] - return [max(chain(head, it), key=key)] - - # When n>=size, it's faster to use sorted() - try: - size = len(iterable) - except (TypeError, AttributeError): - pass - else: - if n >= size: - return sorted(iterable, key=key, reverse=True)[:n] - - # When key is none, use simpler decoration - if key is None: - it = zip(iterable, count(0,-1)) # decorate - result = _nlargest(n, it) - return [r[0] for r in result] # undecorate - - # General case, slowest method - in1, in2 = tee(iterable) - it = zip(map(key, in1), count(0,-1), in2) # decorate - result = _nlargest(n, it) - return [r[2] for r in result] # undecorate - -if __name__ == "__main__": - # Simple sanity test - heap = [] - data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] - for item in data: - heappush(heap, item) - sort = [] - while heap: - sort.append(heappop(heap)) - print(sort) - - import doctest - doctest.testmod() diff --git a/tests/bytecode/pylib-tests/keyword.py b/tests/bytecode/pylib-tests/keyword.py deleted file mode 100644 index dad39cc377..0000000000 --- a/tests/bytecode/pylib-tests/keyword.py +++ /dev/null @@ -1,93 +0,0 @@ -#! /usr/bin/env python3 - -"""Keywords (from "graminit.c") - -This file is automatically generated; please don't muck it up! - -To update the symbols in this file, 'cd' to the top directory of -the python source tree after building the interpreter and run: - - ./python Lib/keyword.py -""" - -__all__ = ["iskeyword", "kwlist"] - -kwlist = [ -#--start keywords-- - 'False', - 'None', - 'True', - 'and', - 'as', - 'assert', - 'break', - 'class', - 'continue', - 'def', - 'del', - 'elif', - 'else', - 'except', - 'finally', - 'for', - 'from', - 'global', - 'if', - 'import', - 'in', - 'is', - 'lambda', - 'nonlocal', - 'not', - 'or', - 'pass', - 'raise', - 'return', - 'try', - 'while', - 'with', - 'yield', -#--end keywords-- - ] - -iskeyword = frozenset(kwlist).__contains__ - -def main(): - import sys, re - - args = sys.argv[1:] - iptfile = args and args[0] or "Python/graminit.c" - if len(args) > 1: optfile = args[1] - else: optfile = "Lib/keyword.py" - - # scan the source file for keywords - with open(iptfile) as fp: - strprog = re.compile('"([^"]+)"') - lines = [] - for line in fp: - if '{1, "' in line: - match = strprog.search(line) - if match: - lines.append(" '" + match.group(1) + "',\n") - lines.sort() - - # load the output skeleton from the target - with open(optfile) as fp: - format = fp.readlines() - - # insert the lines of keywords - try: - start = format.index("#--start keywords--\n") + 1 - end = format.index("#--end keywords--\n") - format[start:end] = lines - except ValueError: - sys.stderr.write("target does not contain format markers\n") - sys.exit(1) - - # write the output file - fp = open(optfile, 'w') - fp.write(''.join(format)) - fp.close() - -if __name__ == "__main__": - main() diff --git a/tests/bytecode/pylib-tests/macurl2path.py b/tests/bytecode/pylib-tests/macurl2path.py deleted file mode 100644 index f22fb207b8..0000000000 --- a/tests/bytecode/pylib-tests/macurl2path.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Macintosh-specific module for conversion between pathnames and URLs. - -Do not import directly; use urllib instead.""" - -import urllib.parse -import os - -__all__ = ["url2pathname","pathname2url"] - -def url2pathname(pathname): - """OS-specific conversion from a relative URL of the 'file' scheme - to a file system path; not recommended for general use.""" - # - # XXXX The .. handling should be fixed... - # - tp = urllib.parse.splittype(pathname)[0] - if tp and tp != 'file': - raise RuntimeError('Cannot convert non-local URL to pathname') - # Turn starting /// into /, an empty hostname means current host - if pathname[:3] == '///': - pathname = pathname[2:] - elif pathname[:2] == '//': - raise RuntimeError('Cannot convert non-local URL to pathname') - components = pathname.split('/') - # Remove . and embedded .. - i = 0 - while i < len(components): - if components[i] == '.': - del components[i] - elif components[i] == '..' and i > 0 and \ - components[i-1] not in ('', '..'): - del components[i-1:i+1] - i = i-1 - elif components[i] == '' and i > 0 and components[i-1] != '': - del components[i] - else: - i = i+1 - if not components[0]: - # Absolute unix path, don't start with colon - rv = ':'.join(components[1:]) - else: - # relative unix path, start with colon. First replace - # leading .. by empty strings (giving ::file) - i = 0 - while i < len(components) and components[i] == '..': - components[i] = '' - i = i + 1 - rv = ':' + ':'.join(components) - # and finally unquote slashes and other funny characters - return urllib.parse.unquote(rv) - -def pathname2url(pathname): - """OS-specific conversion from a file system path to a relative URL - of the 'file' scheme; not recommended for general use.""" - if '/' in pathname: - raise RuntimeError("Cannot convert pathname containing slashes") - components = pathname.split(':') - # Remove empty first and/or last component - if components[0] == '': - del components[0] - if components[-1] == '': - del components[-1] - # Replace empty string ('::') by .. (will result in '/../' later) - for i in range(len(components)): - if components[i] == '': - components[i] = '..' - # Truncate names longer than 31 bytes - components = map(_pncomp2url, components) - - if os.path.isabs(pathname): - return '/' + '/'.join(components) - else: - return '/'.join(components) - -def _pncomp2url(component): - # We want to quote slashes - return urllib.parse.quote(component[:31], safe='') - -def test(): - for url in ["index.html", - "bar/index.html", - "/foo/bar/index.html", - "/foo/bar/", - "/"]: - print('%r -> %r' % (url, url2pathname(url))) - for path in ["drive:", - "drive:dir:", - "drive:dir:file", - "drive:file", - "file", - ":file", - ":dir:", - ":dir:file"]: - print('%r -> %r' % (path, pathname2url(path))) - -if __name__ == '__main__': - test() diff --git a/tests/bytecode/pylib-tests/mimetypes.py b/tests/bytecode/pylib-tests/mimetypes.py deleted file mode 100644 index 2872ee4245..0000000000 --- a/tests/bytecode/pylib-tests/mimetypes.py +++ /dev/null @@ -1,589 +0,0 @@ -"""Guess the MIME type of a file. - -This module defines two useful functions: - -guess_type(url, strict=True) -- guess the MIME type and encoding of a URL. - -guess_extension(type, strict=True) -- guess the extension for a given MIME type. - -It also contains the following, for tuning the behavior: - -Data: - -knownfiles -- list of files to parse -inited -- flag set when init() has been called -suffix_map -- dictionary mapping suffixes to suffixes -encodings_map -- dictionary mapping suffixes to encodings -types_map -- dictionary mapping suffixes to types - -Functions: - -init([files]) -- parse a list of files, default knownfiles (on Windows, the - default values are taken from the registry) -read_mime_types(file) -- parse one file, return a dictionary or None -""" - -import os -import sys -import posixpath -import urllib.parse -try: - import winreg as _winreg -except ImportError: - _winreg = None - -__all__ = [ - "guess_type","guess_extension","guess_all_extensions", - "add_type","read_mime_types","init" -] - -knownfiles = [ - "/etc/mime.types", - "/etc/httpd/mime.types", # Mac OS X - "/etc/httpd/conf/mime.types", # Apache - "/etc/apache/mime.types", # Apache 1 - "/etc/apache2/mime.types", # Apache 2 - "/usr/local/etc/httpd/conf/mime.types", - "/usr/local/lib/netscape/mime.types", - "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 - "/usr/local/etc/mime.types", # Apache 1.3 - ] - -inited = False -_db = None - - -class MimeTypes: - """MIME-types datastore. - - This datastore can handle information from mime.types-style files - and supports basic determination of MIME type from a filename or - URL, and can guess a reasonable extension given a MIME type. - """ - - def __init__(self, filenames=(), strict=True): - if not inited: - init() - self.encodings_map = encodings_map.copy() - self.suffix_map = suffix_map.copy() - self.types_map = ({}, {}) # dict for (non-strict, strict) - self.types_map_inv = ({}, {}) - for (ext, type) in types_map.items(): - self.add_type(type, ext, True) - for (ext, type) in common_types.items(): - self.add_type(type, ext, False) - for name in filenames: - self.read(name, strict) - - def add_type(self, type, ext, strict=True): - """Add a mapping between a type and an extension. - - When the extension is already known, the new - type will replace the old one. When the type - is already known the extension will be added - to the list of known extensions. - - If strict is true, information will be added to - list of standard types, else to the list of non-standard - types. - """ - self.types_map[strict][ext] = type - exts = self.types_map_inv[strict].setdefault(type, []) - if ext not in exts: - exts.append(ext) - - def guess_type(self, url, strict=True): - """Guess the type of a file based on its URL. - - Return value is a tuple (type, encoding) where type is None if - the type can't be guessed (no or unknown suffix) or a string - of the form type/subtype, usable for a MIME Content-type - header; and encoding is None for no encoding or the name of - the program used to encode (e.g. compress or gzip). The - mappings are table driven. Encoding suffixes are case - sensitive; type suffixes are first tried case sensitive, then - case insensitive. - - The suffixes .tgz, .taz and .tz (case sensitive!) are all - mapped to '.tar.gz'. (This is table-driven too, using the - dictionary suffix_map.) - - Optional `strict' argument when False adds a bunch of commonly found, - but non-standard types. - """ - scheme, url = urllib.parse.splittype(url) - if scheme == 'data': - # syntax of data URLs: - # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data - # mediatype := [ type "/" subtype ] *( ";" parameter ) - # data := *urlchar - # parameter := attribute "=" value - # type/subtype defaults to "text/plain" - comma = url.find(',') - if comma < 0: - # bad data URL - return None, None - semi = url.find(';', 0, comma) - if semi >= 0: - type = url[:semi] - else: - type = url[:comma] - if '=' in type or '/' not in type: - type = 'text/plain' - return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) - while ext in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext]) - if ext in self.encodings_map: - encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) - else: - encoding = None - types_map = self.types_map[True] - if ext in types_map: - return types_map[ext], encoding - elif ext.lower() in types_map: - return types_map[ext.lower()], encoding - elif strict: - return None, encoding - types_map = self.types_map[False] - if ext in types_map: - return types_map[ext], encoding - elif ext.lower() in types_map: - return types_map[ext.lower()], encoding - else: - return None, encoding - - def guess_all_extensions(self, type, strict=True): - """Guess the extensions for a file based on its MIME type. - - Return value is a list of strings giving the possible filename - extensions, including the leading dot ('.'). The extension is not - guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). - - Optional `strict' argument when false adds a bunch of commonly found, - but non-standard types. - """ - type = type.lower() - extensions = self.types_map_inv[True].get(type, []) - if not strict: - for ext in self.types_map_inv[False].get(type, []): - if ext not in extensions: - extensions.append(ext) - return extensions - - def guess_extension(self, type, strict=True): - """Guess the extension for a file based on its MIME type. - - Return value is a string giving a filename extension, - including the leading dot ('.'). The extension is not - guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None - is returned. - - Optional `strict' argument when false adds a bunch of commonly found, - but non-standard types. - """ - extensions = self.guess_all_extensions(type, strict) - if not extensions: - return None - return extensions[0] - - def read(self, filename, strict=True): - """ - Read a single mime.types-format file, specified by pathname. - - If strict is true, information will be added to - list of standard types, else to the list of non-standard - types. - """ - with open(filename, encoding='utf-8') as fp: - self.readfp(fp, strict) - - def readfp(self, fp, strict=True): - """ - Read a single mime.types-format file. - - If strict is true, information will be added to - list of standard types, else to the list of non-standard - types. - """ - while 1: - line = fp.readline() - if not line: - break - words = line.split() - for i in range(len(words)): - if words[i][0] == '#': - del words[i:] - break - if not words: - continue - type, suffixes = words[0], words[1:] - for suff in suffixes: - self.add_type(type, '.' + suff, strict) - - def read_windows_registry(self, strict=True): - """ - Load the MIME types database from Windows registry. - - If strict is true, information will be added to - list of standard types, else to the list of non-standard - types. - """ - - # Windows only - if not _winreg: - return - - def enum_types(mimedb): - i = 0 - while True: - try: - ctype = _winreg.EnumKey(mimedb, i) - except EnvironmentError: - break - else: - yield ctype - i += 1 - - with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, - r'MIME\Database\Content Type') as mimedb: - for ctype in enum_types(mimedb): - try: - with _winreg.OpenKey(mimedb, ctype) as key: - suffix, datatype = _winreg.QueryValueEx(key, - 'Extension') - except EnvironmentError: - continue - if datatype != _winreg.REG_SZ: - continue - self.add_type(ctype, suffix, strict) - - -def guess_type(url, strict=True): - """Guess the type of a file based on its URL. - - Return value is a tuple (type, encoding) where type is None if the - type can't be guessed (no or unknown suffix) or a string of the - form type/subtype, usable for a MIME Content-type header; and - encoding is None for no encoding or the name of the program used - to encode (e.g. compress or gzip). The mappings are table - driven. Encoding suffixes are case sensitive; type suffixes are - first tried case sensitive, then case insensitive. - - The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped - to ".tar.gz". (This is table-driven too, using the dictionary - suffix_map). - - Optional `strict' argument when false adds a bunch of commonly found, but - non-standard types. - """ - if _db is None: - init() - return _db.guess_type(url, strict) - - -def guess_all_extensions(type, strict=True): - """Guess the extensions for a file based on its MIME type. - - Return value is a list of strings giving the possible filename - extensions, including the leading dot ('.'). The extension is not - guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None - is returned. - - Optional `strict' argument when false adds a bunch of commonly found, - but non-standard types. - """ - if _db is None: - init() - return _db.guess_all_extensions(type, strict) - -def guess_extension(type, strict=True): - """Guess the extension for a file based on its MIME type. - - Return value is a string giving a filename extension, including the - leading dot ('.'). The extension is not guaranteed to have been - associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. - - Optional `strict' argument when false adds a bunch of commonly found, - but non-standard types. - """ - if _db is None: - init() - return _db.guess_extension(type, strict) - -def add_type(type, ext, strict=True): - """Add a mapping between a type and an extension. - - When the extension is already known, the new - type will replace the old one. When the type - is already known the extension will be added - to the list of known extensions. - - If strict is true, information will be added to - list of standard types, else to the list of non-standard - types. - """ - if _db is None: - init() - return _db.add_type(type, ext, strict) - - -def init(files=None): - global suffix_map, types_map, encodings_map, common_types - global inited, _db - inited = True # so that MimeTypes.__init__() doesn't call us again - db = MimeTypes() - if files is None: - if _winreg: - db.read_windows_registry() - files = knownfiles - for file in files: - if os.path.isfile(file): - db.read(file) - encodings_map = db.encodings_map - suffix_map = db.suffix_map - types_map = db.types_map[True] - common_types = db.types_map[False] - # Make the DB a global variable now that it is fully initialized - _db = db - - -def read_mime_types(file): - try: - f = open(file) - except IOError: - return None - db = MimeTypes() - db.readfp(f, True) - return db.types_map[True] - - -def _default_mime_types(): - global suffix_map - global encodings_map - global types_map - global common_types - - suffix_map = { - '.svgz': '.svg.gz', - '.tgz': '.tar.gz', - '.taz': '.tar.gz', - '.tz': '.tar.gz', - '.tbz2': '.tar.bz2', - '.txz': '.tar.xz', - } - - encodings_map = { - '.gz': 'gzip', - '.Z': 'compress', - '.bz2': 'bzip2', - '.xz': 'xz', - } - - # Before adding new types, make sure they are either registered with IANA, - # at http://www.iana.org/assignments/media-types - # or extensions, i.e. using the x- prefix - - # If you add to these, please keep them sorted! - types_map = { - '.a' : 'application/octet-stream', - '.ai' : 'application/postscript', - '.aif' : 'audio/x-aiff', - '.aifc' : 'audio/x-aiff', - '.aiff' : 'audio/x-aiff', - '.au' : 'audio/basic', - '.avi' : 'video/x-msvideo', - '.bat' : 'text/plain', - '.bcpio' : 'application/x-bcpio', - '.bin' : 'application/octet-stream', - '.bmp' : 'image/x-ms-bmp', - '.c' : 'text/plain', - # Duplicates :( - '.cdf' : 'application/x-cdf', - '.cdf' : 'application/x-netcdf', - '.cpio' : 'application/x-cpio', - '.csh' : 'application/x-csh', - '.css' : 'text/css', - '.dll' : 'application/octet-stream', - '.doc' : 'application/msword', - '.dot' : 'application/msword', - '.dvi' : 'application/x-dvi', - '.eml' : 'message/rfc822', - '.eps' : 'application/postscript', - '.etx' : 'text/x-setext', - '.exe' : 'application/octet-stream', - '.gif' : 'image/gif', - '.gtar' : 'application/x-gtar', - '.h' : 'text/plain', - '.hdf' : 'application/x-hdf', - '.htm' : 'text/html', - '.html' : 'text/html', - '.ico' : 'image/vnd.microsoft.icon', - '.ief' : 'image/ief', - '.jpe' : 'image/jpeg', - '.jpeg' : 'image/jpeg', - '.jpg' : 'image/jpeg', - '.js' : 'application/javascript', - '.ksh' : 'text/plain', - '.latex' : 'application/x-latex', - '.m1v' : 'video/mpeg', - '.m3u' : 'application/vnd.apple.mpegurl', - '.m3u8' : 'application/vnd.apple.mpegurl', - '.man' : 'application/x-troff-man', - '.me' : 'application/x-troff-me', - '.mht' : 'message/rfc822', - '.mhtml' : 'message/rfc822', - '.mif' : 'application/x-mif', - '.mov' : 'video/quicktime', - '.movie' : 'video/x-sgi-movie', - '.mp2' : 'audio/mpeg', - '.mp3' : 'audio/mpeg', - '.mp4' : 'video/mp4', - '.mpa' : 'video/mpeg', - '.mpe' : 'video/mpeg', - '.mpeg' : 'video/mpeg', - '.mpg' : 'video/mpeg', - '.ms' : 'application/x-troff-ms', - '.nc' : 'application/x-netcdf', - '.nws' : 'message/rfc822', - '.o' : 'application/octet-stream', - '.obj' : 'application/octet-stream', - '.oda' : 'application/oda', - '.p12' : 'application/x-pkcs12', - '.p7c' : 'application/pkcs7-mime', - '.pbm' : 'image/x-portable-bitmap', - '.pdf' : 'application/pdf', - '.pfx' : 'application/x-pkcs12', - '.pgm' : 'image/x-portable-graymap', - '.pl' : 'text/plain', - '.png' : 'image/png', - '.pnm' : 'image/x-portable-anymap', - '.pot' : 'application/vnd.ms-powerpoint', - '.ppa' : 'application/vnd.ms-powerpoint', - '.ppm' : 'image/x-portable-pixmap', - '.pps' : 'application/vnd.ms-powerpoint', - '.ppt' : 'application/vnd.ms-powerpoint', - '.ps' : 'application/postscript', - '.pwz' : 'application/vnd.ms-powerpoint', - '.py' : 'text/x-python', - '.pyc' : 'application/x-python-code', - '.pyo' : 'application/x-python-code', - '.qt' : 'video/quicktime', - '.ra' : 'audio/x-pn-realaudio', - '.ram' : 'application/x-pn-realaudio', - '.ras' : 'image/x-cmu-raster', - '.rdf' : 'application/xml', - '.rgb' : 'image/x-rgb', - '.roff' : 'application/x-troff', - '.rtx' : 'text/richtext', - '.sgm' : 'text/x-sgml', - '.sgml' : 'text/x-sgml', - '.sh' : 'application/x-sh', - '.shar' : 'application/x-shar', - '.snd' : 'audio/basic', - '.so' : 'application/octet-stream', - '.src' : 'application/x-wais-source', - '.sv4cpio': 'application/x-sv4cpio', - '.sv4crc' : 'application/x-sv4crc', - '.svg' : 'image/svg+xml', - '.swf' : 'application/x-shockwave-flash', - '.t' : 'application/x-troff', - '.tar' : 'application/x-tar', - '.tcl' : 'application/x-tcl', - '.tex' : 'application/x-tex', - '.texi' : 'application/x-texinfo', - '.texinfo': 'application/x-texinfo', - '.tif' : 'image/tiff', - '.tiff' : 'image/tiff', - '.tr' : 'application/x-troff', - '.tsv' : 'text/tab-separated-values', - '.txt' : 'text/plain', - '.ustar' : 'application/x-ustar', - '.vcf' : 'text/x-vcard', - '.wav' : 'audio/x-wav', - '.wiz' : 'application/msword', - '.wsdl' : 'application/xml', - '.xbm' : 'image/x-xbitmap', - '.xlb' : 'application/vnd.ms-excel', - # Duplicates :( - '.xls' : 'application/excel', - '.xls' : 'application/vnd.ms-excel', - '.xml' : 'text/xml', - '.xpdl' : 'application/xml', - '.xpm' : 'image/x-xpixmap', - '.xsl' : 'application/xml', - '.xwd' : 'image/x-xwindowdump', - '.zip' : 'application/zip', - } - - # These are non-standard types, commonly found in the wild. They will - # only match if strict=0 flag is given to the API methods. - - # Please sort these too - common_types = { - '.jpg' : 'image/jpg', - '.mid' : 'audio/midi', - '.midi': 'audio/midi', - '.pct' : 'image/pict', - '.pic' : 'image/pict', - '.pict': 'image/pict', - '.rtf' : 'application/rtf', - '.xul' : 'text/xul' - } - - -_default_mime_types() - - -if __name__ == '__main__': - import getopt - - USAGE = """\ -Usage: mimetypes.py [options] type - -Options: - --help / -h -- print this message and exit - --lenient / -l -- additionally search of some common, but non-standard - types. - --extension / -e -- guess extension instead of type - -More than one type argument may be given. -""" - - def usage(code, msg=''): - print(USAGE) - if msg: print(msg) - sys.exit(code) - - try: - opts, args = getopt.getopt(sys.argv[1:], 'hle', - ['help', 'lenient', 'extension']) - except getopt.error as msg: - usage(1, msg) - - strict = 1 - extension = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-l', '--lenient'): - strict = 0 - elif opt in ('-e', '--extension'): - extension = 1 - for gtype in args: - if extension: - guess = guess_extension(gtype, strict) - if not guess: print("I don't know anything about type", gtype) - else: print(guess) - else: - guess, encoding = guess_type(gtype, strict) - if not guess: print("I don't know anything about type", gtype) - else: print('type:', guess, 'encoding:', encoding) diff --git a/tests/bytecode/pylib-tests/modulefinder.py b/tests/bytecode/pylib-tests/modulefinder.py deleted file mode 100644 index f90a4327e6..0000000000 --- a/tests/bytecode/pylib-tests/modulefinder.py +++ /dev/null @@ -1,663 +0,0 @@ -"""Find modules used by a script, using introspection.""" - -import dis -import imp -import importlib.machinery -import marshal -import os -import sys -import types -import struct - -# XXX Clean up once str8's cstor matches bytes. -LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) -IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')]) -STORE_NAME = bytes([dis.opname.index('STORE_NAME')]) -STORE_GLOBAL = bytes([dis.opname.index('STORE_GLOBAL')]) -STORE_OPS = [STORE_NAME, STORE_GLOBAL] -HAVE_ARGUMENT = bytes([dis.HAVE_ARGUMENT]) - -# Modulefinder does a good job at simulating Python's, but it can not -# handle __path__ modifications packages make at runtime. Therefore there -# is a mechanism whereby you can register extra paths in this map for a -# package, and it will be honored. - -# Note this is a mapping is lists of paths. -packagePathMap = {} - -# A Public interface -def AddPackagePath(packagename, path): - packagePathMap.setdefault(packagename, []).append(path) - -replacePackageMap = {} - -# This ReplacePackage mechanism allows modulefinder to work around -# situations in which a package injects itself under the name -# of another package into sys.modules at runtime by calling -# ReplacePackage("real_package_name", "faked_package_name") -# before running ModuleFinder. - -def ReplacePackage(oldname, newname): - replacePackageMap[oldname] = newname - - -class Module: - - def __init__(self, name, file=None, path=None): - self.__name__ = name - self.__file__ = file - self.__path__ = path - self.__code__ = None - # The set of global names that are assigned to in the module. - # This includes those names imported through starimports of - # Python modules. - self.globalnames = {} - # The set of starimports this module did that could not be - # resolved, ie. a starimport from a non-Python module. - self.starimports = {} - - def __repr__(self): - s = "Module(%r" % (self.__name__,) - if self.__file__ is not None: - s = s + ", %r" % (self.__file__,) - if self.__path__ is not None: - s = s + ", %r" % (self.__path__,) - s = s + ")" - return s - -class ModuleFinder: - - def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): - if path is None: - path = sys.path - self.path = path - self.modules = {} - self.badmodules = {} - self.debug = debug - self.indent = 0 - self.excludes = excludes - self.replace_paths = replace_paths - self.processed_paths = [] # Used in debugging only - - def msg(self, level, str, *args): - if level <= self.debug: - for i in range(self.indent): - print(" ", end=' ') - print(str, end=' ') - for arg in args: - print(repr(arg), end=' ') - print() - - def msgin(self, *args): - level = args[0] - if level <= self.debug: - self.indent = self.indent + 1 - self.msg(*args) - - def msgout(self, *args): - level = args[0] - if level <= self.debug: - self.indent = self.indent - 1 - self.msg(*args) - - def run_script(self, pathname): - self.msg(2, "run_script", pathname) - with open(pathname) as fp: - stuff = ("", "r", imp.PY_SOURCE) - self.load_module('__main__', fp, pathname, stuff) - - def load_file(self, pathname): - dir, name = os.path.split(pathname) - name, ext = os.path.splitext(name) - with open(pathname) as fp: - stuff = (ext, "r", imp.PY_SOURCE) - self.load_module(name, fp, pathname, stuff) - - def import_hook(self, name, caller=None, fromlist=None, level=-1): - self.msg(3, "import_hook", name, caller, fromlist, level) - parent = self.determine_parent(caller, level=level) - q, tail = self.find_head_package(parent, name) - m = self.load_tail(q, tail) - if not fromlist: - return q - if m.__path__: - self.ensure_fromlist(m, fromlist) - return None - - def determine_parent(self, caller, level=-1): - self.msgin(4, "determine_parent", caller, level) - if not caller or level == 0: - self.msgout(4, "determine_parent -> None") - return None - pname = caller.__name__ - if level >= 1: # relative import - if caller.__path__: - level -= 1 - if level == 0: - parent = self.modules[pname] - assert parent is caller - self.msgout(4, "determine_parent ->", parent) - return parent - if pname.count(".") < level: - raise ImportError("relative importpath too deep") - pname = ".".join(pname.split(".")[:-level]) - parent = self.modules[pname] - self.msgout(4, "determine_parent ->", parent) - return parent - if caller.__path__: - parent = self.modules[pname] - assert caller is parent - self.msgout(4, "determine_parent ->", parent) - return parent - if '.' in pname: - i = pname.rfind('.') - pname = pname[:i] - parent = self.modules[pname] - assert parent.__name__ == pname - self.msgout(4, "determine_parent ->", parent) - return parent - self.msgout(4, "determine_parent -> None") - return None - - def find_head_package(self, parent, name): - self.msgin(4, "find_head_package", parent, name) - if '.' in name: - i = name.find('.') - head = name[:i] - tail = name[i+1:] - else: - head = name - tail = "" - if parent: - qname = "%s.%s" % (parent.__name__, head) - else: - qname = head - q = self.import_module(head, qname, parent) - if q: - self.msgout(4, "find_head_package ->", (q, tail)) - return q, tail - if parent: - qname = head - parent = None - q = self.import_module(head, qname, parent) - if q: - self.msgout(4, "find_head_package ->", (q, tail)) - return q, tail - self.msgout(4, "raise ImportError: No module named", qname) - raise ImportError("No module named " + qname) - - def load_tail(self, q, tail): - self.msgin(4, "load_tail", q, tail) - m = q - while tail: - i = tail.find('.') - if i < 0: i = len(tail) - head, tail = tail[:i], tail[i+1:] - mname = "%s.%s" % (m.__name__, head) - m = self.import_module(head, mname, m) - if not m: - self.msgout(4, "raise ImportError: No module named", mname) - raise ImportError("No module named " + mname) - self.msgout(4, "load_tail ->", m) - return m - - def ensure_fromlist(self, m, fromlist, recursive=0): - self.msg(4, "ensure_fromlist", m, fromlist, recursive) - for sub in fromlist: - if sub == "*": - if not recursive: - all = self.find_all_submodules(m) - if all: - self.ensure_fromlist(m, all, 1) - elif not hasattr(m, sub): - subname = "%s.%s" % (m.__name__, sub) - submod = self.import_module(sub, subname, m) - if not submod: - raise ImportError("No module named " + subname) - - def find_all_submodules(self, m): - if not m.__path__: - return - modules = {} - # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. - # But we must also collect Python extension modules - although - # we cannot separate normal dlls from Python extensions. - suffixes = [] - suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] - suffixes += importlib.machinery.SOURCE_SUFFIXES[:] - suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] - for dir in m.__path__: - try: - names = os.listdir(dir) - except os.error: - self.msg(2, "can't list directory", dir) - continue - for name in names: - mod = None - for suff in suffixes: - n = len(suff) - if name[-n:] == suff: - mod = name[:-n] - break - if mod and mod != "__init__": - modules[mod] = mod - return modules.keys() - - def import_module(self, partname, fqname, parent): - self.msgin(3, "import_module", partname, fqname, parent) - try: - m = self.modules[fqname] - except KeyError: - pass - else: - self.msgout(3, "import_module ->", m) - return m - if fqname in self.badmodules: - self.msgout(3, "import_module -> None") - return None - if parent and parent.__path__ is None: - self.msgout(3, "import_module -> None") - return None - try: - fp, pathname, stuff = self.find_module(partname, - parent and parent.__path__, parent) - except ImportError: - self.msgout(3, "import_module ->", None) - return None - try: - m = self.load_module(fqname, fp, pathname, stuff) - finally: - if fp: - fp.close() - if parent: - setattr(parent, partname, m) - self.msgout(3, "import_module ->", m) - return m - - def load_module(self, fqname, fp, pathname, file_info): - suffix, mode, type = file_info - self.msgin(2, "load_module", fqname, fp and "fp", pathname) - if type == imp.PKG_DIRECTORY: - m = self.load_package(fqname, pathname) - self.msgout(2, "load_module ->", m) - return m - if type == imp.PY_SOURCE: - co = compile(fp.read()+'\n', pathname, 'exec') - elif type == imp.PY_COMPILED: - if fp.read(4) != imp.get_magic(): - self.msgout(2, "raise ImportError: Bad magic number", pathname) - raise ImportError("Bad magic number in %s" % pathname) - fp.read(4) - co = marshal.load(fp) - else: - co = None - m = self.add_module(fqname) - m.__file__ = pathname - if co: - if self.replace_paths: - co = self.replace_paths_in_code(co) - m.__code__ = co - self.scan_code(co, m) - self.msgout(2, "load_module ->", m) - return m - - def _add_badmodule(self, name, caller): - if name not in self.badmodules: - self.badmodules[name] = {} - if caller: - self.badmodules[name][caller.__name__] = 1 - else: - self.badmodules[name]["-"] = 1 - - def _safe_import_hook(self, name, caller, fromlist, level=-1): - # wrapper for self.import_hook() that won't raise ImportError - if name in self.badmodules: - self._add_badmodule(name, caller) - return - try: - self.import_hook(name, caller, level=level) - except ImportError as msg: - self.msg(2, "ImportError:", str(msg)) - self._add_badmodule(name, caller) - else: - if fromlist: - for sub in fromlist: - if sub in self.badmodules: - self._add_badmodule(sub, caller) - continue - try: - self.import_hook(name, caller, [sub], level=level) - except ImportError as msg: - self.msg(2, "ImportError:", str(msg)) - fullname = name + "." + sub - self._add_badmodule(fullname, caller) - - def scan_opcodes(self, co, - unpack = struct.unpack): - # Scan the code, and yield 'interesting' opcode combinations - # Version for Python 2.4 and older - code = co.co_code - names = co.co_names - consts = co.co_consts - while code: - c = code[0] - if c in STORE_OPS: - oparg, = unpack('= HAVE_ARGUMENT: - code = code[3:] - else: - code = code[1:] - - def scan_opcodes_25(self, co, - unpack = struct.unpack): - # Scan the code, and yield 'interesting' opcode combinations - # Python 2.5 version (has absolute and relative imports) - code = co.co_code - names = co.co_names - consts = co.co_consts - LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME - while code: - c = bytes([code[0]]) - if c in STORE_OPS: - oparg, = unpack('= HAVE_ARGUMENT: - code = code[3:] - else: - code = code[1:] - - def scan_code(self, co, m): - code = co.co_code - if sys.version_info >= (2, 5): - scanner = self.scan_opcodes_25 - else: - scanner = self.scan_opcodes - for what, args in scanner(co): - if what == "store": - name, = args - m.globalnames[name] = 1 - elif what == "absolute_import": - fromlist, name = args - have_star = 0 - if fromlist is not None: - if "*" in fromlist: - have_star = 1 - fromlist = [f for f in fromlist if f != "*"] - self._safe_import_hook(name, m, fromlist, level=0) - if have_star: - # We've encountered an "import *". If it is a Python module, - # the code has already been parsed and we can suck out the - # global names. - mm = None - if m.__path__: - # At this point we don't know whether 'name' is a - # submodule of 'm' or a global module. Let's just try - # the full name first. - mm = self.modules.get(m.__name__ + "." + name) - if mm is None: - mm = self.modules.get(name) - if mm is not None: - m.globalnames.update(mm.globalnames) - m.starimports.update(mm.starimports) - if mm.__code__ is None: - m.starimports[name] = 1 - else: - m.starimports[name] = 1 - elif what == "relative_import": - level, fromlist, name = args - if name: - self._safe_import_hook(name, m, fromlist, level=level) - else: - parent = self.determine_parent(m, level=level) - self._safe_import_hook(parent.__name__, None, fromlist, level=0) - else: - # We don't expect anything else from the generator. - raise RuntimeError(what) - - for c in co.co_consts: - if isinstance(c, type(co)): - self.scan_code(c, m) - - def load_package(self, fqname, pathname): - self.msgin(2, "load_package", fqname, pathname) - newname = replacePackageMap.get(fqname) - if newname: - fqname = newname - m = self.add_module(fqname) - m.__file__ = pathname - m.__path__ = [pathname] - - # As per comment at top of file, simulate runtime __path__ additions. - m.__path__ = m.__path__ + packagePathMap.get(fqname, []) - - fp, buf, stuff = self.find_module("__init__", m.__path__) - try: - self.load_module(fqname, fp, buf, stuff) - self.msgout(2, "load_package ->", m) - return m - finally: - if fp: - fp.close() - - def add_module(self, fqname): - if fqname in self.modules: - return self.modules[fqname] - self.modules[fqname] = m = Module(fqname) - return m - - def find_module(self, name, path, parent=None): - if parent is not None: - # assert path is not None - fullname = parent.__name__+'.'+name - else: - fullname = name - if fullname in self.excludes: - self.msgout(3, "find_module -> Excluded", fullname) - raise ImportError(name) - - if path is None: - if name in sys.builtin_module_names: - return (None, None, ("", "", imp.C_BUILTIN)) - - path = self.path - return imp.find_module(name, path) - - def report(self): - """Print a report to stdout, listing the found modules with their - paths, as well as modules that are missing, or seem to be missing. - """ - print() - print(" %-25s %s" % ("Name", "File")) - print(" %-25s %s" % ("----", "----")) - # Print modules found - keys = sorted(self.modules.keys()) - for key in keys: - m = self.modules[key] - if m.__path__: - print("P", end=' ') - else: - print("m", end=' ') - print("%-25s" % key, m.__file__ or "") - - # Print missing modules - missing, maybe = self.any_missing_maybe() - if missing: - print() - print("Missing modules:") - for name in missing: - mods = sorted(self.badmodules[name].keys()) - print("?", name, "imported from", ', '.join(mods)) - # Print modules that may be missing, but then again, maybe not... - if maybe: - print() - print("Submodules thay appear to be missing, but could also be", end=' ') - print("global names in the parent package:") - for name in maybe: - mods = sorted(self.badmodules[name].keys()) - print("?", name, "imported from", ', '.join(mods)) - - def any_missing(self): - """Return a list of modules that appear to be missing. Use - any_missing_maybe() if you want to know which modules are - certain to be missing, and which *may* be missing. - """ - missing, maybe = self.any_missing_maybe() - return missing + maybe - - def any_missing_maybe(self): - """Return two lists, one with modules that are certainly missing - and one with modules that *may* be missing. The latter names could - either be submodules *or* just global names in the package. - - The reason it can't always be determined is that it's impossible to - tell which names are imported when "from module import *" is done - with an extension module, short of actually importing it. - """ - missing = [] - maybe = [] - for name in self.badmodules: - if name in self.excludes: - continue - i = name.rfind(".") - if i < 0: - missing.append(name) - continue - subname = name[i+1:] - pkgname = name[:i] - pkg = self.modules.get(pkgname) - if pkg is not None: - if pkgname in self.badmodules[name]: - # The package tried to import this module itself and - # failed. It's definitely missing. - missing.append(name) - elif subname in pkg.globalnames: - # It's a global in the package: definitely not missing. - pass - elif pkg.starimports: - # It could be missing, but the package did an "import *" - # from a non-Python module, so we simply can't be sure. - maybe.append(name) - else: - # It's not a global in the package, the package didn't - # do funny star imports, it's very likely to be missing. - # The symbol could be inserted into the package from the - # outside, but since that's not good style we simply list - # it missing. - missing.append(name) - else: - missing.append(name) - missing.sort() - maybe.sort() - return missing, maybe - - def replace_paths_in_code(self, co): - new_filename = original_filename = os.path.normpath(co.co_filename) - for f, r in self.replace_paths: - if original_filename.startswith(f): - new_filename = r + original_filename[len(f):] - break - - if self.debug and original_filename not in self.processed_paths: - if new_filename != original_filename: - self.msgout(2, "co_filename %r changed to %r" \ - % (original_filename,new_filename,)) - else: - self.msgout(2, "co_filename %r remains unchanged" \ - % (original_filename,)) - self.processed_paths.append(original_filename) - - consts = list(co.co_consts) - for i in range(len(consts)): - if isinstance(consts[i], type(co)): - consts[i] = self.replace_paths_in_code(consts[i]) - - return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, - co.co_flags, co.co_code, tuple(consts), co.co_names, - co.co_varnames, new_filename, co.co_name, - co.co_firstlineno, co.co_lnotab, - co.co_freevars, co.co_cellvars) - - -def test(): - # Parse command line - import getopt - try: - opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") - except getopt.error as msg: - print(msg) - return - - # Process options - debug = 1 - domods = 0 - addpath = [] - exclude = [] - for o, a in opts: - if o == '-d': - debug = debug + 1 - if o == '-m': - domods = 1 - if o == '-p': - addpath = addpath + a.split(os.pathsep) - if o == '-q': - debug = 0 - if o == '-x': - exclude.append(a) - - # Provide default arguments - if not args: - script = "hello.py" - else: - script = args[0] - - # Set the path based on sys.path and the script directory - path = sys.path[:] - path[0] = os.path.dirname(script) - path = addpath + path - if debug > 1: - print("path:") - for item in path: - print(" ", repr(item)) - - # Create the module finder and turn its crank - mf = ModuleFinder(path, debug, exclude) - for arg in args[1:]: - if arg == '-m': - domods = 1 - continue - if domods: - if arg[-2:] == '.*': - mf.import_hook(arg[:-2], None, ["*"]) - else: - mf.import_hook(arg) - else: - mf.load_file(arg) - mf.run_script(script) - mf.report() - return mf # for -i debugging - - -if __name__ == '__main__': - try: - mf = test() - except KeyboardInterrupt: - print("\n[interrupted]") diff --git a/tests/bytecode/pylib-tests/nturl2path.py b/tests/bytecode/pylib-tests/nturl2path.py deleted file mode 100644 index e0c2f23527..0000000000 --- a/tests/bytecode/pylib-tests/nturl2path.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Convert a NT pathname to a file URL and vice versa.""" - -def url2pathname(url): - """OS-specific conversion from a relative URL of the 'file' scheme - to a file system path; not recommended for general use.""" - # e.g. - # ///C|/foo/bar/spam.foo - # becomes - # C:\foo\bar\spam.foo - import string, urllib.parse - # Windows itself uses ":" even in URLs. - url = url.replace(':', '|') - if '|' not in url: - # No drive specifier, just convert slashes - if url[:4] == '////': - # path is something like ////host/path/on/remote/host - # convert this to \\host\path\on\remote\host - # (notice halving of slashes at the start of the path) - url = url[2:] - components = url.split('/') - # make sure not to convert quoted slashes :-) - return urllib.parse.unquote('\\'.join(components)) - comp = url.split('|') - if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: - error = 'Bad URL: ' + url - raise IOError(error) - drive = comp[0][-1].upper() - components = comp[1].split('/') - path = drive + ':' - for comp in components: - if comp: - path = path + '\\' + urllib.parse.unquote(comp) - # Issue #11474 - handing url such as |c/| - if path.endswith(':') and url.endswith('/'): - path += '\\' - return path - -def pathname2url(p): - """OS-specific conversion from a file system path to a relative URL - of the 'file' scheme; not recommended for general use.""" - # e.g. - # C:\foo\bar\spam.foo - # becomes - # ///C|/foo/bar/spam.foo - import urllib.parse - if ':' not in p: - # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p - components = p.split('\\') - return urllib.parse.quote('/'.join(components)) - comp = p.split(':') - if len(comp) != 2 or len(comp[0]) > 1: - error = 'Bad path: ' + p - raise IOError(error) - - drive = urllib.parse.quote(comp[0].upper()) - components = comp[1].split('\\') - path = '///' + drive + ':' - for comp in components: - if comp: - path = path + '/' + urllib.parse.quote(comp) - return path diff --git a/tests/bytecode/pylib-tests/opcode.py b/tests/bytecode/pylib-tests/opcode.py deleted file mode 100644 index d81b6bc3c9..0000000000 --- a/tests/bytecode/pylib-tests/opcode.py +++ /dev/null @@ -1,185 +0,0 @@ - -""" -opcode module - potentially shared between dis and other modules which -operate on bytecodes (e.g. peephole optimizers). -""" - -__all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"] - -#cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', -# 'is not', 'exception match', 'BAD') - -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasnargs = [] - -opmap = {} -opname = [''] * 256 -for op in range(256): opname[op] = '<%r>' % (op,) -del op - -def def_op(name, op): - opname[op] = name - opmap[name] = op - -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('POP_TOP', 1) -def_op('ROT_TWO', 2) -def_op('ROT_THREE', 3) -def_op('DUP_TOP', 4) -def_op('DUP_TOP_TWO', 5) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_POWER', 19) -def_op('BINARY_MULTIPLY', 20) - -def_op('BINARY_MODULO', 22) -def_op('BINARY_ADD', 23) -def_op('BINARY_SUBTRACT', 24) -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_FLOOR_DIVIDE', 26) -def_op('BINARY_TRUE_DIVIDE', 27) -def_op('INPLACE_FLOOR_DIVIDE', 28) -def_op('INPLACE_TRUE_DIVIDE', 29) - -def_op('STORE_MAP', 54) -def_op('INPLACE_ADD', 55) -def_op('INPLACE_SUBTRACT', 56) -def_op('INPLACE_MULTIPLY', 57) - -def_op('INPLACE_MODULO', 59) -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) -def_op('BINARY_LSHIFT', 62) -def_op('BINARY_RSHIFT', 63) -def_op('BINARY_AND', 64) -def_op('BINARY_XOR', 65) -def_op('BINARY_OR', 66) -def_op('INPLACE_POWER', 67) -def_op('GET_ITER', 68) -def_op('STORE_LOCALS', 69) - -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) -def_op('YIELD_FROM', 72) - -def_op('INPLACE_LSHIFT', 75) -def_op('INPLACE_RSHIFT', 76) -def_op('INPLACE_AND', 77) -def_op('INPLACE_XOR', 78) -def_op('INPLACE_OR', 79) -def_op('BREAK_LOOP', 80) -def_op('WITH_CLEANUP', 81) - -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) - -def_op('YIELD_VALUE', 86) -def_op('POP_BLOCK', 87) -def_op('END_FINALLY', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # Opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries (upto 255) -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list - -jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip -jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code -jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jabs_op('JUMP_ABSOLUTE', 113) # "" -jabs_op('POP_JUMP_IF_FALSE', 114) # "" -jabs_op('POP_JUMP_IF_TRUE', 115) # "" - -name_op('LOAD_GLOBAL', 116) # Index in name list - -jabs_op('CONTINUE_LOOP', 119) # Target address -jrel_op('SETUP_LOOP', 120) # Distance to target address -jrel_op('SETUP_EXCEPT', 121) # "" -jrel_op('SETUP_FINALLY', 122) # "" - -def_op('LOAD_FAST', 124) # Local variable number -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) - -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) -hasnargs.append(131) -def_op('MAKE_FUNCTION', 132) # Number of args with default values -def_op('BUILD_SLICE', 133) # Number of items -def_op('MAKE_CLOSURE', 134) -def_op('LOAD_CLOSURE', 135) -hasfree.append(135) -def_op('LOAD_DEREF', 136) -hasfree.append(136) -def_op('STORE_DEREF', 137) -hasfree.append(137) -def_op('DELETE_DEREF', 138) -hasfree.append(138) - -def_op('CALL_FUNCTION_VAR', 140) # #args + (#kwargs << 8) -hasnargs.append(140) -def_op('CALL_FUNCTION_KW', 141) # #args + (#kwargs << 8) -hasnargs.append(141) -def_op('CALL_FUNCTION_VAR_KW', 142) # #args + (#kwargs << 8) -hasnargs.append(142) - -jrel_op('SETUP_WITH', 143) - -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 - -del def_op, name_op, jrel_op, jabs_op diff --git a/tests/bytecode/pylib-tests/pipes.py b/tests/bytecode/pylib-tests/pipes.py deleted file mode 100644 index f1a16f63de..0000000000 --- a/tests/bytecode/pylib-tests/pipes.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Conversion pipeline templates. - -The problem: ------------- - -Suppose you have some data that you want to convert to another format, -such as from GIF image format to PPM image format. Maybe the -conversion involves several steps (e.g. piping it through compress or -uuencode). Some of the conversion steps may require that their input -is a disk file, others may be able to read standard input; similar for -their output. The input to the entire conversion may also be read -from a disk file or from an open file, and similar for its output. - -The module lets you construct a pipeline template by sticking one or -more conversion steps together. It will take care of creating and -removing temporary files if they are necessary to hold intermediate -data. You can then use the template to do conversions from many -different sources to many different destinations. The temporary -file names used are different each time the template is used. - -The templates are objects so you can create templates for many -different conversion steps and store them in a dictionary, for -instance. - - -Directions: ------------ - -To create a template: - t = Template() - -To add a conversion step to a template: - t.append(command, kind) -where kind is a string of two characters: the first is '-' if the -command reads its standard input or 'f' if it requires a file; the -second likewise for the output. The command must be valid /bin/sh -syntax. If input or output files are required, they are passed as -$IN and $OUT; otherwise, it must be possible to use the command in -a pipeline. - -To add a conversion step at the beginning: - t.prepend(command, kind) - -To convert a file to another file using a template: - sts = t.copy(infile, outfile) -If infile or outfile are the empty string, standard input is read or -standard output is written, respectively. The return value is the -exit status of the conversion pipeline. - -To open a file for reading or writing through a conversion pipeline: - fp = t.open(file, mode) -where mode is 'r' to read the file, or 'w' to write it -- just like -for the built-in function open() or for os.popen(). - -To create a new template object initialized to a given one: - t2 = t.clone() -""" # ' - - -import re -import os -import tempfile -# we import the quote function rather than the module for backward compat -# (quote used to be an undocumented but used function in pipes) -from shlex import quote - -__all__ = ["Template"] - -# Conversion step kinds - -FILEIN_FILEOUT = 'ff' # Must read & write real files -STDIN_FILEOUT = '-f' # Must write a real file -FILEIN_STDOUT = 'f-' # Must read a real file -STDIN_STDOUT = '--' # Normal pipeline element -SOURCE = '.-' # Must be first, writes stdout -SINK = '-.' # Must be last, reads stdin - -stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \ - SOURCE, SINK] - - -class Template: - """Class representing a pipeline template.""" - - def __init__(self): - """Template() returns a fresh pipeline template.""" - self.debugging = 0 - self.reset() - - def __repr__(self): - """t.__repr__() implements repr(t).""" - return '