py: Add MICROPY_USE_SMALL_HEAP_COMPILER option, disabled by default.

This new option allows the original and new parser/compiler to coexist.

py/compile.c

@@ -36,7 +36,7 @@
 #include "py/runtime.h"
 #include "py/asmbase.h"
 
-#if MICROPY_ENABLE_COMPILER
+#if MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER
 
 // TODO need to mangle __attr names
 
@@ -3514,4 +3514,4 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt
     return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
 }
 
-#endif // MICROPY_ENABLE_COMPILER
+#endif // MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER

py/compile2.c

@@ -35,7 +35,7 @@
 #include "py/compile.h"
 #include "py/runtime.h"
 
-#if MICROPY_ENABLE_COMPILER
+#if MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER
 
 #if MICROPY_PY_ASYNC_AWAIT
 #error "async/await syntax not implemented with this parser/compiler"
@@ -3476,4 +3476,4 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt
     return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
 }
 
-#endif // MICROPY_ENABLE_COMPILER
+#endif // MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER

py/mpconfig.h

@@ -358,6 +358,14 @@
 #define MICROPY_COMP_RETURN_IF_EXPR (0)
 #endif
 
+// Whether to use an alternate parser and compiler optimised for small heaps.
+// This parser/compiler uses more code space but a lot less heap when building
+// the parse tree.  But it has the disadvantage that the entire parse tree must
+// fit in a contiguous chunk of memory on the heap.
+#ifndef MICROPY_USE_SMALL_HEAP_COMPILER
+#define MICROPY_USE_SMALL_HEAP_COMPILER (0)
+#endif
+
 /*****************************************************************************/
 /* Internal debugging stuff                                                  */
 
@@ -678,7 +686,7 @@ typedef double mp_float_t;
 
 // Support for async/await/async for/async with
 #ifndef MICROPY_PY_ASYNC_AWAIT
-#define MICROPY_PY_ASYNC_AWAIT (1)
+#define MICROPY_PY_ASYNC_AWAIT (!MICROPY_USE_SMALL_HEAP_COMPILER)
 #endif
 
 // Issue a warning when comparing str and bytes objects

py/parse.c

@@ -41,7 +41,7 @@
 #include "py/objstr.h"
 #include "py/builtin.h"
 
-#if MICROPY_ENABLE_COMPILER
+#if MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER
 
 #define RULE_ACT_ARG_MASK       (0x0f)
 #define RULE_ACT_KIND_MASK      (0x30)
@@ -1081,4 +1081,4 @@ void mp_parse_tree_clear(mp_parse_tree_t *tree) {
     }
 }
 
-#endif // MICROPY_ENABLE_COMPILER
+#endif // MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER

py/parse.h

@@ -23,15 +23,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "py/parse2.h"
 #ifndef MICROPY_INCLUDED_PY_PARSE_H
 #define MICROPY_INCLUDED_PY_PARSE_H
 
 #include <stddef.h>
 #include <stdint.h>
 
+#include "py/parse2.h"
 #include "py/obj.h"
 
+#if !MICROPY_USE_SMALL_HEAP_COMPILER
+
 struct _mp_lexer_t;
 
 // a mp_parse_node_t is:
@@ -105,4 +107,6 @@ typedef struct _mp_parse_t {
 mp_parse_tree_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind);
 void mp_parse_tree_clear(mp_parse_tree_t *tree);
 
+#endif // !MICROPY_USE_SMALL_HEAP_COMPILER
+
 #endif // MICROPY_INCLUDED_PY_PARSE_H

py/parse2.c

@@ -36,6 +36,8 @@
 #include "py/parsenum.h"
 #include "py/smallint.h"
 
+#if MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER
+
 #define RULE_ACT_ARG_MASK       (0x0f)
 #define RULE_ACT_KIND_MASK      (0x30)
 #define RULE_ACT_ALLOW_IDENT    (0x40)
@@ -1445,3 +1447,5 @@ void mp_parse_tree_clear(mp_parse_tree_t *tree) {
         chunk = next;
     }
 }
+
+#endif // MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER

py/parse2.h

@@ -23,14 +23,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef MICROPY_INCLUDED_PY_PARSE_H
-#define MICROPY_INCLUDED_PY_PARSE_H
+#ifndef MICROPY_INCLUDED_PY_PARSE2_H
+#define MICROPY_INCLUDED_PY_PARSE2_H
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include "py/obj.h"
 
+#if MICROPY_USE_SMALL_HEAP_COMPILER
+
 struct _mp_lexer_t;
 
 #define MP_PT_NULL (0)
@@ -42,6 +44,8 @@ struct _mp_lexer_t;
 #define MP_PT_ID_BASE (10) // +16
 #define MP_PT_RULE_BASE (26) // +173-ish
 
+typedef const byte *mp_parse_node_t;
+
 extern const byte pt_const_int0[];
 
 static inline const byte *pt_tok_extract(const byte *p, byte *tok) {
@@ -130,4 +134,6 @@ typedef struct _mp_parse_t {
 mp_parse_tree_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind);
 void mp_parse_tree_clear(mp_parse_tree_t *tree);
 
-#endif // MICROPY_INCLUDED_PY_PARSE_H
+#endif // MICROPY_USE_SMALL_HEAP_COMPILER
+
+#endif // MICROPY_INCLUDED_PY_PARSE2_H

py/py.mk

@@ -117,8 +117,10 @@ PY_O_BASENAME = \
 	mpz.o \
 	reader.o \
 	lexer.o \
+	parse.o \
 	parse2.o \
 	scope.o \
+	compile.o \
 	compile2.o \
 	emitcommon.o \
 	emitbc.o \

py/scope.c

@@ -40,18 +40,25 @@ STATIC const uint8_t scope_simple_name_table[] = {
     [SCOPE_GEN_EXPR] = MP_QSTR__lt_genexpr_gt_,
 };
 
-scope_t *scope_new(scope_kind_t kind, const byte *pn, qstr source_file, mp_uint_t emit_options) {
+scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options) {
     scope_t *scope = m_new0(scope_t, 1);
     scope->kind = kind;
     scope->pn = pn;
     scope->source_file = source_file;
     if (kind == SCOPE_FUNCTION || kind == SCOPE_CLASS) {
+        #if MICROPY_USE_SMALL_HEAP_COMPILER
         qstr id;
         pt_extract_id(pn, &id); // function name
         scope->simple_name = id;
+        #else
+        scope->simple_name = MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t*)pn)->nodes[0]);
+        #endif
     } else {
         scope->simple_name = scope_simple_name_table[kind];
     }
+    #if !MICROPY_USE_SMALL_HEAP_COMPILER
+    scope->raw_code = mp_emit_glue_new_raw_code();
+    #endif
     scope->emit_options = emit_options;
     scope->id_info_alloc = MICROPY_ALLOC_SCOPE_ID_INIT;
     scope->id_info = m_new(id_info_t, scope->id_info_alloc);

py/scope.h

@@ -69,7 +69,10 @@ typedef enum {
 typedef struct _scope_t {
     scope_kind_t kind;
     struct _scope_t *parent;
-    const byte *pn; // points to the node after the scope index node
+    #if !MICROPY_USE_SMALL_HEAP_COMPILER
+    struct _scope_t *next;
+    #endif
+    mp_parse_node_t pn; // for small-heap compiler, points to the node after the scope index node
     uint16_t source_file; // a qstr
     uint16_t simple_name; // a qstr
     mp_raw_code_t *raw_code;
@@ -86,7 +89,7 @@ typedef struct _scope_t {
     id_info_t *id_info;
 } scope_t;
 
-scope_t *scope_new(scope_kind_t kind, const byte *pn, qstr source_file, mp_uint_t emit_options);
+scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options);
 void scope_free(scope_t *scope);
 id_info_t *scope_find_or_add_id(scope_t *scope, qstr qstr, bool *added);
 id_info_t *scope_find(scope_t *scope, qstr qstr);