py/objstr: Add hex/fromhex to bytes/memoryview/bytearray.

These were added in Python 3.5.

Enabled via MICROPY_PY_BUILTINS_BYTES_HEX, and enabled by default for all
ports that currently have ubinascii.

Rework ubinascii to use the implementation of these methods.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

extmod/modubinascii.c

@@ -30,78 +30,21 @@
 
 #include "py/runtime.h"
 #include "py/binary.h"
+#include "py/objstr.h"
 
 #if MICROPY_PY_UBINASCII
 
-STATIC mp_obj_t mod_binascii_hexlify(size_t n_args, const mp_obj_t *args) {
-    // First argument is the data to convert.
-    // Second argument is an optional separator to be used between values.
-    const char *sep = NULL;
-    mp_buffer_info_t bufinfo;
-    mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_READ);
-
-    // Code below assumes non-zero buffer length when computing size with
-    // separator, so handle the zero-length case here.
-    if (bufinfo.len == 0) {
-        return mp_const_empty_bytes;
-    }
-
-    vstr_t vstr;
-    size_t out_len = bufinfo.len * 2;
-    if (n_args > 1) {
-        // 1-char separator between hex numbers
-        out_len += bufinfo.len - 1;
-        sep = mp_obj_str_get_str(args[1]);
-    }
-    vstr_init_len(&vstr, out_len);
-    byte *in = bufinfo.buf, *out = (byte *)vstr.buf;
-    for (mp_uint_t i = bufinfo.len; i--;) {
-        byte d = (*in >> 4);
-        if (d > 9) {
-            d += 'a' - '9' - 1;
-        }
-        *out++ = d + '0';
-        d = (*in++ & 0xf);
-        if (d > 9) {
-            d += 'a' - '9' - 1;
-        }
-        *out++ = d + '0';
-        if (sep != NULL && i != 0) {
-            *out++ = *sep;
-        }
-    }
-    return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+#if MICROPY_PY_BUILTINS_BYTES_HEX
+STATIC mp_obj_t bytes_hex_as_bytes(size_t n_args, const mp_obj_t *args) {
+    return mp_obj_bytes_hex(n_args, args, &mp_type_bytes);
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_binascii_hexlify_obj, 1, 2, mod_binascii_hexlify);
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_hex_as_bytes_obj, 1, 2, bytes_hex_as_bytes);
 
-STATIC mp_obj_t mod_binascii_unhexlify(mp_obj_t data) {
-    mp_buffer_info_t bufinfo;
-    mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ);
-
-    if ((bufinfo.len & 1) != 0) {
-        mp_raise_ValueError(MP_ERROR_TEXT("odd-length string"));
-    }
-    vstr_t vstr;
-    vstr_init_len(&vstr, bufinfo.len / 2);
-    byte *in = bufinfo.buf, *out = (byte *)vstr.buf;
-    byte hex_byte = 0;
-    for (mp_uint_t i = bufinfo.len; i--;) {
-        byte hex_ch = *in++;
-        if (unichar_isxdigit(hex_ch)) {
-            hex_byte += unichar_xdigit_value(hex_ch);
-        } else {
-            mp_raise_ValueError(MP_ERROR_TEXT("non-hex digit found"));
-        }
-        if (i & 1) {
-            hex_byte <<= 4;
-        } else {
-            *out++ = hex_byte;
-            hex_byte = 0;
-        }
-    }
-    return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+STATIC mp_obj_t bytes_fromhex_bytes(mp_obj_t data) {
+    return mp_obj_bytes_fromhex(MP_OBJ_FROM_PTR(&mp_type_bytes), data);
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_binascii_unhexlify_obj, mod_binascii_unhexlify);
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(bytes_fromhex_obj, bytes_fromhex_bytes);
+#endif
 
 // If ch is a character in the base64 alphabet, and is not a pad character, then
 // the corresponding integer between 0 and 63, inclusively, is returned.
@@ -242,8 +185,10 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_binascii_crc32_obj, 1, 2, mod_bin
 
 STATIC const mp_rom_map_elem_t mp_module_binascii_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ubinascii) },
-    { MP_ROM_QSTR(MP_QSTR_hexlify), MP_ROM_PTR(&mod_binascii_hexlify_obj) },
-    { MP_ROM_QSTR(MP_QSTR_unhexlify), MP_ROM_PTR(&mod_binascii_unhexlify_obj) },
+    #if MICROPY_PY_BUILTINS_BYTES_HEX
+    { MP_ROM_QSTR(MP_QSTR_hexlify), MP_ROM_PTR(&bytes_hex_as_bytes_obj) },
+    { MP_ROM_QSTR(MP_QSTR_unhexlify), MP_ROM_PTR(&bytes_fromhex_obj) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_a2b_base64), MP_ROM_PTR(&mod_binascii_a2b_base64_obj) },
     { MP_ROM_QSTR(MP_QSTR_b2a_base64), MP_ROM_PTR(&mod_binascii_b2a_base64_obj) },
     #if MICROPY_PY_UBINASCII_CRC32

ports/cc3200/mpconfigport.h

@@ -78,6 +78,7 @@
 #define MICROPY_VFS_FAT                             (1)
 #define MICROPY_PY_ASYNC_AWAIT (0)
 #define MICROPY_PY_ALL_SPECIAL_METHODS              (1)
+#define MICROPY_PY_BUILTINS_BYTES_HEX               (1)
 #define MICROPY_PY_BUILTINS_INPUT                   (1)
 #define MICROPY_PY_BUILTINS_HELP                    (1)
 #define MICROPY_PY_BUILTINS_HELP_TEXT               cc3200_help_text

ports/javascript/mpconfigport.h

@@ -89,6 +89,7 @@
 #define MICROPY_PY_UTIME_MP_HAL     (1)
 #define MICROPY_REPL_AUTO_INDENT    (1)
 #define MICROPY_PY_FUNCTION_ATTRS   (1)
+#define MICROPY_PY_BUILTINS_BYTES_HEX (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE (1)
 #define MICROPY_PY_BUILTINS_STR_CENTER (1)
 #define MICROPY_PY_BUILTINS_STR_PARTITION (1)

ports/mimxrt/mpconfigport.h

@@ -74,6 +74,7 @@ uint32_t trng_random_u32(void);
 #define MICROPY_PY_DESCRIPTORS              (1)
 #define MICROPY_PY_DELATTR_SETATTR          (1)
 #define MICROPY_PY_FSTRINGS                 (1)
+#define MICROPY_PY_BUILTINS_BYTES_HEX       (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE     (1)
 #define MICROPY_PY_BUILTINS_STR_CENTER      (1)
 #define MICROPY_PY_BUILTINS_STR_PARTITION   (1)

ports/qemu-arm/mpconfigport.h

@@ -28,6 +28,7 @@
 #define MICROPY_PY_ALL_SPECIAL_METHODS (1)
 #define MICROPY_PY_REVERSE_SPECIAL_METHODS (1)
 #define MICROPY_PY_ARRAY_SLICE_ASSIGN (1)
+#define MICROPY_PY_BUILTINS_BYTES_HEX (1)
 #define MICROPY_PY_BUILTINS_FROZENSET (1)
 #define MICROPY_PY_BUILTINS_MEMORYVIEW (1)
 #define MICROPY_PY_BUILTINS_POW3    (1)

ports/samd/mpconfigport.h

@@ -56,6 +56,7 @@
 #define MICROPY_MODULE_WEAK_LINKS           (1)
 // Control over Python builtins
 #define MICROPY_PY_ASYNC_AWAIT              (0)
+#define MICROPY_PY_BUILTINS_BYTES_HEX       (1)
 #define MICROPY_PY_BUILTINS_STR_COUNT       (0)
 #define MICROPY_PY_BUILTINS_MEMORYVIEW      (1)
 #define MICROPY_PY_BUILTINS_SET             (0)

ports/unix/mpconfigport.h

@@ -61,6 +61,7 @@
 #define MICROPY_PY_DESCRIPTORS                  (1)
 #define MICROPY_PY_DELATTR_SETATTR              (1)
 #define MICROPY_PY_FSTRINGS                     (1)
+#define MICROPY_PY_BUILTINS_BYTES_HEX           (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE         (1)
 #define MICROPY_PY_BUILTINS_STR_CENTER          (1)
 #define MICROPY_PY_BUILTINS_STR_PARTITION       (1)

ports/windows/mpconfigport.h

@@ -84,6 +84,7 @@
 #define MICROPY_PY_DESCRIPTORS      (1)
 #define MICROPY_PY_DELATTR_SETATTR  (1)
 #define MICROPY_PY_FSTRINGS         (1)
+#define MICROPY_PY_BUILTINS_BYTES_HEX (1)
 #define MICROPY_PY_BUILTINS_STR_UNICODE (1)
 #define MICROPY_PY_BUILTINS_STR_CENTER (1)
 #define MICROPY_PY_BUILTINS_STR_PARTITION (1)

ports/zephyr/mpconfigport.h

@@ -45,6 +45,7 @@
 #define MICROPY_CPYTHON_COMPAT      (0)
 #define MICROPY_PY_ASYNC_AWAIT      (0)
 #define MICROPY_PY_ATTRTUPLE        (0)
+#define MICROPY_PY_BUILTINS_BYTES_HEX (1)
 #define MICROPY_PY_BUILTINS_ENUMERATE (0)
 #define MICROPY_PY_BUILTINS_FILTER  (0)
 #define MICROPY_PY_BUILTINS_MIN_MAX (0)

py/mpconfig.h

@@ -987,6 +987,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_STR_BYTES_CMP_WARN (0)
 #endif
 
+// Add bytes.hex and bytes.fromhex
+#ifndef MICROPY_PY_BUILTINS_BYTES_HEX
+#define MICROPY_PY_BUILTINS_BYTES_HEX (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
+#endif
+
 // Whether str object is proper unicode
 #ifndef MICROPY_PY_BUILTINS_STR_UNICODE
 #define MICROPY_PY_BUILTINS_STR_UNICODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)

py/objarray.c

@@ -245,6 +245,12 @@ STATIC void memoryview_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
         mp_obj_array_t *self = MP_OBJ_TO_PTR(self_in);
         dest[0] = MP_OBJ_NEW_SMALL_INT(mp_binary_get_size('@', self->typecode & TYPECODE_MASK, NULL));
     }
+    #if MICROPY_PY_BUILTINS_BYTES_HEX
+    else {
+        // Need to forward to locals dict.
+        dest[1] = MP_OBJ_SENTINEL;
+    }
+    #endif
 }
 #endif
 
@@ -607,6 +613,9 @@ const mp_obj_type_t mp_type_memoryview = {
     #if MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE
     .attr = memoryview_attr,
     #endif
+    #if MICROPY_PY_BUILTINS_BYTES_HEX
+    .locals_dict = (mp_obj_dict_t *)&mp_obj_memoryview_locals_dict,
+    #endif
     .subscr = array_subscr,
     .buffer_p = { .get_buffer = array_get_buffer },
 };

py/objstr.c

@@ -1950,6 +1950,84 @@ STATIC mp_obj_t str_encode(size_t n_args, const mp_obj_t *args) {
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_encode_obj, 1, 3, str_encode);
 #endif
 
+#if MICROPY_PY_BUILTINS_BYTES_HEX
+mp_obj_t mp_obj_bytes_hex(size_t n_args, const mp_obj_t *args, const mp_obj_type_t *type) {
+    // First argument is the data to convert.
+    // Second argument is an optional separator to be used between values.
+    const char *sep = NULL;
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_READ);
+
+    // Code below assumes non-zero buffer length when computing size with
+    // separator, so handle the zero-length case here.
+    if (bufinfo.len == 0) {
+        return mp_const_empty_bytes;
+    }
+
+    vstr_t vstr;
+    size_t out_len = bufinfo.len * 2;
+    if (n_args > 1) {
+        // 1-char separator between hex numbers
+        out_len += bufinfo.len - 1;
+        sep = mp_obj_str_get_str(args[1]);
+    }
+    vstr_init_len(&vstr, out_len);
+    byte *in = bufinfo.buf, *out = (byte *)vstr.buf;
+    for (mp_uint_t i = bufinfo.len; i--;) {
+        byte d = (*in >> 4);
+        if (d > 9) {
+            d += 'a' - '9' - 1;
+        }
+        *out++ = d + '0';
+        d = (*in++ & 0xf);
+        if (d > 9) {
+            d += 'a' - '9' - 1;
+        }
+        *out++ = d + '0';
+        if (sep != NULL && i != 0) {
+            *out++ = *sep;
+        }
+    }
+    return mp_obj_new_str_from_vstr(type, &vstr);
+}
+
+mp_obj_t mp_obj_bytes_fromhex(mp_obj_t type_in, mp_obj_t data) {
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ);
+
+    if ((bufinfo.len & 1) != 0) {
+        mp_raise_ValueError(MP_ERROR_TEXT("odd-length string"));
+    }
+    vstr_t vstr;
+    vstr_init_len(&vstr, bufinfo.len / 2);
+    byte *in = bufinfo.buf, *out = (byte *)vstr.buf;
+    byte hex_byte = 0;
+    for (mp_uint_t i = bufinfo.len; i--;) {
+        byte hex_ch = *in++;
+        if (unichar_isxdigit(hex_ch)) {
+            hex_byte += unichar_xdigit_value(hex_ch);
+        } else {
+            mp_raise_ValueError(MP_ERROR_TEXT("non-hex digit found"));
+        }
+        if (i & 1) {
+            hex_byte <<= 4;
+        } else {
+            *out++ = hex_byte;
+            hex_byte = 0;
+        }
+    }
+    return mp_obj_new_str_from_vstr(MP_OBJ_TO_PTR(type_in), &vstr);
+}
+
+STATIC mp_obj_t bytes_hex_as_str(size_t n_args, const mp_obj_t *args) {
+    return mp_obj_bytes_hex(n_args, args, &mp_type_str);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_hex_as_str_obj, 1, 2, bytes_hex_as_str);
+
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(bytes_fromhex_obj, mp_obj_bytes_fromhex);
+STATIC MP_DEFINE_CONST_CLASSMETHOD_OBJ(bytes_fromhex_classmethod_obj, MP_ROM_PTR(&bytes_fromhex_obj));
+#endif // MICROPY_PY_BUILTINS_BYTES_HEX
+
 mp_int_t mp_obj_str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) {
     if (flags == MP_BUFFER_READ) {
         GET_STR_DATA_LEN(self_in, str_data, str_len);
@@ -1970,6 +2048,10 @@ STATIC const mp_rom_map_elem_t array_bytearray_str_bytes_locals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_append), MP_ROM_PTR(&mp_obj_array_append_obj) },
     { MP_ROM_QSTR(MP_QSTR_extend), MP_ROM_PTR(&mp_obj_array_extend_obj) },
     #endif
+    #if MICROPY_PY_BUILTINS_BYTES_HEX
+    { MP_ROM_QSTR(MP_QSTR_hex), MP_ROM_PTR(&bytes_hex_as_str_obj) },
+    { MP_ROM_QSTR(MP_QSTR_fromhex), MP_ROM_PTR(&bytes_fromhex_classmethod_obj) },
+    #endif
     #if MICROPY_CPYTHON_COMPAT
     { MP_ROM_QSTR(MP_QSTR_decode), MP_ROM_PTR(&bytes_decode_obj) },
     #endif
@@ -2018,6 +2100,12 @@ STATIC const mp_rom_map_elem_t array_bytearray_str_bytes_locals_table[] = {
 #define TABLE_ENTRIES_COMPAT 0
 #endif
 
+#if MICROPY_PY_BUILTINS_BYTES_HEX
+#define TABLE_ENTRIES_HEX 2
+#else
+#define TABLE_ENTRIES_HEX 0
+#endif
+
 #if MICROPY_PY_ARRAY || MICROPY_PY_BUILTINS_BYTEARRAY
 #define TABLE_ENTRIES_ARRAY 2
 #else
@@ -2025,8 +2113,8 @@ STATIC const mp_rom_map_elem_t array_bytearray_str_bytes_locals_table[] = {
 #endif
 
 MP_DEFINE_CONST_DICT_WITH_SIZE(mp_obj_str_locals_dict,
-    array_bytearray_str_bytes_locals_table + TABLE_ENTRIES_ARRAY + TABLE_ENTRIES_COMPAT,
-    MP_ARRAY_SIZE(array_bytearray_str_bytes_locals_table) - (TABLE_ENTRIES_ARRAY + TABLE_ENTRIES_COMPAT));
+    array_bytearray_str_bytes_locals_table + TABLE_ENTRIES_ARRAY + TABLE_ENTRIES_HEX + TABLE_ENTRIES_COMPAT,
+    MP_ARRAY_SIZE(array_bytearray_str_bytes_locals_table) - (TABLE_ENTRIES_ARRAY + TABLE_ENTRIES_HEX + TABLE_ENTRIES_COMPAT));
 
 #if TABLE_ENTRIES_COMPAT == 0
 #define mp_obj_bytes_locals_dict mp_obj_str_locals_dict
@@ -2048,6 +2136,12 @@ MP_DEFINE_CONST_DICT_WITH_SIZE(mp_obj_array_locals_dict,
     TABLE_ENTRIES_ARRAY);
 #endif
 
+#if MICROPY_PY_BUILTINS_MEMORYVIEW && MICROPY_PY_BUILTINS_BYTES_HEX
+MP_DEFINE_CONST_DICT_WITH_SIZE(mp_obj_memoryview_locals_dict,
+    array_bytearray_str_bytes_locals_table + TABLE_ENTRIES_ARRAY,
+    1); // Just the "hex" entry.
+#endif
+
 #if !MICROPY_PY_BUILTINS_STR_UNICODE
 STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf);
 

py/objstr.h

@@ -127,8 +127,15 @@ MP_DECLARE_CONST_FUN_OBJ_1(str_isupper_obj);
 MP_DECLARE_CONST_FUN_OBJ_1(str_islower_obj);
 MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_decode_obj);
 
+mp_obj_t mp_obj_bytes_hex(size_t n_args, const mp_obj_t *args, const mp_obj_type_t *type);
+mp_obj_t mp_obj_bytes_fromhex(mp_obj_t type_in, mp_obj_t data);
+
 extern const mp_obj_dict_t mp_obj_str_locals_dict;
 
+#if MICROPY_PY_BUILTINS_MEMORYVIEW && MICROPY_PY_BUILTINS_BYTES_HEX
+extern const mp_obj_dict_t mp_obj_memoryview_locals_dict;
+#endif
+
 #if MICROPY_PY_BUILTINS_BYTEARRAY
 extern const mp_obj_dict_t mp_obj_bytearray_locals_dict;
 #endif