From 9acf412a07d055403d393ef04c0c4d7b62ef3716 Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Wed, 30 Aug 2023 15:58:26 +0200 Subject: [PATCH] feat(tools): move uf2 generation to esptool --- .gitlab/ci/host-test.yml | 6 - .gitlab/ci/rules.yml | 1 - components/esptool_py/project_include.cmake | 25 +++ tools/ci/executable-list.txt | 1 - tools/ci/mypy_ignore_list.txt | 1 - tools/cmake/idf.cmake | 1 - tools/cmake/project.cmake | 3 - tools/cmake/uf2.cmake | 49 ----- tools/idf_py_actions/uf2_ext.py | 19 +- tools/mkuf2.py | 176 ++++++----------- tools/test_build_system/test_build.py | 6 +- tools/test_mkuf2/test_mkuf2.py | 208 -------------------- 12 files changed, 102 insertions(+), 394 deletions(-) delete mode 100644 tools/cmake/uf2.cmake delete mode 100755 tools/test_mkuf2/test_mkuf2.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index c13c6a979a..dde3f469ad 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -208,12 +208,6 @@ test_mkdfu: - cd ${IDF_PATH}/tools/test_mkdfu - ./test_mkdfu.py -test_mkuf2: - extends: .host_test_template - script: - - cd ${IDF_PATH}/tools/test_mkuf2 - - ./test_mkuf2.py - test_sbom: extends: - .host_test_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 2d5d8ac48e..e5b7a61c63 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -193,7 +193,6 @@ - "tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py" - "tools/mkuf2.py" - - "tools/test_mkuf2/test_mkuf2.py" - "tools/split_paths_by_spaces.py" diff --git a/components/esptool_py/project_include.cmake b/components/esptool_py/project_include.cmake index 46c1259113..7c687a50e8 100644 --- a/components/esptool_py/project_include.cmake +++ b/components/esptool_py/project_include.cmake @@ -11,6 +11,7 @@ set(ESPTOOLPY ${python} "$ENV{ESPTOOL_WRAPPER}" "${CMAKE_CURRENT_LIST_DIR}/espto set(ESPSECUREPY ${python} "${CMAKE_CURRENT_LIST_DIR}/esptool/espsecure.py") set(ESPEFUSEPY ${python} "${CMAKE_CURRENT_LIST_DIR}/esptool/espefuse.py") set(ESPMONITOR ${python} -m esp_idf_monitor) +set(ESPMKUF2 ${python} "${idf_path}/tools/mkuf2.py" write --chip ${chip_model}) set(ESPTOOLPY_CHIP "${chip_model}") if(NOT CONFIG_APP_BUILD_TYPE_RAM AND CONFIG_APP_BUILD_GENERATE_BINARIES) @@ -208,6 +209,30 @@ add_custom_target(erase_flash VERBATIM ) +set(UF2_ARGS --json "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json") + +add_custom_target(uf2 + COMMAND ${CMAKE_COMMAND} + -D "IDF_PATH=${idf_path}" + -D "SERIAL_TOOL=${ESPMKUF2}" + -D "SERIAL_TOOL_ARGS=${UF2_ARGS};-o;${CMAKE_CURRENT_BINARY_DIR}/uf2.bin" + -P run_serial_tool.cmake + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + USES_TERMINAL + VERBATIM + ) + +add_custom_target(uf2-app + COMMAND ${CMAKE_COMMAND} + -D "IDF_PATH=${idf_path}" + -D "SERIAL_TOOL=${ESPMKUF2}" + -D "SERIAL_TOOL_ARGS=${UF2_ARGS};-o;${CMAKE_CURRENT_BINARY_DIR}/uf2-app.bin;--bin;app" + -P run_serial_tool.cmake + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + USES_TERMINAL + VERBATIM + ) + add_custom_target(monitor COMMAND ${CMAKE_COMMAND} -D "IDF_PATH=${idf_path}" diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 6d27152c5f..afdeb814fc 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -113,6 +113,5 @@ tools/test_idf_py/test_hints.py tools/test_idf_py/test_idf_py.py tools/test_idf_tools/test_idf_tools.py tools/test_mkdfu/test_mkdfu.py -tools/test_mkuf2/test_mkuf2.py tools/unit-test-app/tools/get_available_configs.sh tools/unit-test-app/unit_test.py diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index c22a0ec64f..74a2079c87 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -95,7 +95,6 @@ tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py tools/test_idf_py/test_idf_py.py tools/test_idf_tools/test_idf_tools.py tools/test_mkdfu/test_mkdfu.py -tools/test_mkuf2/test_mkuf2.py tools/unit-test-app/idf_ext.py tools/unit-test-app/tools/CreateSectionTable.py tools/unit-test-app/tools/UnitTestParser.py diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake index 71cf857369..3d2e96af1b 100644 --- a/tools/cmake/idf.cmake +++ b/tools/cmake/idf.cmake @@ -47,7 +47,6 @@ if(NOT __idf_env_set) include(targets) include(ldgen) include(dfu) - include(uf2) include(version) __build_init("${idf_path}") diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index 290b468f48..7e515d4aaf 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -718,9 +718,6 @@ macro(project project_name) # Add DFU build and flash targets __add_dfu_targets() - # Add UF2 build targets - __add_uf2_targets() - idf_build_executable(${project_elf}) __project_info("${test_components}") diff --git a/tools/cmake/uf2.cmake b/tools/cmake/uf2.cmake deleted file mode 100644 index 97b408dfaf..0000000000 --- a/tools/cmake/uf2.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# Add UF2 build target - -function(__add_uf2_targets) - idf_build_get_property(target IDF_TARGET) - if("${target}" STREQUAL "esp32") - set(uf2_family_id "0x1c5f21b0") - elseif("${target}" STREQUAL "esp32s2") - set(uf2_family_id "0xbfdd4eee") - elseif("${target}" STREQUAL "esp32c3") - set(uf2_family_id "0xd42ba06c") - elseif("${target}" STREQUAL "esp32s3") - set(uf2_family_id "0xc47e5767") - elseif("${target}" STREQUAL "esp32h2") - set(uf2_family_id "0x332726f6") - elseif("${target}" STREQUAL "esp32c2") - set(uf2_family_id "0x2b88d29c") - elseif("${target}" STREQUAL "esp32c6") - set(uf2_family_id "0x540ddf62") - elseif("${target}" STREQUAL "esp32p4") - set(uf2_family_id "0x3d308e94") - elseif("${target}" STREQUAL "linux") - return() - else() - message(FATAL_ERROR "UF2 family identificator is unknown for ${target}") - # Generate an ID and submit a pull request as described here: https://github.com/microsoft/uf2 - endif() - - idf_build_get_property(python PYTHON) - idf_build_get_property(idf_path IDF_PATH) - - add_custom_target(uf2-app - COMMAND ${python} ${idf_path}/tools/mkuf2.py write - -o "${CMAKE_CURRENT_BINARY_DIR}/uf2-app.bin" - --json "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json" - --chip-id "${uf2_family_id}" - --bin app - DEPENDS gen_project_binary - VERBATIM - USES_TERMINAL) - - add_custom_target(uf2 - COMMAND ${python} ${idf_path}/tools/mkuf2.py write - -o "${CMAKE_CURRENT_BINARY_DIR}/uf2.bin" - --json "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json" - --chip-id "${uf2_family_id}" - DEPENDS gen_project_binary bootloader - VERBATIM - USES_TERMINAL) -endfunction() diff --git a/tools/idf_py_actions/uf2_ext.py b/tools/idf_py_actions/uf2_ext.py index 138e849087..a6b2f46700 100644 --- a/tools/idf_py_actions/uf2_ext.py +++ b/tools/idf_py_actions/uf2_ext.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 from typing import Dict, List @@ -7,19 +7,32 @@ from idf_py_actions.tools import PropertyDict, ensure_build_directory, run_targe def action_extensions(base_actions: Dict, project_path: List) -> Dict: - def uf2_target(target_name: str, ctx: Context, args: PropertyDict) -> None: + def uf2_target(target_name: str, ctx: Context, args: PropertyDict, md5_disable: bool) -> None: ensure_build_directory(args, ctx.info_name) - run_target(target_name, args) + extra = list() + if md5_disable: + extra.append('--md5-disable') + run_target(target_name, args, env={'SERIAL_TOOL_EXTRA_ARGS': ' '.join(extra)}) + + uf2_options = [ + { + 'names': ['--md5-disable'], + 'is_flag': True, + 'help': 'Disable MD5 checksum', + }, + ] uf2_actions = { 'actions': { 'uf2': { 'callback': uf2_target, + 'options': uf2_options, 'short_help': 'Generate the UF2 binary with all the binaries included', 'dependencies': ['all'], }, 'uf2-app': { 'callback': uf2_target, + 'options': uf2_options, 'short_help': 'Generate an UF2 binary for the application only', 'dependencies': ['all'], }, diff --git a/tools/mkuf2.py b/tools/mkuf2.py index a81c086358..05d45dfecc 100755 --- a/tools/mkuf2.py +++ b/tools/mkuf2.py @@ -1,112 +1,39 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +# Module was moved to the esptool in ESP-IDF v5.2 and relicensed under GPL v2.0 license. from __future__ import division import argparse -import hashlib import json import os -import struct -from functools import partial -from typing import Dict, List - - -def round_up_int_div(n: int, d: int) -> int: - # equivalent to math.ceil(n / d) - return (n + d - 1) // d - - -class UF2Writer(object): - - # The UF2 format is described here: https://github.com/microsoft/uf2 - UF2_BLOCK_SIZE = 512 - UF2_DATA_SIZE = 476 # max value of CHUNK_SIZE reduced by optional parts. Currently, MD5_PART only. - UF2_MD5_PART_SIZE = 24 - UF2_FIRST_MAGIC = 0x0A324655 - UF2_SECOND_MAGIC = 0x9E5D5157 - UF2_FINAL_MAGIC = 0x0AB16F30 - UF2_FLAG_FAMILYID_PRESENT = 0x00002000 - UF2_FLAG_MD5_PRESENT = 0x00004000 - - def __init__(self, chip_id: int, output_file: os.PathLike, chunk_size: int) -> None: - self.chip_id = chip_id - self.CHUNK_SIZE = self.UF2_DATA_SIZE - self.UF2_MD5_PART_SIZE if chunk_size is None else chunk_size - self.f = open(output_file, 'wb') - - def __enter__(self) -> 'UF2Writer': - return self - - def __exit__(self, exc_type: str, exc_val: int, exc_tb: List) -> None: - if self.f: - self.f.close() - - @staticmethod - def _to_uint32(num: int) -> bytes: - return struct.pack(' None: - assert len_chunk > 0 - assert len_chunk <= self.CHUNK_SIZE - assert block_no < blocks - block = self._to_uint32(self.UF2_FIRST_MAGIC) - block += self._to_uint32(self.UF2_SECOND_MAGIC) - block += self._to_uint32(self.UF2_FLAG_FAMILYID_PRESENT | self.UF2_FLAG_MD5_PRESENT) - block += self._to_uint32(addr) - block += self._to_uint32(len_chunk) - block += self._to_uint32(block_no) - block += self._to_uint32(blocks) - block += self._to_uint32(self.chip_id) - block += chunk - - md5_part = self._to_uint32(addr) - md5_part += self._to_uint32(len_chunk) - md5_part += hashlib.md5(chunk).digest() - assert len(md5_part) == self.UF2_MD5_PART_SIZE - - block += md5_part - block += b'\x00' * (self.UF2_DATA_SIZE - self.UF2_MD5_PART_SIZE - len_chunk) - block += self._to_uint32(self.UF2_FINAL_MAGIC) - assert len(block) == self.UF2_BLOCK_SIZE - self.f.write(block) - - def add_file(self, addr: int, f_path: os.PathLike) -> None: - blocks = round_up_int_div(os.path.getsize(f_path), self.CHUNK_SIZE) - with open(f_path, 'rb') as fin: - a = addr - for i, chunk in enumerate(iter(partial(fin.read, self.CHUNK_SIZE), b'')): - len_chunk = len(chunk) - self._write_block(a, chunk, len_chunk, i, blocks) - a += len_chunk - - -def action_write(args: Dict) -> None: - with UF2Writer(args['chip_id'], args['output_file'], args['chunk_size']) as writer: - for addr, f in args['files']: - print('Adding {} at {:#x}'.format(f, addr)) - writer.add_file(addr, f) - print('"{}" has been written.'.format(args['output_file'])) +import subprocess +import sys def main() -> None: parser = argparse.ArgumentParser() - def four_byte_aligned(integer: int) -> bool: - return integer & 3 == 0 - - def parse_chunk_size(string: str) -> int: - num = int(string, 0) - if not four_byte_aligned(num): - raise argparse.ArgumentTypeError('Chunk size should be a 4-byte aligned number') - return num - - def parse_chip_id(string: str) -> int: - num = int(string, 16) - if num < 0 or num > 0xFFFFFFFF: - raise argparse.ArgumentTypeError('Chip ID should be a 4-byte unsigned integer') - return num + def parse_chip_id(string: str) -> str: + # compatibility layer with old script + print("DEPRECATED option '--chip-id'. Please consider using '--chip' instead") + # DO NOT add new IDs; they are now maintained in esptool. + ids = { + 0x1c5f21b0: 'esp32', + 0xbfdd4eee: 'esp32s2', + 0xd42ba06c: 'esp32c3', + 0xc47e5767: 'esp32s3', + 0x332726f6: 'esp32h2', + 0x2b88d29c: 'esp32c2', + 0x540ddf62: 'esp32c6', + 0x3d308e94: 'esp32p4', + } + try: + return ids[int(string, 16)] + except KeyError: + raise argparse.ArgumentTypeError('Unknown Chip ID') # Provision to add "info" command subparsers = parser.add_subparsers(dest='command') @@ -114,13 +41,17 @@ def main() -> None: write_parser.add_argument('-o', '--output-file', help='Filename for storing the output UF2 image', required=True) - write_parser.add_argument('--chip-id', - required=True, - type=parse_chip_id, - help='Hexa-decimal chip identificator') + group = write_parser.add_mutually_exclusive_group(required=True) + # chip-id used just for backwards compatibility, UF2 family IDs are now stored in esptool + group.add_argument('--chip-id', + type=parse_chip_id, + help=argparse.SUPPRESS) + group.add_argument('--chip', + type=str, + help='Target chip type') write_parser.add_argument('--chunk-size', required=False, - type=parse_chunk_size, + type=int, default=None, help='Specify the used data part of the 512 byte UF2 block. A common value is 256. By ' 'default the largest possible value will be used.') @@ -130,6 +61,9 @@ def main() -> None: help='Use only a subset of binaries from the JSON file, e.g. "partition_table ' 'bootloader app"', nargs='*') + write_parser.add_argument('--md5-disable', + help='Disable MD5 checksum. Useful for compatibility with e.g. TinyUF2', + action='store_true') write_parser.add_argument('files', metavar='
', help='Add at
', nargs='*') @@ -141,15 +75,9 @@ def main() -> None: raise RuntimeError('{} is not a regular file!'.format(file_name)) return file_name - def parse_addr(string: str) -> int: - num = int(string, 0) - if not four_byte_aligned(num): - raise RuntimeError('{} is not a 4-byte aligned valid address'.format(string)) - return num - files = [] if args.files: - files += [(parse_addr(addr), check_file(f_name)) for addr, f_name in zip(args.files[::2], args.files[1::2])] + files += [(addr, check_file(f_name)) for addr, f_name in zip(args.files[::2], args.files[1::2])] if args.json: json_dir = os.path.dirname(os.path.abspath(args.json)) @@ -159,7 +87,7 @@ def main() -> None: The input path is relative to json_dir. This function makes it relative to the current working directory. ''' - return check_file(os.path.relpath(os.path.join(json_dir, path), start=os.curdir)) + return check_file(os.path.abspath(os.path.join(json_dir, path))) with open(args.json) as f: json_content = json.load(f) @@ -176,19 +104,31 @@ def main() -> None: else: flash_dic = json_content['flash_files'] - files += [(parse_addr(addr), process_json_file(f_name)) for addr, f_name in flash_dic.items()] + files += [(addr, process_json_file(f_name)) for addr, f_name in flash_dic.items()] - files = sorted([(addr, f_name) for addr, f_name in dict(files).items()], - key=lambda x: x[0]) # remove possible duplicates and sort based on the address + # remove possible duplicates and sort based on the address + files = sorted([(addr, f_name) for addr, f_name in dict(files).items()], key=lambda x: x[0]) # type: ignore - cmd_args = {'output_file': args.output_file, - 'files': files, - 'chip_id': args.chip_id, - 'chunk_size': args.chunk_size, - } + # list of tuples to simple list + files = [item for t in files for item in t] - {'write': action_write - }[args.command](cmd_args) + cmd = [ + sys.executable, '-m', 'esptool', + '--chip', args.chip_id or args.chip, + 'merge_bin', + '--format', 'uf2', + '-o', args.output_file, + ] + if args.chunk_size: + cmd.extend(['--chunk_size', args.chunk_size]) + + if args.md5_disable: + cmd.append('--md5-disable') + + cmd_str = ' '.join(cmd + files) + print(f'Executing: {cmd_str}') + + sys.exit(subprocess.run(cmd + files).returncode) if __name__ == '__main__': diff --git a/tools/test_build_system/test_build.py b/tools/test_build_system/test_build.py index 2fc19bfc2b..6b206fbd2e 100644 --- a/tools/test_build_system/test_build.py +++ b/tools/test_build_system/test_build.py @@ -158,14 +158,14 @@ def test_build_dfu(idf_py: IdfPyFunc) -> None: def test_build_uf2(idf_py: IdfPyFunc) -> None: logging.info('UF2 build works') ret = idf_py('uf2') - assert 'build/uf2.bin" has been written.' in ret.stdout, 'UF2 build should work for esp32' + assert 'build/uf2.bin, ready to be flashed with any ESP USB Bridge' in ret.stdout, 'UF2 build should work for esp32' assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/uf2.bin']) ret = idf_py('uf2-app') - assert 'build/uf2-app.bin" has been written.' in ret.stdout, 'UF2 build should work for application binary' + assert 'build/uf2-app.bin, ready to be flashed with any ESP USB Bridge' in ret.stdout, 'UF2 build should work for application binary' assert_built(['build/uf2-app.bin']) idf_py('set-target', 'esp32s2') ret = idf_py('uf2') - assert 'build/uf2.bin" has been written.' in ret.stdout, 'UF2 build should work for esp32s2' + assert 'build/uf2.bin, ready to be flashed with any ESP USB Bridge' in ret.stdout, 'UF2 build should work for esp32s2' assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/uf2.bin']) diff --git a/tools/test_mkuf2/test_mkuf2.py b/tools/test_mkuf2/test_mkuf2.py deleted file mode 100755 index b0c59ef650..0000000000 --- a/tools/test_mkuf2/test_mkuf2.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import unicode_literals - -import filecmp -import hashlib -import os -import random -import struct -import sys -import tempfile -import time -import unittest -from functools import partial -from io import open -from itertools import chain - -import pexpect - -current_dir = os.path.dirname(os.path.realpath(__file__)) -mkuf2_dir = os.path.abspath(os.path.join(current_dir, '..')) -mkuf2_path = os.path.join(mkuf2_dir, 'mkuf2.py') - -try: - import mkuf2 -except ImportError: - sys.path.append(mkuf2_dir) - import mkuf2 - - -class UF2Block(object): - def __init__(self, bs): - self.length = len(bs) - - # See https://github.com/microsoft/uf2 for the format - first_part = '<' + 'I' * 8 - # payload is between - last_part = '