From e96da70654e6e43d1f9bd9b518226967eb111c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20Cvetkovi=C4=87?= Date: Fri, 5 Apr 2024 13:27:37 +0100 Subject: [PATCH 1/2] feat(esptool): merge_bin CMake target --- components/esptool_py/project_include.cmake | 23 +++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/components/esptool_py/project_include.cmake b/components/esptool_py/project_include.cmake index 63877c882b..919102fdd3 100644 --- a/components/esptool_py/project_include.cmake +++ b/components/esptool_py/project_include.cmake @@ -94,7 +94,7 @@ if(NOT CONFIG_APP_BUILD_TYPE_RAM AND CONFIG_APP_BUILD_GENERATE_BINARIES) endif() endif() -# We still set "--min-rev" to keep the app compatible with older booloaders where this field is controlled. +# We still set "--min-rev" to keep the app compatible with older bootloaders where this field is controlled. if(CONFIG_IDF_TARGET_ESP32) # for this chip min_rev is major revision math(EXPR min_rev "${CONFIG_ESP_REV_MIN_FULL} / 100") @@ -241,6 +241,21 @@ add_custom_target(uf2-app ) +set(MERGE_BIN_ARGS merge_bin -o "${CMAKE_CURRENT_BINARY_DIR}/merge.bin" "@${CMAKE_CURRENT_BINARY_DIR}/flash_args") + +add_custom_target(merge_bin + COMMAND ${CMAKE_COMMAND} + -D "IDF_PATH=${idf_path}" + -D "SERIAL_TOOL=${ESPTOOLPY}" + -D "SERIAL_TOOL_ARGS=${MERGE_BIN_ARGS}" + -D "WORKING_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}" + -P run_serial_tool.cmake + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + DEPENDS gen_project_binary bootloader + USES_TERMINAL + VERBATIM + ) + set(MONITOR_ARGS "") list(APPEND MONITOR_ARGS "--toolchain-prefix;${_CMAKE_TOOLCHAIN_PREFIX};") @@ -348,7 +363,7 @@ endfunction() # This function takes a fifth optional named parameter: "ALWAYS_PLAINTEXT". As # its name states, it marks whether the image should be flashed as plain text or # not. If build macro CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT is set and -# this parameter is provided, then the image will be flahsed as plain text +# this parameter is provided, then the image will be flashed as plain text # (not encrypted) on the target. This parameter will be ignored if build macro # CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT is not set. function(esptool_py_flash_target_image target_name image_name offset image) @@ -474,7 +489,7 @@ $,\n>") # If we only have encrypted images to flash, we must use legacy # --encrypt parameter. # As the properties ENCRYPTED_IMAGES and NON_ENCRYPTED_IMAGES have not - # been geenrated yet, we must use CMake expression generator to test + # been generated yet, we must use CMake expression generator to test # which esptool.py options we can use. # The variable has_non_encrypted_image will be evaluated to "1" if some @@ -503,7 +518,7 @@ ${non_encrypted_files}\n\ ${if_enc_expr}\ ${encrypted_files}") - # The expression is ready to be geenrated, write it to the file which + # The expression is ready to be generated, write it to the file which # extension is .in file_generate("${CMAKE_CURRENT_BINARY_DIR}/encrypted_${target_name}_args.in" CONTENT "${flash_args_content}") From 0dec6fe65d4de7d99518ba12864cf47a52e3f9e2 Mon Sep 17 00:00:00 2001 From: Jan Beran Date: Mon, 15 Apr 2024 15:58:58 +0200 Subject: [PATCH 2/2] feat(tools): Add idf.py merge-bin command and cmake target --- components/esptool_py/project_include.cmake | 18 ++++- docs/en/api-guides/tools/idf-py.rst | 35 ++++++++ tools/idf_py_actions/serial_ext.py | 90 +++++++++++++++++++-- tools/test_build_system/test_common.py | 12 ++- 4 files changed, 146 insertions(+), 9 deletions(-) diff --git a/components/esptool_py/project_include.cmake b/components/esptool_py/project_include.cmake index 919102fdd3..f4486ba904 100644 --- a/components/esptool_py/project_include.cmake +++ b/components/esptool_py/project_include.cmake @@ -240,10 +240,24 @@ add_custom_target(uf2-app VERBATIM ) +set(MERGE_BIN_ARGS merge_bin) +if(DEFINED ENV{ESP_MERGE_BIN_OUTPUT}) + list(APPEND MERGE_BIN_ARGS "-o" "$ENV{ESP_MERGE_BIN_OUTPUT}") +else() + if(DEFINED ENV{ESP_MERGE_BIN_FORMAT} AND "$ENV{ESP_MERGE_BIN_FORMAT}" STREQUAL "hex") + list(APPEND MERGE_BIN_ARGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/merged-binary.hex") + else() + list(APPEND MERGE_BIN_ARGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/merged-binary.bin") + endif() +endif() -set(MERGE_BIN_ARGS merge_bin -o "${CMAKE_CURRENT_BINARY_DIR}/merge.bin" "@${CMAKE_CURRENT_BINARY_DIR}/flash_args") +if(DEFINED ENV{ESP_MERGE_BIN_FORMAT}) + list(APPEND MERGE_BIN_ARGS "-f" "$ENV{ESP_MERGE_BIN_FORMAT}") +endif() -add_custom_target(merge_bin +list(APPEND MERGE_BIN_ARGS "@${CMAKE_CURRENT_BINARY_DIR}/flash_args") + +add_custom_target(merge-bin COMMAND ${CMAKE_COMMAND} -D "IDF_PATH=${idf_path}" -D "SERIAL_TOOL=${ESPTOOLPY}" diff --git a/docs/en/api-guides/tools/idf-py.rst b/docs/en/api-guides/tools/idf-py.rst index 96404e7092..394d017d57 100644 --- a/docs/en/api-guides/tools/idf-py.rst +++ b/docs/en/api-guides/tools/idf-py.rst @@ -122,6 +122,37 @@ This command automatically builds the project if necessary, and then flash it to Similarly to the ``build`` command, the command can be run with ``app``, ``bootloader`` and ``partition-table`` arguments to flash only the app, bootloader or partition table as applicable. +.. _merging-binaries: + +Merge binaries: ``merge-bin`` +----------------------------- + +.. code-block:: bash + + idf.py merge-bin [-o output-file] [-f format] [] + +There are some situations, e.g. transferring the file to another machine and flashing it without ESP-IDF, where it is convenient to have only one file for flashing instead the several file output of ``idf.py build``. + +The command ``idf.py merge-bin`` will merge the bootloader, partition table, the application itself, and other partitions (if there are any) according to the project configuration and create a single binary file ``merged-binary.[bin|hex]`` in the build folder, which can then be flashed later. + +It is possible to output merged file in binary (raw), IntelHex (hex) and UF2 (uf2) formats. + +The uf2 binary can also be generated by :ref:`idf.py uf2 `. The ``idf.py uf2`` is functionally equivalent to ``idf.py merge-bin -f uf2``. However, the ``idf.py merge-bin`` command provides more flexibility and options for merging binaries into various formats described above. + +Example usage: + +.. code-block:: bash + + idf.py merge-bin -o my-merged-binary.bin -f raw + +There are also some format specific options, which are listed below: + +- Only for raw format: + - ``--flash-offset``: This option will create a merged binary that should be flashed at the specified offset, instead of at the standard offset of 0x0. + - ``--fill-flash-size``: If set, the final binary file will be padded with FF bytes up to this flash size in order to fill the full flash content with the image and re-write the whole flash chip upon flashing. +- Only for uf2 format: + - ``--md5-disable``: This option will disable MD5 checksums at the end of each block. This can be useful for integration with e.g. `tinyuf2 `__. + Hints on How to Resolve Errors ============================== @@ -201,6 +232,8 @@ Clean the Python Byte Code: ``python-clean`` This command deletes generated python byte code from the ESP-IDF directory. The byte code may cause issues when switching between ESP-IDF and Python versions. It is advised to run this target after switching versions of Python. +.. _generate-uf2-binary: + Generate a UF2 Binary: ``uf2`` ------------------------------ @@ -214,6 +247,8 @@ This UF2 file can be copied to a USB mass storage device exposed by another ESP To generate a UF2 binary for the application only (not including the bootloader and partition table), use the ``uf2-app`` command. +The ``idf.py uf2`` command is functionally equivalent to ``idf.py merge-bin -f uf2`` described :ref:`above `. However, the ``idf.py merge-bin`` command provides more flexibility and options for merging binaries into various formats, not only uf2. + .. code-block:: bash idf.py uf2-app diff --git a/tools/idf_py_actions/serial_ext.py b/tools/idf_py_actions/serial_ext.py index 86c663b5d4..cdeaccea4d 100644 --- a/tools/idf_py_actions/serial_ext.py +++ b/tools/idf_py_actions/serial_ext.py @@ -1,17 +1,23 @@ -# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import json import os import shlex import signal import sys -from typing import Any, Dict, List, Optional +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import click from idf_py_actions.global_options import global_options -from idf_py_actions.tools import (PropertyDict, RunTool, ensure_build_directory, get_default_serial_port, - get_sdkconfig_value, run_target) +from idf_py_actions.tools import ensure_build_directory +from idf_py_actions.tools import get_default_serial_port +from idf_py_actions.tools import get_sdkconfig_value +from idf_py_actions.tools import PropertyDict +from idf_py_actions.tools import run_target +from idf_py_actions.tools import RunTool PYTHON = sys.executable @@ -34,7 +40,7 @@ PORT = { } -def yellow_print(message, newline='\n'): # type: (str, Optional[str]) -> None +def yellow_print(message: str, newline: Optional[str]='\n') -> None: """Print a message to stderr with yellow highlighting """ sys.stderr.write('%s%s%s%s' % ('\033[0;33m', message, '\033[0m', newline)) sys.stderr.flush() @@ -212,6 +218,47 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: ensure_build_directory(args, ctx.info_name) run_target(target_name, args, {'ESPBAUD': str(args.baud), 'ESPPORT': args.port}) + def merge_bin(action: str, + ctx: click.core.Context, + args: PropertyDict, + output: str, + format: str, + md5_disable: str, + flash_offset: str, + fill_flash_size: str) -> None: + ensure_build_directory(args, ctx.info_name) + project_desc = _get_project_desc(ctx, args) + merge_bin_args = [PYTHON, '-m', 'esptool'] + target = project_desc['target'] + merge_bin_args += ['--chip', target] + merge_bin_args += ['merge_bin'] # needs to be after the --chip option + if not output: + if format in ('raw', 'uf2'): + output = 'merged-binary.bin' + elif format == 'hex': + output = 'merged-binary.hex' + merge_bin_args += ['-o', output] + if format: + merge_bin_args += ['-f', format] + if md5_disable: + if format != 'uf2': + yellow_print('idf.py merge-bin: --md5-disable is only valid for UF2 format. Option will be ignored.') + else: + merge_bin_args += ['--md5-disable'] + if flash_offset: + if format != 'raw': + yellow_print('idf.py merge-bin: --flash-offset is only valid for RAW format. Option will be ignored.') + else: + merge_bin_args += ['-t', flash_offset] + if fill_flash_size: + if format != 'raw': + yellow_print('idf.py merge-bin: --fill-flash-size is only valid for RAW format, option will be ignored.') + else: + merge_bin_args += ['--fill-flash-size', fill_flash_size] + merge_bin_args += ['@flash_args'] + print(f'Merged binary {output} will be created in the build directory...') + RunTool('merge_bin', merge_bin_args, args.build_dir, build_dir=args.build_dir, hints=not args.no_hints)() + BAUD_AND_PORT = [BAUD_RATE, PORT] flash_options = BAUD_AND_PORT + [ { @@ -252,6 +299,37 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: 'help': 'Erase entire flash chip.', 'options': BAUD_AND_PORT, }, + 'merge-bin': { + 'callback': merge_bin, + 'options': [ + { + 'names': ['--output', '-o'], + 'help': ('Output filename'), + 'type': click.Path(), + }, + { + 'names': ['--format', '-f'], + 'help': ('Format of the output file'), + 'type': click.Choice(['hex', 'uf2', 'raw']), + 'default': 'raw', + }, + { + 'names': ['--md5-disable'], + 'is_flag': True, + 'help': ('[ONLY UF2] Disable MD5 checksum in UF2 output.'), + }, + { + 'names': ['--flash-offset', '-t'], + 'help': ('[ONLY RAW] Flash offset where the output file will be flashed.'), + }, + { + 'names': ['--fill-flash-size'], + 'help': ('[ONLY RAW] If set, the final binary file will be padded with FF bytes up to this flash size.'), + 'type': click.Choice(['256KB', '512KB', '1MB', '2MB', '4MB', '8MB', '16MB', '32MB', '64MB', '128MB']), + }, + ], + 'dependencies': ['all'], # all = build + }, 'monitor': { 'callback': monitor, diff --git a/tools/test_build_system/test_common.py b/tools/test_build_system/test_common.py index 515acce6f9..8633ae2237 100644 --- a/tools/test_build_system/test_common.py +++ b/tools/test_build_system/test_common.py @@ -252,7 +252,7 @@ def test_create_project_with_idf_readonly(idf_copy: Path) -> None: for name in files: path = os.path.join(root, name) if '/bin/' in path: - continue # skip excutables + continue # skip executables os.chmod(os.path.join(root, name), 0o444) # readonly logging.info('Check that command for creating new project will success if the IDF itself is readonly.') change_to_readonly(idf_copy) @@ -308,3 +308,13 @@ def test_save_defconfig_check(idf_py: IdfPyFunc, test_app_copy: Path) -> None: 'Missing CONFIG_IDF_TARGET="esp32c3" in sdkconfig.defaults' assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), \ 'Missing CONFIG_PARTITION_TABLE_OFFSET=0x8001 in sdkconfig.defaults' + + +def test_merge_bin_cmd(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Test if merge-bin command works correctly') + idf_py('merge-bin') + assert (test_app_copy / 'build' / 'merged-binary.bin').is_file() + idf_py('merge-bin', '--output', 'merged-binary-2.bin') + assert (test_app_copy / 'build' / 'merged-binary-2.bin').is_file() + idf_py('merge-bin', '--format', 'hex') + assert (test_app_copy / 'build' / 'merged-binary.hex').is_file()