Merge branch 'feature/add_templates_for_hints' into 'master'

tools: add support of templates for hints

Closes IDF-5612

See merge request espressif/esp-idf!19227

tools/idf_py_actions/hints.yml

@@ -1,58 +1,76 @@
 #  -
 #     re: Regular expression of error to search
-#     hint: Message of the hint. Optionally, it is possible to use '{}' at the place where the matched group from 're' should be inserted. This requires 'match_to_output: True'.
+#     hint: The message of the hint. Optionally, it is possible to use '{}' at the place where the matched group from 're' should be inserted. This requires 'match_to_output: True'. You can use variables with hint messages. For this, you need to add variables and "{}" in a place where you want to put your hint variable, but you can't use 'match_to_output' with variables.
 #     match_to_output: (False by default) see the description of 'hint'.
-
+#     variables:
-
+#       -
+#           re_variables: [set variable for regular expression]
+#           hint_variables: [set variable for hint]
 # Rules to write regex for hints on how to resolve errors
 # - Do not use more than one whitespace in a row. The script automatically merges several whitespaces into one when capturing output
 # - Do not use \n in your regex. They are all automatically deletes by the script when capturing output
+#
+# example of using hints:
+#    -
+#        re: "Error: header {} is missing" (you can use '{1} ... {1}' placeholders in 'hint' and 're', so that you don't have to repeat the same variables, you can use 'hint: 'The {0} (functions/types/macros prefixed with '{1}') has been made into a private API. If users still require usage of the {0} (though this is not recommended), it can be included via  #include "esp_private/{2}.h".' in this file as an example)
+#        hint: "header {} is missing, you need to add dependency on component {}"
+#        variables:
+#            -
+#               re_variables: [Q]
+#               hint_variables: [A, B]
+#            -
+#               re_variables: [W]
+#               hint_variables: [C, D]
+#            -
+#               re_variables: [R]
+#               hint_variables: [E, F]
+#
+#   that example will replace this :
+#   -
+#       re: "Error: header Q is missing"
+#       hint: "header A is missing, you need to add dependency on component B"
+#   -
+#       re: Error: header W is missing"
+#       hint: "header C is missing, you need to add dependency on component D"
+#   -
+#       re: Error: header R is missing"
+#       hint: "header E is missing, you need to add dependency on component F"
 
 -
     re: "warning: passing argument 1 of 'esp_secure_boot_read_key_digests' from incompatible pointer type"
     hint: "The parameter type of the function esp_secure_boot_read_key_digests() has been changed from ets_secure_boot_key_digests_t* to esp_secure_boot_key_digests_t*."
 
 -
-    re: "error: implicit declaration of function 'bootloader_common_get_reset_reason'"
+    re: "error: implicit declaration of function '{}'"
-    hint: "'bootloader_common_get_reset_reason()' has been removed. Please use the function 'esp_rom_get_reset_reason()' in the ROM component."
+    hint: "Function '{}' has been removed. Please use the function {}."
+    variables:
+        -
+            re_variables: [bootloader_common_get_reset_reason]
+            hint_variables: [bootloader_common_get_reset_reason(), "'esp_rom_get_reset_reason()' in the ROM component"]
+        -
+            re_variables: [esp_efuse_get_chip_ver]
+            hint_variables: [esp_efuse_get_chip_ver(), efuse_hal_get_major_chip_version(), efuse_hal_get_minor_chip_version() or efuse_hal_chip_revision() instead]
+        -
+            re_variables: [(esp_spiram_get_chip_size|esp_spiram_get_size)]
+            hint_variables: [esp_spiram_get_chip_size and esp_spiram_get_size, esp_psram_get_size()]
 
 -
     re: "error: implicit declaration of function 'esp_secure_boot_verify_sbv2_signature_block|esp_secure_boot_verify_rsa_signature_block'"
-    hint: "'esp_secure_boot_verify_sbv2_signature_block()' and 'esp_secure_boot_verify_rsa_signature_block()' and has been made private and are no longer available."
+    hint: "'esp_secure_boot_verify_sbv2_signature_block()' and 'esp_secure_boot_verify_rsa_signature_block()' and have been made private and are no longer available."
 
 -
-    re: "error: implicit declaration of function 'esp_cpu_ccount_t'"
+    re: "error: implicit declaration of function '{}'"
-    hint: "Use esp_cpu_cycle_count_t defined in esp_cpu.h instead of esp_cpu_ccount_t."
+    hint: '{0}.h header file is not included by esp_system.h anymore. It shall then be manually included with #include "{0}.h"'
-
+    variables:
--
+        -
-    re: "error: implicit declaration of function 'esp_cpu_(g|s)et_ccount'"
+            re_variables: [(esp_random|esp_fill_random)]
-    hint: "Use esp_cpu_{}et_cycle_count() defined in esp_cpu.h instead."
+            hint_variables: [esp_random]
-    match_to_output: True
+        -
-
+            re_variables: [(esp_base_mac_addr_(s|g)et|esp_efuse_mac_get_(custom|default)|esp_read_mac|esp_derive_local_mac)]
--
+            hint_variables: [esp_mac]
-    re: "error: implicit declaration of function 'esp_efuse_get_chip_ver'"
+        -
-    hint: "Function esp_efuse_get_chip_ver() has been removed. Use efuse_hal_get_major_chip_version(), efuse_hal_get_minor_chip_version() or efuse_hal_chip_revision() instead."
+            re_variables: [esp_chip_info]
-
+            hint_variables: [esp_chip_info]
--
-    re: "error: implicit declaration of function '(esp_random|esp_fill_random)'"
-    hint: "esp_random.h header file is not included by esp_system.h anymore. It shall then be manually included with #include \"esp_random.h\""
-
--
-    re: "error: implicit declaration of function '(esp_base_mac_addr_(s|g)et|esp_efuse_mac_get_(custom|default)|esp_read_mac|esp_derive_local_mac)'"
-    hint: "esp_mac.h header file is not included by esp_system.h anymore. It shall then be manually included with #include \"esp_mac.h\""
-
--
-    re: "error: (implicit declaration of function 'esp_chip_info'|unknown type name 'esp_chip_info_t')"
-    hint: "esp_chip_info.h header file is not included by esp_system.h anymore. It shall then be manually included with #include \"esp_chip_info.h\""
-
--
-    re: "error: implicit declaration of function 'esp_int_wdt_\\w+'"
-    hint: "The Interrupt Watchdog API has been made private, it shall not be used anymore. You can still force its inclusion with #include \"esp_private/esp_int_wdt.h\" (not recommended)"
-
--
-    re: "error: implicit declaration of function '(esp_spiram_get_chip_size|esp_spiram_get_size)'"
-    hint: "{}() has been deleted, please use esp_psram_get_size() instead."
-    match_to_output: True
 
 -
     re: "fatal error: (spiram.h|esp_spiram.h): No such file or directory"
@@ -60,19 +78,39 @@
     match_to_output: True
 
 -
-    re: "fatal error: (soc/cpu.h|compare_set.h): No such file or directory"
+    re: "error: implicit declaration of function '{}'"
-    hint: "{} was removed. Include and use the API function provided by esp_cpu.h instead."
+    hint: "Use {} defined in esp_cpu.h instead of {}."
-    match_to_output: True
+    variables:
+        -
+            re_variables: [esp_cpu_ccount_t]
+            hint_variables: [esp_cpu_cycle_count_t, esp_cpu_ccount_t]
+        -
+            re_variables: [esp_cpu_get_ccount]
+            hint_variables: [esp_cpu_get_cycle_count(), esp_cpu_get_ccount]
+        -
+            re_variables: [esp_cpu_set_ccount]
+            hint_variables: [esp_cpu_set_cycle_count(), esp_cpu_set_ccount]
 
 -
-    re: "fatal error: (esp_intr.h): No such file or directory"
+    re: "fatal error: {}: No such file or directory"
-    hint: "{} was removed. Include esp_intr_alloc.h instead."
+    hint: "{} was removed. Include {} instead."
-    match_to_output: True
+    variables:
+        -
+            re_variables: [esp_intr.h]
+            hint_variables: [esp_intr.h, esp_intr_alloc.h]
+        -
+            re_variables: [soc/cpu.h]
+            hint_variables: [soc/cpu.h, and use the API function provided by esp_cpu.h]
+        -
+            re_variables: [compare_set.h]
+            hint_variables: [compare_set.h, and use the API function provided by esp_cpu.h]
+        -
+            re_variables: [esp_panic.h]
+            hint_variables: [esp_panic.h, use functionalities provided in esp_debug_helpers.h]
 
 -
-    re: "fatal error: (esp_panic.h): No such file or directory"
+    re: "error: implicit declaration of function 'esp_int_wdt_\\w+'"
-    hint: "{} was made private. Use functionalities provided in esp_debug_helpers.h instead."
+    hint: 'The Interrupt Watchdog API has been made private, it shall not be used anymore. You can still force its inclusion with #include "esp_private/esp_int_wdt.h" (not recommended)'
-    match_to_output: True
 
 -
     re: "fatal error: soc/(spinlock.h|clk_ctrl_os.h|rtc_wdt.h): No such file or directory"
@@ -84,9 +122,6 @@
     hint: "{} was renamed and made private. Consider using the logging APIs provided under esp_log.h instead."
     match_to_output: True
 
--
-    re: "fatal error: eh_frame_parser.h: No such file or directory"
-    hint: "Backtrace Parser API (eh_frame_parser.h) has been made private, it shall not be used anymore. You can still force its inclusion with #include \"esp_private/eh_frame_parser.h\" (not recommended)"
 -
     re: "error: unknown type name '(portTickType|xTaskHandle|xQueueHandle|xSemaphoreHandle|xQueueSetHandle|xQueueSetMemberHandle|xTimeOutType|xMemoryRegion|xTaskParameters|xTaskStatusType|xTimerHandle|xCoRoutineHandle|pdTASK_HOOK_CODE|tmrTIMER_CALLBACK|pdTASK_CODE|xListItem|xList)'"
     hint: "You maybe using pre FreeRTOS V8.0.0 data types. The backward compatibility of such data types is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such data types."
@@ -117,19 +152,24 @@
     match_to_output: True
 
 -
-    re: "fatal error: esp32\\w*\\/clk.h: No such file or directory"
+    re: "fatal error: {}.h: No such file or directory"
-    hint: "The ESP Clock API (functions/types/macros prefixed with 'esp_clk') has been made into a private API. If users still require usage of the ESP Clock API (though this is not recommended), it can be included via  '#include \"esp_private/esp_clk.h\"'."
+    hint: 'The {0} (functions/types/macros prefixed with "{1}") has been made into a private API. If users still require usage of the {0} (though this is not recommended), it can be included via  #include "esp_private/{2}.h".'
--
+    variables:
-    re: "fatal error: esp32\\w*\\/cache_err_int.h: No such file or directory"
+        -
-    hint: "The Cache Error Interrupt API (functions/types/macros prefixed with 'esp_cache_err') has been made into a private API. If users still require usage of the Cache Error Interrupt API (though this is not recommended), it can be included via  '#include \"esp_private/cache_err_int.h\"'"
+            re_variables: [esp32\\w*\\/clk]
-
+            hint_variables: [ESP Clock API, esp_clk, esp_clk]
--
+        -
-    re: "fatal error: brownout.h: No such file or directory"
+            re_variables: [esp32\\w*\\/cache_err_int]
-    hint: "The Brownout API (functions/types/macros prefixed with 'esp_brownout') has been made into a private API. If users still require usage of the Brownout API (though this is not recommended), it can be included via  '#include \"esp_private/brownout.h\"'."
+            hint_variables: [Cache Error Interrupt API, esp_cache_err, cache_err_int]
-
+        -
--
+            re_variables: [brownout]
-    re: "fatal error: trax.h: No such file or directory"
+            hint_variables: [Brownout API, esp_brownout, brownout]
-    hint: "The Trax API (functions/types/macros prefixed with 'trax_') has been made into a private API. If users still require usage of the Trax API (though this is not recommended), it can be included via  '#include \"esp_private/trax.h\"'."
+        -
+            re_variables: [trax]
+            hint_variables: [Trax API, trax_, trax]
+        -
+            re_variables: [eh_frame_parser]
+            hint_variables: [Backtrace Parser API, eh_frame_parser, eh_frame_parser]
 
 -
     re: "fatal error: esp_adc_cal.h: No such file or directory"

tools/idf_py_actions/tools.py

@@ -8,11 +8,10 @@ import sys
 from asyncio.subprocess import Process
 from io import open
 from types import FunctionType
-from typing import Any, Dict, List, Optional, TextIO, Tuple, Union
+from typing import Any, Dict, List, Match, Optional, TextIO, Tuple, Union
 
 import click
 import yaml
-from idf_monitor_base.output_helpers import yellow_print
 
 from .constants import GENERATORS
 from .errors import FatalError
@@ -79,6 +78,23 @@ def idf_version() -> Optional[str]:
     return version
 
 
+def color_print(message: str, color: str, newline: Optional[str]='\n') -> None:
+    """ Print a message to stderr with colored highlighting """
+    ansi_normal = '\033[0m'
+    sys.stderr.write('%s%s%s%s' % (color, message, ansi_normal, newline))
+    sys.stderr.flush()
+
+
+def yellow_print(message: str, newline: Optional[str]='\n') -> None:
+    ansi_yellow = '\033[0;33m'
+    color_print(message, ansi_yellow, newline)
+
+
+def red_print(message: str, newline: Optional[str]='\n') -> None:
+    ansi_red = '\033[1;31m'
+    color_print(message, ansi_red, newline)
+
+
 def print_hints(*filenames: str) -> None:
     """Getting output files and printing hints on how to resolve errors based on the output."""
     with open(os.path.join(os.path.dirname(__file__), 'hints.yml'), 'r') as file:
@@ -87,18 +103,39 @@ def print_hints(*filenames: str) -> None:
         with open(file_name, 'r') as file:
             output = ' '.join(line.strip() for line in file if line.strip())
         for hint in hints:
+            variables_list = hint.get('variables')
+            hint_list, hint_vars, re_vars = [], [], []
+            match: Optional[Match[str]] = None
             try:
-                match = re.compile(hint['re']).findall(output)
+                if variables_list:
-            except KeyError:
+                    for variables in variables_list:
-                raise KeyError("Argument 're' missing in {}. Check hints.yml file.".format(hint))
+                        hint_vars = variables['re_variables']
+                        re_vars = variables['hint_variables']
+                        regex = hint['re'].format(*re_vars)
+                        if re.compile(regex).search(output):
+                            try:
+                                hint_list.append(hint['hint'].format(*hint_vars))
+                            except KeyError as e:
+                                red_print('Argument {} missing in {}. Check hints.yml file.'.format(e, hint))
+                                sys.exit(1)
+                else:
+                    match = re.compile(hint['re']).search(output)
+            except KeyError as e:
+                red_print('Argument {} missing in {}. Check hints.yml file.'.format(e, hint))
+                sys.exit(1)
             except re.error as e:
-                raise re.error('{} from hints.yml have {} problem. Check hints.yml file.'.format(hint['re'], e))
+                red_print('{} from hints.yml have {} problem. Check hints.yml file.'.format(hint['re'], e))
-            if match:
+                sys.exit(1)
-                extra_info = ', '.join(match) if hint.get('match_to_output', '') else ''
+            if hint_list:
+                for message in hint_list:
+                    yellow_print('HINT:', message)
+            elif match:
+                extra_info = ', '.join(match.groups()) if hint.get('match_to_output', '') else ''
                 try:
                     yellow_print(' '.join(['HINT:', hint['hint'].format(extra_info)]))
-                except KeyError:
+                except KeyError as e:
-                    raise KeyError("Argument 'hint' missing in {}. Check hints.yml file.".format(hint))
+                    red_print('Argument {} missing in {}. Check hints.yml file.'.format(e, hint))
+                    sys.exit(1)
 
 
 def fit_text_in_terminal(out: str) -> str: