Merge branch 'feature/kconfig_for_reproducible_build' into 'master'

Feature: kconfig for reproducible build

See merge request espressif/esp-idf!15100

.gitlab/ci/host-test.yml

@@ -64,6 +64,23 @@ test_ldgen_on_host:
   variables:
     LC_ALL: C.UTF-8
 
+test_reproducible_build:
+  extends: .host_test_template
+  script:
+    - ./tools/ci/test_reproducible_build.sh
+  artifacts:
+    when: on_failure
+    paths:
+      - "**/sdkconfig"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/*.bin"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    expire_in: 1 week
+
 .host_fuzzer_test_template:
   extends:
     - .host_test_template

.gitlab/ci/rules.yml

@@ -130,6 +130,8 @@
   - "tools/detect_python.sh"
   - "tools/detect_python.fish"
 
+  - "tools/ci/test_reproducible_build.sh"
+
 .patterns-windows: &patterns-windows
   - "tools/windows/**/*"
 

CMakeLists.txt

@@ -147,7 +147,41 @@ endif()
 if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0)
     if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
         list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
-        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=IDF")
+        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF")
+    endif()
+
+    if(CONFIG_APP_REPRODUCIBLE_BUILD)
+        idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit")
+
+        list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+
+        # component dirs
+        idf_build_get_property(python PYTHON)
+        idf_build_get_property(idf_path IDF_PATH)
+        idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS)
+
+        execute_process(
+            COMMAND ${python}
+                "${idf_path}/tools/generate_debug_prefix_map.py"
+                "${BUILD_DIR}"
+                "${component_dirs}"
+            OUTPUT_VARIABLE result
+            RESULT_VARIABLE ret
+        )
+        if(NOT ret EQUAL 0)
+            message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues")
+        endif()
+
+        spaces2list(result)
+        list(LENGTH component_dirs length)
+        math(EXPR max_index "${length} - 1")
+        foreach(index RANGE ${max_index})
+            list(GET component_dirs ${index} folder)
+            list(GET result ${index} after)
+            list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}")
+        endforeach()
     endif()
 endif()
 

Kconfig

@@ -201,6 +201,14 @@ mainmenu "Espressif IoT Development Framework Configuration"
         config APP_BUILD_USE_FLASH_SECTIONS
             bool # Whether to place code/data into memory-mapped flash sections
 
+        config APP_REPRODUCIBLE_BUILD
+            bool "Enable reproducible build"
+            default n
+            select COMPILER_HIDE_PATHS_MACROS
+            help
+                If enabled, all date, time, and path information would be eliminated. A .gdbinit file would be create
+                automatically. (or will be append if you have one already)
+
     endmenu # Build type
 
     source "$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE"

components/app_update/esp_app_desc.c

@@ -32,7 +32,7 @@ const __attribute__((section(".rodata_desc"))) esp_app_desc_t esp_app_desc = {
     .secure_version = 0,
 #endif
 
-#ifdef CONFIG_APP_COMPILE_TIME_DATE
+#if defined(CONFIG_APP_COMPILE_TIME_DATE) && !defined(CONFIG_APP_REPRODUCIBLE_BUILD)
     .time = __TIME__,
     .date = __DATE__,
 #else

components/bootloader_support/src/bootloader_init.c

@@ -91,5 +91,7 @@ void bootloader_enable_random(void)
 void bootloader_print_banner(void)
 {
     ESP_LOGI(TAG, "ESP-IDF %s 2nd stage bootloader", IDF_VER);
+#ifndef CONFIG_APP_REPRODUCIBLE_BUILD
     ESP_LOGI(TAG, "compile time " __TIME__);
+#endif
 }

tools/ci/check_copyright_ignore.txt

@@ -3713,17 +3713,14 @@ tools/kconfig_new/test/gen_kconfig_doc/test_kconfig_out.py
 tools/kconfig_new/test/gen_kconfig_doc/test_target_visibility.py
 tools/ldgen/entity.py
 tools/ldgen/fragments.py
-tools/ldgen/generation.py
 tools/ldgen/ldgen.py
 tools/ldgen/ldgen_common.py
-tools/ldgen/linker_script.py
 tools/ldgen/output_commands.py
 tools/ldgen/samples/template.ld
 tools/ldgen/sdkconfig.py
 tools/ldgen/test/data/linker_script.ld
 tools/ldgen/test/test_entity.py
 tools/ldgen/test/test_fragments.py
-tools/ldgen/test/test_generation.py
 tools/ldgen/test/test_output_commands.py
 tools/mass_mfg/mfg_gen.py
 tools/mkdfu.py

tools/ci/executable-list.txt

@@ -66,6 +66,7 @@ tools/ci/test_autocomplete.py
 tools/ci/test_build_system_cmake.sh
 tools/ci/test_check_kconfigs.py
 tools/ci/test_configure_ci_environment.sh
+tools/ci/test_reproducible_build.sh
 tools/cmake/convert_to_cmake.py
 tools/docker/entrypoint.sh
 tools/docker/hooks/build

tools/ci/test_reproducible_build.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo
+
+for path in \
+  "examples/get-started/hello_world" \
+  "examples/bluetooth/nimble/blecent"; do
+  cd "${IDF_PATH}/${path}"
+
+  echo "CONFIG_APP_REPRODUCIBLE_BUILD=y" >sdkconfig
+
+  idf.py -B build_first fullclean build
+  idf.py -B build_second fullclean build
+
+  for item in \
+    "partition_table/partition-table.bin" \
+    "bootloader/bootloader.bin" \
+    "bootloader/bootloader.elf" \
+    "bootloader/bootloader.map" \
+    "*.bin" \
+    "*.elf" \
+    "*.map"; do
+    diff -s build_first/${item} build_second/${item} # use glob, don't use double quotes
+  done
+
+  # test gdb
+  rm -f gdb.txt
+  elf_file=$(find build_first -maxdepth 1 -iname '*.elf')
+  xtensa-esp32-elf-gdb -x build_first/prefix_map_gdbinit -ex 'set logging on' -ex 'set pagination off' -ex 'list' -ex 'quit' "$elf_file"
+  if grep "No such file or directory" gdb.txt; then
+    exit 1
+  fi
+done

tools/cmake/component.cmake

@@ -189,6 +189,9 @@ function(__component_add component_dir prefix)
 
     # Set Kconfig related properties on the component
     __kconfig_component_init(${component_target})
+
+    # set BUILD_COMPONENT_DIRS build property
+    idf_build_set_property(BUILD_COMPONENT_DIRS ${component_dir} APPEND)
 endfunction()
 
 #

tools/cmake/project.cmake

@@ -111,6 +111,7 @@ function(__project_info test_components)
     include(${sdkconfig_cmake})
     idf_build_get_property(COMPONENT_KCONFIGS KCONFIGS)
     idf_build_get_property(COMPONENT_KCONFIGS_PROJBUILD KCONFIG_PROJBUILDS)
+    idf_build_get_property(debug_prefix_map_gdbinit DEBUG_PREFIX_MAP_GDBINIT)
 
     # Write project description JSON file
     idf_build_get_property(build_dir BUILD_DIR)

tools/cmake/project_description.json.in

@@ -17,5 +17,6 @@
         "COMPONENT_KCONFIGS_PROJBUILD" : "${COMPONENT_KCONFIGS_PROJBUILD}"
     },
     "build_components" : ${build_components_json},
-    "build_component_paths" : ${build_component_paths_json}
+    "build_component_paths" : ${build_component_paths_json},
+    "debug_prefix_map_gdbinit": "${debug_prefix_map_gdbinit}"
 }

tools/generate_debug_prefix_map.py

@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# General Workflow:
+# 1. read all components dirs, a semicolon-separated string (cmake list)
+# 2. map the component dir with a unique prefix /COMPONENT_<NAME>_DIR
+# 2. write the prefix mapping file to $BUILD_DIR/prefix_map_gdbinit
+# 3. print the unique prefix out, a space-separated string, will be used by the build system to add compile options.
+
+import argparse
+import os
+from typing import List
+
+
+def component_name(component_dir: str) -> str:
+    return '/COMPONENT_{}_DIR'.format(os.path.basename(component_dir).upper())
+
+
+GDB_SUBSTITUTE_PATH_FMT = 'set substitute-path {} {}\n'
+
+
+def write_gdbinit(build_dir: str, folders: List[str]) -> None:
+    gdb_init_filepath = os.path.join(build_dir, 'prefix_map_gdbinit')
+
+    with open(gdb_init_filepath, 'w') as fw:
+        for folder in folders:
+            fw.write(f'{GDB_SUBSTITUTE_PATH_FMT.format(component_name(folder), folder)}')
+
+
+def main(build_dir: str, folders: List[str]) -> None:
+    write_gdbinit(build_dir, folders)
+    print(' '.join([component_name(folder) for folder in folders]), end='')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='print the debug-prefix-map and write to '
+                                                 '$BUILD_DIR/prefix_map_gdbinit file')
+
+    parser.add_argument('build_dir',
+                        help='build dir')
+    parser.add_argument('folders',
+                        help='component folders, semicolon separated string')
+    args = parser.parse_args()
+
+    main(args.build_dir, args.folders.split(';'))

tools/idf_py_actions/debug_ext.py

@@ -7,6 +7,7 @@ import sys
 import threading
 import time
 from threading import Thread
+from typing import Any, Dict, List
 
 from idf_py_actions.errors import FatalError
 from idf_py_actions.tools import ensure_build_directory
@@ -83,19 +84,6 @@ def action_extensions(base_actions, project_path):
                 print('Failed to close/kill {}'.format(target))
             processes[target] = None  # to indicate this has ended
 
-    def _get_commandline_options(ctx):
-        """ Return all the command line options up to first action """
-        # This approach ignores argument parsing done Click
-        result = []
-
-        for arg in sys.argv:
-            if arg in ctx.command.commands_with_aliases:
-                break
-
-            result.append(arg)
-
-        return result
-
     def create_local_gdbinit(gdbinit, elf_file):
         with open(gdbinit, 'w') as f:
             if os.name == 'nt':
@@ -188,6 +176,13 @@ def action_extensions(base_actions, project_path):
         processes['openocd_outfile_name'] = openocd_out_name
         print('OpenOCD started as a background task {}'.format(process.pid))
 
+    def get_gdb_args(gdbinit, project_desc: Dict[str, Any]) -> List[str]:
+        args = ['-x={}'.format(gdbinit)]
+        debug_prefix_gdbinit = project_desc.get('debug_prefix_map_gdbinit')
+        if debug_prefix_gdbinit:
+            args.append('-ix={}'.format(debug_prefix_gdbinit))
+        return args
+
     def gdbui(action, ctx, args, gdbgui_port, gdbinit, require_openocd):
         """
         Asynchronous GDB-UI target
@@ -198,7 +193,17 @@ def action_extensions(base_actions, project_path):
         if gdbinit is None:
             gdbinit = os.path.join(local_dir, 'gdbinit')
             create_local_gdbinit(gdbinit, os.path.join(args.build_dir, project_desc['app_elf']))
-        args = ['gdbgui', '-g', gdb, '--gdb-args="-x={}"'.format(gdbinit)]
+
+        # this is a workaround for gdbgui
+        # gdbgui is using shlex.split for the --gdb-args option. When the input is:
+        # - '"-x=foo -x=bar"', would return ['foo bar']
+        # - '-x=foo', would return ['-x', 'foo'] and mess up the former option '--gdb-args'
+        # so for one item, use extra double quotes. for more items, use no extra double quotes.
+        gdb_args = get_gdb_args(gdbinit, project_desc)
+        gdb_args = '"{}"'.format(' '.join(gdb_args)) if len(gdb_args) == 1 else ' '.join(gdb_args)
+        args = ['gdbgui', '-g', gdb, '--gdb-args', gdb_args]
+        print(args)
+
         if gdbgui_port is not None:
             args += ['--port', gdbgui_port]
         gdbgui_out_name = os.path.join(local_dir, GDBGUI_OUT_FILE)
@@ -278,10 +283,10 @@ def action_extensions(base_actions, project_path):
         if gdbinit is None:
             gdbinit = os.path.join(local_dir, 'gdbinit')
             create_local_gdbinit(gdbinit, elf_file)
-        args = [gdb, '-x={}'.format(gdbinit)]
+        args = [gdb, *get_gdb_args(gdbinit, project_desc)]
         if gdb_tui is not None:
             args += ['-tui']
-        t = Thread(target=run_gdb, args=(args, ))
+        t = Thread(target=run_gdb, args=(args,))
         t.start()
         while True:
             try:

tools/ldgen/generation.py

@@ -1,17 +1,6 @@
 #
-# Copyright 2021 Espressif Systems (Shanghai) CO LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 #
 
 import collections
@@ -251,7 +240,7 @@ class EntityNode():
             self.child_placement(entity, sections, target, flags, sections_db)
 
     def get_output_sections(self):
-        return sorted(self.placements.keys(), key=' '.join)
+        return sorted(self.placements.keys(), key=lambda x: sorted(x))  # pylint: disable=W0108
 
 
 class SymbolNode(EntityNode):

tools/ldgen/linker_script.py

@@ -1,17 +1,6 @@
 #
-# Copyright 2021 Espressif Systems (Shanghai) CO LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 #
 
 import collections
@@ -69,8 +58,6 @@ class LinkerScript:
                 target = member.target
                 rules = member.rules
 
-                del rules[:]
-
                 rules.extend(mapping_rules[target])
             except KeyError:
                 message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'."

tools/ldgen/test/test_generation.py

@@ -1,18 +1,7 @@
 #!/usr/bin/env python
 #
-# Copyright 2021 Espressif Systems (Shanghai) CO LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 #
 
 import collections
@@ -102,32 +91,22 @@ class GenerationTest(unittest.TestCase):
     def generate_default_rules(self):
         rules = collections.defaultdict(list)
 
-        rules['flash_text'].append(InputSectionDesc(ROOT, ['.literal', '.literal.*', '.text', '.text.*'], []))
-        rules['flash_rodata'].append(InputSectionDesc(ROOT, ['.rodata', '.rodata.*'], []))
-        rules['dram0_data'].append(InputSectionDesc(ROOT, ['.data', '.data.*'], []))
-        rules['dram0_data'].append(InputSectionDesc(ROOT, ['.dram', '.dram.*'], []))
         rules['dram0_bss'].append(InputSectionDesc(ROOT, ['.bss', '.bss.*'], []))
         rules['dram0_bss'].append(InputSectionDesc(ROOT, ['COMMON'], []))
+        rules['dram0_data'].append(InputSectionDesc(ROOT, ['.data', '.data.*'], []))
+        rules['dram0_data'].append(InputSectionDesc(ROOT, ['.dram', '.dram.*'], []))
+        rules['flash_text'].append(InputSectionDesc(ROOT, ['.literal', '.literal.*', '.text', '.text.*'], []))
+        rules['flash_rodata'].append(InputSectionDesc(ROOT, ['.rodata', '.rodata.*'], []))
         rules['iram0_text'].append(InputSectionDesc(ROOT, ['.iram', '.iram.*'], []))
-        rules['rtc_text'].append(InputSectionDesc(ROOT, ['.rtc.text', '.rtc.literal'], []))
+        rules['rtc_bss'].append(InputSectionDesc(ROOT, ['.rtc.bss'], []))
         rules['rtc_data'].append(InputSectionDesc(ROOT, ['.rtc.data'], []))
         rules['rtc_data'].append(InputSectionDesc(ROOT, ['.rtc.rodata'], []))
-        rules['rtc_bss'].append(InputSectionDesc(ROOT, ['.rtc.bss'], []))
+        rules['rtc_text'].append(InputSectionDesc(ROOT, ['.rtc.text', '.rtc.literal'], []))
 
         return rules
 
     def compare_rules(self, expected, actual):
-        self.assertEqual(set(expected.keys()), set(actual.keys()))
-
-        for target in sorted(actual.keys()):
-            message = 'failed target %s' % target
-            a_cmds = actual[target]
-            e_cmds = expected[target]
-
-            self.assertEqual(len(a_cmds), len(e_cmds), message)
-
-            for a, e in zip(a_cmds, e_cmds):
-                self.assertEqual(a, e, message)
+        self.assertEqual(expected, actual)
 
     def get_default(self, target, rules):
         return rules[target][0]