feat: use standalone project idf-build-apps for find/build apps utils

.gitlab-ci.yml

@@ -56,6 +56,9 @@ variables:
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
   PYTHON_VER: 3.7.10
 
+  CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
+  IDF_BUILD_APPS_PROJ: 2818  # fuhanxi/idf-build-apps
+
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
 
@@ -213,9 +216,27 @@ before_script:
     - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
     # TODO: remove this, IDFCI-1207
     - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
+
+.before_script_build_jobs:
+  before_script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - source tools/ci/setup_python.sh
+    - add_gitlab_ssh_keys
+    - source tools/ci/configure_ci_environment.sh
+    - *setup_tools_unless_target_test
+    - fetch_submodules
+    - *download_test_python_contraint_file
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
+    # TODO: remove this, IDFCI-1207
+    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
+    # not only need pytest related packages, but also needs ttfw requirements
+    - internal_pip_install $IDF_BUILD_APPS_PROJ idf_build_apps
+    - pip install -r tools/ci/python_packages/ttfw_idf/requirements.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
 
 default:
   retry:

.gitlab/CODEOWNERS

@@ -177,13 +177,11 @@
 /examples/zigbee/             @esp-idf-codeowners/ieee802154
 
 /tools/                       @esp-idf-codeowners/tools
-/tools/*_apps.py              @esp-idf-codeowners/ci
 /tools/ble/                   @esp-idf-codeowners/app-utilities
 /tools/catch/                 @esp-idf-codeowners/ci
 /tools/ci/                    @esp-idf-codeowners/ci
 /tools/cmake/                 @esp-idf-codeowners/build-config
 /tools/esp_prov/              @esp-idf-codeowners/app-utilities
-/tools/find_build_apps/       @esp-idf-codeowners/ci
 /tools/idf_size_yaml/         @esp-idf-codeowners/peripherals
 /tools/kconfig*/              @esp-idf-codeowners/build-config
 /tools/ldgen/                 @esp-idf-codeowners/build-config

.gitlab/ci/README.md

@@ -19,6 +19,13 @@
     - [Functions](#functions)
       - [CI Job Related](#ci-job-related)
       - [Shell Script Related](#shell-script-related)
+  - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
+    - [Grammar](#grammar)
+      - [Operands](#operands)
+      - [Operators](#operators)
+      - [Limitation:](#limitation)
+    - [How does it work?](#how-does-it-work)
+    - [Example](#example)
 
 ## General Workflow
 
@@ -224,3 +231,99 @@ To run these commands in shell script locally, place `source tools/ci/utils.sh`
 - `info`: log in green color
 - `run_cmd`: run the command with duration seconds info
 - `retry_failed`: run the command with duration seconds info, retry when failed
+
+## Manifest File to Control the Build/Test apps
+
+`.build-test-rules.yml` file is a manifest file to control if the CI is running the build and test job or not. The Supported Targets table in `README.md` for apps would be auto-generated by `pre-commit` from the app's `.build-test-rules.yml`.
+
+### Grammar
+
+#### Operands
+
+- variables starts with `SOC_`. The value would be parsed from components/soc/[TARGET]/include/soc/soc_caps.h
+- `IDF_TARGET`
+- `INCLUDE_DEFAULT` (The default value of officially supported targets is 1, otherwise is 0)
+- String, must be double-quoted. e.g. `"esp32"`, `"12345"`
+- Integer, support decimal and hex. e.g. `1`, `0xAB`
+- List with String and Integer inside, the type could be mixed. e.g. `["esp32", 1]`
+
+#### Operators
+
+- `==`, `!=`, `>`, `>=`, `<`, `<=`
+- `and`, `or`
+- `in`, `not in` with list
+- parentheses
+
+#### Limitation:
+
+- all operators are binary operator. For more than two operands, you may use nested parentheses trick. For example,
+  - `A == 1 or (B == 2 and C in [1,2,3])`
+  - `(A == 1 and B == 2) or (C not in ["3", "4", 5])`
+
+### How does it work?
+
+By default, we enable build and test jobs for supported targets.
+
+three rules (disable rules are calculated after the `enable` rule):
+- enable: run CI build/test jobs for targets that match any of the specified conditions only
+- disable: will not run CI build/test jobs for targets that match any of the specified conditions
+- disable_test: will not run CI test jobs for targets that match any of the specified conditions
+
+Each key is a test folder. Will apply to all folders inside.
+
+If one sub folder is in a special case, you can overwrite the rules for this folder by adding another entry for this folder itself. Each folder's rules are standalone, and will not inherit its parent's rules. (YAML inheritance is too complicated for reading)
+
+For example in the following codeblock, only `disable` rule exists in `examples/foo/bar`. It's unaware of its parent's `enable` rule. 
+
+```yaml
+examples/foo:
+  enable:
+    - if: IDF_TARGET == "esp32"
+
+examples/foo/bar:
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+```
+
+### Example
+
+```yaml
+examples/foo:
+  enable:
+    - if IDF_TARGET in ["esp32", 1, 2, 3]
+    - if IDF_TARGET not in ["4", "5", 6]
+  # should be run under all targets!
+
+examples/bluetooth:
+  disable:  # disable both build and tests jobs
+    - if: SOC_BT_SUPPORTED != 1
+    # reason is optional if there's no `temporary: true`
+  disable_test:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: lack of ci runners  # required when `temporary: true`
+
+examples/foo:
+  enable:
+    - if IDF_TARGET in ["esp32", 1, 2, 3]
+    - if IDF_TARGET not in ["4", "5", 6]
+  # should be run under all targets!
+
+examples/bluetooth/test_foo:
+  # each folder's settings are standalone
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: no idea
+  # unlike examples/bluetooth, the apps under this folder would not be build nor test for "no idea" under target esp32s2
+
+examples/get-started/hello_world:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: this one only supports linux!
+
+examples/get-started/blink:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux"
+      reason: This one supports all supported targets and linux
+```

.gitlab/ci/assign-test.yml

@@ -7,12 +7,12 @@
     SUBMODULES_TO_FETCH: "none"
   artifacts:
     paths:
-      - ${TEST_DIR}/test_configs
-      - ${BUILD_DIR}/artifact_index.json
+      - ${TEST_DIR}/test_configs/
+      - artifact_index.json
     when: always
     expire_in: 1 week
   script:
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+    - run_cmd python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
 
 assign_example_test:
   extends:
@@ -36,8 +36,7 @@ assign_example_test:
       optional: true
   variables:
     TEST_TYPE: example_test
-    TEST_DIR: ${CI_PROJECT_DIR}/examples
-    BUILD_DIR: ${CI_PROJECT_DIR}/build_examples
+    TEST_DIR: examples
 
 assign_custom_test:
   extends:
@@ -61,8 +60,7 @@ assign_custom_test:
       optional: true
   variables:
     TEST_TYPE: custom_test
-    TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
-    BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps
+    TEST_DIR: tools/test_apps
 
 assign_unit_test:
   extends:
@@ -81,10 +79,7 @@ assign_unit_test:
       optional: true
   variables:
     TEST_TYPE: unit_test
-    TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test
-    BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
-  script:
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+    TEST_DIR: components/idf_test/unit_test
 
 assign_integration_test:
   extends:

.gitlab/ci/build.yml

@@ -15,7 +15,7 @@
 .build_pytest_template:
   extends:
     - .build_template
-    - .before_script_pytest
+    - .before_script_build_jobs
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
@@ -23,139 +23,170 @@
   artifacts:
     paths:
       - "**/build*/size.json"
-      - "**/build*/build.log"
+      - "**/build*/build_log.txt"
       - "**/build*/*.bin"
       - "**/build*/*.elf"
       - "**/build*/*.map"
       - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
       - "**/build*/config/sdkconfig.json"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
       - $SIZE_INFO_LOCATION
     when: always
     expire_in: 3 days
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --pytest-apps
+      --collect-size-info $SIZE_INFO_LOCATION
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
 
 build_pytest_examples_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: examples
 
 build_pytest_examples_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32s2
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
 
 build_pytest_examples_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32s3
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
-
-build_pytest_examples_esp32c2:
-  extends:
-    - .build_pytest_template
-    - .rules:build:example_test-esp32c2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
 
 build_pytest_examples_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32c3
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
 
 build_pytest_components_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: components
 
 build_pytest_components_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: components
 
 build_pytest_components_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: components
 
 build_pytest_components_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: components
 
 build_pytest_components_esp32c2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: components
 
 build_non_test_components_apps:
   extends:
-    - .build_template
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:component_ut
-  variables:
-    IDF_TARGET: all
-    TEST_PREFIX: component_ut
-    TEST_RELATIVE_DIR: component_ut
+  script:
+    - set_component_ut_vars
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
+      -t all
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
 
 build_pytest_test_apps_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
 
 build_pytest_test_apps_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32s2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
 
 build_pytest_test_apps_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32s3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
-
-build_pytest_test_apps_esp32c2:
-  extends:
-    - .build_pytest_template
-    - .rules:build:custom_test-esp32c2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
 
 build_pytest_test_apps_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32c3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
 
 .build_template_app_template:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .before_script_build_jobs
   variables:
     LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
     BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
@@ -177,8 +208,6 @@ build_pytest_test_apps_esp32c3:
     # using on esp-idf. If it doesn't exist then just stick to the default branch
     - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
     - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
     # Only do the default cmake build for each target, remaining part are done in the build_template_app job
     - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
 
@@ -238,33 +267,45 @@ build_ssc_esp32s3:
     TARGET_NAME: "ESP32S3"
 
 .build_esp_idf_tests_cmake_template:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .before_script_build_jobs
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-    - scan_tests
   artifacts:
     paths:
-      - tools/unit-test-app/output/${IDF_TARGET}
-      - tools/unit-test-app/builds/*.json
-      - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json
-      - components/idf_test/unit_test/*.yml
-      - $LOG_PATH
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
       - $SIZE_INFO_LOCATION
+      - components/idf_test/unit_test/*.yml
     when: always
     expire_in: 4 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
-    BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
-    OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
-    BUILD_SYSTEM: "cmake"
-    TEST_TYPE: "unit_test"
     LDGEN_CHECK_MAPPING: 1
   script:
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
-    - cd $CI_PROJECT_DIR/tools/unit-test-app
-    - python tools/UnitTestParser.py ${BUILD_PATH} ${CI_NODE_INDEX:-1}
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
+      -t $IDF_TARGET
+      --config "configs/*="
+      --copy-sdkconfig
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --preserve-all
+    - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
 
 build_esp_idf_tests_cmake_esp32:
   extends:
@@ -303,141 +344,143 @@ build_esp_idf_tests_cmake_esp32c3:
   variables:
     IDF_TARGET: esp32c3
 
-.build_examples_template:
-  extends: .build_template
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
+.build_cmake_template:
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-    - scan_tests
   variables:
-    TEST_PREFIX: examples
-    TEST_RELATIVE_DIR: examples
-    SCAN_TEST_JSON: ${CI_PROJECT_DIR}/${TEST_RELATIVE_DIR}/test_configs/scan_${IDF_TARGET}_${BUILD_SYSTEM}.json
-    TEST_TYPE: example_test
-    LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
-    BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
     LDGEN_CHECK_MAPPING: 1
-  script:
-    # it's not possible to build 100% out-of-tree and have the "artifacts"
-    # mechanism work, but this is the next best thing
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
-
-.build_examples_cmake_template:
-  extends: .build_examples_template
   artifacts:
     paths:
-      - build_${TEST_PREFIX}/list.json
-      - build_${TEST_PREFIX}/list_job_*.json
-      - build_${TEST_PREFIX}/*/*/*/sdkconfig
-      - build_${TEST_PREFIX}/*/*/*/build/size.json
-      - build_${TEST_PREFIX}/*/*/*/build/*.bin
-      - build_${TEST_PREFIX}/*/*/*/build/*.elf
-      - build_${TEST_PREFIX}/*/*/*/build/*.map
-      - build_${TEST_PREFIX}/*/*/*/build/flasher_args.json
-      - build_${TEST_PREFIX}/*/*/*/build/bootloader/*.bin
-      - build_${TEST_PREFIX}/*/*/*/build/partition_table/*.bin
-      - $LOG_PATH
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
       - $SIZE_INFO_LOCATION
     when: always
     expire_in: 4 days
-  variables:
-    BUILD_SYSTEM: cmake
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --copy-sdkconfig
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --extra-preserve-dirs
+        examples/bluetooth/esp_ble_mesh/ble_mesh_console
+        examples/bluetooth/hci/controller_hci_uart_esp32
+        examples/wifi/iperf
 
 build_examples_cmake_esp32:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32
   parallel: 12
   variables:
     IDF_TARGET: esp32
+    TEST_DIR: examples
 
 build_examples_cmake_esp32s2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32s2
   parallel: 8
   variables:
     IDF_TARGET: esp32s2
+    TEST_DIR: examples
 
 build_examples_cmake_esp32s3:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32s3
   parallel: 8
   variables:
     IDF_TARGET: esp32s3
+    TEST_DIR: examples
 
 build_examples_cmake_esp32c2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32c2
   parallel: 8
   variables:
     IDF_TARGET: esp32c2
+    TEST_DIR: examples
 
 build_examples_cmake_esp32c3:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32c3
   parallel: 8
   variables:
     IDF_TARGET: esp32c3
+    TEST_DIR: examples
 
 build_examples_cmake_esp32h2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32h2
   variables:
     IDF_TARGET: esp32h2
-
-.build_test_apps_template:
-  extends: .build_examples_cmake_template
-  variables:
-    TEST_PREFIX: test_apps
-    TEST_RELATIVE_DIR: tools/test_apps
-    TEST_TYPE: custom_test
-  script:
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
+    TEST_DIR: examples
 
 build_test_apps_esp32:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32
   parallel: 2
   variables:
     IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32s2:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32s2
   parallel: 2
   variables:
     IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32s3:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32s3
   parallel: 2
   variables:
     IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32c3:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32c3
   parallel: 2
   variables:
     IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32c2:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32c2
   variables:
     IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
 
 .test_build_system_template:
   stage: host_test

.gitlab/ci/host-test.yml

@@ -436,8 +436,11 @@ test_gen_soc_caps_kconfig:
 test_pytest_qemu:
   extends:
     - .host_test_template
-    - .before_script_pytest
+    - .before_script_build_jobs
   image: $QEMU_IMAGE
   script:
-    - run_cmd python tools/ci/build_pytest_apps.py . --target esp32 -m qemu -vv
+    - run_cmd python tools/ci/ci_build_apps.py . -vv
+      --target esp32
+      --pytest-apps
+      -m qemu
     - pytest --target esp32 -m qemu --embedded-services idf,qemu

.gitlab/ci/pre_check.yml

@@ -153,38 +153,6 @@ check_esp_system:
   script:
     - python components/esp_system/check_system_init_priorities.py
 
-scan_tests:
-  extends:
-    - .pre_check_base_template
-    - .before_script_pytest
-    - .rules:build:target_test
-  image: $TARGET_TEST_ENV_IMAGE
-  tags:
-    - scan_test  # since this job is used for uploading the cache, the runner tags should be unique
-  artifacts:
-    paths:
-      - $EXAMPLE_TEST_OUTPUT_DIR
-      - $TEST_APPS_OUTPUT_DIR
-      - $COMPONENT_UT_OUTPUT_DIR
-    expire_in: 1 week
-  variables:
-    EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
-    EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
-    TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
-    TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
-    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
-    CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
-    EXTRA_TEST_DIRS: >-
-      examples/bluetooth/esp_ble_mesh/ble_mesh_console
-      examples/bluetooth/hci/controller_hci_uart_esp32
-      examples/wifi/iperf
-    EXTRA_EVALUATE_ARGS: '--evaluate-parallel-count --config "sdkconfig.ci=default" --config "sdkconfig.ci.*=" --config "=default"'
-  script:
-    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS $EXTRA_EVALUATE_ARGS
-    - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR $EXTRA_EVALUATE_ARGS
-    - set_component_ut_vars
-    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR --combine-all-targets --except-targets linux $EXTRA_EVALUATE_ARGS
-
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
 check_version_tag:

.gitlab/ci/rules.yml

@@ -41,14 +41,6 @@
   - "components/**/*"
   - "examples/cxx/experimental/experimental_cpp_component/*"
 
-.patterns-build_target_test: &patterns-build_target_test
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
-  - "tools/ci/build_pytest_apps.py"
-
 .patterns-build_system: &patterns-build_system
   - "tools/cmake/**/*"
   - "tools/kconfig_new/**/*"
@@ -56,6 +48,7 @@
   - "tools/requirements.json"
   - "tools/ci/test_build_system*.sh"
   - "tools/ci/test_build_system*.py"
+  - "tools/ci/ci_build_apps.py"
 
 .patterns-custom_test: &patterns-custom_test
   - "components/espcoredump/**/*"
@@ -64,11 +57,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "tools/test_apps/**/*"
   - "tools/ldgen/**/*"
 
@@ -77,11 +65,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "tools/unit-test-app/**/*"
 
   - "components/**/*"
@@ -91,11 +74,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "components/**/*"
 
 .patterns-integration_test: &patterns-integration_test
@@ -485,8 +463,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -508,8 +484,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -525,8 +499,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
 
 .rules:build:component_ut-esp32c3:
   rules:
@@ -543,8 +515,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -563,8 +533,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -583,8 +551,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -603,8 +569,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -629,8 +593,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -648,8 +610,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -664,8 +624,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
 
 .rules:build:custom_test-esp32c3:
   rules:
@@ -680,8 +638,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -698,8 +654,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -716,8 +670,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -734,8 +686,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -772,8 +722,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -801,8 +749,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -824,8 +770,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test-related_changes-bt
     - <<: *if-dev-push
@@ -850,8 +794,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -876,8 +818,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -902,8 +842,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -928,8 +866,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -1012,8 +948,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -1050,8 +984,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1069,8 +1001,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1085,8 +1015,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
 
 .rules:build:unit_test-esp32c3:
   rules:
@@ -1101,8 +1029,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1119,8 +1045,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1137,8 +1061,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1155,8 +1077,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 

.gitlab/ci/static-code-analysis.yml

@@ -10,7 +10,6 @@ clang_tidy_check:
     when: always
     expire_in: 1 day
   variables:
-    CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
     CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
     RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
     OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports

components/cxx/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/cxx/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet

components/driver/.build-test-rules.yml

@@ -0,0 +1,33 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/driver/test_apps/i2s_test_apps:
+  disable:
+    - if: SOC_I2S_SUPPORTED != 1
+
+components/driver/test_apps/i2s_test_apps/legacy_i2s_adc_dac:
+  disable:
+    - if: SOC_I2S_SUPPORTS_ADC_DAC != 1
+
+components/driver/test_apps/legacy_pcnt_driver:
+  disable:
+    - if: SOC_PCNT_SUPPORTED != 1
+
+components/driver/test_apps/legacy_rmt_driver:
+  disable:
+    - if: SOC_RMT_SUPPORTED != 1
+
+components/driver/test_apps/legacy_rtc_temp_driver:
+  disable:
+    - if: SOC_TEMP_SENSOR_SUPPORTED != 1
+
+components/driver/test_apps/pulse_cnt:
+  disable:
+    - if: SOC_PCNT_SUPPORTED != 1
+
+components/driver/test_apps/rmt:
+  disable:
+    - if: SOC_RMT_SUPPORTED != 1
+
+components/driver/test_apps/temperature_sensor:
+  disable:
+    - if: SOC_TEMP_SENSOR_SUPPORTED != 1

components/esp_eth/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_eth/test_apps:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      reason: only test on esp32

components/esp_lcd/.build-test-rules.yml

@@ -0,0 +1,9 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_lcd/test_apps/i80_lcd:
+  disable:
+    - if: SOC_LCD_I80_SUPPORTED != 1
+
+components/esp_lcd/test_apps/rgb_lcd:
+  disable:
+    - if: SOC_LCD_RGB_SUPPORTED != 1

components/esp_netif/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_netif/test_apps:
+  disable_test:
+    - if: IDF_TARGET != "esp32s2"
+      temporary: true
+      reason: lack of runners

components/esp_psram/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_psram/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2", "esp32s3"]
+      reason: only test on esp32, esp32s2, and esp32s3

components/esp_rom/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_rom/host_test/rom_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/esp_system/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_system/test_apps/rtc_power_modes:
+  enable:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: the other targets are not tested yet

components/espcoredump/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/espcoredump/test_apps:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet

components/log/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/log/host_test/log_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/spi_flash/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/spi_flash/host_test/partition_api_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/wear_levelling/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/wear_levelling/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet

docs/en/contribute/esp-idf-tests-with-pytest.rst

@@ -390,11 +390,11 @@ Build Job Names
 Build Job Command
 ^^^^^^^^^^^^^^^^^
 
-The command used by CI to build all the relevant tests is: ``python $IDF_PATH/tools/ci/build_pytest_apps.py <parent_dir> --target <target> -vv``
+The command used by CI to build all the relevant tests is: ``python $IDF_PATH/tools/ci/ci_build_apps.py <parent_dir> --target <target> -vv --pytest-apps``
 
 All apps which supported the specified target would be built with all supported sdkconfig files under ``build_<target>_<config>``.
 
-For example, If you run ``python $IDF_PATH/tools/ci/build_pytest_apps.py $IDF_PATH/examples/system/console/basic --target esp32``, the folder structure would be like this:
+For example, If you run ``python $IDF_PATH/tools/ci/ci_build_apps.py $IDF_PATH/examples/system/console/basic --target esp32 --pytest-apps``, the folder structure would be like this:
 
 .. code:: text
 
@@ -442,7 +442,7 @@ For example, if you want to run all the esp32 tests under the ``$IDF_PATH/exampl
    $ cd $IDF_PATH
    $ . ./export.sh
    $ cd examples/system/console/basic
-   $ python $IDF_PATH/tools/ci/build_pytest_apps.py . --target esp32 -vv
+   $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps
    $ pytest --target esp32
 
 Tips and Tricks

examples/bluetooth/.build-test-rules.yml

@@ -0,0 +1,105 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/bluetooth/bluedroid/ble:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c2", "esp32c3", "esp32h2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/ble_50:
+  enable:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32h2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/ble_50/multi-adv:
+  enable:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/classic_bt:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/coex/a2dp_gatts_coex:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/coex/gattc_gatts_coex:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/blufi:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_ble_mesh:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_ble_mesh/ble_mesh_console:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_hid_device:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_hid_host:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/hci/ble_adv_scan_combined:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/hci/controller_hci_uart_esp32:
+  enable:
+    - if: IDF_TARGET == "esp32"
+
+examples/bluetooth/hci/controller_hci_uart_esp32c3_and_esp32s3:
+  enable:
+    - if: IDF_TARGET in ["esp32c3", "esp32s3"]
+
+examples/bluetooth/hci/controller_vhci_ble_adv:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/nimble:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+
+      reason: not tested yet
+examples/bluetooth/nimble/ble_spp:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/nimble/hci:
+  enable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: the other targets are not tested yet

examples/build_system/.build-test-rules.yml

@@ -0,0 +1,18 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/build_system/cmake/import_lib:
+  disable_test:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/build_system/cmake/linux_host_app:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux
+
+examples/build_system/cmake/plugins:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners

examples/cxx/.build-test-rules.yml

@@ -0,0 +1,41 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/cxx/exceptions:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners
+
+examples/cxx/experimental/esp_modem_cxx:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/cxx/experimental/experimental_cpp_component/host_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux
+
+examples/cxx/experimental/simple_i2c_rw_example:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/cxx/experimental/simple_spi_rw_example:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/cxx/pthread:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners
+examples/cxx/rtti:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners

examples/ethernet/.build-test-rules.yml

@@ -0,0 +1,17 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/ethernet:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/ethernet/iperf:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners

examples/get-started/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/get-started/blink:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners

examples/mesh/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/mesh:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet

examples/network/.build-test-rules.yml

@@ -0,0 +1,17 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/network/network_tests:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/network/simple_sniffer:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: not tested

examples/openthread/.build-test-rules.yml

@@ -0,0 +1,17 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/openthread/ot_br:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/openthread/ot_cli:
+  enable:
+    - if: IDF_TARGET == "esp32h2"
+      reason: only test on esp32h2
+
+examples/openthread/ot_rcp:
+  enable:
+    - if: IDF_TARGET == "esp32h2"
+      reason: only test on esp32h2

examples/peripherals/.build-test-rules.yml

@@ -0,0 +1,234 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/peripherals/adc/dma_read:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/adc/single_read/adc:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/adc/single_read/adc2:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/adc/single_read/single_read:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/gpio/generic_gpio:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/peripherals/gpio/matrix_keyboard:
+  enable:
+    - if: IDF_TARGET == "esp32s2"
+
+examples/peripherals/i2c/i2c_self_test:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/i2c/i2c_simple:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/i2c/i2c_tools:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/peripherals/i2s:
+  disable:
+    - if: SOC_I2S_SUPPORTED != 1
+
+examples/peripherals/i2s/i2s_adc_dac:
+  disable:
+    - if: SOC_I2S_SUPPORTS_ADC_DAC != 1
+
+examples/peripherals/i2s/i2s_audio_recorder_sdcard:
+  enable:
+    - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/i2s/i2s_basic:
+  disable:
+    - if: SOC_I2S_SUPPORTED != 1
+
+examples/peripherals/lcd/i80_controller:
+  disable:
+    - if: SOC_LCD_I80_SUPPORTED != 1
+
+examples/peripherals/lcd/rgb_panel:
+  disable:
+    - if: SOC_LCD_RGB_SUPPORTED != 1
+
+examples/peripherals/mcpwm:
+  disable:
+    - if: SOC_MCPWM_SUPPORTED != 1
+
+examples/peripherals/pcnt:
+  disable:
+    - if: SOC_PCNT_SUPPORTED != 1
+
+examples/peripherals/rmt:
+  disable:
+    - if: SOC_RMT_SUPPORTED != 1
+
+examples/peripherals/rmt/ir_nec_transceiver:
+  disable:
+    - if: SOC_RMT_SUPPORTED != 1
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/peripherals/rmt/musical_buzzer:
+  enable:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/rmt/stepper_motor:
+  enable:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/sdio:
+  disable:
+    - if: SOC_SDIO_SLAVE_SUPPORTED != 1
+
+examples/peripherals/secure_element/atecc608_ecdsa:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/sigmadelta:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/spi_master/hd_eeprom:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/spi_master/lcd:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/spi_slave/receiver:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/spi_slave/sender:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/spi_slave_hd/append_mode/master:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/spi_slave_hd/append_mode/slave:
+  enable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/peripherals/spi_slave_hd/segment_mode/seg_master:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/peripherals/spi_slave_hd/segment_mode/seg_slave:
+  disable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: not tested yet
+
+examples/peripherals/temp_sensor:
+  disable:
+    - if: SOC_TEMP_SENSOR_SUPPORTED != 1
+
+examples/peripherals/touch_sensor:
+  disable:
+    - if: SOC_TOUCH_SENSOR_NUM == 0
+
+examples/peripherals/touch_sensor/touch_element:
+  enable:
+    - if: IDF_TARGET == "esp32s2"
+      reason: only test on esp32s2
+
+examples/peripherals/touch_sensor/touch_sensor_v1:
+  disable:
+    - if: SOC_TOUCH_SENSOR_NUM != 10
+
+examples/peripherals/touch_sensor/touch_sensor_v2:
+  disable:
+    - if: SOC_TOUCH_SENSOR_NUM != 15
+
+examples/peripherals/twai:
+  disable:
+    - if: SOC_TWAI_SUPPORTED != 1
+
+examples/peripherals/twai/twai_alert_and_recovery:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/peripherals/twai/twai_self_test:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/peripherals/uart/uart_echo_rs485:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "esp32h2"
+
+examples/peripherals/usb:
+  disable:
+    - if: SOC_USB_PERIPH_NUM != 1
+
+examples/peripherals/wave_gen:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet

examples/protocols/.build-test-rules.yml

@@ -0,0 +1,253 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/protocols/asio/asio_chat:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/asio/async_request:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/protocols/asio/socks4:
+  enable:
+    - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/protocols/asio/ssl_client_server:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/asio/tcp_echo_server:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/asio/udp_echo_server:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/coap_client:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/coap_server:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/http2_request:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/http_request:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/http_server:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/https_mbedtls:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/https_server/simple:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/https_server/wss_server:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/https_x509_bundle:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/icmp_echo:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/l2tap:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/mdns:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/modbus:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/mqtt/ssl:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/mqtt/ssl_ds:
+  enable:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+
+examples/protocols/mqtt/ssl_mutual_auth:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/mqtt/ssl_psk:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/mqtt/tcp:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/mqtt/ws:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/mqtt/wss:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/slip/slip_udp:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/smtp_client:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/sntp:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/protocols/sockets/non_blocking:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/sockets/tcp_client:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/sockets/tcp_client_multi_net:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/protocols/sockets/tcp_server:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/sockets/udp_client:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/protocols/sockets/udp_server:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners

examples/provisioning/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/provisioning/wifi_prov_mgr:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners

examples/security/.build-test-rules.yml

@@ -0,0 +1,11 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/security/flash_encryption:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners

examples/storage/.build-test-rules.yml

@@ -0,0 +1,155 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/storage/custom_flash_driver:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/storage/ext_flash_fatfs:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/fatfsgen:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/nvs_rw_blob:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/nvs_rw_value:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/nvs_rw_value_cxx:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/partition_api/partition_find:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/partition_api/partition_mmap:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/partition_api/partition_ops:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/parttool:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/sd_card/sdmmc:
+  disable:
+    - if: SOC_SDMMC_HOST_SUPPORTED != 1
+  disable_test:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: lack of runners
+
+examples/storage/sd_card/sdspi:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: lack of runners
+
+examples/storage/semihost_vfs:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/spiffs:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/spiffsgen:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/storage/wear_levelling:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners

examples/system/.build-test-rules.yml

@@ -0,0 +1,221 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/system/app_trace_to_host:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/console/advanced:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/system/console/advanced_usb_cdc:
+  enable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/console/basic:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/system/deep_sleep:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/system/efuse:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: lack of runners
+
+examples/system/esp_event/default_event_loop:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/system/esp_timer:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/flash_suspend:
+  enable:
+    - if: IDF_TARGET == "esp32c3"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/gcov:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/gdbstub:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/system/heap_task_tracking:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/system/himem:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/ipc/ipc_isr:
+  enable:
+    - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/light_sleep:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/system/ota/advanced_https_ota:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/ota/otatool:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/system/ota/pre_encrypted_ota:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/ota/simple_ota_example:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/perfmon:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/select:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/system/startup_time:
+  disable_test:
+    - if: IDF_TARGET == "esp32s3" or IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/sysview_tracing:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2" or IDF_TARGET == "esp32c3"
+      temporary: true
+      reason: lack of runners
+
+examples/system/sysview_tracing_heap_log:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2" or IDF_TARGET == "esp32c3"
+      temporary: true
+      reason: lack of runners
+
+examples/system/task_watchdog:
+  disable_test:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: lack of runners
+
+examples/system/ulp_fsm/ulp:
+  disable:
+    - if: SOC_ULP_SUPPORTED != 1
+
+examples/system/ulp_fsm/ulp_adc:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/ulp_riscv/adc:
+  enable:
+    - if: IDF_TARGET in ["esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/ulp_riscv/ds18b20_onewire:
+  enable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/ulp_riscv/gpio:
+  enable:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/system/ulp_riscv/gpio_interrupt:
+  enable:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet

examples/wifi/.build-test-rules.yml

@@ -0,0 +1,25 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/wifi:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/wifi/ftm:
+  enable:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/wifi/getting_started:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+examples/wifi/iperf:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners

examples/zigbee/.build-test-rules.yml

@@ -0,0 +1,17 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/zigbee/esp_zigbee_gateway:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/zigbee/esp_zigbee_rcp:
+  enable:
+    - if: IDF_TARGET == "esp32h2"
+      reason: only test on esp32h2
+
+examples/zigbee/light_sample:
+  enable:
+    - if: IDF_TARGET == "esp32h2"
+      reason: only test on esp32h2

tools/build_apps.py

@@ -1,171 +0,0 @@
-#!/usr/bin/env python
-
-# coding=utf-8
-#
-# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
-#
-
-import argparse
-import logging
-import os.path
-import re
-import sys
-from typing import List, Optional, TextIO
-
-from find_build_apps import BUILD_SYSTEMS, BuildError, BuildItem, setup_logging
-from find_build_apps.common import SIZE_JSON_FN, rmdir
-
-# This RE will match GCC errors and many other fatal build errors and warnings as well
-LOG_ERROR_WARNING = re.compile(r'(error|warning):', re.IGNORECASE)
-
-# Log this many trailing lines from a failed build log, also
-LOG_DEBUG_LINES = 25
-
-
-def build_apps(
-    build_items: List[BuildItem],
-    parallel_count: int = 1,
-    parallel_index: int = 1,
-    dry_run: bool = False,
-    build_verbose: bool = False,
-    keep_going: bool = False,
-    output_build_list: Optional[TextIO] = None,
-    size_info: Optional[TextIO] = None
-) -> None:
-    if not build_items:
-        logging.warning('Empty build list')
-        sys.exit(0)
-
-    num_builds = len(build_items)
-    num_jobs = parallel_count
-    job_index = parallel_index - 1  # convert to 0-based index
-    num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
-    min_job_index = num_builds_per_job * job_index
-    if min_job_index >= num_builds:
-        logging.warning(
-            f'Nothing to do for job {job_index + 1} (build total: {num_builds}, per job: {num_builds_per_job})')
-        sys.exit(0)
-
-    max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
-    logging.info('Total {} builds, max. {} builds per job, running builds {}-{}'.format(
-        num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))
-
-    builds_for_current_job = build_items[min_job_index:max_job_index + 1]
-    for i, build_item in enumerate(builds_for_current_job):
-        index = i + min_job_index + 1
-        build_item.index = index
-        build_item.dry_run = dry_run
-        build_item.verbose = build_verbose
-        build_item.keep_going = keep_going
-        logging.debug('\tBuild {}: {}'.format(index, repr(build_item)))
-        if output_build_list:
-            output_build_list.write(build_item.to_json_expanded() + '\n')
-
-    failed_builds = []
-    for build_item in builds_for_current_job:
-        logging.info('Running build {}: {}'.format(build_item.index, repr(build_item)))
-        build_system_class = BUILD_SYSTEMS[build_item.build_system]
-        try:
-            build_system_class.build(build_item)
-        except BuildError as e:
-            logging.error(str(e))
-            if build_item.build_log_path:
-                log_filename = os.path.basename(build_item.build_log_path)
-                with open(build_item.build_log_path, 'r') as f:
-                    lines = [line.rstrip() for line in f.readlines() if line.rstrip()]  # non-empty lines
-                    logging.debug('Error and warning lines from {}:'.format(log_filename))
-                    for line in lines:
-                        if LOG_ERROR_WARNING.search(line):
-                            logging.warning('>>> {}'.format(line))
-                    logging.debug('Last {} lines of {}:'.format(LOG_DEBUG_LINES, log_filename))
-                    for line in lines[-LOG_DEBUG_LINES:]:
-                        logging.debug('>>> {}'.format(line))
-            if keep_going:
-                failed_builds.append(build_item)
-            else:
-                sys.exit(1)
-        else:
-            if size_info:
-                build_item.write_size_info(size_info)
-            if not build_item.preserve:
-                logging.info(f'Removing build directory {build_item.build_path}')
-                # we only remove binaries here, log files are still needed by check_build_warnings.py
-                rmdir(build_item.build_path, exclude_file_pattern=SIZE_JSON_FN)
-
-    if failed_builds:
-        logging.error('The following build have failed:')
-        for build in failed_builds:
-            logging.error('\t{}'.format(build))
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='ESP-IDF app builder')
-    parser.add_argument(
-        '-v',
-        '--verbose',
-        action='count',
-        help='Increase the logging level of the script. Can be specified multiple times.',
-    )
-    parser.add_argument(
-        '--build-verbose',
-        action='store_true',
-        help='Enable verbose output from build system.',
-    )
-    parser.add_argument(
-        '--log-file',
-        type=argparse.FileType('w'),
-        help='Write the script log to the specified file, instead of stderr',
-    )
-    parser.add_argument(
-        '--parallel-count',
-        default=1,
-        type=int,
-        help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
-             'it needs to be executed multiple times with same value of --parallel-count and ' +
-             'different values of --parallel-index.',
-    )
-    parser.add_argument(
-        '--parallel-index',
-        default=1,
-        type=int,
-        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
-    )
-    parser.add_argument(
-        '--format',
-        default='json',
-        choices=['json'],
-        help='Format to read the list of builds',
-    )
-    parser.add_argument(
-        '--dry-run',
-        action='store_true',
-        help="Don't actually build, only print the build commands",
-    )
-    parser.add_argument(
-        '--keep-going',
-        action='store_true',
-        help="Don't exit immediately when a build fails.",
-    )
-    parser.add_argument(
-        '--output-build-list',
-        type=argparse.FileType('w'),
-        help='If specified, the list of builds (with all the placeholders expanded) will be written to this file.',
-    )
-    parser.add_argument(
-        '--size-info',
-        type=argparse.FileType('a'),
-        help='If specified, the test case name and size info json will be written to this file'
-    )
-    parser.add_argument(
-        'build_list',
-        type=argparse.FileType('r'),
-        nargs='?',
-        default=sys.stdin,
-        help='Name of the file to read the list of builds from. If not specified, read from stdin.',
-    )
-    args = parser.parse_args()
-    setup_logging(args)
-    items = [BuildItem.from_json(line) for line in args.build_list]
-    build_apps(items, args.parallel_count, args.parallel_index, args.dry_run, args.build_verbose,
-               args.keep_going, args.output_build_list, args.size_info)

tools/ci/build_pytest_apps.py

@@ -1,159 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-
-"""
-This file is used to generate binary files for the given path.
-"""
-
-import argparse
-import copy
-import logging
-import os
-import sys
-from collections import defaultdict
-from typing import List
-
-from idf_ci_utils import IDF_PATH, PytestCase, get_pytest_cases
-
-try:
-    from build_apps import build_apps
-    from find_apps import find_builds_for_app
-    from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging
-except ImportError:
-    sys.path.append(os.path.join(IDF_PATH, 'tools'))
-
-    from build_apps import build_apps
-    from find_apps import find_builds_for_app
-    from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging
-
-
-def main(args: argparse.Namespace) -> None:
-    pytest_cases: List[PytestCase] = []
-    for path in args.paths:
-        pytest_cases += get_pytest_cases(path, args.target, args.marker_expr)
-
-    paths = set()
-    app_configs = defaultdict(set)
-    for case in pytest_cases:
-        for app in case.apps:
-            paths.add(app.path)
-            app_configs[app.path].add(app.config)
-
-    app_dirs = list(paths)
-    if not app_dirs:
-        raise RuntimeError('No apps found')
-
-    logging.info(f'Found {len(app_dirs)} apps')
-    app_dirs.sort()
-
-    # Find compatible configurations of each app, collect them as BuildItems
-    build_items: List[BuildItem] = []
-    config_rules = config_rules_from_str(args.config or [])
-    for app_dir in app_dirs:
-        app_dir = os.path.realpath(app_dir)
-        if args.target in CMakeBuildSystem.supported_targets(app_dir):
-            build_items += find_builds_for_app(
-                app_path=app_dir,
-                work_dir=app_dir,
-                build_dir='build_@t_@w',
-                build_log=f'{app_dir}/build_@t_@w/build.log',
-                target_arg=args.target,
-                build_system='cmake',
-                config_rules=config_rules,
-            )
-
-    modified_build_items = []
-    # auto clean up the binaries if no flag --preserve-all
-    for item in build_items:
-        is_test_related = item.config_name in app_configs[item.app_dir]
-        if args.test_only and not is_test_related:
-            logging.info(f'Skipping non-test app: {item}')
-            continue
-
-        copied_item = copy.deepcopy(item)
-        if not args.preserve_all and not is_test_related:
-            copied_item.preserve = False
-        modified_build_items.append(copied_item)
-
-    logging.info(f'Found {len(modified_build_items)} builds')
-    modified_build_items.sort(key=lambda x: x.build_path)  # type: ignore
-
-    build_apps(
-        build_items=modified_build_items,
-        parallel_count=args.parallel_count,
-        parallel_index=args.parallel_index,
-        dry_run=False,
-        build_verbose=args.build_verbose,
-        keep_going=True,
-        output_build_list=None,
-        size_info=args.size_info,
-    )
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description='Build all the pytest apps under specified paths. Will auto remove those non-test apps binaries'
-    )
-    parser.add_argument(
-        '-t', '--target', required=True, help='Build apps for given target.'
-    )
-    parser.add_argument(
-        '-m',
-        '--marker-expr',
-        default='not host_test',  # host_test apps would be built and tested under the same job
-        help='only build tests matching given mark expression. For example: -m "host_test and generic".',
-    )
-    parser.add_argument(
-        '--config',
-        default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'],
-        action='append',
-        help='Adds configurations (sdkconfig file names) to build. This can either be '
-        + 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
-        + 'relative to the project directory, to be used. Optional NAME can be specified, '
-        + 'which can be used as a name of this configuration. FILEPATTERN is the name of '
-        + 'the sdkconfig file, relative to the project directory, with at most one wildcard. '
-        + 'The part captured by the wildcard is used as the name of the configuration.',
-    )
-    parser.add_argument(
-        'paths',
-        nargs='+',
-        help='One or more app paths. Will use the current path if not specified.',
-    )
-    parser.add_argument(
-        '--parallel-count', default=1, type=int, help='Number of parallel build jobs.'
-    )
-    parser.add_argument(
-        '--parallel-index',
-        default=1,
-        type=int,
-        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
-    )
-    parser.add_argument(
-        '--size-info',
-        type=argparse.FileType('a'),
-        help='If specified, the test case name and size info json will be written to this file',
-    )
-    parser.add_argument(
-        '-v',
-        '--verbose',
-        action='count',
-        help='Increase the logging level of the script. Can be specified multiple times.',
-    )
-    parser.add_argument(
-        '--build-verbose',
-        action='store_true',
-        help='Enable verbose output from build system.',
-    )
-    parser.add_argument(
-        '--preserve-all',
-        action='store_true',
-        help='Preserve the binaries for all apps when specified.',
-    )
-    parser.add_argument(
-        '--test-only',
-        action='store_true',
-        help='Build only test related app when specified.',
-    )
-    arguments = parser.parse_args()
-    setup_logging(arguments)
-    main(arguments)

tools/ci/build_template_app.sh

@@ -6,9 +6,6 @@
 #     the fast build will be built.
 #
 # Needs to be called under IDF root folder
-#
-# This script will call find_apps.py with the following arguments:
-# - CMake build arguments: --work-dir {BUILD_PATH}/cmake --build-dir ${BUILD_DIR} --build-log ${BUILD_LOG_CMAKE}
 
 set -euo pipefail
 
@@ -50,55 +47,36 @@ get_config_str() {
     echo ${CONFIG_STR}
 }
 
-search_cmake() {
-    TARGET=$1
-    shift
-    CONFIG_STR=$*
-    tools/find_apps.py -vv --format json --work-dir ${BUILD_PATH}/cmake --build-dir ${BUILD_DIR} --build-log ${BUILD_LOG_CMAKE} -p esp-idf-template --build-system cmake ${CONFIG_STR} --target ${TARGET} --output scan_temp.json
-    cat scan_temp.json >> scan.json
-    rm scan_temp.json
-}
-
-build() {
-    tools/build_apps.py -vv --format json --keep-going --parallel-count 1 --parallel-index 1 --size-info ${SIZE_INFO_LOCATION} scan.json
-    rm scan.json
-}
-
 build_stage2() {
     CONFIG_STR=$(get_config_str sdkconfig.ci.*=)
-    search_cmake esp32 ${CONFIG_STR}
-    search_cmake esp32s2 ${CONFIG_STR}
-    search_cmake esp32s3 ${CONFIG_STR}
-    search_cmake esp32c3 ${CONFIG_STR}
-    search_cmake esp32h2 ${CONFIG_STR}
-    search_cmake esp32c2 ${CONFIG_STR}
-
-    build build_list_1.json
-
-    CONFIG_STR=$(get_config_str sdkconfig.ci3.*=)
-    search_cmake esp32 ${CONFIG_STR}
-    search_cmake esp32s2 ${CONFIG_STR}
-    search_cmake esp32s3 ${CONFIG_STR}
-    search_cmake esp32c3 ${CONFIG_STR}
-    search_cmake esp32h2 ${CONFIG_STR}
-    search_cmake esp32c2 ${CONFIG_STR}
 
     # Override EXTRA_CFLAGS and EXTRA_CXXFLAGS in the environment
     export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
     export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
-    build
+    python -m idf_build_apps build -vv \
+        -p esp-idf-template \
+        -t all \
+        ${CONFIG_STR} \
+        --work-dir ${BUILD_PATH}/cmake \
+        --build-dir ${BUILD_DIR} \
+        --build-log ${BUILD_LOG_CMAKE} \
+        --size-file size.json \
+        --collect-size-info size_info.txt \
+        --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3 # add esp32h2 back after IDF-5541
 }
 
 build_stage1() {
     CONFIG_STR=$(get_config_str sdkconfig.ci2.*=)
-    search_cmake esp32 ${CONFIG_STR}
-    search_cmake esp32s2 ${CONFIG_STR}
-    search_cmake esp32s3 ${CONFIG_STR}
-    search_cmake esp32c3 ${CONFIG_STR}
-    search_cmake esp32h2 ${CONFIG_STR}
-    search_cmake esp32c2 ${CONFIG_STR}
-
-    build
+    python -m idf_build_apps build -vv \
+        -p esp-idf-template \
+        -t all \
+        ${CONFIG_STR} \
+        --work-dir ${BUILD_PATH}/cmake \
+        --build-dir ${BUILD_DIR} \
+        --build-log ${BUILD_LOG_CMAKE} \
+        --size-file size.json \
+        --collect-size-info size_info.txt \
+        --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32h2
 }
 
 # Default arguments

tools/ci/check_build_test_rules.py

@@ -0,0 +1,422 @@
+#!/usr/bin/env python
+
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import inspect
+import os
+import re
+import sys
+from io import StringIO
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from idf_ci_utils import IDF_PATH, get_pytest_cases, get_ttfw_cases
+
+YES = u'\u2713'
+NO = u'\u2717'
+
+# | Supported Target | ... |
+# | ---------------- | --- |
+SUPPORTED_TARGETS_TABLE_REGEX = re.compile(
+    r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE
+)
+
+USUAL_TO_FORMAL = {
+    'esp32': 'ESP32',
+    'esp32s2': 'ESP32-S2',
+    'esp32s3': 'ESP32-S3',
+    'esp32c3': 'ESP32-C3',
+    'esp32h2': 'ESP32-H2',
+    'esp32c2': 'ESP32-C2',
+    'linux': 'Linux',
+}
+
+FORMAL_TO_USUAL = {
+    'ESP32': 'esp32',
+    'ESP32-S2': 'esp32s2',
+    'ESP32-S3': 'esp32s3',
+    'ESP32-C3': 'esp32c3',
+    'ESP32-H2': 'esp32h2',
+    'ESP32-C2': 'esp32c2',
+    'Linux': 'linux',
+}
+
+
+def doublequote(s: str) -> str:
+    if s.startswith('"') and s.endswith('"'):
+        return s
+
+    return f'"{s}"'
+
+
+def check_readme(paths: List[str]) -> None:
+    from idf_build_apps import App, find_apps
+    from idf_build_apps.constants import SUPPORTED_TARGETS
+
+    def get_readme_path(_app: App) -> Optional[str]:
+        _readme_path = os.path.join(_app.app_dir, 'README.md')
+
+        if not os.path.isfile(_readme_path):
+            _readme_path = os.path.join(_app.app_dir, '..', 'README.md')
+
+            if not os.path.isfile(_readme_path):
+                _readme_path = None  # type: ignore
+
+        return _readme_path
+
+    def _generate_new_support_table_str(_app: App) -> str:
+        # extra space here
+        table_headers = [
+            f'{USUAL_TO_FORMAL[target]}' for target in _app.supported_targets
+        ]
+        table_headers = ['Supported Targets'] + table_headers
+
+        res = '| ' + ' | '.join(table_headers) + ' |\n'
+        res += '| ' + ' | '.join(['-' * len(item) for item in table_headers]) + ' |'
+
+        return res
+
+    def _parse_existing_support_table_str(_app: App) -> Tuple[Optional[str], List[str]]:
+        _readme_path = get_readme_path(_app)
+        if not _readme_path:
+            return None, SUPPORTED_TARGETS
+
+        with open(_readme_path) as _fr:
+            _readme_str = _fr.read()
+
+        support_string = SUPPORTED_TARGETS_TABLE_REGEX.findall(_readme_str)
+        if not support_string:
+            return None, SUPPORTED_TARGETS
+
+        # old style
+        parts = [
+            part.strip()
+            for part in support_string[0].split('\n', 1)[0].split('|')
+            if part.strip()
+        ]
+        return support_string[0].strip(), [FORMAL_TO_USUAL[part] for part in parts[1:]]
+
+    def check_enable_build(_app: App, _old_supported_targets: List[str]) -> bool:
+        if _app.supported_targets == sorted(_old_supported_targets):
+            return True
+
+        _readme_path = get_readme_path(_app)
+        if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(_old_supported_targets)])}]'
+
+        print(
+            inspect.cleandoc(
+                f'''
+            {_app.app_dir}:
+            - enable build targets according to the manifest file: {_app.supported_targets}
+            - enable build targets according to the old Supported Targets table under readme "{_readme_path}": {_old_supported_targets}
+
+            If you want to disable some targets, please use the following snippet:
+
+            # Please combine this with the original one
+            #
+            # Notes:
+            # - please keep in mind to avoid duplicated folders as yaml keys
+            # - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands
+            {_app.app_dir}:
+              enable:
+                - if: {if_clause}
+                  temporary: true
+                  reason: <why only enable build jobs for these targets>
+
+            '''
+            )
+        )
+
+        return False
+
+    apps = sorted(
+        find_apps(
+            paths,
+            'all',
+            recursive=True,
+            manifest_files=[
+                str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')
+            ],
+        )
+    )
+    exit_code = 0
+
+    checked_app_dirs = set()
+    for app in apps:
+        if app.app_dir not in checked_app_dirs:
+            checked_app_dirs.add(app.app_dir)
+        else:
+            continue
+
+        replace_str, old_supported_targets = _parse_existing_support_table_str(app)
+        success = check_enable_build(app, old_supported_targets)
+        if not success:
+            print(f'check_enable_build failed for app: {app}')
+            print('-' * 80)
+            exit_code = 1
+
+        readme_path = get_readme_path(app)
+        # no readme, create a new file
+        if not readme_path:
+            with open(os.path.join(app.app_dir, 'README.md'), 'w') as fw:
+                fw.write(_generate_new_support_table_str(app) + '\n')
+            print(f'Added new README file: {os.path.join(app.app_dir, "README.md")}')
+            print('-' * 80)
+            exit_code = 1
+        # has old table, but different string
+        elif replace_str and replace_str != _generate_new_support_table_str(app):
+            with open(readme_path) as fr:
+                readme_str = fr.read()
+
+            with open(readme_path, 'w') as fw:
+                fw.write(
+                    readme_str.replace(
+                        replace_str, _generate_new_support_table_str(app)
+                    )
+                )
+                print(f'Modified README file: {readme_path}')
+            print('-' * 80)
+            exit_code = 1
+        # does not have old table
+        elif not replace_str:
+            with open(readme_path) as fr:
+                readme_str = fr.read()
+
+            with open(readme_path, 'w') as fw:
+                fw.write(
+                    _generate_new_support_table_str(app) + '\n\n' + readme_str
+                )  # extra new line
+
+            print(f'Modified README file: {readme_path}')
+            print('-' * 80)
+            exit_code = 1
+
+    sys.exit(exit_code)
+
+
+def check_test_scripts(paths: List[str]) -> None:
+    from idf_build_apps import App, find_apps
+
+    # takes long time, run only in CI
+    # dict:
+    # {
+    #      app_dir: {
+    #          'script_path': 'path/to/script',
+    #          'targets': ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32h2', 'esp32c2', 'linux'],
+    #      }
+    # }
+    def check_enable_test(
+        _app: App,
+        _pytest_app_dir_targets_dict: Dict[str, Dict[str, str]],
+        _ttfw_app_dir_targets_dict: Dict[str, Dict[str, str]],
+    ) -> bool:
+        if _app.app_dir in _pytest_app_dir_targets_dict:
+            test_script_path = _pytest_app_dir_targets_dict[_app.app_dir]['script_path']
+            actual_verified_targets = sorted(
+                set(_pytest_app_dir_targets_dict[_app.app_dir]['targets'])
+            )
+        elif _app.app_dir in _ttfw_app_dir_targets_dict:
+            test_script_path = _ttfw_app_dir_targets_dict[_app.app_dir]['script_path']
+            actual_verified_targets = sorted(
+                set(_ttfw_app_dir_targets_dict[_app.app_dir]['targets'])
+            )
+        else:
+            return True  # no test case
+
+        if (
+            _app.app_dir in _pytest_app_dir_targets_dict
+            and _app.app_dir in _ttfw_app_dir_targets_dict
+        ):
+            print(
+                f'''
+            Both pytest and ttfw test cases are found for {_app.app_dir},
+            please remove one of them.
+            pytest script: {_pytest_app_dir_targets_dict[_app.app_dir]['script_path']}
+            ttfw script: {_ttfw_app_dir_targets_dict[_app.app_dir]['script_path']}
+            '''
+            )
+            return False
+
+        actual_extra_tested_targets = set(actual_verified_targets) - set(
+            _app.verified_targets
+        )
+        if actual_extra_tested_targets:
+            print(
+                inspect.cleandoc(
+                    f'''
+                {_app.app_dir}:
+                - enable test targets according to the manifest file: {_app.verified_targets}
+                - enable test targets according to the test scripts: {actual_verified_targets}
+
+                test scripts enabled targets should be a subset of the manifest file declared ones.
+                Please check the test script: {test_script_path}.
+
+                '''
+                )
+            )
+            return False
+
+        if actual_verified_targets == _app.verified_targets:
+            return True
+
+        if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(set(_app.verified_targets) - set(actual_verified_targets))])}]'
+
+        print(
+            inspect.cleandoc(
+                f'''
+            {_app.app_dir}:
+            - enable test targets according to the manifest file: {_app.verified_targets}
+            - enable test targets according to the test scripts: {actual_verified_targets}
+
+            the test scripts enabled test targets should be the same with the manifest file enabled ones. Please check
+            the test script manually: {test_script_path}.
+
+            If you want to enable test targets in the pytest test scripts, please add `@pytest.mark.MISSING_TARGET`
+            marker above the test case function.
+
+            If you want to enable test targets in the ttfw test scripts, please add/extend the keyword `targets` in
+            the ttfw decorator, e.g. `@ttfw_idf.idf_example_test(..., target=['esp32', 'MISSING_TARGET'])`
+
+            If you want to disable the test targets in the manifest file, please modify your manifest file with
+            the following code snippet:
+
+            # Please combine this with the original one
+            #
+            # Notes:
+            # - please keep in mind to avoid duplicated folders as yaml keys
+            # - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands
+            {_app.app_dir}:
+              disable_test:
+                - if: {if_clause}
+                  temporary: true
+                  reason: <why you disable this test>
+
+            '''
+            )
+        )
+        return False
+
+    apps = sorted(
+        find_apps(
+            paths,
+            'all',
+            recursive=True,
+            manifest_files=[
+                str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')
+            ],
+        )
+    )
+    exit_code = 0
+
+    pytest_cases = get_pytest_cases(paths)
+    ttfw_cases = get_ttfw_cases(paths)
+
+    pytest_app_dir_targets_dict = {}
+    ttfw_app_dir_targets_dict = {}
+    for case in pytest_cases:
+        for pytest_app in case.apps:
+            app_dir = os.path.relpath(pytest_app.path, IDF_PATH)
+            if app_dir not in pytest_app_dir_targets_dict:
+                pytest_app_dir_targets_dict[app_dir] = {
+                    'script_path': case.path,
+                    'targets': [pytest_app.target],
+                }
+            else:
+                pytest_app_dir_targets_dict[app_dir]['targets'].append(
+                    pytest_app.target
+                )
+
+    for case in ttfw_cases:
+        app_dir = case.case_info['app_dir']
+        if app_dir not in ttfw_app_dir_targets_dict:
+            ttfw_app_dir_targets_dict[app_dir] = {
+                'script_path': case.case_info['script_path'],
+                'targets': [case.case_info['target'].lower()],
+            }
+        else:
+            ttfw_app_dir_targets_dict[app_dir]['targets'].append(
+                case.case_info['target'].lower()
+            )
+
+    checked_app_dirs = set()
+    for app in apps:
+        if app.app_dir not in checked_app_dirs:
+            checked_app_dirs.add(app.app_dir)
+        else:
+            continue
+
+        success = check_enable_test(
+            app, pytest_app_dir_targets_dict, ttfw_app_dir_targets_dict
+        )
+        if not success:
+            print(f'check_enable_test failed for app: {app}')
+            print('-' * 80)
+            exit_code = 1
+            continue
+
+    sys.exit(exit_code)
+
+
+def sort_yaml(files: List[str]) -> None:
+    from ruamel.yaml import YAML, CommentedMap
+
+    yaml = YAML()
+    yaml.indent(mapping=2, sequence=4, offset=2)
+    yaml.width = 4096  # avoid wrap lines
+
+    exit_code = 0
+    for f in files:
+        with open(f) as fr:
+            file_s = fr.read()
+            fr.seek(0)
+            file_d: CommentedMap = yaml.load(fr)
+
+        sorted_yaml = CommentedMap(dict(sorted(file_d.items())))
+        file_d.copy_attributes(sorted_yaml)
+
+        with StringIO() as s:
+            yaml.dump(sorted_yaml, s)
+
+            string = s.getvalue()
+            if string != file_s:
+                with open(f, 'w') as fw:
+                    fw.write(string)
+                print(
+                    f'Sorted yaml file {f}. Please take a look. sometimes the format is a bit messy'
+                )
+                exit_code = 1
+
+    sys.exit(exit_code)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='ESP-IDF apps build/test checker')
+    action = parser.add_subparsers(dest='action')
+
+    _check_readme = action.add_parser('check-readmes')
+    _check_readme.add_argument('paths', nargs='+', help='check under paths')
+
+    _check_test_scripts = action.add_parser('check-test-scripts')
+    _check_test_scripts.add_argument('paths', nargs='+', help='check under paths')
+
+    _sort_yaml = action.add_parser('sort-yaml')
+    _sort_yaml.add_argument('files', nargs='+', help='all specified yaml files')
+
+    arg = parser.parse_args()
+
+    if arg.action == 'sort-yaml':
+        sort_yaml(arg.files)
+    else:
+        check_dirs = set()
+        for path in arg.paths:
+            if os.path.isfile(path):
+                check_dirs.add(os.path.dirname(path))
+            else:
+                check_dirs.add(path)
+
+        if arg.action == 'check-readmes':
+            check_readme(list(check_dirs))
+        elif arg.action == 'check-test-scripts':
+            check_test_scripts(list(check_dirs))

tools/ci/check_copyright_ignore.txt

@@ -2078,10 +2078,8 @@ examples/wifi/wps/main/wps.c
 tools/ble/lib_ble_client.py
 tools/ble/lib_gap.py
 tools/ble/lib_gatt.py
-tools/build_apps.py
 tools/catch/catch.hpp
 tools/esp_app_trace/test/sysview/blink.c
-tools/find_apps.py
 tools/find_build_apps/__init__.py
 tools/find_build_apps/cmake.py
 tools/find_build_apps/common.py

tools/ci/check_executables.py

@@ -5,9 +5,14 @@
 
 import argparse
 import os
-from sys import exit
+import sys
 
-from idf_ci_utils import is_executable
+try:
+    from idf_ci_utils import is_executable
+except ImportError:
+    sys.path.append(os.path.join(os.path.dirname(__file__)))
+
+    from idf_ci_utils import is_executable
 
 
 def _strip_each_item(iterable):
@@ -65,4 +70,4 @@ def main():
 
 
 if __name__ == '__main__':
-    exit(main())
+    sys.exit(main())

tools/ci/ci_build_apps.py

@@ -0,0 +1,240 @@
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+This file is used in CI generate binary files for different kinds of apps
+"""
+
+import argparse
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Set
+
+from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging
+from idf_ci_utils import IDF_PATH, get_pytest_app_paths, get_pytest_cases, get_ttfw_app_paths
+
+
+def get_pytest_apps(
+    paths: List[str],
+    target: str,
+    config_rules_str: List[str],
+    marker_expr: str,
+    preserve_all: bool = False,
+) -> List[App]:
+    pytest_cases = get_pytest_cases(paths, target, marker_expr)
+
+    _paths: Set[str] = set()
+    app_configs = defaultdict(set)
+    for case in pytest_cases:
+        for app in case.apps:
+            _paths.add(app.path)
+            app_configs[app.path].add(app.config)
+
+    app_dirs = list(_paths)
+    if not app_dirs:
+        raise RuntimeError('No apps found')
+
+    LOGGER.info(f'Found {len(app_dirs)} apps')
+    app_dirs.sort()
+
+    apps = find_apps(
+        app_dirs,
+        target=target,
+        build_dir='build_@t_@w',
+        config_rules_str=config_rules_str,
+        build_log_path='build_log.txt',
+        size_json_path='size.json',
+        check_warnings=True,
+        manifest_files=[
+            str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')
+        ],
+    )
+
+    for app in apps:
+        is_test_related = app.config_name in app_configs[app.app_dir]
+        if not preserve_all and not is_test_related:
+            app.preserve = False
+
+    return apps  # type: ignore
+
+
+def get_cmake_apps(
+    paths: List[str],
+    target: str,
+    config_rules_str: List[str],
+    preserve_all: bool = False,
+) -> List[App]:
+    ttfw_app_dirs = get_ttfw_app_paths(paths, target)
+    apps = find_apps(
+        paths,
+        recursive=True,
+        target=target,
+        build_dir='build_@t_@w',
+        config_rules_str=config_rules_str,
+        build_log_path='build_log.txt',
+        size_json_path='size.json',
+        check_warnings=True,
+        preserve=False,
+        manifest_files=[
+            str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')
+        ],
+    )
+
+    apps_for_build = []
+    pytest_app_dirs = get_pytest_app_paths(paths, target)
+    for app in apps:
+        if preserve_all or app.app_dir in ttfw_app_dirs:  # relpath
+            app.preserve = True
+
+        if os.path.realpath(app.app_dir) in pytest_app_dirs:
+            LOGGER.debug('Skipping build app with pytest scripts: %s', app)
+            continue
+
+        apps_for_build.append(app)
+
+    return apps_for_build
+
+
+APPS_BUILD_PER_JOB = 30
+
+
+def main(args: argparse.Namespace) -> None:
+    if args.pytest_apps:
+        LOGGER.info('Only build apps with pytest scripts')
+        apps = get_pytest_apps(
+            args.paths, args.target, args.config, args.marker_expr, args.preserve_all
+        )
+    else:
+        LOGGER.info('build apps. will skip pytest apps with pytest scripts')
+        apps = get_cmake_apps(args.paths, args.target, args.config, args.preserve_all)
+
+    LOGGER.info('Found %d apps after filtering', len(apps))
+    LOGGER.info(
+        'Suggest setting the parallel count to %d for this build job',
+        len(apps) // APPS_BUILD_PER_JOB + 1,
+    )
+
+    if args.extra_preserve_dirs:
+        for app in apps:
+            if app.preserve:
+                continue
+            for extra_preserve_dir in args.extra_preserve_dirs:
+                if Path(extra_preserve_dir).resolve() in Path(app.app_dir).resolve().parents:
+                    app.preserve = True
+
+    ret_code = build_apps(
+        apps,
+        parallel_count=args.parallel_count,
+        parallel_index=args.parallel_index,
+        dry_run=False,
+        build_verbose=args.build_verbose,
+        keep_going=True,
+        collect_size_info=args.collect_size_info,
+        collect_app_info=args.collect_app_info,
+        ignore_warning_strs=args.ignore_warning_str,
+        ignore_warning_file=args.ignore_warning_file,
+        copy_sdkconfig=args.copy_sdkconfig,
+    )
+
+    sys.exit(ret_code)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Build all the apps for different test types. Will auto remove those non-test apps binaries',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument('paths', nargs='+', help='Paths to the apps to build.')
+    parser.add_argument(
+        '-t',
+        '--target',
+        required=True,
+        help='Build apps for given target. could pass "all" to get apps for all targets',
+    )
+    parser.add_argument(
+        '--config',
+        default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'],
+        action='append',
+        help='Adds configurations (sdkconfig file names) to build. This can either be '
+        'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
+        'relative to the project directory, to be used. Optional NAME can be specified, '
+        'which can be used as a name of this configuration. FILEPATTERN is the name of '
+        'the sdkconfig file, relative to the project directory, with at most one wildcard. '
+        'The part captured by the wildcard is used as the name of the configuration.',
+    )
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        action='count',
+        help='Increase the LOGGER level of the script. Can be specified multiple times.',
+    )
+    parser.add_argument(
+        '--build-verbose',
+        action='store_true',
+        help='Enable verbose output from build system.',
+    )
+    parser.add_argument(
+        '--preserve-all',
+        action='store_true',
+        help='Preserve the binaries for all apps when specified.',
+    )
+    parser.add_argument(
+        '--parallel-count', default=1, type=int, help='Number of parallel build jobs.'
+    )
+    parser.add_argument(
+        '--parallel-index',
+        default=1,
+        type=int,
+        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
+    )
+    parser.add_argument(
+        '--collect-size-info',
+        type=argparse.FileType('w'),
+        help='If specified, the test case name and size info json will be written to this file',
+    )
+    parser.add_argument(
+        '--collect-app-info',
+        type=argparse.FileType('w'),
+        help='If specified, the test case name and app info json will be written to this file',
+    )
+    parser.add_argument(
+        '--ignore-warning-str',
+        action='append',
+        help='Ignore the warning string that match the specified regex in the build output. '
+        'Can be specified multiple times.',
+    )
+    parser.add_argument(
+        '--ignore-warning-file',
+        default=os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt'),
+        type=argparse.FileType('r'),
+        help='Ignore the warning strings in the specified file. Each line should be a regex string.',
+    )
+    parser.add_argument(
+        '--copy-sdkconfig',
+        action='store_true',
+        help='Copy the sdkconfig file to the build directory.',
+    )
+    parser.add_argument(
+        '--extra-preserve-dirs', nargs='+',
+        help='also preserve binaries of the apps under the specified dirs'
+    )
+
+    parser.add_argument(
+        '--pytest-apps',
+        action='store_true',
+        help='Only build apps with pytest scripts. Will build apps without pytest scripts if this flag is unspecified.',
+    )
+    parser.add_argument(
+        '-m',
+        '--marker-expr',
+        default='not host_test',  # host_test apps would be built and tested under the same job
+        help='only build tests matching given mark expression. For example: -m "host_test and generic". Works only'
+        'for pytest',
+    )
+
+    arguments = parser.parse_args()
+
+    setup_logging(arguments.verbose)
+    main(arguments)

tools/ci/configure_ci_environment.sh

@@ -31,12 +31,6 @@ fi
 # Set ccache base directory to the project checkout path, to cancel out differences between runners
 export CCACHE_BASEDIR="${CI_PROJECT_DIR}"
 
-# In tools/ci/find_apps_build_apps.sh, we use --work-dir argument to copy apps to a separate location
-# before building them. This results in cache misses, even though the same code is compiled.
-# To solve this issue, we can disable 'hash_dir' option of ccache by setting CCACHE_NOHASHDIR env variable.
-# Note, this can result in issues with debug information, see:
-#   https://ccache.dev/manual/4.5.html#_compiling_in_different_directories
-#
 # 'CI_CCACHE_DISABLE_NOHASHDIR' variable can be used at project level to revert to hash_dir=true, in
 # case we start seeing failures due to false cache hits.
 if [ "${CI_CCACHE_DISABLE_NOHASHDIR}" != "1" ]; then

tools/ci/exclude_check_tools_files.txt

@@ -22,6 +22,7 @@ tools/ci/executable-list.txt
 tools/ci/fix_empty_prototypes.sh
 tools/ci/get-full-sources.sh
 tools/ci/idf_ci_utils.py
+tools/ci/ignore_build_warnings.txt
 tools/ci/mirror-submodule-update.sh
 tools/ci/multirun_with_pyenv.sh
 tools/ci/mypy_ignore_list.txt

tools/ci/executable-list.txt

@@ -47,11 +47,10 @@ examples/system/ota/otatool/otatool_example.py
 examples/system/ota/otatool/otatool_example.sh
 install.fish
 install.sh
-tools/build_apps.py
 tools/check_python_dependencies.py
 tools/ci/build_template_app.sh
 tools/ci/check_api_violation.sh
-tools/ci/check_build_warnings.py
+tools/ci/check_build_test_rules.py
 tools/ci/check_callgraph.py
 tools/ci/check_codeowners.py
 tools/ci/check_copyright.py
@@ -70,7 +69,6 @@ tools/ci/check_type_comments.py
 tools/ci/checkout_project_ref.py
 tools/ci/deploy_docs.py
 tools/ci/envsubst.py
-tools/ci/find_apps_build_apps.sh
 tools/ci/fix_empty_prototypes.sh
 tools/ci/get-full-sources.sh
 tools/ci/get_supported_examples.sh
@@ -88,7 +86,6 @@ tools/esp_app_trace/logtrace_proc.py
 tools/esp_app_trace/sysviewtrace_proc.py
 tools/esp_app_trace/test/logtrace/test.sh
 tools/esp_app_trace/test/sysview/test.sh
-tools/find_apps.py
 tools/format.sh
 tools/gen_esp_err_to_name.py
 tools/gen_soc_caps_kconfig/gen_soc_caps_kconfig.py

tools/ci/find_apps_build_apps.sh

@@ -1,175 +0,0 @@
-#!/usr/bin/env bash
-#
-# Find apps and build apps for example_test, custom_test, and unit_test
-#
-# Runs as part of CI process.
-#
-
-# -----------------------------------------------------------------------------
-# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
-
-if [[ -n ${DEBUG_SHELL} ]]; then
-  set -x # Activate the expand mode if DEBUG is anything but empty.
-fi
-
-if [ -z ${CI_NODE_TOTAL} ]; then
-  CI_NODE_TOTAL=1
-  echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
-fi
-
-if [ -z ${CI_NODE_INDEX} ]; then
-  # Gitlab uses a 1-based index
-  CI_NODE_INDEX=1
-  echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
-fi
-
-set -o errexit  # Exit if command failed.
-set -o pipefail # Exit if pipe failed.
-set -o nounset  # Exit if variable not set.
-
-export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
-
-# -----------------------------------------------------------------------------
-
-die() {
-  echo "${1:-"Unknown Error"}" 1>&2
-  exit 1
-}
-
-[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
-[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
-[ -f ${SIZE_INFO_LOCATION} ] && rm ${SIZE_INFO_LOCATION}
-
-export REALPATH=realpath
-if [ "$(uname -s)" = "Darwin" ]; then
-  export REALPATH=grealpath
-fi
-
-# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose.
-BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH})
-LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH})
-
-ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
-JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
-
-# -----------------------------------------------------------------------------
-# common variables, will specify special cases later
-WORK_DIR="--work-dir ${BUILD_PATH}/@f/@w/@t"
-BUILD_DIR="build"
-BUILD_LOG="${LOG_PATH}/@f_@w.txt"
-CONFIG="--config sdkconfig.ci=default
-        --config sdkconfig.ci.*=
-        --config =default"
-
-export EXTRA_CFLAGS="${PEDANTIC_CFLAGS}"
-export EXTRA_CXXFLAGS="${PEDANTIC_CXXFLAGS}"
-
-# --config rules above explained:
-# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
-# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
-# 3. If none of the above exist, build the default configuration under the name "default"
-# --work-dir and --build-log above uses "placeholders" @x:
-# - @f: full path to the test with slashes replaced with underscores
-# - @w: wildcard used as config name
-# - @t: target name
-# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32
-
-# -----------------------------------------------------------------------------
-# Example tests specific settings
-if [ "${TEST_TYPE}" = "example_test" ]; then
-  export EXTRA_CFLAGS="${PEDANTIC_CFLAGS:-}"
-  export EXTRA_CXXFLAGS="${PEDANTIC_CXXFLAGS:-}"
-
-  EXTRA_ARGS="--app-list ${SCAN_TEST_JSON}"
-# -----------------------------------------------------------------------------
-# Custom tests specific settings
-elif [ "${TEST_TYPE}" = "custom_test" ]; then
-  EXTRA_ARGS="--app-list ${SCAN_TEST_JSON}"
-# -----------------------------------------------------------------------------
-# Unit tests specific settings
-elif [ "${TEST_TYPE}" = "unit_test" ]; then
-  ALL_BUILD_LIST_JSON="${BUILD_PATH}/${IDF_TARGET}/list.json"
-  JOB_BUILD_LIST_JSON="${BUILD_PATH}/${IDF_TARGET}/list_job_${CI_NODE_INDEX}.json"
-
-  mkdir -p ${BUILD_PATH}/${IDF_TARGET}
-  mkdir -p ${OUTPUT_PATH}/${IDF_TARGET}
-
-  WORK_DIR=""
-  BUILD_DIR="builds/@t/@w"
-  BUILD_LOG="${LOG_PATH}/@w.txt"
-
-  APP_FOLDER="tools/unit-test-app"
-
-  EXTRA_ARGS="
-  -p ${APP_FOLDER}
-  --build-system ${BUILD_SYSTEM}
-  --target ${IDF_TARGET}
-  --recursive
-  "
-  CONFIG="--config configs/*="
-# -----------------------------------------------------------------------------
-else
-  die "TEST_TYPE should only be one of {example_test, custom_test, unit_test}"
-fi
-
-echo "$TEST_TYPE running for target $IDF_TARGET"
-cd ${IDF_PATH}
-
-# This part of the script produces the same result for all the build jobs.
-#
-# It may be moved to a separate stage (pre-build) later, then the build jobs
-# will receive ${BUILD_LIST_JSON} file as an artifact.
-#
-# If changing the work-dir or build-dir, remember to update the "artifacts" in
-# gitlab-ci configs, and IDFApp.py.
-
-${IDF_PATH}/tools/find_apps.py \
-  -vv \
-  --format json \
-  ${WORK_DIR} \
-  --build-dir ${BUILD_DIR} \
-  --build-log ${BUILD_LOG} \
-  --output ${ALL_BUILD_LIST_JSON} \
-  ${EXTRA_ARGS} \
-  ${CONFIG}
-
-# The part below is where the actual builds happen
-
-${IDF_PATH}/tools/build_apps.py \
-  -vv \
-  --format json \
-  --keep-going \
-  --parallel-count ${CI_NODE_TOTAL} \
-  --parallel-index ${CI_NODE_INDEX} \
-  --output-build-list ${JOB_BUILD_LIST_JSON} \
-  --size-info ${SIZE_INFO_LOCATION} \
-  ${ALL_BUILD_LIST_JSON}
-
-# Check for build warnings
-${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}
-
-if [ "${TEST_TYPE}" = "unit_test" ]; then
-  # Copy build artifacts to output directory
-  build_names=$(
-    cd ${BUILD_PATH}/${IDF_TARGET}
-    find . -maxdepth 1 \! -name . -prune -type d | cut -c 3-
-  )
-  for build_name in $build_names; do
-    src=${BUILD_PATH}/${IDF_TARGET}/${build_name}
-    dst=${OUTPUT_PATH}/${IDF_TARGET}/${build_name}
-    echo "Copying artifacts from ${src} to ${dst}"
-
-    rm -rf ${dst}
-    mkdir -p ${dst}
-    cp ${src}/{*.bin,*.elf,*.map,sdkconfig,flasher_args.json,flash_project_args} ${dst}/
-
-    mkdir -p ${dst}/bootloader
-    cp ${src}/bootloader/*.bin ${dst}/bootloader/
-
-    mkdir -p ${dst}/partition_table
-    cp ${src}/partition_table/*.bin ${dst}/partition_table/
-  done
-
-  # Copy app list json files to build path
-  mv ${BUILD_PATH}/${IDF_TARGET}/*.json ${BUILD_PATH}
-fi

tools/ci/idf_ci_utils.py

@@ -4,7 +4,7 @@
 # SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
-
+import contextlib
 import io
 import logging
 import os
@@ -12,7 +12,14 @@ import subprocess
 import sys
 from contextlib import redirect_stdout
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, List, Optional, Set
+from typing import TYPE_CHECKING, Any, List, Optional, Set, Union
+
+try:
+    from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
+except ImportError:
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+    from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
 
 if TYPE_CHECKING:
     from _pytest.python import Function
@@ -125,6 +132,9 @@ def to_list(s: Any) -> List[Any]:
     return [s]
 
 
+####################
+# Pytest Utilities #
+####################
 @dataclass
 class PytestApp:
     path: str
@@ -195,37 +205,110 @@ class PytestCollectPlugin:
 
 
 def get_pytest_cases(
-    folder: str, target: str, marker_expr: Optional[str] = None
+    paths: Union[str, List[str]], target: str = 'all', marker_expr: Optional[str] = None
 ) -> List[PytestCase]:
     import pytest
     from _pytest.config import ExitCode
 
-    collector = PytestCollectPlugin(target)
-    if marker_expr:
-        marker_expr = f'{target} and ({marker_expr})'
+    if target == 'all':
+        targets = SUPPORTED_TARGETS + PREVIEW_TARGETS
     else:
-        marker_expr = target  # target is also a marker
+        targets = [target]
 
-    with io.StringIO() as buf:
-        with redirect_stdout(buf):
-            res = pytest.main(
-                ['--collect-only', folder, '-q', '-m', marker_expr], plugins=[collector]
-            )
-        if res.value != ExitCode.OK:
-            if res.value == ExitCode.NO_TESTS_COLLECTED:
-                print(
-                    f'WARNING: no pytest app found for target {target} under folder {folder}'
-                )
-            else:
-                print(buf.getvalue())
-                raise RuntimeError('pytest collection failed')
+    cases = []
+    for t in targets:
+        collector = PytestCollectPlugin(t)
+        if marker_expr:
+            _marker_expr = f'{t} and ({marker_expr})'
+        else:
+            _marker_expr = t  # target is also a marker
 
-    return collector.cases
+        for path in to_list(paths):
+            with io.StringIO() as buf:
+                with redirect_stdout(buf):
+                    cmd = ['--collect-only', path, '-q', '-m', _marker_expr]
+                    res = pytest.main(cmd, plugins=[collector])
+                if res.value != ExitCode.OK:
+                    if res.value == ExitCode.NO_TESTS_COLLECTED:
+                        print(
+                            f'WARNING: no pytest app found for target {t} under path {path}'
+                        )
+                    else:
+                        print(buf.getvalue())
+                        raise RuntimeError(
+                            f'pytest collection failed at {path} with command \"{" ".join(cmd)}\"'
+                        )
+
+        cases.extend(collector.cases)
+
+    return cases
 
 
 def get_pytest_app_paths(
-    folder: str, target: str, marker_expr: Optional[str] = None
+    paths: Union[str, List[str]], target: str, marker_expr: Optional[str] = None
 ) -> Set[str]:
-    cases = get_pytest_cases(folder, target, marker_expr)
+    cases = get_pytest_cases(paths, target, marker_expr)
 
     return set({app.path for case in cases for app in case.apps})
+
+
+##################
+# TTFW Utilities #
+##################
+def get_ttfw_cases(paths: Union[str, List[str]]) -> List[Any]:
+    """
+    Get the test cases from ttfw_idf under the given paths
+
+    :param paths: list of paths to search
+    """
+    try:
+        from ttfw_idf.IDFAssignTest import IDFAssignTest
+    except ImportError:
+        sys.path.append(os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
+
+        from ttfw_idf.IDFAssignTest import IDFAssignTest
+
+    # mock CI_JOB_ID if not exists
+    if not os.environ.get('CI_JOB_ID'):
+        os.environ['CI_JOB_ID'] = '1'
+
+    cases = []
+    for path in to_list(paths):
+        assign = IDFAssignTest(
+            path, os.path.join(IDF_PATH, '.gitlab', 'ci', 'target-test.yml')
+        )
+        with contextlib.redirect_stdout(None):  # swallow stdout
+            try:
+                cases += assign.search_cases()
+            except ImportError as e:
+                logging.error(str(e))
+
+    return cases
+
+
+def get_ttfw_app_paths(
+    paths: Union[str, List[str]], target: Optional[str] = None
+) -> Set[str]:
+    """
+    Get the app paths from ttfw_idf under the given paths
+    """
+    from idf_build_apps import CMakeApp
+
+    cases = get_ttfw_cases(paths)
+    res: Set[str] = set()
+    for case in cases:
+        if not target or target == case.case_info['target'].lower():
+            # ttfw has no good way to detect the app path for master-slave tests
+            # the apps real location may be the sub folder of the test script path
+            # check if the current folder is an app, if it's not, add all its subfolders if they are apps
+            # only one level down
+            _app_dir = case.case_info['app_dir']
+            if CMakeApp.is_app(_app_dir):
+                res.add(_app_dir)
+            else:
+                for child in os.listdir(_app_dir):
+                    sub_path = os.path.join(_app_dir, child)
+                    if os.path.isdir(sub_path) and CMakeApp.is_app(sub_path):
+                        res.add(sub_path)
+
+    return res

tools/ci/ignore_build_warnings.txt

@@ -0,0 +1,16 @@
+library/error\.o
+/.*error\S*\.o
+.*error.*\.c\.obj
+.*error.*\.c
+.*error.*\.cpp\.obj
+.*error.*\.cxx\.obj
+.*error.*\.cc\.obj
+-Werror
+error\.d
+/.*error\S*.d
+reassigning to symbol
+changes choice state
+crosstool_version_check\.cmake
+CryptographyDeprecationWarning
+Warning: \d+/\d+ app partitions are too small for binary
+CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\)

tools/ci/mypy_ignore_list.txt

@@ -201,8 +201,6 @@ tools/esp_prov/transport/transport.py
 tools/esp_prov/transport/transport_ble.py
 tools/esp_prov/transport/transport_console.py
 tools/esp_prov/transport/transport_http.py
-tools/find_apps.py
-tools/find_build_apps/common.py
 tools/gen_esp_err_to_name.py
 tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py
 tools/kconfig_new/confgen.py

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -1,272 +0,0 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import argparse
-import errno
-import json
-import logging
-import os
-from collections import defaultdict
-from copy import deepcopy
-from typing import Any
-
-from ci.idf_ci_utils import get_pytest_app_paths
-from find_apps import find_apps, find_builds_for_app
-from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, config_rules_from_str
-from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
-from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest
-
-TEST_LABELS = {
-    'example_test': 'BOT_LABEL_EXAMPLE_TEST',
-    'test_apps': 'BOT_LABEL_CUSTOM_TEST',
-    'component_ut': ['BOT_LABEL_UNIT_TEST',
-                     'BOT_LABEL_UNIT_TEST_32',
-                     'BOT_LABEL_UNIT_TEST_S2',
-                     'BOT_LABEL_UNIT_TEST_C3'],
-}
-
-BUILD_ALL_LABELS = [
-    'BOT_LABEL_BUILD',
-    'BOT_LABEL_BUILD_ALL_APPS',
-    'BOT_LABEL_REGULAR_TEST',
-    'BOT_LABEL_WEEKEND_TEST',
-    'NIGHTLY_RUN',
-    'BOT_LABEL_NIGHTLY_RUN',
-]
-
-BUILD_PER_JOB = 30  # each build takes 1 mins around
-
-SCAN_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
-
-
-def _has_build_all_label():  # type: () -> bool
-    for label in BUILD_ALL_LABELS:
-        if os.getenv(label):
-            return True
-    return False
-
-
-def _judge_build_or_not(action, build_all):  # type: (str, bool) -> tuple[bool, bool]
-    """
-    :return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps)
-    """
-    if build_all or _has_build_all_label() or (not os.getenv('BOT_TRIGGER_WITH_LABEL')):
-        logging.info('Build all apps')
-        return True, True
-
-    labels = TEST_LABELS[action]
-    if not isinstance(labels, list):
-        labels = [labels]  # type: ignore
-
-    for label in labels:
-        if os.getenv(label):
-            logging.info('Build only test cases apps')
-            return True, False
-    logging.info('Skip all')
-    return False, False
-
-
-def output_json(apps_dict_list, target, build_system, output_dir):  # type: (list, str, str, str) -> None
-    output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system))
-    with open(output_path, 'w') as fw:
-        fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list])
-
-
-# we might need artifacts to run test cases locally.
-# So we need to save artifacts which have test case not executed by CI.
-class _ExampleAssignTest(ExampleAssignTest):
-    DEFAULT_FILTER = {}  # type: dict[str, Any]
-
-
-class _TestAppsAssignTest(TestAppsAssignTest):
-    DEFAULT_FILTER = {}  # type: dict[str, Any]
-
-
-def main():  # type: () -> None
-    parser = argparse.ArgumentParser(description='Scan the required build tests')
-    parser.add_argument('test_type',
-                        choices=TEST_LABELS.keys(),
-                        help='Scan test type')
-    parser.add_argument('paths', nargs='+',
-                        help='One or more app paths')
-    parser.add_argument('-b', '--build-system',
-                        choices=BUILD_SYSTEMS.keys(),
-                        default=BUILD_SYSTEM_CMAKE)
-    parser.add_argument('-c', '--ci-config-file',
-                        required=True,
-                        help='gitlab ci config target-test file')
-    parser.add_argument('-o', '--output-path',
-                        required=True,
-                        help='output path of the scan result')
-    parser.add_argument('--exclude', nargs='*',
-                        help='Ignore specified directory. Can be used multiple times.')
-    parser.add_argument('--extra_test_dirs', nargs='*',
-                        help='Additional directories to preserve artifacts for local tests')
-    parser.add_argument('--preserve_all', action='store_true',
-                        help='add this flag to preserve artifacts for all apps')
-    parser.add_argument('--build-all', action='store_true',
-                        help='add this flag to build all apps')
-    parser.add_argument('--combine-all-targets', action='store_true',
-                        help='add this flag to combine all target jsons into one')
-    parser.add_argument('--except-targets', nargs='+',
-                        help='only useful when "--combine-all-targets". Specified targets would be skipped.')
-    parser.add_argument(
-        '--config',
-        action='append',
-        help='Only useful when "--evaluate-parallel-count" is flagged.'
-             'Adds configurations (sdkconfig file names) to build. This can either be ' +
-             'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
-             'relative to the project directory, to be used. Optional NAME can be specified, ' +
-             'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
-             'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
-             'The part captured by the wildcard is used as the name of the configuration.',
-    )
-    parser.add_argument('--evaluate-parallel-count', action='store_true',
-                        help='suggest parallel count according to build items')
-
-    args = parser.parse_args()
-    build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
-
-    if not os.path.exists(args.output_path):
-        try:
-            os.makedirs(args.output_path)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise e
-
-    if (not build_standalone_apps) and (not build_test_case_apps):
-        for target in SCAN_TARGETS:
-            output_json([], target, args.build_system, args.output_path)
-            SystemExit(0)
-
-    idf_path = str(os.getenv('IDF_PATH'))
-    paths = set([os.path.join(idf_path, path) if not os.path.isabs(path) else path for path in args.paths])
-
-    test_cases = []
-    for path in paths:
-        if args.test_type == 'example_test':
-            assign = _ExampleAssignTest(path, args.ci_config_file)
-        elif args.test_type in ['test_apps', 'component_ut']:
-            assign = _TestAppsAssignTest(path, args.ci_config_file)
-        else:
-            raise SystemExit(1)  # which is impossible
-        test_cases.extend(assign.search_cases())
-
-    '''
-    {
-        <target>: {
-            'test_case_apps': [<app_dir>],   # which is used in target tests
-            'standalone_apps': [<app_dir>],  # which is not
-        },
-        ...
-    }
-    '''
-    scan_info_dict = defaultdict(dict)  # type: dict[str, dict]
-    # store the test cases dir, exclude these folders when scan for standalone apps
-    default_exclude = args.exclude if args.exclude else []
-
-    build_system = args.build_system.lower()
-    build_system_class = BUILD_SYSTEMS[build_system]
-
-    for target in SCAN_TARGETS:
-        exclude_apps = deepcopy(default_exclude)
-
-        if build_test_case_apps:
-            scan_info_dict[target]['test_case_apps'] = set()
-            test_dirs = args.extra_test_dirs if args.extra_test_dirs else []
-            for case in test_cases:
-                if case.case_info['target'].lower() == target.lower():
-                    test_dirs.append(case.case_info['app_dir'])
-            for app_dir in test_dirs:
-                app_dir = os.path.join(idf_path, app_dir) if not os.path.isabs(app_dir) else app_dir
-                _apps = find_apps(build_system_class, app_dir, True, exclude_apps, target.lower())
-                if _apps:
-                    scan_info_dict[target]['test_case_apps'].update(_apps)
-                    exclude_apps.extend(_apps)
-        else:
-            scan_info_dict[target]['test_case_apps'] = set()
-
-        if build_standalone_apps:
-            scan_info_dict[target]['standalone_apps'] = set()
-            for path in paths:
-                scan_info_dict[target]['standalone_apps'].update(
-                    find_apps(build_system_class, path, True, exclude_apps, target.lower()))
-        else:
-            scan_info_dict[target]['standalone_apps'] = set()
-    test_case_apps_preserve_default = True if build_system == 'cmake' else False
-    output_files = []
-    build_items_total_count = 0
-    for target in SCAN_TARGETS:
-        # get pytest apps paths
-        pytest_app_paths = set()
-        for path in paths:
-            pytest_app_paths.update(get_pytest_app_paths(path, target))
-
-        apps = []
-        for app_dir in scan_info_dict[target]['test_case_apps']:
-            if app_dir in pytest_app_paths:
-                print(f'WARNING: has pytest script: {app_dir}')
-                continue
-            apps.append({
-                'app_dir': app_dir,
-                'build_system': args.build_system,
-                'target': target,
-                'preserve': args.preserve_all or test_case_apps_preserve_default
-            })
-        for app_dir in scan_info_dict[target]['standalone_apps']:
-            if app_dir in pytest_app_paths:
-                print(f'Skipping pytest app: {app_dir}')
-                continue
-            apps.append({
-                'app_dir': app_dir,
-                'build_system': args.build_system,
-                'target': target,
-                'preserve': args.preserve_all
-            })
-        output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
-        with open(output_path, 'w') as fw:
-            if args.evaluate_parallel_count:
-                build_items = []
-                config_rules = config_rules_from_str(args.config or [])
-                for app in apps:
-                    build_items += find_builds_for_app(
-                        app['app_dir'],
-                        app['app_dir'],
-                        'build',
-                        '',
-                        app['target'],
-                        app['build_system'],
-                        config_rules,
-                        app['preserve'],
-                    )
-                print('Found {} builds'.format(len(build_items)))
-                if args.combine_all_targets:
-                    if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
-                            or (not args.except_targets):
-                        build_items_total_count += len(build_items)
-                else:
-                    print(f'suggest set parallel count for target {target} to {len(build_items) // BUILD_PER_JOB + 1}')
-            fw.writelines([json.dumps(app) + '\n' for app in apps])
-
-            if args.combine_all_targets:
-                if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
-                        or (not args.except_targets):
-                    output_files.append(output_path)
-                else:
-                    print(f'skipping combining target {target}')
-
-    if args.combine_all_targets:
-        scan_all_json = os.path.join(args.output_path, f'scan_all_{build_system}.json')
-        lines = []
-        for file in output_files:
-            with open(file) as fr:
-                lines.extend([line for line in fr.readlines() if line.strip()])
-        with open(scan_all_json, 'w') as fw:
-            fw.writelines(lines)
-        print(f'combined into file: {scan_all_json}')
-
-        if args.evaluate_parallel_count:
-            print(f'Total build: {build_items_total_count}. Suggest set parallel count for all target to {build_items_total_count // BUILD_PER_JOB + 1}')
-
-
-if __name__ == '__main__':
-    main()

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -157,7 +157,7 @@ class Artifacts(object):
         self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
 
     def _download_sdkconfig_file(self, base_path, job_id):  # type: (str, str) -> None
-        self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), 'sdkconfig')],
+        self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')],
                                            self.dest_root_path)
 
     def download_artifacts(self):  # type: () -> Any
@@ -185,7 +185,7 @@ class Artifacts(object):
             self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
 
             # download sdkconfig file
-            self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), 'sdkconfig')],
+            self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')],
                                                self.dest_root_path)
         else:
             base_path = None
@@ -448,31 +448,24 @@ class Example(IDFApp):
             target = 'esp32'
         super(Example, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
 
-    def _get_sdkconfig_paths(self):  # type: () -> List[str]
-        """
-        overrides the parent method to provide exact path of sdkconfig for example tests
-        """
-        return [os.path.join(self.binary_path, '..', 'sdkconfig')]
-
     def _try_get_binary_from_local_fs(self):  # type: () -> Optional[str]
         # build folder of example path
         path = os.path.join(self.idf_path, self.app_path, 'build')
         if os.path.exists(path):
             return path
 
+        # new style build dir
+        path = os.path.join(self.idf_path, self.app_path, f'build_{self.target}_{self.config_name}')
+        if os.path.exists(path):
+            return path
+
         # Search for CI build folders.
-        # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
-        # (see tools/ci/build_examples.sh)
-        # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
-        app_path_underscored = self.app_path.replace(os.path.sep, '_')
-        example_path = os.path.join(self.idf_path, self.case_group.LOCAL_BUILD_DIR)
-        for dirpath in os.listdir(example_path):
-            if os.path.basename(dirpath) == app_path_underscored:
-                path = os.path.join(example_path, dirpath, self.config_name, self.target, 'build')
-                if os.path.exists(path):
-                    return path
-                else:
-                    return None
+        # Path format: $IDF_PATH/<app_dir>/build_<target>_<config>
+        build_dir = f'build_{self.target}_{self.config_name}'
+        example_path = os.path.join(self.idf_path, self.app_path, build_dir)
+        if os.path.exists(example_path):
+            return path
+
         return None
 
 

tools/ci/python_packages/ttfw_idf/IDFAssignTest.py

@@ -4,10 +4,10 @@
 Command line tool to assign tests to CI test jobs.
 """
 import argparse
-import errno
 import json
 import os
 import re
+import sys
 from copy import deepcopy
 
 import yaml
@@ -23,21 +23,20 @@ from tiny_test_fw.Utility import CIAssignTest
 try:
     from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
 except ImportError:
-    SUPPORTED_TARGETS = []
-    PREVIEW_TARGETS = []
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
+
+    from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
 
 IDF_PATH_FROM_ENV = os.getenv('IDF_PATH', '')
 
 
 class IDFCaseGroup(CIAssignTest.Group):
-    LOCAL_BUILD_DIR = None
     BUILD_JOB_NAMES = None
 
     @classmethod
     def get_artifact_index_file(cls):
-        assert cls.LOCAL_BUILD_DIR
         if IDF_PATH_FROM_ENV:
-            artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, cls.LOCAL_BUILD_DIR, 'artifact_index.json')
+            artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, 'artifact_index.json')
         else:
             artifact_index_file = 'artifact_index.json'
         return artifact_index_file
@@ -55,7 +54,7 @@ class IDFAssignTest(CIAssignTest.AssignTest):
         super(IDFAssignTest, self).__init__(test_case_path, ci_config_file, case_group)
 
     def format_build_log_path(self, parallel_num):
-        return '{}/list_job_{}.json'.format(self.case_group.LOCAL_BUILD_DIR, parallel_num)
+        return 'list_job_{}.json'.format(parallel_num)
 
     def create_artifact_index_file(self, project_id=None, pipeline_id=None):
         if project_id is None:
@@ -76,12 +75,6 @@ class IDFAssignTest(CIAssignTest.AssignTest):
                     build_info['ci_job_id'] = job_info['id']
                     artifact_index_list.append(build_info)
         artifact_index_file = self.case_group.get_artifact_index_file()
-        try:
-            os.makedirs(os.path.dirname(artifact_index_file))
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise e
-
         with open(artifact_index_file, 'w') as f:
             json.dump(artifact_index_list, f)
 
@@ -95,19 +88,16 @@ class IDFAssignTest(CIAssignTest.AssignTest):
 class ExampleGroup(IDFCaseGroup):
     SORT_KEYS = CI_JOB_MATCH_KEYS = ['env_tag', 'target']
 
-    LOCAL_BUILD_DIR = 'build_examples'  # type: ignore
     EXAMPLE_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
     BUILD_JOB_NAMES = ['build_examples_cmake_{}'.format(target) for target in EXAMPLE_TARGETS]  # type: ignore
 
 
 class TestAppsGroup(ExampleGroup):
-    LOCAL_BUILD_DIR = 'build_test_apps'
     TEST_APP_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
     BUILD_JOB_NAMES = ['build_test_apps_{}'.format(target) for target in TEST_APP_TARGETS]  # type: ignore
 
 
 class ComponentUTGroup(TestAppsGroup):
-    LOCAL_BUILD_DIR = 'build_component_ut'
     UNIT_TEST_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
     BUILD_JOB_NAMES = ['build_component_ut_{}'.format(target) for target in UNIT_TEST_TARGETS]  # type: ignore
 
@@ -116,7 +106,6 @@ class UnitTestGroup(IDFCaseGroup):
     SORT_KEYS = ['test environment', 'tags', 'chip_target']
     CI_JOB_MATCH_KEYS = ['test environment']
 
-    LOCAL_BUILD_DIR = 'tools/unit-test-app/builds'  # type: ignore
     UNIT_TEST_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
     BUILD_JOB_NAMES = ['build_esp_idf_tests_cmake_{}'.format(target) for target in UNIT_TEST_TARGETS]  # type: ignore
 

tools/ci/setup_python.sh

@@ -47,4 +47,4 @@ else
 fi
 
 # add esp-idf local package path to PYTHONPATH so it can be imported directly
-export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"

tools/find_apps.py

@@ -1,317 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-#
-# ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps.
-# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
-
-import argparse
-import glob
-import json
-import logging
-import os
-import re
-import sys
-import typing
-
-from find_build_apps import (BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, DEFAULT_TARGET, BuildItem, BuildSystem, ConfigRule,
-                             config_rules_from_str, setup_logging)
-
-
-# Helper functions
-def dict_from_sdkconfig(path):
-    """
-    Parse the sdkconfig file at 'path', return name:value pairs as a dict
-    """
-    regex = re.compile(r'^([^#=]+)=(.+)$')
-    result = {}
-    with open(path) as f:
-        for line in f:
-            m = regex.match(line)
-            if m:
-                val = m.group(2)
-                if val.startswith('"') and val.endswith('"'):
-                    val = val[1:-1]
-                result[m.group(1)] = val
-    return result
-
-
-# Main logic: enumerating apps and builds
-
-
-def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
-                        build_system, config_rules, preserve_artifacts=True):
-    # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem]
-    """
-    Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
-    :param app_path: app directory (can be / usually will be a relative path)
-    :param work_dir: directory where the app should be copied before building.
-                     May contain env. variables and placeholders.
-    :param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders.
-    :param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go
-                      into stdout/stderr.
-    :param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with
-                       a different CONFIG_IDF_TARGET value.
-    :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
-    :param config_rules: mapping of sdkconfig file name patterns to configuration names
-    :param preserve_artifacts: determine if the built binary will be uploaded as artifacts.
-    :return: list of BuildItems representing build configuration of the app
-    """
-    build_items = []  # type: typing.List[BuildItem]
-    default_config_name = ''
-
-    for rule in config_rules:
-        if not rule.file_name:
-            default_config_name = rule.config_name
-            continue
-
-        sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name))
-        sdkconfig_paths = sorted(sdkconfig_paths)
-        for sdkconfig_path in sdkconfig_paths:
-
-            # Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument.
-            sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path)
-            target_from_config = sdkconfig_dict.get('CONFIG_IDF_TARGET')
-            if target_from_config is not None and target_from_config != target_arg:
-                logging.debug('Skipping sdkconfig {} which requires target {}'.format(
-                    sdkconfig_path, target_from_config))
-                continue
-
-            # Figure out the config name
-            config_name = rule.config_name or ''
-            if '*' in rule.file_name:
-                # convert glob pattern into a regex
-                regex_str = r'.*' + rule.file_name.replace('.', r'\.').replace('*', r'(.*)')
-                groups = re.match(regex_str, sdkconfig_path)
-                assert groups
-                config_name = groups.group(1)
-
-            sdkconfig_path = os.path.relpath(sdkconfig_path, app_path)
-            logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format(
-                app_path, sdkconfig_path, config_name))
-            build_items.append(
-                BuildItem(
-                    app_path,
-                    work_dir,
-                    build_dir,
-                    build_log,
-                    target_arg,
-                    sdkconfig_path,
-                    config_name,
-                    build_system,
-                    preserve_artifacts,
-                ))
-
-    if not build_items:
-        logging.debug('\tAdding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name))
-        return [
-            BuildItem(
-                app_path,
-                work_dir,
-                build_dir,
-                build_log,
-                target_arg,
-                None,
-                default_config_name,
-                build_system,
-                preserve_artifacts,
-            )
-        ]
-
-    return build_items
-
-
-def find_apps(build_system_class, path, recursive, exclude_list, target):
-    # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
-    """
-    Find app directories in path (possibly recursively), which contain apps for the given build system, compatible
-    with the given target.
-    :param build_system_class: class derived from BuildSystem, representing the build system in use
-    :param path: path where to look for apps
-    :param recursive: whether to recursively descend into nested directories if no app is found
-    :param exclude_list: list of paths to be excluded from the recursive search
-    :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped.
-    :return: list of paths of the apps found
-    """
-    build_system_name = build_system_class.NAME
-    logging.debug('Looking for {} apps in {}{}'.format(build_system_name, path, ' recursively' if recursive else ''))
-
-    apps_found = []  # type: typing.List[str]
-    for root, dirs, _ in os.walk(path, topdown=True):
-        logging.debug('Entering {}'.format(root))
-        if root in exclude_list:
-            logging.debug('Skipping {} (excluded)'.format(root))
-            del dirs[:]
-            continue
-
-        if build_system_class.is_app(root):
-            logging.debug('Found {} app in {}'.format(build_system_name, root))
-            # Don't recurse into app subdirectories
-            del dirs[:]
-
-            supported_targets = build_system_class.supported_targets(root)
-            if supported_targets and (target in supported_targets):
-                apps_found.append(root)
-            else:
-                if supported_targets:
-                    logging.debug('Skipping, app only supports targets: ' + ', '.join(supported_targets))
-                else:
-                    logging.debug('Skipping, app has no supported targets')
-                continue
-
-        if not recursive:
-            if not apps_found:
-                logging.warning('Path {} specified without --recursive flag, but no {} app found there'.format(
-                    path, build_system_name))
-            break  # only check the top-most dir if "recursive" is unflagged
-
-    return apps_found
-
-
-def main():
-    parser = argparse.ArgumentParser(description='Tool to generate build steps for IDF apps')
-    parser.add_argument(
-        '-v',
-        '--verbose',
-        action='count',
-        help='Increase the logging level of the script. Can be specified multiple times.',
-    )
-    parser.add_argument(
-        '--log-file',
-        type=argparse.FileType('w'),
-        help='Write the script log to the specified file, instead of stderr',
-    )
-    parser.add_argument(
-        '--recursive',
-        action='store_true',
-        help='Look for apps in the specified directories recursively.',
-    )
-    parser.add_argument(
-        '--build-system',
-        choices=BUILD_SYSTEMS.keys()
-    )
-    parser.add_argument(
-        '--work-dir',
-        help='If set, the app is first copied into the specified directory, and then built.' +
-             'If not set, the work directory is the directory of the app.',
-    )
-    parser.add_argument(
-        '--config',
-        action='append',
-        help='Adds configurations (sdkconfig file names) to build. This can either be ' +
-             'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
-             'relative to the project directory, to be used. Optional NAME can be specified, ' +
-             'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
-             'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
-             'The part captured by the wildcard is used as the name of the configuration.',
-    )
-    parser.add_argument(
-        '--build-dir',
-        help='If set, specifies the build directory name. Can expand placeholders. Can be either a ' +
-             'name relative to the work directory, or an absolute path.',
-    )
-    parser.add_argument(
-        '--build-log',
-        help='If specified, the build log will be written to this file. Can expand placeholders.',
-    )
-    parser.add_argument('--target', help='Build apps for given target.')
-    parser.add_argument(
-        '--format',
-        default='json',
-        choices=['json'],
-        help='Format to write the list of builds as',
-    )
-    parser.add_argument(
-        '--exclude',
-        action='append',
-        help='Ignore specified directory (if --recursive is given). Can be used multiple times.',
-    )
-    parser.add_argument(
-        '-o',
-        '--output',
-        type=argparse.FileType('w'),
-        help='Output the list of builds to the specified file',
-    )
-    parser.add_argument(
-        '--app-list',
-        default=None,
-        help='Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. '
-             'If the file does not exist, will build all apps and upload all artifacts.'
-    )
-    parser.add_argument(
-        '-p', '--paths',
-        nargs='+',
-        help='One or more app paths.'
-    )
-    args = parser.parse_args()
-    setup_logging(args)
-
-    # Arguments Validation
-    if args.app_list:
-        conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths]
-        if any(conflict_args):
-            raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not '
-                             'be specified with "app_list"')
-        if not os.path.exists(args.app_list):
-            raise OSError('File not found {}'.format(args.app_list))
-    else:
-        # If the build target is not set explicitly, get it from the environment or use the default one (esp32)
-        if not args.target:
-            env_target = os.environ.get('IDF_TARGET')
-            if env_target:
-                logging.info('--target argument not set, using IDF_TARGET={} from the environment'.format(env_target))
-                args.target = env_target
-            else:
-                logging.info('--target argument not set, using IDF_TARGET={} as the default'.format(DEFAULT_TARGET))
-                args.target = DEFAULT_TARGET
-        if not args.build_system:
-            logging.info('--build-system argument not set, using {} as the default'.format(BUILD_SYSTEM_CMAKE))
-            args.build_system = BUILD_SYSTEM_CMAKE
-        required_args = [args.build_system, args.target, args.paths]
-        if not all(required_args):
-            raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.')
-
-    # Prepare the list of app paths, try to read from the scan_tests result.
-    # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again.
-    # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts.
-    if args.app_list:
-        apps = [json.loads(line) for line in open(args.app_list)]
-    else:
-        app_dirs = []
-        build_system_class = BUILD_SYSTEMS[args.build_system]
-        for path in args.paths:
-            app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
-        apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs]
-
-    if not apps:
-        logging.warning('No apps found')
-        SystemExit(0)
-
-    logging.info('Found {} apps'.format(len(apps)))
-    apps.sort(key=lambda x: x['app_dir'])
-
-    # Find compatible configurations of each app, collect them as BuildItems
-    build_items = []  # type: typing.List[BuildItem]
-    config_rules = config_rules_from_str(args.config or [])
-    for app in apps:
-        build_items += find_builds_for_app(
-            app['app_dir'],
-            args.work_dir,
-            args.build_dir,
-            args.build_log,
-            args.target or app['target'],
-            args.build_system or app['build_system'],
-            config_rules,
-            app['preserve'],
-        )
-    logging.info('Found {} builds'.format(len(build_items)))
-
-    # Write out the BuildItems. Only JSON supported now (will add YAML later).
-    if args.format != 'json':
-        raise NotImplementedError()
-
-    out = args.output or sys.stdout
-    out.writelines([item.to_json() + '\n' for item in build_items])
-
-
-if __name__ == '__main__':
-    main()

tools/find_build_apps/__init__.py

@@ -1,20 +0,0 @@
-from .cmake import BUILD_SYSTEM_CMAKE, CMakeBuildSystem
-from .common import (DEFAULT_TARGET, BuildError, BuildItem, BuildSystem, ConfigRule, config_rules_from_str,
-                     setup_logging)
-
-BUILD_SYSTEMS = {
-    BUILD_SYSTEM_CMAKE: CMakeBuildSystem,
-}
-
-__all__ = [
-    'BuildItem',
-    'BuildSystem',
-    'BuildError',
-    'ConfigRule',
-    'config_rules_from_str',
-    'setup_logging',
-    'DEFAULT_TARGET',
-    'CMakeBuildSystem',
-    'BUILD_SYSTEM_CMAKE',
-    'BUILD_SYSTEMS',
-]

tools/find_build_apps/cmake.py

@@ -1,100 +0,0 @@
-import logging
-import os
-import shutil
-import subprocess
-import sys
-
-from .common import BuildError, BuildItem, BuildSystem
-
-try:
-    from typing import Any, Optional
-except ImportError:
-    pass
-
-BUILD_SYSTEM_CMAKE = 'cmake'
-IDF_PY = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf.py')
-
-# While ESP-IDF component CMakeLists files can be identified by the presence of 'idf_component_register' string,
-# there is no equivalent for the project CMakeLists files. This seems to be the best option...
-CMAKE_PROJECT_LINE = r'include($ENV{IDF_PATH}/tools/cmake/project.cmake)'
-
-
-class CMakeBuildSystem(BuildSystem):
-    NAME = BUILD_SYSTEM_CMAKE
-
-    @classmethod
-    def build(cls, build_item):  # type: (BuildItem) -> None
-        build_path, work_path, extra_cmakecache_items = cls.build_prepare(build_item)
-        # Prepare the build arguments
-        args = [
-            sys.executable,
-            IDF_PY,
-            '-B',
-            build_path,
-            '-C',
-            work_path,
-            '-DIDF_TARGET=' + build_item.target,
-        ]
-        if extra_cmakecache_items:
-            for key, val in extra_cmakecache_items.items():
-                args.append('-D{}={}'.format(key, val))
-            if 'TEST_EXCLUDE_COMPONENTS' in extra_cmakecache_items \
-                    and 'TEST_COMPONENTS' not in extra_cmakecache_items:
-                args.append('-DTESTS_ALL=1')
-        if build_item.verbose:
-            args.append('-v')
-        if 'CONFIG_APP_BUILD_BOOTLOADER' in extra_cmakecache_items:
-            # In case if secure_boot is enabled then for bootloader build need to add `bootloader` cmd
-            args.append('bootloader')
-        args.append('build')
-        cmdline = format(' '.join(args))
-        logging.info('Running {}'.format(cmdline))
-
-        if build_item.dry_run:
-            return
-
-        log_file = None
-        build_stdout = sys.stdout
-        build_stderr = sys.stderr
-        if build_item.build_log_path:
-            logging.info('Writing build log to {}'.format(build_item.build_log_path))
-            log_file = open(build_item.build_log_path, 'w')
-            build_stdout = log_file
-            build_stderr = log_file
-
-        try:
-            os.environ['IDF_TARGET'] = build_item.target
-            subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
-        except subprocess.CalledProcessError as e:
-            raise BuildError('Build failed with exit code {}'.format(e.returncode))
-        else:
-            # Also save the sdkconfig file in the build directory
-            shutil.copyfile(
-                os.path.join(work_path, 'sdkconfig'),
-                os.path.join(build_path, 'sdkconfig'),
-            )
-            build_item.size_json_fp = build_item.get_size_json_fp()
-        finally:
-            if log_file:
-                log_file.close()
-
-    @staticmethod
-    def _read_cmakelists(app_path):  # type: (str) -> Optional[str]
-        cmakelists_path = os.path.join(app_path, 'CMakeLists.txt')
-        if not os.path.exists(cmakelists_path):
-            return None
-        with open(cmakelists_path, 'r') as cmakelists_file:
-            return cmakelists_file.read()
-
-    @staticmethod
-    def is_app(path):  # type: (str) -> bool
-        cmakelists_file_content = CMakeBuildSystem._read_cmakelists(path)
-        if not cmakelists_file_content:
-            return False
-        if CMAKE_PROJECT_LINE not in cmakelists_file_content:
-            return False
-        return True
-
-    @classmethod
-    def supported_targets(cls, app_path):  # type: (str) -> Any
-        return cls._supported_targets(app_path)

tools/find_build_apps/common.py

@@ -1,473 +0,0 @@
-# coding=utf-8
-import fnmatch
-import json
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import typing
-from abc import abstractmethod
-from collections import namedtuple
-from io import open
-
-DEFAULT_TARGET = 'esp32'
-
-TARGET_PLACEHOLDER = '@t'
-WILDCARD_PLACEHOLDER = '@w'
-NAME_PLACEHOLDER = '@n'
-FULL_NAME_PLACEHOLDER = '@f'
-INDEX_PLACEHOLDER = '@i'
-
-IDF_SIZE_PY = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_size.py')
-SIZE_JSON_FN = 'size.json'
-
-SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$")
-
-# If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake
-SDKCONFIG_TEST_OPTS = [
-    'EXCLUDE_COMPONENTS',
-    'TEST_EXCLUDE_COMPONENTS',
-    'TEST_COMPONENTS',
-]
-
-# These keys in sdkconfig.defaults are not propagated to the final sdkconfig file:
-SDKCONFIG_IGNORE_OPTS = [
-    'TEST_GROUPS'
-]
-
-# ConfigRule represents one --config argument of find_apps.py.
-# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
-# file_name can also be empty to indicate that the default configuration of the app should be used.
-# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used.
-# For example:
-#   filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
-#   filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
-ConfigRule = namedtuple('ConfigRule', ['file_name', 'config_name'])
-
-
-def config_rules_from_str(rule_strings):  # type: (typing.List[str]) -> typing.List[ConfigRule]
-    """
-    Helper function to convert strings like 'file_name=config_name' into ConfigRule objects
-    :param rule_strings: list of rules as strings
-    :return: list of ConfigRules
-    """
-    rules = []  # type: typing.List[ConfigRule]
-    for rule_str in rule_strings:
-        items = rule_str.split('=', 2)
-        rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
-    return rules
-
-
-def find_first_match(pattern, path):
-    for root, _, files in os.walk(path):
-        res = fnmatch.filter(files, pattern)
-        if res:
-            return os.path.join(root, res[0])
-    return None
-
-
-def rmdir(path, exclude_file_pattern=None):
-    if not exclude_file_pattern:
-        shutil.rmtree(path, ignore_errors=True)
-        return
-
-    for root, dirs, files in os.walk(path, topdown=False):
-        for f in files:
-            if not fnmatch.fnmatch(f, exclude_file_pattern):
-                os.remove(os.path.join(root, f))
-        for d in dirs:
-            try:
-                os.rmdir(os.path.join(root, d))
-            except OSError:
-                pass
-
-
-class BuildItem(object):
-    """
-    Instance of this class represents one build of an application.
-    The parameters which distinguish the build are passed to the constructor.
-    """
-
-    def __init__(
-            self,
-            app_path,
-            work_dir,
-            build_path,
-            build_log_path,
-            target,
-            sdkconfig_path,
-            config_name,
-            build_system,
-            preserve_artifacts,
-    ):
-        # These internal variables store the paths with environment variables and placeholders;
-        # Public properties with similar names use the _expand method to get the actual paths.
-        self._app_dir = app_path
-        self._work_dir = work_dir
-        self._build_dir = build_path
-        self._build_log_path = build_log_path
-
-        self.sdkconfig_path = sdkconfig_path
-        self.config_name = config_name
-        self.target = target
-        self.build_system = build_system
-
-        self.preserve = preserve_artifacts
-
-        self._app_name = os.path.basename(os.path.normpath(app_path))
-        self.size_json_fp = None
-
-        # Some miscellaneous build properties which are set later, at the build stage
-        self.index = None
-        self.verbose = False
-        self.dry_run = False
-        self.keep_going = False
-
-        self.work_path = self.work_dir or self.app_dir
-        if not self.build_dir:
-            self.build_path = os.path.join(self.work_path, 'build')
-        elif os.path.isabs(self.build_dir):
-            self.build_path = self.build_dir
-        else:
-            self.build_path = os.path.normpath(os.path.join(self.work_path, self.build_dir))
-
-    @property
-    def app_dir(self):
-        """
-        :return: directory of the app
-        """
-        return self._expand(self._app_dir)
-
-    @property
-    def work_dir(self):
-        """
-        :return: directory where the app should be copied to, prior to the build. Can be None, which means that the app
-                 directory should be used.
-        """
-        return self._expand(self._work_dir)
-
-    @property
-    def build_dir(self):
-        """
-        :return: build directory, either relative to the work directory (if relative path is used) or absolute path.
-        """
-        return self._expand(self._build_dir)
-
-    @property
-    def build_log_path(self):
-        """
-        :return: path of the build log file
-        """
-        return self._expand(self._build_log_path)
-
-    def __repr__(self):
-        return '({}) Build app {} for target {}, sdkconfig {} in {}'.format(
-            self.build_system,
-            self.app_dir,
-            self.target,
-            self.sdkconfig_path or '(default)',
-            self.build_dir,
-        )
-
-    def to_json(self):  # type: () -> str
-        """
-        :return: JSON string representing this object
-        """
-        return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path)
-
-    def to_json_expanded(self):  # type: () -> str
-        """
-        :return: JSON string representing this object, with all placeholders in paths expanded
-        """
-        return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path)
-
-    def _to_json(self, app_dir, work_dir, build_dir, build_log_path):  # type: (str, str, str, str) -> str
-        """
-        Internal function, called by to_json and to_json_expanded
-        """
-        return json.dumps({
-            'build_system': self.build_system,
-            'app_dir': app_dir,
-            'work_dir': work_dir,
-            'build_dir': build_dir,
-            'build_log_path': build_log_path,
-            'sdkconfig': self.sdkconfig_path,
-            'config': self.config_name,
-            'target': self.target,
-            'verbose': self.verbose,
-            'preserve': self.preserve,
-        })
-
-    @staticmethod
-    def from_json(json_str):  # type: (typing.Text) -> BuildItem
-        """
-        :return: Get the BuildItem from a JSON string
-        """
-        d = json.loads(str(json_str))
-        result = BuildItem(
-            app_path=d['app_dir'],
-            work_dir=d['work_dir'],
-            build_path=d['build_dir'],
-            build_log_path=d['build_log_path'],
-            sdkconfig_path=d['sdkconfig'],
-            config_name=d['config'],
-            target=d['target'],
-            build_system=d['build_system'],
-            preserve_artifacts=d['preserve']
-        )
-        result.verbose = d['verbose']
-        return result
-
-    def _expand(self, path):  # type: (str) -> str
-        """
-        Internal method, expands any of the placeholders in {app,work,build} paths.
-        """
-        if not path:
-            return path
-
-        if self.index is not None:
-            path = path.replace(INDEX_PLACEHOLDER, str(self.index))
-        path = path.replace(TARGET_PLACEHOLDER, self.target)
-        path = path.replace(NAME_PLACEHOLDER, self._app_name)
-        if (FULL_NAME_PLACEHOLDER in path):  # to avoid recursion to the call to app_dir in the next line:
-            path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, '_'))
-        wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
-        if wildcard_pos != -1:
-            if self.config_name:
-                # if config name is defined, put it in place of the placeholder
-                path = path.replace(WILDCARD_PLACEHOLDER, self.config_name)
-            else:
-                # otherwise, remove the placeholder and one character on the left
-                # (which is usually an underscore, dash, or other delimiter)
-                left_of_wildcard = max(0, wildcard_pos - 1)
-                right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER)
-                path = path[0:left_of_wildcard] + path[right_of_wildcard:]
-        path = os.path.expandvars(path)
-        return path
-
-    def get_size_json_fp(self):
-        if self.size_json_fp and os.path.exists(self.size_json_fp):
-            return self.size_json_fp
-
-        assert os.path.exists(self.build_path)
-        assert os.path.exists(self.work_path)
-
-        map_file = find_first_match('*.map', self.build_path)
-        if not map_file:
-            logging.info('.map file not found under "{}"'.format(self.build_path))
-            return None
-
-        size_json_fp = os.path.join(self.build_path, SIZE_JSON_FN)
-        idf_size_args = [
-            sys.executable,
-            IDF_SIZE_PY,
-            '--json',
-            '-o', size_json_fp,
-            map_file
-        ]
-        subprocess.check_call(idf_size_args)
-        return size_json_fp
-
-    def write_size_info(self, size_info_fs):
-        if not self.size_json_fp or (not os.path.exists(self.size_json_fp)):
-            logging.info(f'No size info found for app {self._app_name}')
-            return
-        size_info_dict = {
-            'app_name': self._app_name,
-            'config_name': self.config_name,
-            'target': self.target,
-            'path': self.size_json_fp,
-        }
-        size_info_fs.write(json.dumps(size_info_dict) + '\n')
-
-
-class BuildSystem:
-    """
-    Class representing a build system.
-    Derived classes implement the methods below.
-    Objects of these classes aren't instantiated, instead the class (type object) is used.
-    """
-    NAME = 'undefined'
-    SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[ |]+(?:[0-9a-zA-Z\-]+))+)')
-
-    FORMAL_TO_USUAL = {
-        'ESP32': 'esp32',
-        'ESP32-S2': 'esp32s2',
-        'ESP32-S3': 'esp32s3',
-        'ESP32-C3': 'esp32c3',
-        'ESP32-H2': 'esp32h2',
-        'ESP32-C2': 'esp32c2',
-        'Linux': 'linux',
-    }
-
-    # ESP32H2-TODO: IDF-4559
-    # Build only apps who has ESP32-H2 in the README.md supported targets table.
-    DEFAULT_TARGETS = ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux']
-
-    @classmethod
-    def build_prepare(cls, build_item):
-        app_path = build_item.app_dir
-        work_path = build_item.work_path
-        build_path = build_item.build_path
-
-        if work_path != app_path:
-            if os.path.exists(work_path):
-                logging.debug('Work directory {} exists, removing'.format(work_path))
-                if not build_item.dry_run:
-                    shutil.rmtree(work_path)
-            logging.debug('Copying app from {} to {}'.format(app_path, work_path))
-            if not build_item.dry_run:
-                shutil.copytree(app_path, work_path)
-
-        if os.path.exists(build_path):
-            logging.debug('Build directory {} exists, removing'.format(build_path))
-            if not build_item.dry_run:
-                shutil.rmtree(build_path)
-
-        if not build_item.dry_run:
-            os.makedirs(build_path)
-
-        # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
-        # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
-        #
-        # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
-        # CMake variable. However here we do this manually to perform environment variable expansion in the
-        # sdkconfig files.
-        sdkconfig_defaults_list = ['sdkconfig.defaults', 'sdkconfig.defaults.' + build_item.target]
-        if build_item.sdkconfig_path:
-            sdkconfig_defaults_list.append(build_item.sdkconfig_path)
-
-        sdkconfig_file = os.path.join(work_path, 'sdkconfig')
-        if os.path.exists(sdkconfig_file):
-            logging.debug('Removing sdkconfig file: {}'.format(sdkconfig_file))
-            if not build_item.dry_run:
-                os.unlink(sdkconfig_file)
-
-        logging.debug('Creating sdkconfig file: {}'.format(sdkconfig_file))
-        extra_cmakecache_items = {}
-        if not build_item.dry_run:
-            with open(sdkconfig_file, 'w') as f_out:
-                for sdkconfig_name in sdkconfig_defaults_list:
-                    sdkconfig_path = os.path.join(work_path, sdkconfig_name)
-                    if not sdkconfig_path or not os.path.exists(sdkconfig_path):
-                        continue
-                    logging.debug('Appending {} to sdkconfig'.format(sdkconfig_name))
-                    with open(sdkconfig_path, 'r') as f_in:
-                        for line in f_in:
-                            if not line.endswith('\n'):
-                                line += '\n'
-                            if cls.NAME == 'cmake':
-                                m = SDKCONFIG_LINE_REGEX.match(line)
-                                key = m.group(1) if m else None
-                                if key in SDKCONFIG_TEST_OPTS:
-                                    extra_cmakecache_items[key] = m.group(2)
-                                    continue
-                                if key in SDKCONFIG_IGNORE_OPTS:
-                                    continue
-                            f_out.write(os.path.expandvars(line))
-        else:
-            for sdkconfig_name in sdkconfig_defaults_list:
-                sdkconfig_path = os.path.join(app_path, sdkconfig_name)
-                if not sdkconfig_path:
-                    continue
-                logging.debug('Considering sdkconfig {}'.format(sdkconfig_path))
-                if not os.path.exists(sdkconfig_path):
-                    continue
-                logging.debug('Appending {} to sdkconfig'.format(sdkconfig_name))
-
-        # The preparation of build is finished. Implement the build part in sub classes.
-        if cls.NAME == 'cmake':
-            return build_path, work_path, extra_cmakecache_items
-        else:
-            return build_path, work_path
-
-    @staticmethod
-    @abstractmethod
-    def build(build_item):
-        pass
-
-    @staticmethod
-    @abstractmethod
-    def is_app(path):
-        pass
-
-    @staticmethod
-    def _read_readme(app_path):
-        # Markdown supported targets should be:
-        # e.g. | Supported Targets | ESP32 |
-        #      | ----------------- | ----- |
-        # reStructuredText supported targets should be:
-        # e.g. ================= =====
-        #      Supported Targets ESP32
-        #      ================= =====
-        def get_md_or_rst(app_path):
-            readme_path = os.path.join(app_path, 'README.md')
-            if not os.path.exists(readme_path):
-                readme_path = os.path.join(app_path, 'README.rst')
-                if not os.path.exists(readme_path):
-                    return None
-            return readme_path
-
-        readme_path = get_md_or_rst(app_path)
-        # Handle sub apps situation, e.g. master-slave
-        if not readme_path:
-            readme_path = get_md_or_rst(os.path.dirname(app_path))
-        if not readme_path:
-            return None
-        with open(readme_path, 'r', encoding='utf8') as readme_file:
-            return readme_file.read()
-
-    @classmethod
-    def _supported_targets(cls, app_path):
-        readme_file_content = BuildSystem._read_readme(app_path)
-        if not readme_file_content:
-            return cls.DEFAULT_TARGETS  # supports all default targets if no readme found
-        match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
-        if not match:
-            return cls.DEFAULT_TARGETS  # supports all default targets if no such header in readme
-        if len(match) > 1:
-            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
-        support_str = match[0].strip()
-
-        targets = []
-        for part in support_str.split('|'):
-            for inner in part.split(' '):
-                inner = inner.strip()
-                if not inner:
-                    continue
-                elif inner in cls.FORMAL_TO_USUAL:
-                    targets.append(cls.FORMAL_TO_USUAL[inner])
-                else:
-                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
-                                              .format(inner, app_path, ', '.join(cls.FORMAL_TO_USUAL.keys())))
-        return targets
-
-    @classmethod
-    @abstractmethod
-    def supported_targets(cls, app_path):
-        pass
-
-
-class BuildError(RuntimeError):
-    pass
-
-
-def setup_logging(args):
-    """
-    Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument.
-    :param args: namespace obtained from argparse
-    """
-    if not args.verbose:
-        log_level = logging.WARNING
-    elif args.verbose == 1:
-        log_level = logging.INFO
-    else:
-        log_level = logging.DEBUG
-
-    logging.basicConfig(
-        format='%(levelname)s: %(message)s',
-        stream=getattr(args, 'log_file', None) or sys.stderr,
-        level=log_level,
-    )

tools/test_apps/.build-test-rules.yml

@@ -0,0 +1,143 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+tools/test_apps/build_system/ldgen_test:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+tools/test_apps/peripherals/usb:
+  enable:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/phy/phy_multi_init_data_test:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+tools/test_apps/protocols/esp_netif/build_config:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/protocols/mdns:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2" or IDF_TARGET == "esp32c3"
+      temporary: true
+      reason: lack of runners
+
+tools/test_apps/protocols/mqtt/publish_connect_test:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2" or IDF_TARGET == "esp32c3"
+      temporary: true
+      reason: lack of runners
+
+tools/test_apps/security/secure_boot:
+#  disable:
+#    - if: SOC_SECURE_BOOT_SUPPORTED != 1
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+tools/test_apps/system/bootloader_sections:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+tools/test_apps/system/build_test:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+tools/test_apps/system/cxx_no_except:
+  enable:
+    - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32c3"
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/system/eh_frame:
+  enable:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32h2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/system/flash_psram:
+  enable:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/system/g0_components:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "esp32h2"
+
+tools/test_apps/system/g1_components:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+tools/test_apps/system/gdb_loadable_elf:
+  disable_test:
+    - if: IDF_TARGET != "esp32"
+      temporary: true
+      reason: lack of runners
+
+tools/test_apps/system/longjmp_test:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: lack of runners
+
+tools/test_apps/system/memprot:
+  enable:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/system/monitor_ide_integration:
+  enable:
+    - if: IDF_TARGET == "esp32" or IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: the other targets are not tested yet
+  disable_test:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: lack of runners
+
+tools/test_apps/system/no_embedded_paths:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+tools/test_apps/system/panic:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "esp32h2"
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32s2"]
+      temporary: true
+      reason: lack of runners

tools/unit-test-app/tools/UnitTestParser.py

@@ -134,7 +134,7 @@ class Parser(object):
             * first tag is always group of test cases, it's mandatory
             * the rest tags should be [type=value].
                 * if the type have default value, then [type] equal to [type=default_value].
-                * if the type don't don't exist, then equal to [type=omitted_value]
+                * if the type don't exist, then equal to [type=omitted_value]
             default_value and omitted_value are defined in TagDefinition.yml
         :param tags_raw: raw tag string
         :return: tag dict
@@ -287,12 +287,15 @@ class Parser(object):
         """ parse test cases from multiple built unit test apps """
         test_cases = []
 
-        output_folder = os.path.join(self.idf_path, self.ut_bin_folder, self.idf_target)
         configs_folder = os.path.join(self.idf_path, self.UT_CONFIG_FOLDER)
-        test_configs = [item for item in os.listdir(output_folder)
-                        if os.path.isdir(os.path.join(output_folder, item))]
+        config_output_prefix = f'build_{self.idf_target}_'
+        test_configs = []
+        for item in os.listdir(self.ut_bin_folder):
+            if os.path.isdir(os.path.join(self.ut_bin_folder, item)) and item.startswith(config_output_prefix):
+                test_configs.append(item.split(config_output_prefix)[1])
+
         for config in test_configs:
-            config_output_folder = os.path.join(output_folder, config)
+            config_output_folder = os.path.join(self.ut_bin_folder, f'{config_output_prefix}{config}')
             if os.path.exists(config_output_folder):
                 test_cases.extend(self.parse_test_cases_for_one_config(configs_folder, config_output_folder, config))
         test_cases.sort(key=lambda x: x['config'] + x['summary'])