Merge branch 'feat/add_ci_related_requirements_txt' into 'master'

Feat/add ci related requirements txt

Closes IDF-4590 and IDFCI-1389

See merge request espressif/esp-idf!19246

.gitlab-ci.yml

@@ -211,7 +211,7 @@ before_script:
     # This adds tools (compilers) and the version-specific Python environment to PATH
     - *setup_tools_unless_target_test
     # Install packages required by CI scripts into IDF Python environment
-    - pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt
+    - pip install -r $IDF_PATH/tools/requirements/requirements.ci.txt
     - source tools/ci/configure_ci_environment.sh
     # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
     - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
@@ -228,9 +228,8 @@ before_script:
     - *setup_tools_unless_target_test
     - fetch_submodules
     - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
-    # TODO: remove this, IDFCI-1207
-    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
     - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
 
 .before_script_build_jobs:
@@ -244,13 +243,9 @@ before_script:
     - *setup_tools_unless_target_test
     - fetch_submodules
     - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
-    # TODO: remove this, IDFCI-1207
-    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
     - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
-    # not only need pytest related packages, but also needs ttfw requirements
-    - internal_pip_install $IDF_BUILD_APPS_PROJ idf_build_apps
-    - pip install -r tools/ci/python_packages/ttfw_idf/requirements.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
 

.gitlab/ci/build.yml

@@ -300,11 +300,11 @@ build_ssc_esp32s3:
       -t $IDF_TARGET
       --config "configs/*="
       --copy-sdkconfig
+      --preserve-all
       --collect-size-info $SIZE_INFO_LOCATION
       --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
-      --preserve-all
     - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
 
 build_esp_idf_tests_cmake_esp32:

.gitlab/ci/docs.yml

@@ -62,7 +62,7 @@ check_docs_lang_sync:
   dependencies: []
   script:
     - cd docs
-    - pip install -U -r requirements.txt
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
   parallel:
     matrix:
@@ -76,7 +76,7 @@ check_docs_gh_links:
     - .doc-rules:build:docs
   script:
     - cd docs
-    - pip install -U -r requirements.txt
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs gh-linkcheck
 
 # stage: build_doc
@@ -161,7 +161,7 @@ build_docs_pdf:
   script:
     - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
     - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --)
-    - pip install -U -r docs/requirements.txt
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - deploy-docs
 
 # stage: test_deploy
@@ -227,5 +227,5 @@ check_doc_links:
   allow_failure: true
   script:
     - cd docs
-    - pip install -U -r requirements.txt
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs -t $DOCTGT -l $DOCLANG linkcheck

.gitlab/ci/upload_cache.yml

@@ -24,9 +24,7 @@ upload-pip-cache:
     - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
     - source tools/ci/setup_python.sh
     - rm -rf .cache/pip  # clear old packages
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
-    # TODO: remove this, IDFCI-1207
-    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
   parallel:
     matrix:
       - GEO: [ 'shiny',  'brew' ]

docs/requirements.txt

@@ -1 +0,0 @@
-esp-docs~=1.1.0

examples/README.md

@@ -42,14 +42,15 @@ Building an example is the same as building any other project:
 
 ## Running Test Python Script (ttfw)
 
-Some of the examples have `..._test.py` scripts that are used to test that the example works as expected. These scripts run automatically in the internal test queue. They are not intended to be run by ESP-IDF users but sometimes you may want to run them locally. The following requirements must be met in the IDF python virtual environment.
+Some of the examples have `..._test.py` scripts that are used to test that the example works as expected. These scripts run automatically in the internal test queue. They are not intended to be run by ESP-IDF users but sometimes you may want to run them locally.
 
-- ttfw needs to be in the `PYTHONPATH`. Add it like this: `export PYTHONPATH=$PYTHONPATH:$IDF_PATH/tools/ci/python_packages`
-- Install all requirements from `tools/ci/python_packages/ttfw_idf/requirements.txt`: `python -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt`
+Install Python dependencies and export the Python path where the IDF CI Python modules are found with the following commands:
 
-These commands help solve the issue with `ModuleNotFoundError: No module named 'ttfw_idf'` and `ModuleNotFoundError: No module named 'tiny_test_fw'`.
-
-Some examples might fail due to other missing packages. You might need to install them manually: `pip install websocket`.
+```bash
+bash install.sh --enable-ttfw
+source export.sh
+export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$IDF_PATH/tools:$PYTHONPATH
+```
 
 ## Running Test Python Script (pytest)
 

examples/bluetooth/nimble/ble_l2cap_coc/coc_blecent/README.md

@@ -21,7 +21,7 @@ Maximum data of 512 bytes can be transferred over L2CAP when MTU is set to 512 a
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
+* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw).
 * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to Use Example

examples/bluetooth/nimble/ble_l2cap_coc/coc_bleprph/README.md

@@ -21,8 +21,8 @@ Maximum data of 512 bytes can be transferred over L2CAP when MTU is set to 512 a
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
-* Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
+* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw).
+* * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to Use Example
 

examples/bluetooth/nimble/ble_spp/spp_client/README.md

@@ -76,7 +76,7 @@ It performs three GATT operations against the specified peer:
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
+* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw).
 * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to use example

examples/bluetooth/nimble/ble_spp/spp_server/README.md

@@ -76,7 +76,7 @@ It performs three GATT operations against the specified peer:
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
+* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw).
 * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to use example

examples/bluetooth/nimble/blecent/README.md

@@ -27,7 +27,7 @@ A Python based utility `blecent_test.py` is also provided (which will run as a B
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
+* To install the dependency packages needed, please refer to the top level [README file](../../../README.md#running-test-python-script-ttfw).
 * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to Use Example

examples/bluetooth/nimble/blehr/README.md

@@ -17,7 +17,7 @@ A Python based utility `blehr_test.py` is also provided (which will run as a BLE
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
+* To install the dependency packages needed, please refer to the top level [README file](../../../README.md#running-test-python-script-ttfw).
 * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to Use Example

examples/bluetooth/nimble/bleprph/README.md

@@ -21,7 +21,7 @@ A Python based utility `bleprph_test.py` is also provided (which will run as a B
 
 Note :
 
-* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed.
+* To install the dependency packages needed, please refer to the top level [README file](../../../README.md#running-test-python-script-ttfw).
 * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus).
 
 ## How to Use Example

examples/provisioning/wifi_prov_mgr/README.md

@@ -45,7 +45,7 @@ For iOS, a provisioning application along with source code is available on GitHu
 
 #### Platform : Linux / Windows / macOS
 
-To provision the device running this example, the `esp_prov.py` script needs to be run (found under `$IDF_PATH/tools/esp_prov`). Make sure to satisfy all the dependencies prior to running the script (check out the `requirements.txt` file in `esp_prov` directory).
+To install the dependency packages needed, please refer to the top level [README file](../../README.md#running-test-python-script-ttfw).
 
 `esp_prov` supports BLE and SoftAP transport for Linux, MacOS and Windows platforms. For BLE, however, if dependencies are not met, the script falls back to console mode and requires another application through which the communication can take place. The `esp_prov` console will guide you through the provisioning process of locating the correct BLE GATT services and characteristics, the values to write, and input read values.
 

pytest.ini

@@ -23,6 +23,7 @@ markers =
   esp32s3: support esp32s3 target
   esp32c3: support esp32c3 target
   esp32c2: support esp32c2 target
+  esp32h2: support esp32h2 target
   supported_targets: support all supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2')
   preview_targets: support all preview targets ('linux', 'esp32h2')
   all_targets: support all targets, including supported ones and preview ones

tools/ble/requirements.txt

@@ -1,3 +0,0 @@
-future
-dbus-python
-pygobject

tools/ci/python_packages/idf_iperf_test_util/requirements.txt

@@ -1 +0,0 @@
-pyecharts

tools/ci/python_packages/tiny_test_fw/requirements.txt

@@ -1,4 +0,0 @@
-pyserial
-pyyaml
-junit_xml
-netifaces

tools/ci/python_packages/ttfw_idf/requirements.txt

@@ -1,4 +0,0 @@
--r ../tiny_test_fw/requirements.txt
-pexpect
-python-gitlab
-pygdbmi>=0.9.0.0

tools/esp_prov/README.md

@@ -90,16 +90,17 @@ For Android, a provisioning tool along with source code is available [here](http
 
 ## Dependencies
 
-This requires the following python libraries to run (included in requirements.txt):
+This requires the following python libraries to run:
 * `bleak`
 * `future`
 * `protobuf`
 * `cryptography`
 
-Run `pip install -r $IDF_PATH/tools/esp_prov/requirements.txt`
+To install the dependency packages needed, please run the following command:
 
-Note :
-* The packages listed in requirements.txt are limited only to the ones needed AFTER fully satisfying the requirements of ESP-IDF
+```shell
+bash install.sh --enable-ttfw
+```
 
 # EXAMPLE USAGE
 

tools/esp_prov/requirements.txt

@@ -1,4 +0,0 @@
-bleak
-future
-cryptography
-protobuf

tools/requirements.json

@@ -18,6 +18,24 @@
             "description": "Packages for CI with pytest",
             "optional": true,
             "requirement_path": "tools/requirements/requirements.pytest.txt"
+        },
+        {
+            "name": "ttfw",
+            "description": "Packages for CI with ttfw",
+            "optional": true,
+            "requirement_path": "tools/requirements/requirements.ttfw.txt"
+        },
+        {
+            "name": "ci",
+            "description": "Packages for ESP-IDF CI scripts",
+            "optional": true,
+            "requirement_path": "tools/requirements/requirements.ci.txt"
+        },
+        {
+            "name": "docs",
+            "description": "Packages for building ESP-IDF documentation",
+            "optional": true,
+            "requirement_path": "tools/requirements/requirements.docs.txt"
         }
     ]
 }

tools/requirements/requirements.ci.txt

@@ -0,0 +1,9 @@
+# Python package requirements for CI in ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-ci"
+
+# ci
+coverage
+idf-build-apps
+junit_xml
+python-gitlab
+pyyaml

tools/requirements/requirements.docs.txt

@@ -0,0 +1,4 @@
+# Python package requirements for building the documentation of ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-docs"
+
+esp-docs

tools/requirements/requirements.pytest.txt

@@ -1,7 +1,15 @@
+# Python package requirements for pytest in ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-pytest"
+
 pytest-embedded-serial-esp
 pytest-embedded-idf
 pytest-embedded-qemu
 pytest-rerunfailures
+
+# build
+idf-build-apps
+
+# dependencies in pytest test scripts
 scapy
 websocket-client
 netifaces

tools/requirements/requirements.ttfw.txt

@@ -0,0 +1,35 @@
+# Python package requirements for CI in ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-ttfw"
+
+# build
+idf-build-apps
+
+# ttfw
+pyserial
+pyyaml
+junit_xml
+netifaces
+
+# ttfw-idf
+pexpect
+python-gitlab
+pygdbmi
+
+# ble
+future
+dbus-python; sys_platform != 'win32'
+pygobject; sys_platform != 'win32'
+
+# iperf_test_util
+pyecharts
+
+# esp_prov
+bleak
+# future  # addressed before under ble
+protobuf
+
+# tools/test_apps/system/monitor_ide_integration
+SimpleWebSocketServer
+
+# py_debug_backend
+debug_backend

tools/test_apps/README.md

@@ -63,12 +63,14 @@ All the following instructions are general. Part of them may be complemented by
 
 ### Requirements
 
-The following requirements need to be satisfied in the IDF python virtual environment.
+Install Python dependencies and export the Python path where the IDF CI Python modules are found with the following commands:
 
-- ttfw needs to be in the `PYTHONPATH`. Add it like this: `export PYTHONPATH=$PYTHONPATH:$IDF_PATH/tools/ci/python_packages`
-- Install all requirements from `tools/ci/python_packages/ttfw_idf/requirements.txt`: `pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt`
-
-You should also set the port via the environment variable ESPPORT to prevent the tools from looking and iterating over all serial ports. The latter causes much trouble, currently:
+```bash
+bash install.sh --enable-ttfw
+source export.sh
+export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$PYTHONPATH
+```
+You should also set the port via the environment variable `ESPPORT` to prevent the tools from looking and iterating over all serial ports. The latter causes much trouble, currently:
 
 ```
 export ESPPORT=/dev/ttyUSB<X>

tools/unit-test-app/README.md

@@ -39,7 +39,7 @@ Unit test uses 3 stages in CI: `build`, `assign_test`, `unit_test`.
 
 ### Build Stage:
 
-`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts.
+`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/build_<target>_<config>`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts.
 
 When we add new test case, it will construct a structure to save case data during build. We'll parse the test case from this structure. The description (defined in test case: `TEST_CASE("name", "description")`) is used to extend test case definition. The format of test description is a list of tags:
 
@@ -63,7 +63,7 @@ We will build unit-test-app with different sdkconfigs. Some config items require
 
 ### Assign Test Stage:
 
-`assign_test` job will try to assign all cases to test jobs defined in `.gitlab-ci.yml`, according to test environment and tags. For each job, one config file with same name of test job will be generated in `components/idf_test/unit_test/CIConfigs/`(this folder will be passed to test jobs as artifacts). These config files will tell test jobs which cases it need to run, and pass some extra configs (like if the case will reset) of test case to runner.
+`assign_unit_test` job will try to assign all cases to test jobs defined in `.gitlab-ci.yml`, according to test environment and tags. For each job, one config file with same name of test job will be generated in `components/idf_test/unit_test/CIConfigs/`(this folder will be passed to test jobs as artifacts). These config files will tell test jobs which cases it need to run, and pass some extra configs (like if the case will reset) of test case to runner.
 
 Please check related document in tiny-test-fw for details.
 
@@ -110,7 +110,7 @@ First you can check the logs. It's saved as unit test job artifacts. You can dow
 
 If you want to reproduce locally, you need to:
 
-1. Download artifacts of `build_esp_idf_tests`. The built binary is in `tools/unit-test-app/output` folder.
+1. Download artifacts of `build_esp_idf_tests`. The built binary is in `tools/unit-test-app/build_<target>_<config>` folder.
     * Built binary in CI could be slightly different from locally built binary with the same revision, some cases might only fails with CI built binary.
 2. Check the following print in CI job to get the config name: `Running unit test for config: config_name`. Then flash the binary of this config to your board.
 3. Run the failed case on your board (refer to Running Unit Tests section).
@@ -124,8 +124,9 @@ If you want to reproduce locally, you need to:
 First, install Python dependencies and export the Python path where the IDF CI Python modules are found:
 
 ```bash
-pip install -r $IDF_PATH/tools/ci/python_packages/tiny_test_fw/requirements.txt
-export PYTHONPATH=$IDF_PATH/tools/ci/python_packages
+bash install.sh --enable-ttfw
+source export.sh
+export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$PYTHONPATH
 ```
 
 Change to the unit test app directory, configure the app as needed and build it in the default "build" directory. For example:
@@ -138,6 +139,20 @@ idf.py build -T vfs
 
 (Instead of these steps, you can do whatever is needed to configure & build a unit test app with the tests and config that you need.)
 
+If you want to build exactly the same binary files under the same location as they are in CI pipelines, you may run:
+
+```bash
+cd $IDF_PATH
+python tools/ci/ci_build_apps.py tools/unit-test-app -v -t $IDF_TARGET --config "configs/*=" --copy-sdkconfig --preserve-all
+```
+
+This would build all configs. if you want to build only one config (let's take `psram` as an example), you may use:
+
+```bash
+cd $IDF_PATH
+python tools/ci/ci_build_apps.py tools/unit-test-app -v -t $IDF_TARGET --config "configs/psram=" --copy-sdkconfig --preserve-all
+```
+
 ### run a single test case by name
 
 ```bash