Tools: Improve the Python package system

Introduce features into the Python package management system & manage
package versions outside of ESP-IDF repo.

.gitlab-ci.yml

@@ -72,6 +72,13 @@ variables:
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
   PYTEST_EMBEDDED_TAG: "v0.4.5"
 
+  # cache python dependencies
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+cache:
+  paths:
+    - .cache/pip
+
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
   export IDF_MIRROR_PREFIX_MAP=
@@ -95,6 +102,7 @@ before_script:
   - source tools/ci/configure_ci_environment.sh
   - *setup_tools_unless_target_test
   - fetch_submodules
+  - $IDF_PATH/tools/idf_tools.py install-python-env
 
 # used for check scripts which we want to run unconditionally
 .before_script_no_sync_submodule:
@@ -103,6 +111,7 @@ before_script:
     - source tools/ci/utils.sh
     - source tools/ci/setup_python.sh
     - source tools/ci/configure_ci_environment.sh
+    - $IDF_PATH/tools/idf_tools.py install-python-env
 
 .before_script_minimal:
   before_script:
@@ -133,6 +142,7 @@ before_script:
     - source tools/ci/configure_ci_environment.sh
     - *setup_tools_unless_target_test
     - fetch_submodules
+    - $IDF_PATH/tools/idf_tools.py install-python-env
     - cd /tmp
     - retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
     - cd pytest-embedded && bash foreach.sh install

.gitlab/CODEOWNERS

@@ -199,7 +199,7 @@
 
 /tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
-requirements.txt              @esp-idf-codeowners/tools
+requirements.*.txt              @esp-idf-codeowners/tools
 
 # sort-order-reset
 

.gitlab/ci/host-test.yml

@@ -215,8 +215,7 @@ test_idf_tools:
     - cd ${IDF_PATH}/tools/test_idf_tools
     - ./test_idf_tools.py
     # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
-    - cd ${IDF_PATH}/tools
+    - python3 ./test_idf_tools_python_env.py
-    - python3 ./idf_tools.py install-python-env
 
 .test_efuse_table_on_host_template:
   extends: .host_test_template

.gitlab/ci/rules.yml

@@ -124,6 +124,7 @@
   - "tools/tools_schema.json"
   - "tools/idf_tools.py"
   - "tools/test_idf_tools/**/*"
+  - "tools/install_util.py"
 
   - "tools/mkdfu.py"
   - "tools/test_mkdfu/**/*"

docs/en/api-guides/jtag-debugging/using-debugger.rst

@@ -228,7 +228,7 @@ It is also possible to execute the described debugging tools conveniently from `
 
 4.  ``idf.py gdbgui``
 
-    Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window.
+    Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window. Please run the install script with the "--enable-gdbgui" argument in order to make this option supported, e.g. ``install.sh --enable-gdbgui``.
 
 
     It is possible to combine these debugging actions on a single command line allowing convenient setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list.

docs/en/api-guides/tools/idf-tools.rst

@@ -101,6 +101,10 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
 
 * ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
 
+* ``install-python-env``: Create Python virtual environment and install the required Python packages.
+
+* ``check-python-dependencies``: Checks if all required Python packages are installed.
+
 .. _idf-tools-install:
 
 Install scripts

export.bat

@@ -50,7 +50,7 @@ DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $*
 DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*
 
 echo Checking if Python packages are up to date...
-python.exe "%IDF_PATH%\tools\check_python_dependencies.py"
+python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies
 if %errorlevel% neq 0 goto :__end
 
 echo.

export.fish

@@ -19,7 +19,7 @@ function __main
     eval "$idf_exports"
 
     echo "Checking if Python packages are up to date..."
-    python "$IDF_PATH"/tools/check_python_dependencies.py || return 1
+    python "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1
 
     # Allow calling some IDF python tools without specifying the full path
     # "$IDF_PATH"/tools is already added by 'idf_tools.py export'

export.ps1

@@ -69,7 +69,7 @@ if ($dif_Path -ne $null) {
 
 Write-Output "Checking if Python packages are up to date..."
 
-Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/check_python_dependencies.py`""
+Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies"
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
 
 Write-Output "

export.sh

@@ -96,7 +96,7 @@ __main() {
 
     __verbose "Using Python interpreter in $(which python)"
     __verbose "Checking if Python packages are up to date..."
-    python "${IDF_PATH}/tools/check_python_dependencies.py" || return 1
+    python "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1
 
 
     # Allow calling some IDF python tools without specifying the full path

install.bat

@@ -17,15 +17,16 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :error_missing_requirements
 set IDF_PATH=%~dp0
 set IDF_PATH=%IDF_PATH:~0,-1%
 
-set TARGETS="all"
+for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract targets "%*"') do set TARGETS=%%i
-if NOT "%1"=="" set TARGETS=%*
 
 echo Installing ESP-IDF tools
 python.exe "%IDF_PATH%\tools\idf_tools.py" install --targets=%TARGETS%
 if %errorlevel% neq 0 goto :end
 
+for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract features "%*"') do set FEATURES=%%i
+
 echo Setting up Python environment
-python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env
+python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env --features=%FEATURES%
 if %errorlevel% neq 0 goto :end
 
 echo All done! You can now run:

install.fish

@@ -7,17 +7,16 @@ set -x IDF_PATH $basedir
 echo "Detecting the Python interpreter"
 source "$IDF_PATH"/tools/detect_python.fish
 
-if not set -q argv[1]
+set TARGETS ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract targets $argv) || exit 1
-    set TARGETS "all"
+
-else
-    set TARGETS $argv[1]
-end
 echo "Installing ESP-IDF tools"
 "$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS
 or exit 1
 
+set FEATURES ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract features $argv) || exit 1
+
 echo "Installing Python environment and packages"
-"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env
+"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env --features=$FEATURES
 
 echo "All done! You can now run:"
 echo ""

install.ps1

@@ -1,18 +1,16 @@
 #!/usr/bin/env pwsh
 $IDF_PATH = $PSScriptRoot
 
-if($args.count -eq 0){
+$TARGETS = (python "$IDF_PATH/tools/install_util.py" extract targets "$args")
-    $TARGETS = "all"
+
-}else
-{
-    $TARGETS = $args[0] -join ','
-}
 Write-Output "Installing ESP-IDF tools"
 Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}"
 if ($LASTEXITCODE -ne 0)  { exit $LASTEXITCODE } # if error
 
+$FEATURES = (python "$IDF_PATH/tools/install_util.py" extract features "$args")
+
 Write-Output "Setting up Python environment"
-Start-Process -Wait  -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env"
+Start-Process -Wait  -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env --features=${FEATURES}"
 if ($LASTEXITCODE -ne 0)  { exit $LASTEXITCODE} # if error
 
 

install.sh

@@ -10,16 +10,15 @@ export IDF_PATH
 echo "Detecting the Python interpreter"
 . "${IDF_PATH}/tools/detect_python.sh"
 
-if [ "$#" -eq 0 ]; then
+TARGETS=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract targets "$@"`
-  TARGETS="all"
+
-else
-  TARGETS=$1
-fi
 echo "Installing ESP-IDF tools"
 "${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install --targets=${TARGETS}
 
+FEATURES=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract features "$@"`
+
 echo "Installing Python environment and packages"
-"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env
+"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env --features=${FEATURES}
 
 echo "All done! You can now run:"
 echo ""

requirements.core.txt

@@ -0,0 +1,23 @@
+# Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems.
+
+setuptools
+click
+pyserial
+future
+cryptography
+pyparsing
+pyelftools
+idf-component-manager
+
+# esptool dependencies (see components/esptool_py/esptool/setup.py)
+reedsolo
+bitstring
+ecdsa
+
+# espcoredump dependencies
+construct
+pygdbmi
+
+# kconfig and menuconfig dependencies
+kconfiglib
+windows-curses; sys_platform == 'win32'

requirements.gdbgui.txt

@@ -0,0 +1,4 @@
+# Python package requirements for gdbgui support ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-gdbgui"
+
+gdbgui

requirements.txt

@@ -1,41 +0,0 @@
-# This is a list of python packages needed for ESP-IDF. This file is used with pip.
-# Please see the Get Started section of the ESP-IDF Programming Guide for further information.
-#
-setuptools>=21
-# The setuptools package is required to install source distributions and on some systems is not installed by default.
-# Please keep it as the first item of this list. Version 21 is required to handle PEP 508 environment markers.
-#
-click>=7.0
-pyserial>=3.3
-future>=0.15.2
-
-cryptography>=2.1.4
---only-binary cryptography
-# Only binary for cryptography is here to make it work on ARMv7 architecture
-# We do have cryptography binary on https://dl.espressif.com/pypi for ARM
-# On https://pypi.org/ are no ARM binaries as standard now
-
-pyparsing>=3.0.3  # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3
-pyelftools>=0.22
-idf-component-manager>=0.2.99-beta
-
-gdbgui==0.13.2.0
-# 0.13.2.1 supports Python 3.6+ only
-# Windows is not supported since 0.14.0.0. See https://github.com/cs01/gdbgui/issues/348
-pygdbmi<=0.9.0.2
-# The pygdbmi required max version 0.9.0.2 since 0.9.0.3 is not compatible with latest gdbgui (>=0.13.2.0)
-# A compatible Socket.IO should be used. See https://github.com/miguelgrinberg/python-socketio/issues/578
-python-socketio<5
-
-# esptool requirements (see components/esptool_py/esptool/setup.py)
-reedsolo>=1.5.3,<=1.5.4
-bitstring>=3.1.6
-ecdsa>=0.16.0
-
-# espcoredump requirements
-# This is the last version supports both 2.7 and 3.4
-construct==2.10.54
-
-# kconfig & menuconfig support
-kconfiglib==13.7.1
-windows-curses; sys_platform == 'win32'

tools/check_python_dependencies.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
@@ -8,69 +8,80 @@ import os
 import re
 import sys
 
+PYTHON_PACKAGE_RE = re.compile(r'[^<>=~]+')
+
 try:
     import pkg_resources
-except Exception:
+except ImportError:
     print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a '
           'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for '
           'setting up the required packages.')
     sys.exit(1)
 
 
-def escape_backslash(path):
-    if sys.platform == 'win32':
-        # escaped backslashes are necessary in order to be able to copy-paste the printed path
-        return path.replace('\\', '\\\\')
-    else:
-        return path
-
-
 if __name__ == '__main__':
-    idf_path = os.getenv('IDF_PATH')
-
-    default_requirements_path = os.path.join(idf_path, 'requirements.txt')  # type: ignore
-
     parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker')
     parser.add_argument('--requirements', '-r',
-                        help='Path to the requirements file',
+                        help='Path to a requirements file (can be used multiple times)',
-                        default=default_requirements_path)
+                        action='append', default=[])
+    parser.add_argument('--constraints', '-c', default=[],
+                        help='Path to a constraints file (can be used multiple times)',
+                        action='append')
     args = parser.parse_args()
 
+    required_set = set()
+    for req_path in args.requirements:
+        with open(req_path) as f:
+            required_set |= set(i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#'))
+
+    constr_dict = {}  # for example package_name -> package_name==1.0
+    for const_path in args.constraints:
+        with open(const_path) as f:
+            for con in [i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#')]:
+                if con.startswith('file://'):
+                    con = os.path.basename(con)
+                elif con.startswith('--only-binary'):
+                    continue
+                elif con.startswith('-e') and '#egg=' in con:  # version control URLs, take the egg= part at the end only
+                    con_m = re.search(r'#egg=([^\s]+)', con)
+                    if not con_m:
+                        print('Malformed input. Cannot find name in {}'.format(con))
+                        sys.exit(1)
+                    con = con_m[1]
+
+                name_m = PYTHON_PACKAGE_RE.search(con)
+                if not name_m:
+                    print('Malformed input. Cannot find name in {}'.format(con))
+                    sys.exit(1)
+                constr_dict[name_m[0]] = con
+
+    # We need to constrain package dependencies as well. So all installed packages need to be checked.
+    # For example package A requires package B. We have only A in our requirements. But the newest version of B could
+    # broke at some time and in that case we add a constraint for B (on the server) but don't have to update the
+    # requirement file (in the ESP-IDF repo).
+    required_set |= set(i.key for i in pkg_resources.working_set)
+
     not_satisfied = []
-    with open(args.requirements) as f:
+    for requirement in required_set:
-        for line in f:
+        # If there is a version-specific constraint for the requirement then use it. Otherwise, just use the
-            line = line.strip()
+        # requirement as is.
-            # pkg_resources.require() cannot handle the full requirements file syntax so we need to make
+        to_require = constr_dict.get(requirement, requirement)
-            # adjustments for options which we use.
+        try:
-            if line.startswith('file://'):
+            pkg_resources.require(to_require)
-                line = os.path.basename(line)
+        except pkg_resources.ResolutionError:
-            if line.startswith('--only-binary'):
+            not_satisfied.append(to_require)
-                continue
-            if line.startswith('-e') and '#egg=' in line:  # version control URLs, take the egg= part at the end only
-                line = re.search(r'#egg=([^\s]+)', line).group(1)  # type: ignore
-            try:
-                pkg_resources.require(line)
-            except Exception:
-                not_satisfied.append(line)
 
     if len(not_satisfied) > 0:
         print('The following Python requirements are not satisfied:')
-        for requirement in not_satisfied:
+        print(os.linesep.join(not_satisfied))
-            print(requirement)
+        if 'IDF_PYTHON_ENV_PATH' in os.environ:
-        if os.path.realpath(args.requirements) != os.path.realpath(default_requirements_path):
-            # we're using this script to check non-default requirements.txt, so tell the user to run pip
-            print('Please check the documentation for the feature you are using, or run "%s -m pip install -r %s"' % (sys.executable, args.requirements))
-        elif os.environ.get('IDF_PYTHON_ENV_PATH'):
             # We are running inside a private virtual environment under IDF_TOOLS_PATH,
             # ask the user to run install.bat again.
-            if sys.platform == 'win32':
+            install_script = 'install.bat' if sys.platform == 'win32' else 'install.sh'
-                install_script = 'install.bat'
+            print('To install the missing packages, please run "{}"'.format(install_script))
-            else:
-                install_script = 'install.sh'
-            print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script))  # type: ignore
         else:
             print('Please follow the instructions found in the "Set up the tools" section of '
-                  'ESP-IDF Getting Started Guide')
+                  'ESP-IDF Getting Started Guide.')
 
         print('Diagnostic information:')
         idf_python_env_path = os.environ.get('IDF_PYTHON_ENV_PATH')
@@ -81,4 +92,4 @@ if __name__ == '__main__':
             print('    PATH: {}'.format(os.getenv('PATH')))
         sys.exit(1)
 
-    print('Python requirements from {} are satisfied.'.format(args.requirements))
+    print('Python requirements are satisfied.')

tools/ci/mypy_ignore_list.txt

@@ -148,7 +148,6 @@ examples/wifi/iperf/iperf_test.py
 tools/ble/lib_ble_client.py
 tools/ble/lib_gap.py
 tools/ble/lib_gatt.py
-tools/check_python_dependencies.py
 tools/check_term.py
 tools/ci/check_artifacts_expire_time.py
 tools/ci/check_callgraph.py

tools/cmake/build.cmake

@@ -282,7 +282,7 @@ function(__build_check_python)
         idf_build_get_property(python PYTHON)
         idf_build_get_property(idf_path IDF_PATH)
         message(STATUS "Checking Python dependencies...")
-        execute_process(COMMAND "${python}" "${idf_path}/tools/check_python_dependencies.py"
+        execute_process(COMMAND "${python}" "${idf_path}/tools/idf_tools.py" "check-python-dependencies"
             RESULT_VARIABLE result)
         if(result EQUAL 1)
             # check_python_dependencies returns error code 1 on failure

tools/idf.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -95,7 +95,8 @@ def check_environment():
         out = subprocess.check_output(
             [
                 os.environ['PYTHON'],
-                os.path.join(os.environ['IDF_PATH'], 'tools', 'check_python_dependencies.py'),
+                os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_tools.py'),
+                'check-python-dependencies',
             ],
             env=os.environ,
         )

tools/idf_py_actions/debug_ext.py

@@ -217,7 +217,8 @@ def action_extensions(base_actions, project_path):
             process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1, env=env)
         except Exception as e:
             print(e)
-            raise FatalError('Error starting gdbgui. Please make sure gdbgui can be started', ctx)
+            raise FatalError('Error starting gdbgui. Please make sure gdbgui has been installed with '
+                             '"install.{sh,bat,ps1,fish} --enable-gdbgui" and can be started.', ctx)
 
         processes['gdbgui'] = process
         processes['gdbgui_outfile'] = gdbgui_out

tools/idf_tools.py

@@ -88,6 +88,7 @@ DOWNLOAD_RETRY_COUNT = 3
 URL_PREFIX_MAP_SEPARATOR = ','
 IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
 IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
+IDF_DL_URL = 'https://dl.espressif.com/dl/esp-idf'
 
 PYTHON_PLATFORM = platform.system() + '-' + platform.machine()
 
@@ -361,6 +362,31 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None):
     return result
 
 
+def download(url, destination):  # type: (str, str) -> None
+    info('Downloading {} to {}'.format(os.path.basename(url), destination))
+    try:
+        ctx = None
+        # For dl.espressif.com, add the ISRG x1 root certificate.
+        # This works around the issue with outdated certificate stores in some installations.
+        if 'dl.espressif.com' in url:
+            try:
+                ctx = ssl.create_default_context()
+                ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
+            except AttributeError:
+                # no ssl.create_default_context or load_verify_locations cadata argument
+                # in Python <=2.7.8
+                pass
+
+        urlretrieve_ctx(url, destination, report_progress if not global_non_interactive else None, context=ctx)
+        sys.stdout.write('\rDone\n')
+    except Exception as e:
+        # urlretrieve could throw different exceptions, e.g. IOError when the server is down
+        # Errors are ignored because the downloaded file is checked a couple of lines later.
+        warn('Download failure {}'.format(e))
+    finally:
+        sys.stdout.flush()
+
+
 # Sometimes renaming a directory on Windows (randomly?) causes a PermissionError.
 # This is confirmed to be a workaround:
 # https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118
@@ -680,29 +706,9 @@ class IDFTool(object):
                 return
 
         downloaded = False
+        local_temp_path = local_path + '.tmp'
         for retry in range(DOWNLOAD_RETRY_COUNT):
-            local_temp_path = local_path + '.tmp'
+            download(url, local_temp_path)
-            info('Downloading {} to {}'.format(archive_name, local_temp_path))
-            try:
-                ctx = None
-                # For dl.espressif.com, add the ISRG x1 root certificate.
-                # This works around the issue with outdated certificate stores in some installations.
-                if 'dl.espressif.com' in url:
-                    try:
-                        ctx = ssl.create_default_context()
-                        ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
-                    except AttributeError:
-                        # no ssl.create_default_context or load_verify_locations cadata argument
-                        # in Python <=2.7.8
-                        pass
-
-                urlretrieve_ctx(url, local_temp_path, report_progress if not global_non_interactive else None, context=ctx)
-                sys.stdout.write('\rDone\n')
-            except Exception as e:
-                # urlretrieve could throw different exceptions, e.g. IOError when the server is down
-                # Errors are ignored because the downloaded file is checked a couple of lines later.
-                warn('Download failure {}'.format(e))
-            sys.stdout.flush()
             if not os.path.isfile(local_temp_path) or not self.check_download_file(download_obj, local_temp_path):
                 warn('Failed to download {} to {}'.format(url, local_temp_path))
                 continue
@@ -969,7 +975,7 @@ def dump_tools_json(tools_info):  # type: ignore
     return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True)
 
 
-def get_python_env_path():  # type: () -> Tuple[str, str, str]
+def get_python_env_path():  # type: () -> Tuple[str, str, str, str]
     python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
 
     version_file_path = os.path.join(global_idf_path, 'version.txt')  # type: ignore
@@ -1020,7 +1026,7 @@ def get_python_env_path():  # type: () -> Tuple[str, str, str]
     idf_python_export_path = os.path.join(idf_python_env_path, subdir)
     virtualenv_python = os.path.join(idf_python_export_path, python_exe)
 
-    return idf_python_env_path, idf_python_export_path, virtualenv_python
+    return idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version
 
 
 def get_idf_env():  # type: () -> Any
@@ -1037,29 +1043,34 @@ def get_idf_env():  # type: () -> Any
             os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending)))
 
         info('Creating {}' .format(idf_env_file_path))
-        return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': {}}}}
+        return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': []}}}
 
 
-def export_targets_to_idf_env_json(targets):  # type: (list[str]) -> None
+def export_into_idf_env_json(targets, features):  # type: (Optional[list[str]], Optional[list[str]]) -> None
     idf_env_json = get_idf_env()
-    targets = list(set(targets + get_user_defined_targets()))
+    targets = list(set(targets + get_requested_targets_and_features()[0])) if targets else None
 
     for env in idf_env_json['idfInstalled']:
         if env == idf_env_json['idfSelectedId']:
-            idf_env_json['idfInstalled'][env]['targets'] = targets
+            update_with = []
+            if targets:
+                update_with += [('targets', targets)]
+            if features:
+                update_with += [('features', features)]
+            idf_env_json['idfInstalled'][env].update(update_with)
             break
 
     try:
         if global_idf_tools_path:  # mypy fix for Optional[str] in the next call
             # the directory doesn't exist if this is run on a clean system the first time
             mkdir_p(global_idf_tools_path)
-        with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w:  # type: ignore
+            with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w:
-            json.dump(idf_env_json, w, indent=4)
+                json.dump(idf_env_json, w, indent=4)
     except (IOError, OSError):
         warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE)))  # type: ignore
 
 
-def clean_targets(targets_str):  # type: (str) -> list[str]
+def add_and_save_targets(targets_str):  # type: (str) -> list[str]
     targets_from_tools_json = get_all_targets_from_tools_json()
     invalid_targets = []
 
@@ -1072,26 +1083,44 @@ def clean_targets(targets_str):  # type: (str) -> list[str]
             raise SystemExit(1)
         # removing duplicates
         targets = list(set(targets))
-        export_targets_to_idf_env_json(targets)
+        export_into_idf_env_json(targets, None)
     else:
-        export_targets_to_idf_env_json(targets_from_tools_json)
+        export_into_idf_env_json(targets_from_tools_json, None)
     return targets
 
 
-def get_user_defined_targets():  # type: () -> list[str]
+def feature_to_requirements_path(feature):  # type: (str) -> str
+    return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature))
+
+
+def add_and_save_features(features_str):  # type: (str) -> list[str]
+    _, features = get_requested_targets_and_features()
+    for new_feature_candidate in features_str.split(','):
+        if os.path.isfile(feature_to_requirements_path(new_feature_candidate)):
+            features += [new_feature_candidate]
+
+    features = list(set(features + ['core']))  # remove duplicates
+    export_into_idf_env_json(None, features)
+    return features
+
+
+def get_requested_targets_and_features():  # type: () -> tuple[list[str], list[str]]
     try:
         with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file:  # type: ignore
             idf_env_json = json.load(idf_env_file)
     except OSError:
         # warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE)))  # type: ignore
-        return []
+        return [], []
 
     targets = []
+    features = []
     for env in idf_env_json['idfInstalled']:
         if env == idf_env_json['idfSelectedId']:
-            targets = idf_env_json['idfInstalled'][env]['targets']
+            env_dict = idf_env_json['idfInstalled'][env]
+            targets = env_dict.get('targets', [])
+            features = env_dict.get('features', [])
             break
-    return targets
+    return targets, features
 
 
 def get_all_targets_from_tools_json():  # type: () -> list[str]
@@ -1108,7 +1137,7 @@ def get_all_targets_from_tools_json():  # type: () -> list[str]
 
 
 def filter_tools_info(tools_info):  # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool]
-    targets = get_user_defined_targets()
+    targets, _ = get_requested_targets_and_features()
     if not targets:
         return tools_info
     else:
@@ -1240,7 +1269,7 @@ def action_export(args):  # type: ignore
                 export_vars[k] = v
 
     current_path = os.getenv('PATH')
-    idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path()
+    idf_python_env_path, idf_python_export_path, virtualenv_python, _ = get_python_env_path()
     if os.path.exists(virtualenv_python):
         idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0]
         if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path:
@@ -1349,7 +1378,7 @@ def action_download(args):  # type: ignore
     targets = []  # type: list[str]
     # Installing only single tools, no targets are specified.
     if 'required' in tools_spec:
-        targets = clean_targets(args.targets)
+        targets = add_and_save_targets(args.targets)
 
     if args.platform not in PLATFORM_FROM_NAME:
         fatal('unknown platform: {}' % args.platform)
@@ -1409,8 +1438,8 @@ def action_install(args):  # type: ignore
     targets = []  # type: list[str]
     # Installing only single tools, no targets are specified.
     if 'required' in tools_spec:
-        targets = clean_targets(args.targets)
+        targets = add_and_save_targets(args.targets)
-        info('Selected targets are: {}' .format(', '.join(get_user_defined_targets())))
+        info('Selected targets are: {}' .format(', '.join(get_requested_targets_and_features()[0])))
 
     if not tools_spec or 'required' in tools_spec:
         # Installing tools for all ESP_targets required by the operating system.
@@ -1475,9 +1504,42 @@ def get_wheels_dir():  # type: () -> Optional[str]
     return wheels_dir
 
 
+def get_requirements(new_features):  # type: (str) -> list[str]
+    features = add_and_save_features(new_features)
+    return [feature_to_requirements_path(feature) for feature in features]
+
+
+def get_constraints(idf_version):  # type: (str) -> str
+    constraint_file = 'espidf.constraints.v{}.txt'.format(idf_version)
+    constraint_path = os.path.join(os.path.expanduser(IDF_TOOLS_PATH_DEFAULT), constraint_file)
+    constraint_url = '/'.join([IDF_DL_URL, constraint_file])
+    temp_path = constraint_path + '.tmp'
+
+    mkdir_p(os.path.dirname(temp_path))
+
+    for _ in range(DOWNLOAD_RETRY_COUNT):
+        download(constraint_url, temp_path)
+        if not os.path.isfile(temp_path):
+            warn('Failed to download {} to {}'.format(constraint_url, temp_path))
+            continue
+        if os.path.isfile(constraint_path):
+            # Windows cannot rename to existing file. It needs to be deleted.
+            os.remove(constraint_path)
+        rename_with_retry(temp_path, constraint_path)
+        return constraint_path
+
+    if os.path.isfile(constraint_path):
+        warn('Failed to download, retry count has expired, using a previously downloaded version')
+        return constraint_path
+    else:
+        fatal('Failed to download, and retry count has expired')
+        raise DownloadError()
+
+
 def action_install_python_env(args):  # type: ignore
+    use_constraints = not args.no_constraints
     reinstall = args.reinstall
-    idf_python_env_path, _, virtualenv_python = get_python_env_path()
+    idf_python_env_path, _, virtualenv_python, idf_version = get_python_env_path()
 
     is_virtualenv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)
     if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall):
@@ -1549,8 +1611,12 @@ def action_install_python_env(args):  # type: ignore
         warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.')
         env_copy['PIP_USER'] = 'no'
     run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location']
-    requirements_txt = os.path.join(global_idf_path, 'requirements.txt')
+    requirements_file_list = get_requirements(args.features)
-    run_args += ['-r', requirements_txt]
+    for requirement_file in requirements_file_list:
+        run_args += ['-r', requirement_file]
+    if use_constraints:
+        constraint_file = get_constraints(idf_version)
+        run_args += ['--upgrade', '--constraint', constraint_file]
     if args.extra_wheels_dir:
         run_args += ['--find-links', args.extra_wheels_dir]
     if args.no_index:
@@ -1562,10 +1628,58 @@ def action_install_python_env(args):  # type: ignore
     if wheels_dir is not None:
         run_args += ['--find-links', wheels_dir]
 
-    info('Installing Python packages from {}'.format(requirements_txt))
+    info('Installing Python packages')
+    if use_constraints:
+        info(' Constraint file: {}'.format(constraint_file))
+    info(' Requirement files:')
+    info(os.linesep.join('  - {}'.format(path) for path in requirements_file_list))
     subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr, env=env_copy)
 
 
+def action_check_python_dependencies(args):  # type: ignore
+    use_constraints = not args.no_constraints
+    req_paths = get_requirements('')  # no new features -> just detect the existing ones
+
+    _, _, virtualenv_python, idf_version = get_python_env_path()
+
+    if not os.path.isfile(virtualenv_python):
+        fatal('{} doesn\'t exist! Please run the install script or "idf_tools.py install-python-env" in order to '
+              'create it'.format(virtualenv_python))
+        raise SystemExit(1)
+
+    if use_constraints:
+        constr_path = get_constraints(idf_version)
+        info('Constraint file: {}'.format(constr_path))
+
+    info('Requirement files:')
+    info(os.linesep.join(' - {}'.format(path) for path in req_paths))
+
+    info('Python being checked: {}'.format(virtualenv_python))
+
+    # The dependency checker will be invoked with virtualenv_python. idf_tools.py could have been invoked with a
+    # different one, therefore, importing is not a suitable option.
+    dep_check_cmd = [virtualenv_python,
+                     os.path.join(global_idf_path,
+                                  'tools',
+                                  'check_python_dependencies.py')]
+
+    if use_constraints:
+        dep_check_cmd += ['-c', constr_path]
+
+    for req_path in req_paths:
+        dep_check_cmd += ['-r', req_path]
+
+    try:
+        ret = subprocess.run(dep_check_cmd)
+        if ret and ret.returncode:
+            # returncode is a negative number and system exit output is usually expected be positive.
+            raise SystemExit(-ret.returncode)
+    except FileNotFoundError:
+        # Python environment not yet created
+        fatal('Requirements are not satisfied!')
+        raise SystemExit(1)
+
+
 def action_add_version(args):  # type: ignore
     tools_info = load_tools_info()
     tool_name = args.tool
@@ -1771,6 +1885,11 @@ def main(argv):  # type: (list[str]) -> None
                                     'to use during installation')
     install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default='https://dl.espressif.com/pypi')
     install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index')
+    install_python_env.add_argument('--features', default='core', help='A comma separated list of desired features for installing.'
+                                                                       ' It defaults to installing just the core funtionality.')
+    install_python_env.add_argument('--no-constraints', action='store_true', default=False,
+                                    help='Disable constraint settings. Use with care and only when you want to manage '
+                                         'package versions by yourself.')
 
     if IDF_MAINTAINER:
         add_version = subparsers.add_parser('add-version', help='Add or update download info for a version')
@@ -1790,6 +1909,12 @@ def main(argv):  # type: (list[str]) -> None
                              help='Output file name')
         gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~')
 
+    check_python_dependencies = subparsers.add_parser('check-python-dependencies',
+                                                      help='Check that all required Python packages are installed.')
+    check_python_dependencies.add_argument('--no-constraints', action='store_true', default=False,
+                                           help='Disable constraint settings. Use with care and only when you want '
+                                                'to manage package versions by yourself.')
+
     args = parser.parse_args(argv)
 
     if args.action is None:

tools/install_util.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This script is used from the $IDF_PATH/install.* scripts. This way the argument parsing can be done at one place and
+# doesn't have to be implemented for all shells.
+
+import argparse
+from itertools import chain
+
+try:
+    import python_version_checker
+
+    # check the Python version before it will fail with an exception on syntax or package incompatibility.
+    python_version_checker.check()
+except RuntimeError as e:
+    print(e)
+    raise SystemExit(1)
+
+
+def action_extract_features(args: str) -> None:
+    """
+    Command line arguments starting with "--enable-" are features. This function selects those and prints them.
+    """
+    features = ['core']  # "core" features should be always installed
+
+    if args:
+        arg_prefix = '--enable-'
+        features += [arg[len(arg_prefix):] for arg in args.split() if arg.startswith(arg_prefix)]
+
+    print(','.join(features))
+
+
+def action_extract_targets(args: str) -> None:
+    """
+    Command line arguments starting with "esp" are chip targets. This function selects those and prints them.
+    """
+    target_sep = ','
+    targets = []
+
+    if args:
+        target_args = (arg for arg in args.split() if arg.lower().startswith('esp'))
+        # target_args can be comma-separated lists of chip targets
+
+        targets = list(chain.from_iterable(commalist.split(target_sep) for commalist in target_args))
+
+    print(target_sep.join(targets or ['all']))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(dest='action', required=True)
+    extract = subparsers.add_parser('extract', help='Process arguments and extract part of it')
+
+    extract.add_argument('type', choices=['targets', 'features'])
+    extract.add_argument('str-to-parse', nargs='?')
+
+    args, unknown_args = parser.parse_known_args()
+    # standalone "--enable-" won't be included into str-to-parse
+
+    action_func = globals()['action_{}_{}'.format(args.action, args.type)]
+    str_to_parse = vars(args)['str-to-parse'] or ''
+    action_func(' '.join(chain([str_to_parse], unknown_args)))
+
+
+if __name__ == '__main__':
+    main()

tools/test_idf_tools/test_idf_tools_python_env.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+from typing import List
+
+try:
+    import idf_tools
+except ImportError:
+    sys.path.append('..')
+    import idf_tools
+
+IDF_PATH = os.environ.get('IDF_PATH', '../..')
+TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
+PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env')
+REQ_SATISFIED = 'Python requirements are satisfied'
+REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH)
+REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH)
+CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)
+
+
+class TestPythonInstall(unittest.TestCase):
+
+    def setUp(self):  # type: () -> None
+        if os.path.isdir(PYTHON_DIR):
+            shutil.rmtree(PYTHON_DIR)
+        if os.path.isfile(os.path.join(TOOLS_DIR, 'idf-env.json')):
+            os.remove(os.path.join(TOOLS_DIR, 'idf-env.json'))
+
+    def run_idf_tools(self, extra_args):  # type: (List[str]) -> str
+        args = [sys.executable, '../idf_tools.py'] + extra_args
+        ret = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
+        return ret.stdout.decode('utf-8', 'ignore')
+
+    def test_default_arguments(self):  # type: () -> None
+        output = self.run_idf_tools(['check-python-dependencies'])
+        self.assertNotIn(REQ_SATISFIED, output)
+        self.assertIn('bin/python doesn\'t exist', output)
+
+        output = self.run_idf_tools(['install-python-env'])
+        self.assertIn(CONSTR, output)
+        self.assertIn(REQ_CORE, output)
+        self.assertNotIn(REQ_GDBGUI, output)
+
+        output = self.run_idf_tools(['check-python-dependencies'])
+        self.assertIn(REQ_SATISFIED, output)
+
+    def test_opt_argument(self):  # type: () -> None
+        output = self.run_idf_tools(['install-python-env', '--features', 'gdbgui'])
+        self.assertIn(CONSTR, output)
+        self.assertIn(REQ_CORE, output)
+        self.assertIn(REQ_GDBGUI, output)
+
+        output = self.run_idf_tools(['install-python-env'])
+        # The gdbgui should be installed as well because the feature is is stored in the JSON file
+        self.assertIn(CONSTR, output)
+        self.assertIn(REQ_CORE, output)
+        self.assertIn(REQ_GDBGUI, output)
+
+    def test_no_constraints(self):  # type: () -> None
+        output = self.run_idf_tools(['install-python-env', '--no-constraints'])
+        self.assertNotIn(CONSTR, output)
+        self.assertIn(REQ_CORE, output)
+
+
+if __name__ == '__main__':
+    unittest.main()