From b28d7e6850936a9e5bbf542fa1d55adf6f9ae2f3 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Thu, 16 Sep 2021 16:48:03 +0200 Subject: [PATCH] Tools: Improve the Python package system Introduce features into the Python package management system & manage package versions outside of ESP-IDF repo. --- .gitlab-ci.yml | 10 + .gitlab/CODEOWNERS | 2 +- .gitlab/ci/host-test.yml | 3 +- .gitlab/ci/rules.yml | 1 + .../jtag-debugging/using-debugger.rst | 2 +- docs/en/api-guides/tools/idf-tools.rst | 4 + export.bat | 2 +- export.fish | 2 +- export.ps1 | 2 +- export.sh | 2 +- install.bat | 7 +- install.fish | 11 +- install.ps1 | 12 +- install.sh | 11 +- requirements.core.txt | 23 ++ requirements.gdbgui.txt | 4 + requirements.txt | 41 ---- tools/check_python_dependencies.py | 99 ++++---- tools/ci/mypy_ignore_list.txt | 1 - tools/cmake/build.cmake | 2 +- tools/idf.py | 5 +- tools/idf_py_actions/debug_ext.py | 3 +- tools/idf_tools.py | 217 ++++++++++++++---- tools/install_util.py | 70 ++++++ .../test_idf_tools_python_env.py | 71 ++++++ 25 files changed, 441 insertions(+), 166 deletions(-) create mode 100644 requirements.core.txt create mode 100644 requirements.gdbgui.txt delete mode 100644 requirements.txt create mode 100644 tools/install_util.py create mode 100644 tools/test_idf_tools/test_idf_tools_python_env.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dd4974b908..4490038dfe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,6 +72,13 @@ variables: CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1" PYTEST_EMBEDDED_TAG: "v0.4.5" + # cache python dependencies + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +cache: + paths: + - .cache/pip + .setup_tools_unless_target_test: &setup_tools_unless_target_test | if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then export IDF_MIRROR_PREFIX_MAP= @@ -95,6 +102,7 @@ before_script: - source tools/ci/configure_ci_environment.sh - *setup_tools_unless_target_test - fetch_submodules + - $IDF_PATH/tools/idf_tools.py install-python-env # used for check scripts which we want to run unconditionally .before_script_no_sync_submodule: @@ -103,6 +111,7 @@ before_script: - source tools/ci/utils.sh - source tools/ci/setup_python.sh - source tools/ci/configure_ci_environment.sh + - $IDF_PATH/tools/idf_tools.py install-python-env .before_script_minimal: before_script: @@ -133,6 +142,7 @@ before_script: - source tools/ci/configure_ci_environment.sh - *setup_tools_unless_target_test - fetch_submodules + - $IDF_PATH/tools/idf_tools.py install-python-env - cd /tmp - retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git - cd pytest-embedded && bash foreach.sh install diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 9c87aa7de8..02b1466521 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -199,7 +199,7 @@ /tools/unit-test-app/ @esp-idf-codeowners/system @esp-idf-codeowners/tools -requirements.txt @esp-idf-codeowners/tools +requirements.*.txt @esp-idf-codeowners/tools # sort-order-reset diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index d545664c9e..02de85ab79 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -215,8 +215,7 @@ test_idf_tools: - cd ${IDF_PATH}/tools/test_idf_tools - ./test_idf_tools.py # Test for create virtualenv. It must be invoked from Python, not from virtualenv. - - cd ${IDF_PATH}/tools - - python3 ./idf_tools.py install-python-env + - python3 ./test_idf_tools_python_env.py .test_efuse_table_on_host_template: extends: .host_test_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 6e8ef42bd4..a09574919e 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -124,6 +124,7 @@ - "tools/tools_schema.json" - "tools/idf_tools.py" - "tools/test_idf_tools/**/*" + - "tools/install_util.py" - "tools/mkdfu.py" - "tools/test_mkdfu/**/*" diff --git a/docs/en/api-guides/jtag-debugging/using-debugger.rst b/docs/en/api-guides/jtag-debugging/using-debugger.rst index 12920f4e32..de618be4f4 100644 --- a/docs/en/api-guides/jtag-debugging/using-debugger.rst +++ b/docs/en/api-guides/jtag-debugging/using-debugger.rst @@ -228,7 +228,7 @@ It is also possible to execute the described debugging tools conveniently from ` 4. ``idf.py gdbgui`` - Starts `gdbgui `_ debugger frontend enabling out-of-the-box debugging in a browser window. + Starts `gdbgui `_ debugger frontend enabling out-of-the-box debugging in a browser window. Please run the install script with the "--enable-gdbgui" argument in order to make this option supported, e.g. ``install.sh --enable-gdbgui``. It is possible to combine these debugging actions on a single command line allowing convenient setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list. diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 7c2cfe2044..0120cfcae1 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -101,6 +101,10 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo * ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``. +* ``install-python-env``: Create Python virtual environment and install the required Python packages. + +* ``check-python-dependencies``: Checks if all required Python packages are installed. + .. _idf-tools-install: Install scripts diff --git a/export.bat b/export.bat index f7f54f676c..c54f35049d 100644 --- a/export.bat +++ b/export.bat @@ -50,7 +50,7 @@ DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $* DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $* echo Checking if Python packages are up to date... -python.exe "%IDF_PATH%\tools\check_python_dependencies.py" +python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies if %errorlevel% neq 0 goto :__end echo. diff --git a/export.fish b/export.fish index 85fd69b73c..799fa5e872 100644 --- a/export.fish +++ b/export.fish @@ -19,7 +19,7 @@ function __main eval "$idf_exports" echo "Checking if Python packages are up to date..." - python "$IDF_PATH"/tools/check_python_dependencies.py || return 1 + python "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1 # Allow calling some IDF python tools without specifying the full path # "$IDF_PATH"/tools is already added by 'idf_tools.py export' diff --git a/export.ps1 b/export.ps1 index ce92c15108..f6fbb9eed0 100644 --- a/export.ps1 +++ b/export.ps1 @@ -69,7 +69,7 @@ if ($dif_Path -ne $null) { Write-Output "Checking if Python packages are up to date..." -Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/check_python_dependencies.py`"" +Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error Write-Output " diff --git a/export.sh b/export.sh index c10ee3f56f..dc7170bd9f 100644 --- a/export.sh +++ b/export.sh @@ -96,7 +96,7 @@ __main() { __verbose "Using Python interpreter in $(which python)" __verbose "Checking if Python packages are up to date..." - python "${IDF_PATH}/tools/check_python_dependencies.py" || return 1 + python "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1 # Allow calling some IDF python tools without specifying the full path diff --git a/install.bat b/install.bat index 7dc15be373..48d1d5accc 100644 --- a/install.bat +++ b/install.bat @@ -17,15 +17,16 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :error_missing_requirements set IDF_PATH=%~dp0 set IDF_PATH=%IDF_PATH:~0,-1% -set TARGETS="all" -if NOT "%1"=="" set TARGETS=%* +for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract targets "%*"') do set TARGETS=%%i echo Installing ESP-IDF tools python.exe "%IDF_PATH%\tools\idf_tools.py" install --targets=%TARGETS% if %errorlevel% neq 0 goto :end +for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract features "%*"') do set FEATURES=%%i + echo Setting up Python environment -python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env +python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env --features=%FEATURES% if %errorlevel% neq 0 goto :end echo All done! You can now run: diff --git a/install.fish b/install.fish index 6f044ec15e..47b9aeb592 100755 --- a/install.fish +++ b/install.fish @@ -7,17 +7,16 @@ set -x IDF_PATH $basedir echo "Detecting the Python interpreter" source "$IDF_PATH"/tools/detect_python.fish -if not set -q argv[1] - set TARGETS "all" -else - set TARGETS $argv[1] -end +set TARGETS ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract targets $argv) || exit 1 + echo "Installing ESP-IDF tools" "$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS or exit 1 +set FEATURES ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract features $argv) || exit 1 + echo "Installing Python environment and packages" -"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env +"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env --features=$FEATURES echo "All done! You can now run:" echo "" diff --git a/install.ps1 b/install.ps1 index bcd020b95c..01347b3eda 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1,18 +1,16 @@ #!/usr/bin/env pwsh $IDF_PATH = $PSScriptRoot -if($args.count -eq 0){ - $TARGETS = "all" -}else -{ - $TARGETS = $args[0] -join ',' -} +$TARGETS = (python "$IDF_PATH/tools/install_util.py" extract targets "$args") + Write-Output "Installing ESP-IDF tools" Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error +$FEATURES = (python "$IDF_PATH/tools/install_util.py" extract features "$args") + Write-Output "Setting up Python environment" -Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env" +Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env --features=${FEATURES}" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE} # if error diff --git a/install.sh b/install.sh index b45ade95af..0036023908 100755 --- a/install.sh +++ b/install.sh @@ -10,16 +10,15 @@ export IDF_PATH echo "Detecting the Python interpreter" . "${IDF_PATH}/tools/detect_python.sh" -if [ "$#" -eq 0 ]; then - TARGETS="all" -else - TARGETS=$1 -fi +TARGETS=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract targets "$@"` + echo "Installing ESP-IDF tools" "${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install --targets=${TARGETS} +FEATURES=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract features "$@"` + echo "Installing Python environment and packages" -"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env +"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env --features=${FEATURES} echo "All done! You can now run:" echo "" diff --git a/requirements.core.txt b/requirements.core.txt new file mode 100644 index 0000000000..5554dd3b9e --- /dev/null +++ b/requirements.core.txt @@ -0,0 +1,23 @@ +# Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems. + +setuptools +click +pyserial +future +cryptography +pyparsing +pyelftools +idf-component-manager + +# esptool dependencies (see components/esptool_py/esptool/setup.py) +reedsolo +bitstring +ecdsa + +# espcoredump dependencies +construct +pygdbmi + +# kconfig and menuconfig dependencies +kconfiglib +windows-curses; sys_platform == 'win32' diff --git a/requirements.gdbgui.txt b/requirements.gdbgui.txt new file mode 100644 index 0000000000..d060b779b7 --- /dev/null +++ b/requirements.gdbgui.txt @@ -0,0 +1,4 @@ +# Python package requirements for gdbgui support ESP-IDF. +# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-gdbgui" + +gdbgui diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5141f4481b..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,41 +0,0 @@ -# This is a list of python packages needed for ESP-IDF. This file is used with pip. -# Please see the Get Started section of the ESP-IDF Programming Guide for further information. -# -setuptools>=21 -# The setuptools package is required to install source distributions and on some systems is not installed by default. -# Please keep it as the first item of this list. Version 21 is required to handle PEP 508 environment markers. -# -click>=7.0 -pyserial>=3.3 -future>=0.15.2 - -cryptography>=2.1.4 ---only-binary cryptography -# Only binary for cryptography is here to make it work on ARMv7 architecture -# We do have cryptography binary on https://dl.espressif.com/pypi for ARM -# On https://pypi.org/ are no ARM binaries as standard now - -pyparsing>=3.0.3 # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3 -pyelftools>=0.22 -idf-component-manager>=0.2.99-beta - -gdbgui==0.13.2.0 -# 0.13.2.1 supports Python 3.6+ only -# Windows is not supported since 0.14.0.0. See https://github.com/cs01/gdbgui/issues/348 -pygdbmi<=0.9.0.2 -# The pygdbmi required max version 0.9.0.2 since 0.9.0.3 is not compatible with latest gdbgui (>=0.13.2.0) -# A compatible Socket.IO should be used. See https://github.com/miguelgrinberg/python-socketio/issues/578 -python-socketio<5 - -# esptool requirements (see components/esptool_py/esptool/setup.py) -reedsolo>=1.5.3,<=1.5.4 -bitstring>=3.1.6 -ecdsa>=0.16.0 - -# espcoredump requirements -# This is the last version supports both 2.7 and 3.4 -construct==2.10.54 - -# kconfig & menuconfig support -kconfiglib==13.7.1 -windows-curses; sys_platform == 'win32' diff --git a/tools/check_python_dependencies.py b/tools/check_python_dependencies.py index 75184943d8..8a74572384 100755 --- a/tools/check_python_dependencies.py +++ b/tools/check_python_dependencies.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse @@ -8,69 +8,80 @@ import os import re import sys +PYTHON_PACKAGE_RE = re.compile(r'[^<>=~]+') + try: import pkg_resources -except Exception: +except ImportError: print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a ' 'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for ' 'setting up the required packages.') sys.exit(1) -def escape_backslash(path): - if sys.platform == 'win32': - # escaped backslashes are necessary in order to be able to copy-paste the printed path - return path.replace('\\', '\\\\') - else: - return path - - if __name__ == '__main__': - idf_path = os.getenv('IDF_PATH') - - default_requirements_path = os.path.join(idf_path, 'requirements.txt') # type: ignore - parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker') parser.add_argument('--requirements', '-r', - help='Path to the requirements file', - default=default_requirements_path) + help='Path to a requirements file (can be used multiple times)', + action='append', default=[]) + parser.add_argument('--constraints', '-c', default=[], + help='Path to a constraints file (can be used multiple times)', + action='append') args = parser.parse_args() + required_set = set() + for req_path in args.requirements: + with open(req_path) as f: + required_set |= set(i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#')) + + constr_dict = {} # for example package_name -> package_name==1.0 + for const_path in args.constraints: + with open(const_path) as f: + for con in [i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#')]: + if con.startswith('file://'): + con = os.path.basename(con) + elif con.startswith('--only-binary'): + continue + elif con.startswith('-e') and '#egg=' in con: # version control URLs, take the egg= part at the end only + con_m = re.search(r'#egg=([^\s]+)', con) + if not con_m: + print('Malformed input. Cannot find name in {}'.format(con)) + sys.exit(1) + con = con_m[1] + + name_m = PYTHON_PACKAGE_RE.search(con) + if not name_m: + print('Malformed input. Cannot find name in {}'.format(con)) + sys.exit(1) + constr_dict[name_m[0]] = con + + # We need to constrain package dependencies as well. So all installed packages need to be checked. + # For example package A requires package B. We have only A in our requirements. But the newest version of B could + # broke at some time and in that case we add a constraint for B (on the server) but don't have to update the + # requirement file (in the ESP-IDF repo). + required_set |= set(i.key for i in pkg_resources.working_set) + not_satisfied = [] - with open(args.requirements) as f: - for line in f: - line = line.strip() - # pkg_resources.require() cannot handle the full requirements file syntax so we need to make - # adjustments for options which we use. - if line.startswith('file://'): - line = os.path.basename(line) - if line.startswith('--only-binary'): - continue - if line.startswith('-e') and '#egg=' in line: # version control URLs, take the egg= part at the end only - line = re.search(r'#egg=([^\s]+)', line).group(1) # type: ignore - try: - pkg_resources.require(line) - except Exception: - not_satisfied.append(line) + for requirement in required_set: + # If there is a version-specific constraint for the requirement then use it. Otherwise, just use the + # requirement as is. + to_require = constr_dict.get(requirement, requirement) + try: + pkg_resources.require(to_require) + except pkg_resources.ResolutionError: + not_satisfied.append(to_require) if len(not_satisfied) > 0: print('The following Python requirements are not satisfied:') - for requirement in not_satisfied: - print(requirement) - if os.path.realpath(args.requirements) != os.path.realpath(default_requirements_path): - # we're using this script to check non-default requirements.txt, so tell the user to run pip - print('Please check the documentation for the feature you are using, or run "%s -m pip install -r %s"' % (sys.executable, args.requirements)) - elif os.environ.get('IDF_PYTHON_ENV_PATH'): + print(os.linesep.join(not_satisfied)) + if 'IDF_PYTHON_ENV_PATH' in os.environ: # We are running inside a private virtual environment under IDF_TOOLS_PATH, # ask the user to run install.bat again. - if sys.platform == 'win32': - install_script = 'install.bat' - else: - install_script = 'install.sh' - print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script)) # type: ignore + install_script = 'install.bat' if sys.platform == 'win32' else 'install.sh' + print('To install the missing packages, please run "{}"'.format(install_script)) else: print('Please follow the instructions found in the "Set up the tools" section of ' - 'ESP-IDF Getting Started Guide') + 'ESP-IDF Getting Started Guide.') print('Diagnostic information:') idf_python_env_path = os.environ.get('IDF_PYTHON_ENV_PATH') @@ -81,4 +92,4 @@ if __name__ == '__main__': print(' PATH: {}'.format(os.getenv('PATH'))) sys.exit(1) - print('Python requirements from {} are satisfied.'.format(args.requirements)) + print('Python requirements are satisfied.') diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index f50e618abb..fba2ac2cd0 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -148,7 +148,6 @@ examples/wifi/iperf/iperf_test.py tools/ble/lib_ble_client.py tools/ble/lib_gap.py tools/ble/lib_gatt.py -tools/check_python_dependencies.py tools/check_term.py tools/ci/check_artifacts_expire_time.py tools/ci/check_callgraph.py diff --git a/tools/cmake/build.cmake b/tools/cmake/build.cmake index 94a2f8ccc5..273f751e6e 100644 --- a/tools/cmake/build.cmake +++ b/tools/cmake/build.cmake @@ -282,7 +282,7 @@ function(__build_check_python) idf_build_get_property(python PYTHON) idf_build_get_property(idf_path IDF_PATH) message(STATUS "Checking Python dependencies...") - execute_process(COMMAND "${python}" "${idf_path}/tools/check_python_dependencies.py" + execute_process(COMMAND "${python}" "${idf_path}/tools/idf_tools.py" "check-python-dependencies" RESULT_VARIABLE result) if(result EQUAL 1) # check_python_dependencies returns error code 1 on failure diff --git a/tools/idf.py b/tools/idf.py index 96ea928af3..e8950fd5cf 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # @@ -95,7 +95,8 @@ def check_environment(): out = subprocess.check_output( [ os.environ['PYTHON'], - os.path.join(os.environ['IDF_PATH'], 'tools', 'check_python_dependencies.py'), + os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_tools.py'), + 'check-python-dependencies', ], env=os.environ, ) diff --git a/tools/idf_py_actions/debug_ext.py b/tools/idf_py_actions/debug_ext.py index f534b03d87..b3242dcdb8 100644 --- a/tools/idf_py_actions/debug_ext.py +++ b/tools/idf_py_actions/debug_ext.py @@ -217,7 +217,8 @@ def action_extensions(base_actions, project_path): process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1, env=env) except Exception as e: print(e) - raise FatalError('Error starting gdbgui. Please make sure gdbgui can be started', ctx) + raise FatalError('Error starting gdbgui. Please make sure gdbgui has been installed with ' + '"install.{sh,bat,ps1,fish} --enable-gdbgui" and can be started.', ctx) processes['gdbgui'] = process processes['gdbgui_outfile'] = gdbgui_out diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 1a1d8b8825..9ce3f9f64b 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -88,6 +88,7 @@ DOWNLOAD_RETRY_COUNT = 3 URL_PREFIX_MAP_SEPARATOR = ',' IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD') IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD') +IDF_DL_URL = 'https://dl.espressif.com/dl/esp-idf' PYTHON_PLATFORM = platform.system() + '-' + platform.machine() @@ -361,6 +362,31 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None): return result +def download(url, destination): # type: (str, str) -> None + info('Downloading {} to {}'.format(os.path.basename(url), destination)) + try: + ctx = None + # For dl.espressif.com, add the ISRG x1 root certificate. + # This works around the issue with outdated certificate stores in some installations. + if 'dl.espressif.com' in url: + try: + ctx = ssl.create_default_context() + ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT) + except AttributeError: + # no ssl.create_default_context or load_verify_locations cadata argument + # in Python <=2.7.8 + pass + + urlretrieve_ctx(url, destination, report_progress if not global_non_interactive else None, context=ctx) + sys.stdout.write('\rDone\n') + except Exception as e: + # urlretrieve could throw different exceptions, e.g. IOError when the server is down + # Errors are ignored because the downloaded file is checked a couple of lines later. + warn('Download failure {}'.format(e)) + finally: + sys.stdout.flush() + + # Sometimes renaming a directory on Windows (randomly?) causes a PermissionError. # This is confirmed to be a workaround: # https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118 @@ -680,29 +706,9 @@ class IDFTool(object): return downloaded = False + local_temp_path = local_path + '.tmp' for retry in range(DOWNLOAD_RETRY_COUNT): - local_temp_path = local_path + '.tmp' - info('Downloading {} to {}'.format(archive_name, local_temp_path)) - try: - ctx = None - # For dl.espressif.com, add the ISRG x1 root certificate. - # This works around the issue with outdated certificate stores in some installations. - if 'dl.espressif.com' in url: - try: - ctx = ssl.create_default_context() - ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT) - except AttributeError: - # no ssl.create_default_context or load_verify_locations cadata argument - # in Python <=2.7.8 - pass - - urlretrieve_ctx(url, local_temp_path, report_progress if not global_non_interactive else None, context=ctx) - sys.stdout.write('\rDone\n') - except Exception as e: - # urlretrieve could throw different exceptions, e.g. IOError when the server is down - # Errors are ignored because the downloaded file is checked a couple of lines later. - warn('Download failure {}'.format(e)) - sys.stdout.flush() + download(url, local_temp_path) if not os.path.isfile(local_temp_path) or not self.check_download_file(download_obj, local_temp_path): warn('Failed to download {} to {}'.format(url, local_temp_path)) continue @@ -969,7 +975,7 @@ def dump_tools_json(tools_info): # type: ignore return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True) -def get_python_env_path(): # type: () -> Tuple[str, str, str] +def get_python_env_path(): # type: () -> Tuple[str, str, str, str] python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) version_file_path = os.path.join(global_idf_path, 'version.txt') # type: ignore @@ -1020,7 +1026,7 @@ def get_python_env_path(): # type: () -> Tuple[str, str, str] idf_python_export_path = os.path.join(idf_python_env_path, subdir) virtualenv_python = os.path.join(idf_python_export_path, python_exe) - return idf_python_env_path, idf_python_export_path, virtualenv_python + return idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version def get_idf_env(): # type: () -> Any @@ -1037,29 +1043,34 @@ def get_idf_env(): # type: () -> Any os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending))) info('Creating {}' .format(idf_env_file_path)) - return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': {}}}} + return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': []}}} -def export_targets_to_idf_env_json(targets): # type: (list[str]) -> None +def export_into_idf_env_json(targets, features): # type: (Optional[list[str]], Optional[list[str]]) -> None idf_env_json = get_idf_env() - targets = list(set(targets + get_user_defined_targets())) + targets = list(set(targets + get_requested_targets_and_features()[0])) if targets else None for env in idf_env_json['idfInstalled']: if env == idf_env_json['idfSelectedId']: - idf_env_json['idfInstalled'][env]['targets'] = targets + update_with = [] + if targets: + update_with += [('targets', targets)] + if features: + update_with += [('features', features)] + idf_env_json['idfInstalled'][env].update(update_with) break try: if global_idf_tools_path: # mypy fix for Optional[str] in the next call # the directory doesn't exist if this is run on a clean system the first time mkdir_p(global_idf_tools_path) - with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w: # type: ignore - json.dump(idf_env_json, w, indent=4) + with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w: + json.dump(idf_env_json, w, indent=4) except (IOError, OSError): warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore -def clean_targets(targets_str): # type: (str) -> list[str] +def add_and_save_targets(targets_str): # type: (str) -> list[str] targets_from_tools_json = get_all_targets_from_tools_json() invalid_targets = [] @@ -1072,26 +1083,44 @@ def clean_targets(targets_str): # type: (str) -> list[str] raise SystemExit(1) # removing duplicates targets = list(set(targets)) - export_targets_to_idf_env_json(targets) + export_into_idf_env_json(targets, None) else: - export_targets_to_idf_env_json(targets_from_tools_json) + export_into_idf_env_json(targets_from_tools_json, None) return targets -def get_user_defined_targets(): # type: () -> list[str] +def feature_to_requirements_path(feature): # type: (str) -> str + return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature)) + + +def add_and_save_features(features_str): # type: (str) -> list[str] + _, features = get_requested_targets_and_features() + for new_feature_candidate in features_str.split(','): + if os.path.isfile(feature_to_requirements_path(new_feature_candidate)): + features += [new_feature_candidate] + + features = list(set(features + ['core'])) # remove duplicates + export_into_idf_env_json(None, features) + return features + + +def get_requested_targets_and_features(): # type: () -> tuple[list[str], list[str]] try: with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file: # type: ignore idf_env_json = json.load(idf_env_file) except OSError: # warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore - return [] + return [], [] targets = [] + features = [] for env in idf_env_json['idfInstalled']: if env == idf_env_json['idfSelectedId']: - targets = idf_env_json['idfInstalled'][env]['targets'] + env_dict = idf_env_json['idfInstalled'][env] + targets = env_dict.get('targets', []) + features = env_dict.get('features', []) break - return targets + return targets, features def get_all_targets_from_tools_json(): # type: () -> list[str] @@ -1108,7 +1137,7 @@ def get_all_targets_from_tools_json(): # type: () -> list[str] def filter_tools_info(tools_info): # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool] - targets = get_user_defined_targets() + targets, _ = get_requested_targets_and_features() if not targets: return tools_info else: @@ -1240,7 +1269,7 @@ def action_export(args): # type: ignore export_vars[k] = v current_path = os.getenv('PATH') - idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path() + idf_python_env_path, idf_python_export_path, virtualenv_python, _ = get_python_env_path() if os.path.exists(virtualenv_python): idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0] if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path: @@ -1349,7 +1378,7 @@ def action_download(args): # type: ignore targets = [] # type: list[str] # Installing only single tools, no targets are specified. if 'required' in tools_spec: - targets = clean_targets(args.targets) + targets = add_and_save_targets(args.targets) if args.platform not in PLATFORM_FROM_NAME: fatal('unknown platform: {}' % args.platform) @@ -1409,8 +1438,8 @@ def action_install(args): # type: ignore targets = [] # type: list[str] # Installing only single tools, no targets are specified. if 'required' in tools_spec: - targets = clean_targets(args.targets) - info('Selected targets are: {}' .format(', '.join(get_user_defined_targets()))) + targets = add_and_save_targets(args.targets) + info('Selected targets are: {}' .format(', '.join(get_requested_targets_and_features()[0]))) if not tools_spec or 'required' in tools_spec: # Installing tools for all ESP_targets required by the operating system. @@ -1475,9 +1504,42 @@ def get_wheels_dir(): # type: () -> Optional[str] return wheels_dir +def get_requirements(new_features): # type: (str) -> list[str] + features = add_and_save_features(new_features) + return [feature_to_requirements_path(feature) for feature in features] + + +def get_constraints(idf_version): # type: (str) -> str + constraint_file = 'espidf.constraints.v{}.txt'.format(idf_version) + constraint_path = os.path.join(os.path.expanduser(IDF_TOOLS_PATH_DEFAULT), constraint_file) + constraint_url = '/'.join([IDF_DL_URL, constraint_file]) + temp_path = constraint_path + '.tmp' + + mkdir_p(os.path.dirname(temp_path)) + + for _ in range(DOWNLOAD_RETRY_COUNT): + download(constraint_url, temp_path) + if not os.path.isfile(temp_path): + warn('Failed to download {} to {}'.format(constraint_url, temp_path)) + continue + if os.path.isfile(constraint_path): + # Windows cannot rename to existing file. It needs to be deleted. + os.remove(constraint_path) + rename_with_retry(temp_path, constraint_path) + return constraint_path + + if os.path.isfile(constraint_path): + warn('Failed to download, retry count has expired, using a previously downloaded version') + return constraint_path + else: + fatal('Failed to download, and retry count has expired') + raise DownloadError() + + def action_install_python_env(args): # type: ignore + use_constraints = not args.no_constraints reinstall = args.reinstall - idf_python_env_path, _, virtualenv_python = get_python_env_path() + idf_python_env_path, _, virtualenv_python, idf_version = get_python_env_path() is_virtualenv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix) if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall): @@ -1549,8 +1611,12 @@ def action_install_python_env(args): # type: ignore warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.') env_copy['PIP_USER'] = 'no' run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location'] - requirements_txt = os.path.join(global_idf_path, 'requirements.txt') - run_args += ['-r', requirements_txt] + requirements_file_list = get_requirements(args.features) + for requirement_file in requirements_file_list: + run_args += ['-r', requirement_file] + if use_constraints: + constraint_file = get_constraints(idf_version) + run_args += ['--upgrade', '--constraint', constraint_file] if args.extra_wheels_dir: run_args += ['--find-links', args.extra_wheels_dir] if args.no_index: @@ -1562,10 +1628,58 @@ def action_install_python_env(args): # type: ignore if wheels_dir is not None: run_args += ['--find-links', wheels_dir] - info('Installing Python packages from {}'.format(requirements_txt)) + info('Installing Python packages') + if use_constraints: + info(' Constraint file: {}'.format(constraint_file)) + info(' Requirement files:') + info(os.linesep.join(' - {}'.format(path) for path in requirements_file_list)) subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr, env=env_copy) +def action_check_python_dependencies(args): # type: ignore + use_constraints = not args.no_constraints + req_paths = get_requirements('') # no new features -> just detect the existing ones + + _, _, virtualenv_python, idf_version = get_python_env_path() + + if not os.path.isfile(virtualenv_python): + fatal('{} doesn\'t exist! Please run the install script or "idf_tools.py install-python-env" in order to ' + 'create it'.format(virtualenv_python)) + raise SystemExit(1) + + if use_constraints: + constr_path = get_constraints(idf_version) + info('Constraint file: {}'.format(constr_path)) + + info('Requirement files:') + info(os.linesep.join(' - {}'.format(path) for path in req_paths)) + + info('Python being checked: {}'.format(virtualenv_python)) + + # The dependency checker will be invoked with virtualenv_python. idf_tools.py could have been invoked with a + # different one, therefore, importing is not a suitable option. + dep_check_cmd = [virtualenv_python, + os.path.join(global_idf_path, + 'tools', + 'check_python_dependencies.py')] + + if use_constraints: + dep_check_cmd += ['-c', constr_path] + + for req_path in req_paths: + dep_check_cmd += ['-r', req_path] + + try: + ret = subprocess.run(dep_check_cmd) + if ret and ret.returncode: + # returncode is a negative number and system exit output is usually expected be positive. + raise SystemExit(-ret.returncode) + except FileNotFoundError: + # Python environment not yet created + fatal('Requirements are not satisfied!') + raise SystemExit(1) + + def action_add_version(args): # type: ignore tools_info = load_tools_info() tool_name = args.tool @@ -1771,6 +1885,11 @@ def main(argv): # type: (list[str]) -> None 'to use during installation') install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default='https://dl.espressif.com/pypi') install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index') + install_python_env.add_argument('--features', default='core', help='A comma separated list of desired features for installing.' + ' It defaults to installing just the core funtionality.') + install_python_env.add_argument('--no-constraints', action='store_true', default=False, + help='Disable constraint settings. Use with care and only when you want to manage ' + 'package versions by yourself.') if IDF_MAINTAINER: add_version = subparsers.add_parser('add-version', help='Add or update download info for a version') @@ -1790,6 +1909,12 @@ def main(argv): # type: (list[str]) -> None help='Output file name') gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~') + check_python_dependencies = subparsers.add_parser('check-python-dependencies', + help='Check that all required Python packages are installed.') + check_python_dependencies.add_argument('--no-constraints', action='store_true', default=False, + help='Disable constraint settings. Use with care and only when you want ' + 'to manage package versions by yourself.') + args = parser.parse_args(argv) if args.action is None: diff --git a/tools/install_util.py b/tools/install_util.py new file mode 100644 index 0000000000..5b66e09fa2 --- /dev/null +++ b/tools/install_util.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 + +# This script is used from the $IDF_PATH/install.* scripts. This way the argument parsing can be done at one place and +# doesn't have to be implemented for all shells. + +import argparse +from itertools import chain + +try: + import python_version_checker + + # check the Python version before it will fail with an exception on syntax or package incompatibility. + python_version_checker.check() +except RuntimeError as e: + print(e) + raise SystemExit(1) + + +def action_extract_features(args: str) -> None: + """ + Command line arguments starting with "--enable-" are features. This function selects those and prints them. + """ + features = ['core'] # "core" features should be always installed + + if args: + arg_prefix = '--enable-' + features += [arg[len(arg_prefix):] for arg in args.split() if arg.startswith(arg_prefix)] + + print(','.join(features)) + + +def action_extract_targets(args: str) -> None: + """ + Command line arguments starting with "esp" are chip targets. This function selects those and prints them. + """ + target_sep = ',' + targets = [] + + if args: + target_args = (arg for arg in args.split() if arg.lower().startswith('esp')) + # target_args can be comma-separated lists of chip targets + + targets = list(chain.from_iterable(commalist.split(target_sep) for commalist in target_args)) + + print(target_sep.join(targets or ['all'])) + + +def main() -> None: + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(dest='action', required=True) + extract = subparsers.add_parser('extract', help='Process arguments and extract part of it') + + extract.add_argument('type', choices=['targets', 'features']) + extract.add_argument('str-to-parse', nargs='?') + + args, unknown_args = parser.parse_known_args() + # standalone "--enable-" won't be included into str-to-parse + + action_func = globals()['action_{}_{}'.format(args.action, args.type)] + str_to_parse = vars(args)['str-to-parse'] or '' + action_func(' '.join(chain([str_to_parse], unknown_args))) + + +if __name__ == '__main__': + main() diff --git a/tools/test_idf_tools/test_idf_tools_python_env.py b/tools/test_idf_tools/test_idf_tools_python_env.py new file mode 100644 index 0000000000..9f602fc92e --- /dev/null +++ b/tools/test_idf_tools/test_idf_tools_python_env.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import subprocess +import sys +import unittest +from typing import List + +try: + import idf_tools +except ImportError: + sys.path.append('..') + import idf_tools + +IDF_PATH = os.environ.get('IDF_PATH', '../..') +TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT) +PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env') +REQ_SATISFIED = 'Python requirements are satisfied' +REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH) +REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH) +CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR) + + +class TestPythonInstall(unittest.TestCase): + + def setUp(self): # type: () -> None + if os.path.isdir(PYTHON_DIR): + shutil.rmtree(PYTHON_DIR) + if os.path.isfile(os.path.join(TOOLS_DIR, 'idf-env.json')): + os.remove(os.path.join(TOOLS_DIR, 'idf-env.json')) + + def run_idf_tools(self, extra_args): # type: (List[str]) -> str + args = [sys.executable, '../idf_tools.py'] + extra_args + ret = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120) + return ret.stdout.decode('utf-8', 'ignore') + + def test_default_arguments(self): # type: () -> None + output = self.run_idf_tools(['check-python-dependencies']) + self.assertNotIn(REQ_SATISFIED, output) + self.assertIn('bin/python doesn\'t exist', output) + + output = self.run_idf_tools(['install-python-env']) + self.assertIn(CONSTR, output) + self.assertIn(REQ_CORE, output) + self.assertNotIn(REQ_GDBGUI, output) + + output = self.run_idf_tools(['check-python-dependencies']) + self.assertIn(REQ_SATISFIED, output) + + def test_opt_argument(self): # type: () -> None + output = self.run_idf_tools(['install-python-env', '--features', 'gdbgui']) + self.assertIn(CONSTR, output) + self.assertIn(REQ_CORE, output) + self.assertIn(REQ_GDBGUI, output) + + output = self.run_idf_tools(['install-python-env']) + # The gdbgui should be installed as well because the feature is is stored in the JSON file + self.assertIn(CONSTR, output) + self.assertIn(REQ_CORE, output) + self.assertIn(REQ_GDBGUI, output) + + def test_no_constraints(self): # type: () -> None + output = self.run_idf_tools(['install-python-env', '--no-constraints']) + self.assertNotIn(CONSTR, output) + self.assertIn(REQ_CORE, output) + + +if __name__ == '__main__': + unittest.main()