diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index a2edac37e5..485726059c 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -108,6 +108,11 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo * ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/tools/requirements/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day. +* ``uninstall``: Print and remove tools, that are currently not used by active ESP-IDF version. + + - ``--dry-run`` Print installed unused tools. + - ``--remove-archives`` Additionally remove all older versions of previously downloaded installation packages. + .. _idf-tools-install: Install scripts diff --git a/export.bat b/export.bat index 1faf0a0678..f5a46cee06 100644 --- a/export.bat +++ b/export.bat @@ -56,6 +56,11 @@ echo Checking if Python packages are up to date... python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies if %errorlevel% neq 0 goto :__end +python.exe "%IDF_PATH%\tools\idf_tools.py" uninstall --dry-run > UNINSTALL_OUTPUT +SET /p UNINSTALL= Tuple[str, str]: + if sys.platform == 'win32': + subdir = 'Scripts' + python_exe = 'python.exe' + else: + subdir = 'bin' + python_exe = 'python' + return python_exe, subdir + + def get_python_env_path(): # type: () -> Tuple[str, str, str, str] python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) @@ -1018,13 +1028,7 @@ def get_python_env_path(): # type: () -> Tuple[str, str, str, str] idf_python_env_path = os.path.join(global_idf_tools_path, 'python_env', # type: ignore 'idf{}_py{}_env'.format(idf_version, python_ver_major_minor)) - if sys.platform == 'win32': - subdir = 'Scripts' - python_exe = 'python.exe' - else: - subdir = 'bin' - python_exe = 'python' - + python_exe, subdir = get_python_exe_and_subdir() idf_python_export_path = os.path.join(idf_python_env_path, subdir) virtualenv_python = os.path.join(idf_python_export_path, python_exe) @@ -1374,39 +1378,53 @@ def apply_github_assets_option(tool_download_obj): # type: ignore tool_download_obj.url = new_url +def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec, + quiet=False): # type: (str, list[str], list[str], bool) -> Tuple[list[str], Dict[str, IDFTool]] + if selected_platform not in PLATFORM_FROM_NAME: + fatal(f'unknown platform: {selected_platform}') + raise SystemExit(1) + selected_platform = PLATFORM_FROM_NAME[selected_platform] + + # If this function is not called from action_download, but is used just for detecting active tools, info about downloading is unwanted. + global global_quiet + try: + old_global_quiet = global_quiet + global_quiet = quiet + tools_info = load_tools_info() + tools_info_for_platform = OrderedDict() + for name, tool_obj in tools_info.items(): + tool_for_platform = tool_obj.copy_for_platform(selected_platform) + tools_info_for_platform[name] = tool_for_platform + + if not tools_spec or 'required' in tools_spec: + # Downloading tools for all ESP_targets required by the operating system. + tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS] + # Filtering tools user defined list of ESP_targets + if 'all' not in targets: + def is_tool_selected(tool): # type: (IDFTool) -> bool + supported_targets = tool.get_supported_targets() + return (any(item in targets for item in supported_targets) or supported_targets == ['all']) + tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])] + info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec))) + + # Downloading tools for all ESP_targets (MacOS, Windows, Linux) + elif 'all' in tools_spec: + tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER] + info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec))) + finally: + global_quiet = old_global_quiet + + return tools_spec, tools_info_for_platform + + def action_download(args): # type: ignore - tools_info = load_tools_info() tools_spec = args.tools targets = [] # type: list[str] # Installing only single tools, no targets are specified. if 'required' in tools_spec: targets = add_and_save_targets(args.targets) - if args.platform not in PLATFORM_FROM_NAME: - fatal('unknown platform: {}' % args.platform) - raise SystemExit(1) - platform = PLATFORM_FROM_NAME[args.platform] - - tools_info_for_platform = OrderedDict() - for name, tool_obj in tools_info.items(): - tool_for_platform = tool_obj.copy_for_platform(platform) - tools_info_for_platform[name] = tool_for_platform - - if not tools_spec or 'required' in tools_spec: - # Downloading tools for all ESP_targets required by the operating system. - tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS] - # Filtering tools user defined list of ESP_targets - if 'all' not in targets: - def is_tool_selected(tool): # type: (IDFTool) -> bool - supported_targets = tool.get_supported_targets() - return (any(item in targets for item in supported_targets) or supported_targets == ['all']) - tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])] - info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec))) - - # Downloading tools for all ESP_targets (MacOS, Windows, Linux) - elif 'all' in tools_spec: - tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER] - info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec))) + tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(args.platform, targets, args.tools) for tool_spec in tools_spec: if '@' not in tool_spec: @@ -1747,6 +1765,75 @@ def action_rewrite(args): # type: ignore info('Wrote output to {}'.format(args.output)) +def action_uninstall(args): # type: (Any) -> None + """ Print or remove installed tools, that are currently not used by active ESP-IDF version. + Additionally remove all older versions of previously downloaded archives. + """ + + def is_tool_selected(tool): # type: (IDFTool) -> bool + supported_targets = tool.get_supported_targets() + return (supported_targets == ['all'] or any(item in targets for item in supported_targets)) + + tools_info = load_tools_info() + targets, _ = get_requested_targets_and_features() + tools_path = os.path.join(global_idf_tools_path or '', 'tools') + dist_path = os.path.join(global_idf_tools_path or '', 'dist') + used_tools = [k for k, v in tools_info.items() if (v.get_install_type() == IDFTool.INSTALL_ALWAYS and is_tool_selected(tools_info[k]))] + installed_tools = os.listdir(tools_path) if os.path.isdir(tools_path) else [] + unused_tools = [tool for tool in installed_tools if tool not in used_tools] + # Keeping tools added by windows installer + KEEP_WIN_TOOLS = ['idf-git', 'idf-python'] + for tool in KEEP_WIN_TOOLS: + if tool in unused_tools: + unused_tools.remove(tool) + + # Print unused tools. + if args.dry_run: + if unused_tools: + print('For removing {} use command \'{} {} {}\''.format(', '.join(unused_tools), get_python_exe_and_subdir()[0], + os.path.join(global_idf_path or '', 'tools', 'idf_tools.py'), 'uninstall')) + return + + # Remove installed tools that are not used by current ESP-IDF version. + for tool in unused_tools: + try: + shutil.rmtree(os.path.join(tools_path, tool)) + info(os.path.join(tools_path, tool) + ' was removed.') + except OSError as error: + warn(f'{error.filename} can not be removed because {error.strerror}.') + + # Remove old archives versions and archives that are not used by the current ESP-IDF version. + if args.remove_archives: + targets, _ = get_requested_targets_and_features() + tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(CURRENT_PLATFORM, targets, ['required'], quiet=True) + used_archives = [] + + # Detect used active archives + for tool_spec in tools_spec: + if '@' not in tool_spec: + tool_name = tool_spec + tool_version = None + else: + tool_name, tool_version = tool_spec.split('@', 1) + tool_obj = tools_info_for_platform[tool_name] + if tool_version is None: + tool_version = tool_obj.get_recommended_version() + # mypy-checks + if tool_version is not None: + archive_version = tool_obj.versions[tool_version].get_download_for_platform(CURRENT_PLATFORM) + if archive_version is not None: + archive_version_url = archive_version.url + + archive = os.path.basename(archive_version_url) + used_archives.append(archive) + + downloaded_archives = os.listdir(dist_path) + for archive in downloaded_archives: + if archive not in used_archives: + os.remove(os.path.join(dist_path, archive)) + info(os.path.join(dist_path, archive) + ' was removed.') + + def action_validate(args): # type: ignore try: import jsonschema @@ -1882,6 +1969,10 @@ def main(argv): # type: (list[str]) -> None download.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' + ' It defaults to installing all supported targets.') + uninstall = subparsers.add_parser('uninstall', help='Remove installed tools, that are not used by current version of ESP-IDF.') + uninstall.add_argument('--dry-run', help='Print unused tools.', action='store_true') + uninstall.add_argument('--remove-archives', help='Remove old archive versions and archives from unused tools.', action='store_true') + if IDF_MAINTAINER: for subparser in [download, install]: subparser.add_argument('--mirror-prefix-map', nargs='*', diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py index b5deaedc68..192cc3a918 100755 --- a/tools/test_idf_tools/test_idf_tools.py +++ b/tools/test_idf_tools/test_idf_tools.py @@ -92,6 +92,7 @@ class TestUsage(unittest.TestCase): print('Using IDF_TOOLS_PATH={}'.format(cls.temp_tools_dir)) os.environ['IDF_TOOLS_PATH'] = cls.temp_tools_dir + cls.idf_env_json = os.path.join(cls.temp_tools_dir, 'idf-env.json') @classmethod def tearDownClass(cls): @@ -104,8 +105,8 @@ class TestUsage(unittest.TestCase): if os.path.isdir(os.path.join(self.temp_tools_dir, 'tools')): shutil.rmtree(os.path.join(self.temp_tools_dir, 'tools')) - if os.path.isfile(os.path.join(self.temp_tools_dir, 'idf-env.json')): - os.remove(os.path.join(self.temp_tools_dir, 'idf-env.json')) + if os.path.isfile(self.idf_env_json): + os.remove(self.idf_env_json) def assert_tool_installed(self, output, tool, tool_version, tool_archive_name=None): if tool_archive_name is None: @@ -314,6 +315,23 @@ class TestUsage(unittest.TestCase): self.assertNotIn('%s/tools/xtensa-esp32s2-elf/%s/xtensa-esp32s2-elf/bin' % (self.temp_tools_dir, XTENSA_ESP32S2_ELF_VERSION), output) + def test_uninstall_option(self): + self.run_idf_tools_with_action(['install', '--targets=esp32,esp32c3']) + output = self.run_idf_tools_with_action(['uninstall', '--dry-run']) + self.assertEqual(output, '') + + with open(self.idf_env_json, 'r') as idf_env_file: + idf_env_json = json.load(idf_env_file) + idf_env_json['idfInstalled'][idf_env_json['idfSelectedId']]['targets'].remove('esp32') + with open(self.idf_env_json, 'w') as w: + json.dump(idf_env_json, w) + + output = self.run_idf_tools_with_action(['uninstall']) + self.assertIn(XTENSA_ESP32_ELF, output) + self.assertIn(ESP32ULP, output) + output = self.run_idf_tools_with_action(['uninstall', '--dry-run']) + self.assertEqual(output, '') + class TestMaintainer(unittest.TestCase):