From c8dffc9378380f6d0ef34bfe152fee6f49fdb71c Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 12 Jan 2024 14:42:00 +0100 Subject: [PATCH] ci: fix collect multi-dut test case with markers issue While collecting, we stop registering the new plugin. Otherwise the new created plugin will override the one we passed also run the tests inside idf. the behavior is different. --- conftest.py | 16 ++++++------- pytest.ini | 1 + .../templates/.dynamic_jobs.yml | 1 - tools/ci/idf_pytest/plugin.py | 23 ++++++++++++------ tools/ci/idf_pytest/tests/conftest.py | 15 ++++++++++-- .../ci/idf_pytest/tests/test_get_all_apps.py | 24 +++++++++++++++++-- .../idf_pytest/tests/test_get_pytest_cases.py | 8 ++++--- 7 files changed, 65 insertions(+), 23 deletions(-) diff --git a/conftest.py b/conftest.py index f212ff4cc3..92a6fceb44 100644 --- a/conftest.py +++ b/conftest.py @@ -1,14 +1,13 @@ # SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # pylint: disable=W0621 # redefined-outer-name - +# # IDF is using [pytest](https://github.com/pytest-dev/pytest) and # [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its test framework. - +# # if you found any bug or have any question, # please report to https://github.com/espressif/pytest-embedded/issues # or discuss at https://github.com/espressif/pytest-embedded/discussions - import os import sys @@ -432,11 +431,12 @@ def pytest_configure(config: Config) -> None: for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): apps.extend(import_apps_from_txt(f)) - config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( - target=target, - apps=apps, - ) - config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) + if '--collect-only' not in config.invocation_params.args: + config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( + target=target, + apps=apps, + ) + config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) def pytest_unconfigure(config: Config) -> None: diff --git a/pytest.ini b/pytest.ini index 08ef25dfe7..82e028d894 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,6 +13,7 @@ addopts = --logfile-extension ".txt" --check-duplicates y --ignore-glob */managed_components/* + --ignore pytest_embedded_log # ignore DeprecationWarning filterwarnings = diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 6ee7569c94..e41f2d9980 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -82,6 +82,5 @@ --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern "list_job_*.txt" after_script: - python tools/ci/artifacts_handler.py upload --type logs junit_reports diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index af9e61e247..7cf1a4b774 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import os import typing as t from collections import defaultdict @@ -16,11 +15,20 @@ from idf_build_apps import App from idf_build_apps.constants import BuildStatus from pytest_embedded import Dut from pytest_embedded.plugin import parse_multi_dut_args -from pytest_embedded.utils import find_by_suffix, to_list -from pytest_ignore_test_results.ignore_results import ChildCase, ChildCasesStashKey +from pytest_embedded.utils import find_by_suffix +from pytest_embedded.utils import to_list +from pytest_ignore_test_results.ignore_results import ChildCase +from pytest_ignore_test_results.ignore_results import ChildCasesStashKey -from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, CollectMode, PytestApp, PytestCase -from .utils import comma_sep_str_to_list, format_case_id, merge_junit_files +from .constants import CollectMode +from .constants import DEFAULT_SDKCONFIG +from .constants import PREVIEW_TARGETS +from .constants import PytestApp +from .constants import PytestCase +from .constants import SUPPORTED_TARGETS +from .utils import comma_sep_str_to_list +from .utils import format_case_id +from .utils import merge_junit_files IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']() ITEM_FAILED_CASES_KEY = pytest.StashKey[list]() @@ -43,9 +51,10 @@ class IdfPytestEmbedded: apps: t.Optional[t.List[App]] = None, ): if isinstance(target, str): - self.target = sorted(comma_sep_str_to_list(target)) + # sequence also matters + self.target = comma_sep_str_to_list(target) else: - self.target = sorted(target) + self.target = target if not self.target: raise ValueError('`target` should not be empty') diff --git a/tools/ci/idf_pytest/tests/conftest.py b/tools/ci/idf_pytest/tests/conftest.py index bfd05d8dfb..b2244bd5b6 100644 --- a/tools/ci/idf_pytest/tests/conftest.py +++ b/tools/ci/idf_pytest/tests/conftest.py @@ -1,10 +1,12 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import os import sys +import tempfile from pathlib import Path +import pytest + tools_ci_dir = os.path.join(os.path.dirname(__file__), '..', '..') if tools_ci_dir not in sys.path: sys.path.append(tools_ci_dir) @@ -13,6 +15,8 @@ tools_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..') if tools_dir not in sys.path: sys.path.append(tools_dir) +from idf_ci_utils import IDF_PATH # noqa: E402 + def create_project(name: str, folder: Path) -> Path: p = folder / name @@ -46,3 +50,10 @@ void app_main(void) {} ) return p + + +@pytest.fixture +def tmp_path() -> Path: + os.makedirs(os.path.join(IDF_PATH, 'pytest_embedded_log'), exist_ok=True) + + return Path(tempfile.mkdtemp(prefix=os.path.join(IDF_PATH, 'pytest_embedded_log') + os.sep)) diff --git a/tools/ci/idf_pytest/tests/test_get_all_apps.py b/tools/ci/idf_pytest/tests/test_get_all_apps.py index e2aa2c95b2..d4d53baee8 100644 --- a/tools/ci/idf_pytest/tests/test_get_all_apps.py +++ b/tools/ci/idf_pytest/tests/test_get_all_apps.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - from pathlib import Path -from idf_pytest.script import SUPPORTED_TARGETS, get_all_apps +from idf_pytest.script import get_all_apps +from idf_pytest.script import SUPPORTED_TARGETS from conftest import create_project @@ -38,6 +38,26 @@ def test_foo(dut): assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 +def test_get_all_apps_multi_dut_with_markers_test_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + + (tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_with_markers_test_script.py').write_text( + """import pytest + +@pytest.mark.esp32 +@pytest.mark.parametrize('count', [2, 3], indirect=True) +def test_foo(dut): + pass +""", + encoding='utf-8', + ) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + + assert len(test_related_apps) == 1 + assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 1 + + def test_get_all_apps_multi_dut_test_script(tmp_path: Path) -> None: create_project('foo', tmp_path) with open(tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw: diff --git a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py index e078146ce4..e221fef935 100644 --- a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py +++ b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py @@ -1,6 +1,5 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - from pathlib import Path from idf_pytest.constants import CollectMode @@ -45,11 +44,14 @@ def test_get_pytest_cases_single_specific(tmp_path: Path) -> None: def test_get_pytest_cases_multi_specific(tmp_path: Path) -> None: script = tmp_path / 'pytest_get_pytest_cases_multi_specific.py' script.write_text(TEMPLATE_SCRIPT) - cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2, esp32s2') + cases = get_pytest_cases([str(tmp_path)], 'esp32s2,esp32s2, esp32s3') assert len(cases) == 1 assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2,esp32s2') # order matters + assert len(cases) == 0 + def test_get_pytest_cases_multi_all(tmp_path: Path) -> None: script = tmp_path / 'pytest_get_pytest_cases_multi_all.py'