kopia lustrzana https://github.com/espressif/esp-idf
Merge branch 'ci/improve_build_scripts_for_multi_targets' into 'master'
CI: improve build scripts for multi targets See merge request espressif/esp-idf!17895pull/8962/head
Fu Hanxi
commit
38345d9cd8
|
|
@ -73,7 +73,7 @@ variables:
|
|||
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
|
||||
CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
|
||||
CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
|
||||
PYTEST_EMBEDDED_VERSION: "0.6.0"
|
||||
PYTEST_EMBEDDED_VERSION: "0.7.0"
|
||||
|
||||
# cache python dependencies
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
|
|
|
|||
|
|
@ -111,6 +111,16 @@ example_test_pytest_esp32_flash_encryption:
|
|||
TARGET: ESP32
|
||||
ENV_MARKER: flash_encryption
|
||||
|
||||
example_test_pytest_esp32_multi_dut_generic:
|
||||
extends:
|
||||
- .pytest_examples_dir_template
|
||||
- .rules:test:example_test-esp32
|
||||
needs:
|
||||
- build_pytest_examples_esp32
|
||||
variables:
|
||||
TARGET: ESP32
|
||||
ENV_MARKER: multi_dut_generic
|
||||
|
||||
example_test_pytest_esp32c3_flash_encryption:
|
||||
extends:
|
||||
- .pytest_examples_dir_template
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from _pytest.python import Function
|
|||
from _pytest.reports import TestReport
|
||||
from _pytest.runner import CallInfo
|
||||
from _pytest.terminal import TerminalReporter
|
||||
from pytest_embedded.plugin import apply_count, parse_configuration
|
||||
from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture
|
||||
from pytest_embedded.utils import find_by_suffix
|
||||
|
||||
SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2']
|
||||
|
|
@ -75,7 +75,7 @@ def session_tempdir() -> str:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
@parse_configuration
|
||||
@multi_dut_argument
|
||||
def config(request: FixtureRequest) -> str:
|
||||
return getattr(request, 'param', None) or DEFAULT_SDKCONFIG
|
||||
|
||||
|
|
@ -91,7 +91,7 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
@apply_count
|
||||
@multi_dut_fixture
|
||||
def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str:
|
||||
"""
|
||||
Check local build dir with the following priority:
|
||||
|
|
@ -137,7 +137,7 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st
|
|||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@apply_count
|
||||
@multi_dut_fixture
|
||||
def junit_properties(
|
||||
test_case_name: str, record_xml_attribute: Callable[[str, object], None]
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,11 @@ In ESP-IDF, we use the following plugins by default:
|
|||
|
||||
All the introduced concepts and usages are based on the default behavior in ESP-IDF. Not all of them are available in vanilla pytest.
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
``$ pip install -U pytest-embedded-serial-esp~=0.7.0 pytest-embedded-idf~=0.7.0``
|
||||
|
||||
Basic Concepts
|
||||
--------------
|
||||
|
||||
|
|
@ -95,12 +100,12 @@ Pytest Execution Process
|
|||
|
||||
1. Get all the python files with the prefix ``pytest_``
|
||||
2. Get all the test functions with the prefix ``test_``
|
||||
3. Apply the `params <https://docs.pytest.org/en/latest/how-to/parametrize.html>`__, duplicate the test functions.
|
||||
4. Filter the test cases with CLI options. Introduced detail usages `here <#filter-the-test-cases>`__
|
||||
3. Apply the `params <https://docs.pytest.org/en/latest/how-to/parametrize.html>`__, and duplicate the test functions.
|
||||
4. Filter the test cases with CLI options. Introduced detailed usages `here <#filter-the-test-cases>`__
|
||||
|
||||
3. Test Running Phase
|
||||
|
||||
1. Construct the `fixtures <https://docs.pytest.org/en/latest/how-to/fixtures.html>`__. In ESP-IDF, the common fixtures are initialized with this order:
|
||||
1. Construct the `fixtures <https://docs.pytest.org/en/latest/how-to/fixtures.html>`__. In ESP-IDF, the common fixtures are initialized in this order:
|
||||
|
||||
1. ``pexpect_proc``: `pexpect <https://github.com/pexpect/pexpect>`__ instance
|
||||
|
||||
|
|
@ -116,7 +121,7 @@ Pytest Execution Process
|
|||
|
||||
2. Run the real test function
|
||||
|
||||
3. Deconstruct the fixtures with this order:
|
||||
3. Deconstruct the fixtures in this order:
|
||||
|
||||
1. ``dut``
|
||||
|
||||
|
|
@ -160,7 +165,7 @@ This code example is taken from :idf_file:`pytest_console_basic.py <examples/sys
|
|||
elif config == 'nohistory':
|
||||
dut.expect('Command history disabled')
|
||||
|
||||
.. note::
|
||||
.. note::
|
||||
|
||||
Using ``expect_exact`` is better here. For further reading about the different types of ``expect`` functions, please refer to the `pytest-embedded Expecting documentation <https://docs.espressif.com/projects/pytest-embedded/en/latest/expecting>`__.
|
||||
|
||||
|
|
@ -198,7 +203,7 @@ You can use ``pytest.mark.parametrize`` with “config” to apply the same test
|
|||
'nohistory', # <-- run with app built by sdkconfig.ci.nohistory
|
||||
], indirect=True) # <-- `indirect=True` is required
|
||||
|
||||
Overall, this test case would be duplicated to 4 test functions:
|
||||
Overall, this test function would be replicated to 4 test cases:
|
||||
|
||||
- esp32.history.test_console_advanced
|
||||
- esp32.nohistory.test_console_advanced
|
||||
|
|
@ -208,6 +213,80 @@ Overall, this test case would be duplicated to 4 test functions:
|
|||
Advanced Examples
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Multi Dut Tests with the Same App
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This code example is taken from :idf_file:`pytest_usb_host.py <tools/test_apps/peripherals/usb/pytest_usb_host.py>`.
|
||||
|
||||
.. code:: python
|
||||
|
||||
@pytest.mark.esp32s2
|
||||
@pytest.mark.esp32s3
|
||||
@pytest.mark.usb_host
|
||||
@pytest.mark.parametrize('count', [
|
||||
2,
|
||||
], indirect=True)
|
||||
def test_usb_host(dut: Tuple[IdfDut, IdfDut]) -> None:
|
||||
device = dut[0] # <-- assume the first dut is the device
|
||||
host = dut[1] # <-- and the second dut is the host
|
||||
...
|
||||
|
||||
After setting the param ``count`` to 2, all these fixtures are changed into tuples.
|
||||
|
||||
Multi Dut Tests with Different Apps
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This code example is taken from :idf_file:`pytest_wifi_getting_started.py <examples/wifi/getting_started/pytest_wifi_getting_started.py>`.
|
||||
|
||||
.. code:: python
|
||||
|
||||
@pytest.mark.esp32
|
||||
@pytest.mark.multi_dut_generic
|
||||
@pytest.mark.parametrize(
|
||||
'count, app_path', [
|
||||
(2,
|
||||
f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'),
|
||||
], indirect=True
|
||||
)
|
||||
def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None:
|
||||
softap = dut[0]
|
||||
station = dut[1]
|
||||
...
|
||||
|
||||
Here the first dut was flashed with the app :idf_file:`softap <examples/wifi/getting_started/softAP/main/softap_example_main.c>`, and the second dut was flashed with the app :idf_file:`station <examples/wifi/getting_started/station/main/station_example_main.c>`.
|
||||
|
||||
.. note::
|
||||
|
||||
Here the ``app_path`` should be set with absolute path. the ``__file__`` macro in python would return the absolute path of the test script itself.
|
||||
|
||||
Multi Dut Tests with Different Apps, and Targets
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This code example is taken from :idf_file:`pytest_wifi_getting_started.py <examples/wifi/getting_started/pytest_wifi_getting_started.py>`. As the comment says, for now it's not running in the ESP-IDF CI.
|
||||
|
||||
.. code:: python
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'count, app_path, target', [
|
||||
(2,
|
||||
f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}',
|
||||
'esp32|esp32s2'),
|
||||
(2,
|
||||
f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}',
|
||||
'esp32s2|esp32'),
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None:
|
||||
softap = dut[0]
|
||||
station = dut[1]
|
||||
...
|
||||
|
||||
Overall, this test function would be replicated to 2 test cases:
|
||||
|
||||
- softap with esp32 target, and station with esp32s2 target
|
||||
- softap with esp32s2 target, and station with esp32 target
|
||||
|
||||
Support different targets with different sdkconfig files
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
|
@ -278,7 +357,7 @@ Mark Known Failure Cases
|
|||
Sometimes a test couldn't pass for the following reasons:
|
||||
|
||||
- Has a bug
|
||||
- Success ratio too low because of environment issue, such as network issue. Retry couldn't help
|
||||
- The success ratio is too low because of environment issue, such as network issue. Retry couldn't help
|
||||
|
||||
Now you may mark this test case with marker `xfail <https://docs.pytest.org/en/latest/how-to/skipping.html#xfail-mark-test-functions-as-expected-to-fail>`__ with a user-friendly readable reason.
|
||||
|
||||
|
|
@ -397,7 +476,7 @@ You can call pytest with ``--junitxml <filepath>`` to generate the JUnit report.
|
|||
Skip Auto Flash Binary
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Skipping auto-flash binary everytime would be useful when you're debugging your test script.
|
||||
Skipping auto-flash binary every time would be useful when you're debugging your test script.
|
||||
|
||||
You can call pytest with ``--skip-autoflash y`` to achieve it.
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,15 @@
|
|||
from __future__ import print_function
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import esp_prov
|
||||
except ImportError:
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tools'))
|
||||
import esp_prov
|
||||
|
||||
import esp_prov
|
||||
import pytest
|
||||
from pytest_embedded import Dut
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
import os.path
|
||||
from typing import Tuple
|
||||
|
||||
import pytest
|
||||
from pytest_embedded_idf.dut import IdfDut
|
||||
|
||||
# @pytest.mark.supported_targets
|
||||
# This test should support all targets, even between different target types
|
||||
# For now our CI only support multi dut with esp32
|
||||
# If you want to enable different target type, please use the following param
|
||||
# @pytest.mark.parametrize(
|
||||
# 'count, app_path, target', [
|
||||
# (2,
|
||||
# f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}',
|
||||
# 'esp32|esp32s2'),
|
||||
# ],
|
||||
# indirect=True,
|
||||
# )
|
||||
|
||||
|
||||
@pytest.mark.esp32
|
||||
@pytest.mark.multi_dut_generic
|
||||
@pytest.mark.parametrize(
|
||||
'count, app_path', [
|
||||
(2,
|
||||
f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'),
|
||||
], indirect=True
|
||||
)
|
||||
def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None:
|
||||
softap = dut[0]
|
||||
station = dut[1]
|
||||
|
||||
ssid = 'myssid'
|
||||
password = 'mypassword'
|
||||
tag = 'wifi station'
|
||||
|
||||
station.expect(f'{tag}: got ip:', timeout=60)
|
||||
station.expect(f'{tag}: connected to ap SSID:{ssid} password:{password}', timeout=60)
|
||||
softap.expect('station .+ join, AID=', timeout=60)
|
||||
|
|
@ -35,6 +35,9 @@ markers =
|
|||
flash_encryption: Flash Encryption runners
|
||||
ir_transceiver: runners with a pair of IR transmitter and receiver
|
||||
|
||||
## multi-dut markers
|
||||
multi_dut_generic: tests should be run on generic runners, at least have two duts connected.
|
||||
|
||||
# log related
|
||||
log_cli = True
|
||||
log_cli_level = INFO
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import sys
|
|||
from collections import defaultdict
|
||||
from typing import List
|
||||
|
||||
from idf_ci_utils import IDF_PATH, get_pytest_cases
|
||||
from idf_ci_utils import IDF_PATH, PytestCase, get_pytest_cases
|
||||
|
||||
try:
|
||||
from build_apps import build_apps
|
||||
|
|
@ -28,15 +28,16 @@ except ImportError:
|
|||
|
||||
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
pytest_cases = []
|
||||
pytest_cases: List[PytestCase] = []
|
||||
for path in args.paths:
|
||||
pytest_cases += get_pytest_cases(path, args.target)
|
||||
|
||||
paths = set()
|
||||
app_configs = defaultdict(set)
|
||||
for case in pytest_cases:
|
||||
paths.add(case.app_path)
|
||||
app_configs[case.app_path].add(case.config)
|
||||
for app in case.apps:
|
||||
paths.add(app.path)
|
||||
app_configs[app.path].add(app.config)
|
||||
|
||||
app_dirs = list(paths)
|
||||
if not app_dirs:
|
||||
|
|
|
|||
|
|
@ -1,19 +1,22 @@
|
|||
# internal use only for CI
|
||||
# some CI related util functions
|
||||
#
|
||||
# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import redirect_stdout
|
||||
from typing import TYPE_CHECKING, List
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, List, Set
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _pytest.nodes import Function
|
||||
from _pytest.python import Function
|
||||
|
||||
|
||||
IDF_PATH = os.path.abspath(
|
||||
os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
|
|
@ -113,39 +116,83 @@ def is_in_directory(file_path: str, folder: str) -> bool:
|
|||
return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)
|
||||
|
||||
|
||||
class PytestCase:
|
||||
def __init__(self, test_path: str, target: str, config: str, case: str):
|
||||
self.app_path = os.path.dirname(test_path)
|
||||
self.test_path = test_path
|
||||
self.target = target
|
||||
self.config = config
|
||||
self.case = case
|
||||
def to_list(s: Any) -> List[Any]:
|
||||
if isinstance(s, (set, tuple)):
|
||||
return list(s)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.test_path}: {self.target}.{self.config}.{self.case}'
|
||||
if isinstance(s, list):
|
||||
return s
|
||||
|
||||
return [s]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PytestApp:
|
||||
path: str
|
||||
target: str
|
||||
config: str
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.path, self.target, self.config))
|
||||
|
||||
|
||||
@dataclass
|
||||
class PytestCase:
|
||||
path: str
|
||||
name: str
|
||||
apps: Set[PytestApp]
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.path, self.name, self.apps))
|
||||
|
||||
|
||||
class PytestCollectPlugin:
|
||||
def __init__(self, target: str) -> None:
|
||||
self.target = target
|
||||
self.nodes: List[PytestCase] = []
|
||||
self.cases: List[PytestCase] = []
|
||||
|
||||
@staticmethod
|
||||
def get_param(item: 'Function', key: str, default: Any = None) -> Any:
|
||||
if not hasattr(item, 'callspec'):
|
||||
raise ValueError(f'Function {item} does not have params')
|
||||
|
||||
return item.callspec.params.get(key, default) or default
|
||||
|
||||
def pytest_collection_modifyitems(self, items: List['Function']) -> None:
|
||||
from pytest_embedded.plugin import parse_multi_dut_args
|
||||
|
||||
for item in items:
|
||||
try:
|
||||
file_path = str(item.path)
|
||||
except AttributeError:
|
||||
# pytest 6.x
|
||||
file_path = item.fspath
|
||||
|
||||
target = self.target
|
||||
if hasattr(item, 'callspec'):
|
||||
config = item.callspec.params.get('config', 'default')
|
||||
else:
|
||||
config = 'default'
|
||||
count = 1
|
||||
case_path = str(item.path)
|
||||
case_name = item.originalname
|
||||
target = self.target
|
||||
# funcargs is not calculated while collection
|
||||
if hasattr(item, 'callspec'):
|
||||
count = item.callspec.params.get('count', 1)
|
||||
app_paths = to_list(
|
||||
parse_multi_dut_args(
|
||||
count,
|
||||
self.get_param(item, 'app_path', os.path.dirname(case_path)),
|
||||
)
|
||||
)
|
||||
configs = to_list(
|
||||
parse_multi_dut_args(
|
||||
count, self.get_param(item, 'config', 'default')
|
||||
)
|
||||
)
|
||||
targets = to_list(
|
||||
parse_multi_dut_args(count, self.get_param(item, 'target', target))
|
||||
)
|
||||
else:
|
||||
app_paths = [os.path.dirname(case_path)]
|
||||
configs = ['default']
|
||||
targets = [target]
|
||||
|
||||
self.nodes.append(PytestCase(file_path, target, config, case_name))
|
||||
case_apps = set()
|
||||
for i in range(count):
|
||||
case_apps.add(PytestApp(app_paths[i], targets[i], configs[i]))
|
||||
|
||||
self.cases.append(PytestCase(case_path, case_name, case_apps))
|
||||
|
||||
|
||||
def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
|
||||
|
|
@ -156,18 +203,23 @@ def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
|
|||
|
||||
with io.StringIO() as buf:
|
||||
with redirect_stdout(buf):
|
||||
res = pytest.main(['--collect-only', folder, '-q', '--target', target], plugins=[collector])
|
||||
res = pytest.main(
|
||||
['--collect-only', folder, '-q', '--target', target],
|
||||
plugins=[collector],
|
||||
)
|
||||
if res.value != ExitCode.OK:
|
||||
if res.value == ExitCode.NO_TESTS_COLLECTED:
|
||||
print(f'WARNING: no pytest app found for target {target} under folder {folder}')
|
||||
print(
|
||||
f'WARNING: no pytest app found for target {target} under folder {folder}'
|
||||
)
|
||||
else:
|
||||
print(buf.getvalue())
|
||||
raise RuntimeError('pytest collection failed')
|
||||
|
||||
return collector.nodes
|
||||
return collector.cases
|
||||
|
||||
|
||||
def get_pytest_app_paths(folder: str, target: str) -> List[str]:
|
||||
nodes = get_pytest_cases(folder, target)
|
||||
def get_pytest_app_paths(folder: str, target: str) -> Set[str]:
|
||||
cases = get_pytest_cases(folder, target)
|
||||
|
||||
return list({node.app_path for node in nodes})
|
||||
return set({app.path for case in cases for app in case.apps})
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ class PanicTestDut(IdfDut):
|
|||
"""Extract the core dump from flash, run espcoredump on it"""
|
||||
coredump_file_name = os.path.join(self.logdir, 'coredump_data.bin')
|
||||
logging.info('Writing flash binary core dump to %s', coredump_file_name)
|
||||
self.serial.dump_flash(coredump_file_name, partition='coredump')
|
||||
self.serial.dump_flash(partition='coredump', output=coredump_file_name)
|
||||
|
||||
output_file_name = os.path.join(self.logdir, 'coredump_flash_result.txt')
|
||||
self._call_espcoredump(
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue