From 4746a71028b4b3810c8797ba0110cf49ae412ec4 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 18 May 2022 14:59:34 +0800 Subject: [PATCH] ci: add qemu example --- .gitlab-ci.yml | 5 ++++- .gitlab/ci/host-test.yml | 9 ++++++++ conftest.py | 18 +++++++++++++++ examples/get-started/hello_world/README.md | 8 +++---- .../get-started/hello_world/example_test.py | 20 ----------------- .../hello_world/pytest_hello_world.py | 22 +++++++++++++++++++ pytest.ini | 12 +++++++--- tools/ci/build_pytest_apps.py | 12 ++++++++-- tools/ci/idf_ci_utils.py | 22 ++++++++++++------- 9 files changed, 89 insertions(+), 39 deletions(-) delete mode 100644 examples/get-started/hello_world/example_test.py create mode 100644 examples/get-started/hello_world/pytest_hello_world.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4401e10dd8..438c5231ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,6 +62,7 @@ variables: AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:2-1" CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:2-1" TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2" + QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:2-20210826" SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3" LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells-v5.0:2" @@ -209,13 +210,15 @@ before_script: - fetch_submodules - *download_test_python_contraint_file - $IDF_PATH/tools/idf_tools.py install-python-env + # TODO: remove this, IDFCI-1207 + - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - pip install "pytest-embedded-serial-esp~=$PYTEST_EMBEDDED_VERSION" "pytest-embedded-idf~=$PYTEST_EMBEDDED_VERSION" + "pytest-embedded-qemu~=$PYTEST_EMBEDDED_VERSION" pytest-rerunfailures scapy google-api-python-client - - cd $IDF_PATH - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index a7baf3531d..29783e2201 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -434,3 +434,12 @@ test_gen_soc_caps_kconfig: script: - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/ - ./test/test_gen_soc_caps_kconfig.py + +test_pytest_qemu: + extends: + - .host_test_template + - .before_script_pytest + image: $QEMU_IMAGE + script: + - run_cmd python tools/ci/build_pytest_apps.py . --target esp32 -m qemu -vv + - pytest --target esp32 -m qemu --embedded-services idf,qemu diff --git a/conftest.py b/conftest.py index b2730058de..29c08a87f1 100644 --- a/conftest.py +++ b/conftest.py @@ -31,6 +31,7 @@ from _pytest.runner import CallInfo from _pytest.terminal import TerminalReporter from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture from pytest_embedded.utils import find_by_suffix +from pytest_embedded_idf.dut import IdfDut SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2'] PREVIEW_TARGETS = ['linux', 'esp32h2'] @@ -74,6 +75,23 @@ def session_tempdir() -> str: return _TEST_SESSION_TMPDIR +@pytest.fixture() +def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]: + def real_func() -> None: + res = dut.expect(r'Minimum free heap size: (\d+) bytes') + logging.info('\n------ heap size info ------\n' + '[app_name] {}\n' + '[config_name] {}\n' + '[target] {}\n' + '[minimum_free_heap_size] {} Bytes\n' + '------ heap size end ------'.format(os.path.basename(dut.app.app_path), + config, + dut.target, + res.group(1).decode('utf8'))) + + return real_func + + @pytest.fixture @multi_dut_argument def config(request: FixtureRequest) -> str: diff --git a/examples/get-started/hello_world/README.md b/examples/get-started/hello_world/README.md index 7afca17e2f..d47aefec11 100644 --- a/examples/get-started/hello_world/README.md +++ b/examples/get-started/hello_world/README.md @@ -24,12 +24,10 @@ Below is short explanation of remaining files in the project folder. ``` ├── CMakeLists.txt -├── example_test.py Python script used for automated example testing +├── pytest_hello_world.py Python script used for automated testing ├── main -│   ├── CMakeLists.txt -│   ├── component.mk Component make file -│   └── hello_world_main.c -├── Makefile Makefile used by legacy GNU Make +│ ├── CMakeLists.txt +│ └── hello_world_main.c └── README.md This is the file you are currently reading ``` diff --git a/examples/get-started/hello_world/example_test.py b/examples/get-started/hello_world/example_test.py deleted file mode 100644 index 9901d5c1ca..0000000000 --- a/examples/get-started/hello_world/example_test.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python - -from __future__ import division, print_function, unicode_literals - -import ttfw_idf - - -@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2', 'esp32c3'], ci_target=['esp32']) -def test_examples_hello_world(env, extra_data): - app_name = 'hello_world' - dut = env.get_dut(app_name, 'examples/get-started/hello_world') - dut.start_app() - res = dut.expect(ttfw_idf.MINIMUM_FREE_HEAP_SIZE_RE) - if not res: - raise ValueError('Maximum heap size info not found') - ttfw_idf.print_heap_size(app_name, dut.app.config_name, dut.TARGET, res[0]) - - -if __name__ == '__main__': - test_examples_hello_world() diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py new file mode 100644 index 0000000000..3cb161c774 --- /dev/null +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +from typing import Callable + +import pytest +from pytest_embedded_idf.dut import IdfDut +from pytest_embedded_qemu.dut import QemuDut + + +@pytest.mark.supported_targets +@pytest.mark.generic +def test_hello_world(dut: IdfDut, log_minimum_free_heap_size: Callable[..., None]) -> None: + dut.expect('Hello world!') + log_minimum_free_heap_size() + + +@pytest.mark.esp32 # we only support qemu on esp32 for now +@pytest.mark.host_test +@pytest.mark.qemu +def test_hello_world_host(dut: QemuDut) -> None: + dut.expect('Hello world!') diff --git a/pytest.ini b/pytest.ini index 35ba5c550e..1ee9797cc3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,14 +7,16 @@ python_files = pytest_*.py addopts = -s --embedded-services esp,idf - -W ignore::_pytest.warning_types.PytestExperimentalApiWarning --tb short # ignore DeprecationWarning filterwarnings = - ignore:Call to deprecated create function (.*)\(\):DeprecationWarning + ignore::DeprecationWarning:matplotlib.*: + ignore::DeprecationWarning:google.protobuf.*: + ignore::_pytest.warning_types.PytestExperimentalApiWarning markers = + # target markers esp32: support esp32 target esp32s2: support esp32s2 target esp32s3: support esp32s3 target @@ -36,9 +38,13 @@ markers = ir_transceiver: runners with a pair of IR transmitter and receiver wifi: wifi runner - ## multi-dut markers + # multi-dut markers multi_dut_generic: tests should be run on generic runners, at least have two duts connected. + # host_test markers + host_test: tests which shouldn't be built at the build stage, and instead built in host_test stage. + qemu: build and test using qemu-system-xtensa, not real target. + # log related log_cli = True log_cli_level = INFO diff --git a/tools/ci/build_pytest_apps.py b/tools/ci/build_pytest_apps.py index c8ed196f7d..7e57cdb3f0 100644 --- a/tools/ci/build_pytest_apps.py +++ b/tools/ci/build_pytest_apps.py @@ -30,7 +30,7 @@ except ImportError: def main(args: argparse.Namespace) -> None: pytest_cases: List[PytestCase] = [] for path in args.paths: - pytest_cases += get_pytest_cases(path, args.target) + pytest_cases += get_pytest_cases(path, args.target, args.marker_expr) paths = set() app_configs = defaultdict(set) @@ -94,7 +94,15 @@ if __name__ == '__main__': parser = argparse.ArgumentParser( description='Build all the pytest apps under specified paths. Will auto remove those non-test apps binaries' ) - parser.add_argument('--target', required=True, help='Build apps for given target.') + parser.add_argument( + '-t', '--target', required=True, help='Build apps for given target.' + ) + parser.add_argument( + '-m', + '--marker-expr', + default='not host_test', # host_test apps would be built and tested under the same job + help='only build tests matching given mark expression. For example: -m "host_test and generic".', + ) parser.add_argument( '--config', default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 460cf640a0..2b5ae27940 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -12,12 +12,11 @@ import subprocess import sys from contextlib import redirect_stdout from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, List, Set +from typing import TYPE_CHECKING, Any, List, Optional, Set if TYPE_CHECKING: from _pytest.python import Function - IDF_PATH = os.path.abspath( os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')) ) @@ -158,7 +157,7 @@ class PytestCollectPlugin: return item.callspec.params.get(key, default) or default - def pytest_collection_modifyitems(self, items: List['Function']) -> None: + def pytest_report_collectionfinish(self, items: List['Function']) -> None: from pytest_embedded.plugin import parse_multi_dut_args for item in items: @@ -195,17 +194,22 @@ class PytestCollectPlugin: self.cases.append(PytestCase(case_path, case_name, case_apps)) -def get_pytest_cases(folder: str, target: str) -> List[PytestCase]: +def get_pytest_cases( + folder: str, target: str, marker_expr: Optional[str] = None +) -> List[PytestCase]: import pytest from _pytest.config import ExitCode collector = PytestCollectPlugin(target) + if marker_expr: + marker_expr = f'{target} and ({marker_expr})' + else: + marker_expr = target # target is also a marker with io.StringIO() as buf: with redirect_stdout(buf): res = pytest.main( - ['--collect-only', folder, '-q', '--target', target], - plugins=[collector], + ['--collect-only', folder, '-q', '-m', marker_expr], plugins=[collector] ) if res.value != ExitCode.OK: if res.value == ExitCode.NO_TESTS_COLLECTED: @@ -219,7 +223,9 @@ def get_pytest_cases(folder: str, target: str) -> List[PytestCase]: return collector.cases -def get_pytest_app_paths(folder: str, target: str) -> Set[str]: - cases = get_pytest_cases(folder, target) +def get_pytest_app_paths( + folder: str, target: str, marker_expr: Optional[str] = None +) -> Set[str]: + cases = get_pytest_cases(folder, target, marker_expr) return set({app.path for case in cases for app in case.apps})