From b5e6b883a5bf3a99a13d78c4d92ee884ae5ba717 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 16 Feb 2024 09:32:19 +0100 Subject: [PATCH] ci: fix test related apps are force been built --- .gitlab/ci/build.yml | 2 - .../scripts/generate_build_child_pipeline.py | 51 ++++++++++++++----- tools/ci/idf_pytest/script.py | 3 +- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 06d4d7bd91..5e8307530d 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -266,8 +266,6 @@ generate_build_child_pipeline: when: always script: - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} build_child_pipeline: stage: build diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py index c9b8d842de..9c77e83c7f 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -1,27 +1,47 @@ # SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - """This file is used for generating the child pipeline for build jobs.""" - import argparse import os import typing as t import __init__ # noqa: F401 # inject the system path import yaml -from dynamic_pipelines.constants import (DEFAULT_APPS_BUILD_PER_JOB, DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, - DEFAULT_TEST_PATHS, NON_TEST_RELATED_APPS_FILENAME, - NON_TEST_RELATED_BUILD_JOB_NAME, TEST_RELATED_APPS_FILENAME, - TEST_RELATED_BUILD_JOB_NAME) -from dynamic_pipelines.models import BuildJob, EmptyJob +from dynamic_pipelines.constants import DEFAULT_APPS_BUILD_PER_JOB +from dynamic_pipelines.constants import DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH +from dynamic_pipelines.constants import DEFAULT_TEST_PATHS +from dynamic_pipelines.constants import NON_TEST_RELATED_APPS_FILENAME +from dynamic_pipelines.constants import NON_TEST_RELATED_BUILD_JOB_NAME +from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME +from dynamic_pipelines.constants import TEST_RELATED_BUILD_JOB_NAME +from dynamic_pipelines.models import BuildJob +from dynamic_pipelines.models import EmptyJob from dynamic_pipelines.utils import dump_jobs_to_yaml from idf_build_apps.utils import semicolon_separated_str_to_list from idf_ci.app import dump_apps_to_txt from idf_ci_utils import IDF_PATH -from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR, DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, CollectMode +from idf_pytest.constants import CollectMode +from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR +from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_FILEPATTERNS from idf_pytest.script import get_all_apps +def _separate_str_to_list(s: str) -> t.List[str]: + """ + Gitlab env file will escape the doublequotes in the env file, so we need to remove them + + For example, + + in pipeline.env file we have + + MR_MODIFIED_COMPONENTS="app1;app2" + MR_MODIFIED_FILES="main/app1.c;main/app2.c" + + gitlab will load the doublequotes as well, so we need to remove the doublequotes + """ + return semicolon_separated_str_to_list(s.strip('"')) # type: ignore + + def main(arguments: argparse.Namespace) -> None: # load from default build test rules config file extra_default_build_targets: t.List[str] = [] @@ -146,7 +166,8 @@ if __name__ == '__main__': ) parser.add_argument( '--modified-components', - type=semicolon_separated_str_to_list, + type=_separate_str_to_list, + default=os.getenv('MR_MODIFIED_COMPONENTS'), help='semicolon-separated string which specifies the modified components. ' 'app with `depends_components` set in the corresponding manifest files would only be built ' 'if depends on any of the specified components. ' @@ -155,7 +176,8 @@ if __name__ == '__main__': ) parser.add_argument( '--modified-files', - type=semicolon_separated_str_to_list, + type=_separate_str_to_list, + default=os.getenv('MR_MODIFIED_FILES'), help='semicolon-separated string which specifies the modified files. ' 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' 'if any of the specified file pattern matches any of the specified modified files. ' @@ -165,7 +187,7 @@ if __name__ == '__main__': parser.add_argument( '-if', '--ignore-app-dependencies-filepatterns', - type=semicolon_separated_str_to_list, + type=_separate_str_to_list, help='semicolon-separated string which specifies the file patterns used for ' 'ignoring checking the app dependencies. ' 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' @@ -188,7 +210,12 @@ if __name__ == '__main__': args.modified_files = None args.ignore_app_dependencies_filepatterns = None else: - print('Build and run only test cases matching the modified components and files') + print( + f'Build and run only test cases matching:\n' + f'- modified components: {args.modified_components}\n' + f'- modified files: {args.modified_files}' + ) + if args.modified_files and not args.ignore_app_dependencies_filepatterns: # setting default values args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index 6dabe65863..93dd6b1a67 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -213,8 +213,7 @@ def get_all_apps( elif app.build_status != BuildStatus.SKIPPED: if case := pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)): test_related_apps.add(app) - # should be built if - app.build_status = BuildStatus.SHOULD_BE_BUILT + # build or not should be decided by the build stage app.preserve = True logging.debug('Found test-related app: %s - required by %s', app, case.path) else: