ci: fix test related apps are force been built

.gitlab/ci/build.yml

@@ -266,8 +266,6 @@ generate_build_child_pipeline:
     when: always
   script:
     - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --modified-files ${MR_MODIFIED_FILES}
 
 build_child_pipeline:
   stage: build

tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py

@@ -1,27 +1,47 @@
 # SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 """This file is used for generating the child pipeline for build jobs."""
-
 import argparse
 import os
 import typing as t
 
 import __init__  # noqa: F401 # inject the system path
 import yaml
-from dynamic_pipelines.constants import (DEFAULT_APPS_BUILD_PER_JOB, DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH,
-                                         DEFAULT_TEST_PATHS, NON_TEST_RELATED_APPS_FILENAME,
-                                         NON_TEST_RELATED_BUILD_JOB_NAME, TEST_RELATED_APPS_FILENAME,
-                                         TEST_RELATED_BUILD_JOB_NAME)
-from dynamic_pipelines.models import BuildJob, EmptyJob
+from dynamic_pipelines.constants import DEFAULT_APPS_BUILD_PER_JOB
+from dynamic_pipelines.constants import DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH
+from dynamic_pipelines.constants import DEFAULT_TEST_PATHS
+from dynamic_pipelines.constants import NON_TEST_RELATED_APPS_FILENAME
+from dynamic_pipelines.constants import NON_TEST_RELATED_BUILD_JOB_NAME
+from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME
+from dynamic_pipelines.constants import TEST_RELATED_BUILD_JOB_NAME
+from dynamic_pipelines.models import BuildJob
+from dynamic_pipelines.models import EmptyJob
 from dynamic_pipelines.utils import dump_jobs_to_yaml
 from idf_build_apps.utils import semicolon_separated_str_to_list
 from idf_ci.app import dump_apps_to_txt
 from idf_ci_utils import IDF_PATH
-from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR, DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, CollectMode
+from idf_pytest.constants import CollectMode
+from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR
+from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
 from idf_pytest.script import get_all_apps
 
 
+def _separate_str_to_list(s: str) -> t.List[str]:
+    """
+    Gitlab env file will escape the doublequotes in the env file, so we need to remove them
+
+    For example,
+
+    in pipeline.env file we have
+
+    MR_MODIFIED_COMPONENTS="app1;app2"
+    MR_MODIFIED_FILES="main/app1.c;main/app2.c"
+
+    gitlab will load the doublequotes as well, so we need to remove the doublequotes
+    """
+    return semicolon_separated_str_to_list(s.strip('"'))  # type: ignore
+
+
 def main(arguments: argparse.Namespace) -> None:
     # load from default build test rules config file
     extra_default_build_targets: t.List[str] = []
@@ -146,7 +166,8 @@ if __name__ == '__main__':
     )
     parser.add_argument(
         '--modified-components',
-        type=semicolon_separated_str_to_list,
+        type=_separate_str_to_list,
+        default=os.getenv('MR_MODIFIED_COMPONENTS'),
         help='semicolon-separated string which specifies the modified components. '
         'app with `depends_components` set in the corresponding manifest files would only be built '
         'if depends on any of the specified components. '
@@ -155,7 +176,8 @@ if __name__ == '__main__':
     )
     parser.add_argument(
         '--modified-files',
-        type=semicolon_separated_str_to_list,
+        type=_separate_str_to_list,
+        default=os.getenv('MR_MODIFIED_FILES'),
         help='semicolon-separated string which specifies the modified files. '
         'app with `depends_filepatterns` set in the corresponding manifest files would only be built '
         'if any of the specified file pattern matches any of the specified modified files. '
@@ -165,7 +187,7 @@ if __name__ == '__main__':
     parser.add_argument(
         '-if',
         '--ignore-app-dependencies-filepatterns',
-        type=semicolon_separated_str_to_list,
+        type=_separate_str_to_list,
         help='semicolon-separated string which specifies the file patterns used for '
         'ignoring checking the app dependencies. '
         'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored '
@@ -188,7 +210,12 @@ if __name__ == '__main__':
         args.modified_files = None
         args.ignore_app_dependencies_filepatterns = None
     else:
-        print('Build and run only test cases matching the modified components and files')
+        print(
+            f'Build and run only test cases matching:\n'
+            f'- modified components: {args.modified_components}\n'
+            f'- modified files: {args.modified_files}'
+        )
+
         if args.modified_files and not args.ignore_app_dependencies_filepatterns:
             # setting default values
             args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS

tools/ci/idf_pytest/script.py

@@ -213,8 +213,7 @@ def get_all_apps(
         elif app.build_status != BuildStatus.SKIPPED:
             if case := pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)):
                 test_related_apps.add(app)
-                # should be built if
-                app.build_status = BuildStatus.SHOULD_BE_BUILT
+                # build or not should be decided by the build stage
                 app.preserve = True
                 logging.debug('Found test-related app: %s - required by %s', app, case.path)
             else: