ci: check warnings while generating the target test jobs

.gitlab/ci/build.yml

@@ -287,6 +287,7 @@ generate_build_child_pipeline:
   dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - pipeline_variables
+    - check_test_cases_env_markers_and_required_runners
   artifacts:
     paths:
       - build_child_pipeline.yml

.gitlab/ci/pre_check.yml

@@ -170,3 +170,9 @@ pipeline_variables:
   artifacts:
     reports:
       dotenv: pipeline.env
+
+check_test_cases_env_markers_and_required_runners:
+  extends:
+    - .pre_check_template
+  script:
+    - python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py --check

tools/ci/dynamic_pipelines/constants.py

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 
 from idf_ci_utils import IDF_PATH
@@ -31,3 +30,7 @@ REPORT_TEMPLATE_FILEPATH = os.path.join(
 )
 
 BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests'
+
+KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
+    IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
+)

tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py

@@ -1,28 +1,36 @@
 # SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 """This file is used for generating the child pipeline for target test jobs.
 
 1. Check the build jobs' artifacts to get the built apps' information.
 2. Post the Build Report if it's running in an MR pipeline.
 3. Generate the child pipeline for target test jobs.
 """
-
 import argparse
 import glob
+import logging
 import os
 import typing as t
-from collections import Counter, defaultdict
+from collections import Counter
+from collections import defaultdict
 
 import __init__  # noqa: F401 # inject the system path
-from dynamic_pipelines.constants import (BUILD_ONLY_LABEL, DEFAULT_CASES_TEST_PER_JOB,
-                                         DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, DEFAULT_TEST_PATHS)
-from dynamic_pipelines.models import EmptyJob, Job, TargetTestJob
+import yaml
+from dynamic_pipelines.constants import BUILD_ONLY_LABEL
+from dynamic_pipelines.constants import DEFAULT_CASES_TEST_PER_JOB
+from dynamic_pipelines.constants import DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH
+from dynamic_pipelines.constants import DEFAULT_TEST_PATHS
+from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH
+from dynamic_pipelines.models import EmptyJob
+from dynamic_pipelines.models import Job
+from dynamic_pipelines.models import TargetTestJob
 from dynamic_pipelines.utils import dump_jobs_to_yaml
 from gitlab.v4.objects import Project
 from gitlab_api import Gitlab
 from idf_build_apps import App
+from idf_build_apps.constants import BuildStatus
 from idf_ci.app import import_apps_from_txt
+from idf_pytest.script import get_all_apps
 from idf_pytest.script import get_pytest_cases
 
 
@@ -41,16 +49,23 @@ def get_tags_with_amount(s: str) -> t.List[str]:
     return sorted(res)
 
 
-def get_target_test_jobs(project: Project, paths: str, apps: t.List[App]) -> t.Tuple[t.List[Job], t.List[str]]:
+def get_target_test_jobs(
+    project: Project, paths: str, apps: t.List[App]
+) -> t.Tuple[t.List[Job], t.List[str], t.Dict[str, t.List[str]]]:
     """
     Return the target test jobs and the extra yaml files to include
     """
+    issues: t.Dict[str, t.List[str]] = {
+        'no_env_marker_test_cases': [],
+        'no_runner_tags': [],
+    }
+
     if mr_labels := os.getenv('CI_MERGE_REQUEST_LABELS'):
         print(f'MR labels: {mr_labels}')
 
         if BUILD_ONLY_LABEL in mr_labels.split(','):
             print('MR has build only label, skip generating target test child pipeline')
-            return [EmptyJob()], []
+            return [EmptyJob()], [], issues
 
     pytest_cases = get_pytest_cases(
         paths,
@@ -61,7 +76,7 @@ def get_target_test_jobs(project: Project, paths: str, apps: t.List[App]) -> t.T
     res = defaultdict(list)
     for case in pytest_cases:
         if not case.env_markers:
-            print(f'No env markers found for {case.item.originalname} in {case.path}. Ignoring...')
+            issues['no_env_marker_test_cases'].append(case.item.nodeid)
             continue
 
         res[(case.target_selector, tuple(sorted(case.env_markers)))].append(case)
@@ -72,9 +87,10 @@ def get_target_test_jobs(project: Project, paths: str, apps: t.List[App]) -> t.T
         # we don't need to get all runner, as long as we get one runner, it's fine
         runner_list = project.runners.list(status='online', tag_list=','.join(runner_tags), get_all=False)
         if not runner_list:
-            print(f'WARNING: No runner found with tag {",".join(runner_tags)}, ignoring the following test cases:')
+            issues['no_runner_tags'].append(','.join(runner_tags))
+            logging.warning(f'No runner found for {",".join(runner_tags)}, required by cases:')
             for case in cases:
-                print(f'  - {case.name}')
+                logging.warning(f'  - {case.item.nodeid}')
             continue
 
         target_test_job = TargetTestJob(
@@ -95,11 +111,48 @@ def get_target_test_jobs(project: Project, paths: str, apps: t.List[App]) -> t.T
     else:
         extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml']
 
-    return target_test_jobs, extra_include_yml
+    issues['no_env_marker_test_cases'] = sorted(issues['no_env_marker_test_cases'])
+    issues['no_runner_tags'] = sorted(issues['no_runner_tags'])
+
+    return target_test_jobs, extra_include_yml, issues
 
 
-def generate_target_test_child_pipeline(project: Project, paths: str, apps: t.List[App], output_filepath: str) -> None:
-    target_test_jobs, extra_include_yml = get_target_test_jobs(project, paths, apps)
+def generate_target_test_child_pipeline(
+    project: Project,
+    paths: str,
+    apps: t.List[App],
+    output_filepath: str,
+) -> None:
+    target_test_jobs, extra_include_yml, issues = get_target_test_jobs(project, paths, apps)
+
+    with open(KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH) as fr:
+        known_warnings_dict = yaml.safe_load(fr) or dict()
+
+    failed = False
+    known_no_env_marker_test_cases = set(known_warnings_dict.get('no_env_marker_test_cases', []))
+    no_env_marker_test_cases = set(issues['no_env_marker_test_cases'])
+
+    if no_env_marker_test_cases - known_no_env_marker_test_cases:
+        print('ERROR: NEW "no_env_marker_test_cases" detected:')
+        for case in no_env_marker_test_cases - known_no_env_marker_test_cases:
+            print(f'  - {case}')
+        failed = True
+
+    known_no_runner_tags = set(known_warnings_dict.get('no_runner_tags', []))
+    no_runner_tags = set(issues['no_runner_tags'])
+
+    if no_runner_tags - known_no_runner_tags:
+        print('ERROR: NEW "no_runner_tags" detected:')
+        for tag in no_runner_tags - known_no_runner_tags:
+            print(f'  - {tag}')
+        failed = True
+
+    if failed:
+        raise SystemExit(
+            f'Please fix the issue, '
+            f'or update the known warnings file: {KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH}'
+        )
+
     dump_jobs_to_yaml(target_test_jobs, output_filepath, extra_include_yml)
     print(f'Generate child pipeline yaml file {output_filepath} with {sum(j.parallel for j in target_test_jobs)} jobs')
 
@@ -134,13 +187,37 @@ if __name__ == '__main__':
         default=DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH,
         help='Output child pipeline file path',
     )
+    parser.add_argument(
+        '--check',
+        action='store_true',
+        help='Check if the child pipeline could be generated successfully. '
+             'test cases without env marker or required unset runner will be printed out. '
+             '(Note: All apps and test cases will be checked)',
+    )
+    parser.add_argument(
+        '--app-info-filepattern',
+        default='list_job_*.txt',
+        help='glob pattern to specify the files that include built app info generated by '
+        '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
+        'paths not exist in local file system if not listed recorded in the app info.',
+    )
 
     args = parser.parse_args()
 
-    app_list_filepattern = 'list_job_*.txt'
-    apps = []
-    for f in glob.glob(app_list_filepattern):
-        apps.extend(import_apps_from_txt(f))
-
     gl_project = Gitlab(args.project_id).project
-    generate_target_test_child_pipeline(gl_project, args.paths, apps, args.output)
+
+    if args.check:
+        apps = list(get_all_apps(args.paths)[0])  # test related apps only
+        for app in apps:
+            app.build_status = BuildStatus.SUCCESS  # pretend they are built successfully
+    else:
+        apps = []
+        for f in glob.glob(args.app_info_filepattern):
+            apps.extend(import_apps_from_txt(f))
+
+    generate_target_test_child_pipeline(
+        gl_project,
+        args.paths,
+        apps,
+        args.output,
+    )

tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml

@@ -0,0 +1,2 @@
+no_env_marker_test_cases: []
+no_runner_tags: []