diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index b97925dbb3..06d4d7bd91 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -32,6 +32,8 @@ # keep the size info to help track the binary size - size_info.txt - "**/build*/size.json" + expire_in: 1 week + when: always script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v @@ -85,6 +87,8 @@ - log_template_app/* - size_info.txt - build_template_app/**/size.json + expire_in: 1 week + when: always script: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} @@ -178,6 +182,14 @@ build_clang_test_apps_esp32c6: - job: fast_template_app artifacts: false optional: true + artifacts: + reports: + junit: XUNIT_RESULT.xml + paths: + - XUNIT_RESULT.xml + - test_build_system + expire_in: 1 week + when: always script: - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - cd ${IDF_PATH}/tools/test_build_system @@ -189,13 +201,6 @@ build_clang_test_apps_esp32c6: pytest_build_system: extends: .test_build_system_template parallel: 3 - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml pytest_build_system_macos: extends: @@ -205,13 +210,6 @@ pytest_build_system_macos: tags: - macos_shell parallel: 3 - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml build_docker: extends: @@ -264,6 +262,8 @@ generate_build_child_pipeline: - build_child_pipeline.yml - test_related_apps.txt - non_test_related_apps.txt + expire_in: 1 week + when: always script: - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py --modified-components ${MR_MODIFIED_COMPONENTS} diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index e0d994c288..80aeda6ea6 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -353,9 +353,12 @@ default: - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - fetch_submodules - artifacts: - expire_in: 1 week - when: always +# gitlab bug, setting them here doesn't work +# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563 +# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672 +# artifacts: +# expire_in: 1 week +# when: always retry: max: 2 when: diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 0442e5dac1..d98a7d4046 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -96,6 +96,9 @@ check_docs_lang_sync: - if [ -n "${BREATHE_ALT_INSTALL_URL}" ]; then pip uninstall -y breathe && pip install -U ${BREATHE_ALT_INSTALL_URL}; fi - cd docs - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build + artifacts: + expire_in: 4 days + when: always parallel: matrix: - DOCLANG: ["en", "zh_CN"] @@ -124,7 +127,6 @@ build_docs_html_full: paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* - expire_in: 4 days variables: DOC_BUILDERS: "html" @@ -137,7 +139,6 @@ build_docs_html_full_prod: paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* - expire_in: 4 days variables: DOC_BUILDERS: "html" @@ -153,7 +154,6 @@ build_docs_html_partial: paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* - expire_in: 4 days variables: DOC_BUILDERS: "html" parallel: @@ -174,7 +174,6 @@ build_docs_pdf: artifacts: paths: - docs/_build/*/*/latex/* - expire_in: 4 days variables: DOC_BUILDERS: "latex" @@ -186,7 +185,6 @@ build_docs_pdf_prod: artifacts: paths: - docs/_build/*/*/latex/* - expire_in: 4 days variables: DOC_BUILDERS: "latex" diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 9da0101494..9b023405f8 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -14,6 +14,9 @@ optional: true artifacts: false - pipeline_variables + artifacts: + expire_in: 1 week + when: always test_nvs_on_host: extends: .host_test_template diff --git a/.gitlab/ci/integration_test.yml b/.gitlab/ci/integration_test.yml index 5064a6bd1c..c14e515de6 100644 --- a/.gitlab/ci/integration_test.yml +++ b/.gitlab/ci/integration_test.yml @@ -46,6 +46,7 @@ gen_integration_pipeline: paths: - idf-integration-ci/child_pipeline/ expire_in: 2 weeks + when: always script: - add_gitlab_ssh_keys - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 0de62d1be8..104377d311 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -72,6 +72,7 @@ check_chip_support_components: paths: - esp_hw_support_part.h - bootloader_support_part.h + expire_in: 1 week script: - python tools/ci/check_soc_headers_leak.py - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py @@ -85,6 +86,7 @@ check_esp_err_to_name: when: on_failure paths: - components/esp_common/esp_err_to_name.c + expire_in: 1 week script: - cd ${IDF_PATH}/tools/ - ./gen_esp_err_to_name.py @@ -157,6 +159,10 @@ pipeline_variables: artifacts: reports: dotenv: pipeline.env + paths: + - pipeline.env + expire_in: 1 week + when: always check_test_cases_env_markers_and_required_runners: extends: diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index e1ef1d0d54..d22f524d54 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -6,6 +6,8 @@ clang_tidy_check: artifacts: paths: - clang_tidy_reports/ + expire_in: 1 week + when: always variables: IDF_TOOLCHAIN: clang script: @@ -23,6 +25,10 @@ check_pylint: artifacts: reports: codequality: pylint.json + paths: + - pylint.json + expire_in: 1 week + when: always script: - | if [ -n "$CI_MERGE_REQUEST_IID" ]; then @@ -70,6 +76,8 @@ check_pylint: artifacts: paths: - $REPORT_PATTERN + expire_in: 1 week + when: always dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. - clang_tidy_check diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 9550889fc3..6949dda5ad 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -26,6 +26,7 @@ test_cli_installer_win: paths: - tools/tools.new.json - tools/test_idf_tools/test_python_env_logs.txt + expire_in: 1 week variables: IDF_PATH: "$CI_PROJECT_DIR" script: @@ -46,6 +47,8 @@ test_tools_win: - ${IDF_PATH}/XUNIT_*.xml reports: junit: ${IDF_PATH}/XUNIT_*.xml + expire_in: 1 week + when: always variables: LC_ALL: C.UTF-8 PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" @@ -90,3 +93,4 @@ pytest_build_system_win: expire_in: 2 days reports: junit: XUNIT_RESULT.xml + when: always diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py index e68cfd83da..baac4ee2e3 100755 --- a/tools/ci/gitlab_yaml_linter.py +++ b/tools/ci/gitlab_yaml_linter.py @@ -104,6 +104,21 @@ class YmlLinter: f'Please set `dependencies:` (to null) explicitly to avoid missing artifacts issue' ) + def _lint_artifacts_expire_in_and_when(self) -> None: + """ + Set `artifacts: expire_in` and `artifacts: when` together since gitlab has bugs: + + - https://gitlab.com/gitlab-org/gitlab/-/issues/404563 (expire_in) + - https://gitlab.com/gitlab-org/gitlab/-/issues/440672 (when) + """ + for job_name, d in self.yml_config.jobs.items(): + if 'artifacts' in d: + if 'expire_in' not in d['artifacts']: + self._errors.append(f'job {job_name} missing `artifacts: expire_in`. (suggest to set to `1 week`)') + + if 'when' not in d['artifacts']: + self._errors.append(f'job {job_name} missing `artifacts: when`. (suggest to set to `always`)') + if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index d8d33d4635..c865690a19 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -202,22 +202,34 @@ class GitlabYmlConfig: for k in self._job_keys: self._expand_extends(k) + def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any: + for k, v in d2.items(): + if k in d1: + if isinstance(v, dict) and isinstance(d1[k], dict): + d1[k] = self._merge_dict(d1[k], v) + else: + d1[k] = v + else: + d1[k] = v + + return d1 + def _expand_extends(self, name: str) -> t.Dict[str, t.Any]: extends = to_list(self.config[name].pop('extends', None)) - original_d = self.config[name].copy() if not extends: return self.config[name] # type: ignore + original_d = self.config[name].copy() d = {} while extends: - self._used_template_keys.update(extends) + self._used_template_keys.update(extends) # for tracking for i in extends: d.update(self._expand_extends(i)) extends = to_list(self.config[name].pop('extends', None)) - self.config[name] = {**d, **original_d} + self.config[name] = self._merge_dict(d, original_d) return self.config[name] # type: ignore