ci: optimize git strategy

pull/12666/head
Fu Hanxi 2023-11-01 12:43:19 +01:00
rodzic 6ab440168a
commit a6ccc2e18e
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 19399699CF3C4B16
10 zmienionych plików z 200 dodań i 78 usunięć

Wyświetl plik

@ -3,8 +3,8 @@ workflow:
# Disable those non-protected push triggered pipelines
- if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
when: never
# when running merged result pipelines, it would create a temp commit id. use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA instead of $CI_COMMIT_SHA.
# Please use PIPELINE_COMMIT_SHA at all places that require a commit sha
# when running merged result pipelines, CI_COMMIT_SHA represents the temp commit it created.
# Please use PIPELINE_COMMIT_SHA at all places that require a commit sha of the original commit.
- if: $CI_OPEN_MERGE_REQUESTS != null
variables:
PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA

Wyświetl plik

@ -21,8 +21,7 @@
needs:
- job: fast_template_app
artifacts: false
- job: mr_variables
optional: true # only MR pipelines would have this
- pipeline_variables
artifacts:
paths:
# The other artifacts patterns are defined under tools/ci/artifacts_handler.py
@ -46,8 +45,8 @@
examples/bluetooth/esp_ble_mesh/ble_mesh_console
examples/bluetooth/hci/controller_hci_uart_esp32
examples/wifi/iperf
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
# for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server
- python tools/ci/artifacts_handler.py upload
@ -65,8 +64,8 @@
--copy-sdkconfig
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
$TEST_BUILD_OPTS_EXTRA
- python tools/ci/artifacts_handler.py upload
@ -82,8 +81,8 @@
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- python tools/ci/artifacts_handler.py upload
.build_pytest_no_jtag_template:
@ -98,8 +97,8 @@
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- python tools/ci/artifacts_handler.py upload
.build_pytest_jtag_template:
@ -114,8 +113,8 @@
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- python tools/ci/artifacts_handler.py upload
build_pytest_examples_esp32:
@ -264,8 +263,8 @@ build_only_components_apps:
-t all
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- python tools/ci/artifacts_handler.py upload
build_pytest_test_apps_esp32:
@ -336,8 +335,8 @@ build_only_tools_test_apps:
-t all
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- python tools/ci/artifacts_handler.py upload
.build_template_app_template:

Wyświetl plik

@ -21,15 +21,20 @@ variables:
# GitLab-CI environment
# XXX_ATTEMPTS variables (https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts) are not defined here.
# Use values from "CI / CD Settings" - "Variables".
# GIT_STRATEGY is not defined here.
# Use an option from "CI / CD Settings" - "General pipelines".
# now we have pack-objects cache, so clone strategy is faster than fetch
GIT_STRATEGY: clone
# we will download archive for each submodule instead of clone.
# we don't do "recursive" when fetch submodule as they're not used in CI now.
GIT_SUBMODULE_STRATEGY: none
# since we're using merged-result pipelines, the last commit should work for most cases
GIT_DEPTH: 1
# --no-recurse-submodules: we use cache for submodules
# --prune --prune-tags: in case remote branch or tag is force pushed
GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
# we're using .cache folder for caches
GIT_CLEAN_FLAGS: -ffdx -e .cache/
LATEST_GIT_TAG: v5.3-dev
SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
# by default we will fetch all submodules
# jobs can overwrite this variable to only fetch submodules they required
@ -39,20 +44,17 @@ variables:
IDF_SKIP_CHECK_SUBMODULES: 1
IDF_PATH: "$CI_PROJECT_DIR"
BATCH_BUILD: "1"
V: "0"
CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
PYTHON_VER: 3.8.17
# Docker images
ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.3:1"
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.3:1-1"
ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.3:1"
ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.3:1-1"
QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.3:1-20230522"
TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.3:1"
TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.3:1"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
# target test repo parameters
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
@ -203,6 +205,111 @@ variables:
- *show_ccache_statistics
- *upload_failed_job_log_artifacts
##############################
# Git Strategy Job Templates #
##############################
.git_init: &git_init |
mkdir -p "${CI_PROJECT_DIR}"
cd "${CI_PROJECT_DIR}"
git init
.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists |
# check if set mirror
if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then
MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"
elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then
MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
fi
# fetch from mirror first if set
if [ -n "${MIRROR_REPO_URL:-}" ]; then
if git remote -v | grep origin; then
git remote set-url origin "${MIRROR_REPO_URL}"
else
git remote add origin "${MIRROR_REPO_URL}"
fi
git fetch origin --no-recurse-submodules
fi
# set remote url to CI_REPOSITORY_URL
if git remote -v | grep origin; then
git remote set-url origin "${CI_REPOSITORY_URL}"
else
git remote add origin "${CI_REPOSITORY_URL}"
fi
.git_checkout_fetch_head: &git_checkout_fetch_head |
git checkout FETCH_HEAD
git clean ${GIT_CLEAN_FLAGS}
# git diff requires two commits, with different CI env var
#
# By default, we use git strategy "clone" with depth 1 to speed up the clone process.
# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs.
#
# Since there's no way to get the correct git_depth before the job starts,
# we can't set `GIT_DEPTH` in the job definition.
#
# Set git strategy to "none" and fetch manually instead.
.before_script:fetch:git_diff:
variables:
GIT_STRATEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- |
# merged results pipelines, by default
if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA)
# merge request pipelines, when the mr got conflicts
elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA)
# other pipelines, like the protected branches pipelines
else
git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA)
fi
- *git_checkout_fetch_head
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# git describe requires commit history until the latest tag
.before_script:fetch:git_describe:
variables:
GIT_STRAEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- |
git fetch origin refs/tags/"${LATEST_GIT_TAG}":refs/tags/"${LATEST_GIT_TAG}" --depth=1
git repack -d
git fetch origin $CI_COMMIT_SHA --shallow-since=$(git log -1 --format=%as "${LATEST_GIT_TAG}")
- *git_checkout_fetch_head
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# target test runners may locate in different places
# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit
.before_script:fetch:target_test:
variables:
GIT_STRATEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
- *git_checkout_fetch_head
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# no submodules
#############
# `default` #
#############

Wyświetl plik

@ -1,8 +1,7 @@
.deploy_job_template:
stage: deploy
image: $ESP_ENV_IMAGE
tags:
- deploy
tags: [ deploy ]
# Check this before push_to_github
check_submodule_sync:
@ -10,11 +9,11 @@ check_submodule_sync:
- .deploy_job_template
- .rules:test:submodule
stage: test_deploy
tags:
- github_sync
tags: [ brew, github_sync ]
retry: 2
variables:
GIT_STRATEGY: clone
# for brew runners, we always set GIT_STRATEGY to fetch
GIT_STRATEGY: fetch
SUBMODULES_TO_FETCH: "none"
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
dependencies: []
@ -35,6 +34,12 @@ push_to_github:
- .rules:push_to_github
needs:
- check_submodule_sync
tags: [ brew, github_sync ]
variables:
# for brew runners, we always set GIT_STRATEGY to fetch
GIT_STRATEGY: fetch
# github also need full record of commits
GIT_DEPTH: 0
script:
- add_github_ssh_keys
- git remote remove github &>/dev/null || true
@ -47,6 +52,9 @@ deploy_update_SHA_in_esp-dockerfiles:
- .before_script:minimal
- .rules:protected-no_label-always
dependencies: []
variables:
GIT_DEPTH: 2
tags: [ shiny, build ]
script:
- 'curl --header "PRIVATE-TOKEN: ${ESPCI_SCRIPTS_TOKEN}" -o create_MR_in_esp_dockerfile.sh $GITLAB_HTTP_SERVER/api/v4/projects/1260/repository/files/create_MR_in_esp_dockerfile%2Fcreate_MR_in_esp_dockerfile.sh/raw\?ref\=master'
- chmod +x create_MR_in_esp_dockerfile.sh

Wyświetl plik

@ -196,6 +196,8 @@ build_docs_pdf_prod:
.deploy_docs_template:
image: $ESP_IDF_DOC_ENV_IMAGE
extends:
- .before_script:fetch:git_describe
variables:
DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
PYTHONUNBUFFERED: 1

Wyświetl plik

@ -13,8 +13,7 @@
- job: upload-submodules-cache
optional: true
artifacts: false
- job: mr_variables
optional: true # only MR pipelines would have this
- pipeline_variables
test_nvs_on_host:
extends: .host_test_template
@ -314,8 +313,8 @@ test_pytest_qemu:
--pytest-apps
-m qemu
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
- run_cmd pytest
--target $IDF_TARGET
@ -344,8 +343,8 @@ test_pytest_linux:
--pytest-apps
-m host_test
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
--modified-components ${MODIFIED_COMPONENTS}
--modified-files ${MODIFIED_FILES}
- retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
- run_cmd pytest
--target linux

Wyświetl plik

@ -5,27 +5,16 @@
- host_test
dependencies: []
.check_pre_commit_template:
check_pre_commit:
extends:
- .pre_check_template
- .before_script:minimal
image: $PRE_COMMIT_IMAGE
check_pre_commit_master_release:
extends:
- .check_pre_commit_template
- .rules:protected
needs:
- pipeline_variables
script:
- fetch_submodules
- git diff-tree --no-commit-id --name-only -r $PIPELINE_COMMIT_SHA | xargs pre-commit run --files
check_pre_commit_MR:
extends:
- .check_pre_commit_template
- .rules:mr
script:
- fetch_submodules
- python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs pre-commit run --files
- pre-commit run --files $MODIFIED_FILES
check_MR_style_dangerjs:
extends:
@ -60,6 +49,7 @@ check_version:
extends:
- .pre_check_template
- .rules:protected
- .before_script:fetch:git_describe
script:
- export IDF_PATH=$PWD
- tools/ci/check_idf_version.sh
@ -148,10 +138,12 @@ check_esp_system:
# For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
# the version returned by 'git describe'
# Don't forget to update the env var `LATEST_GIT_TAG` in .gitlab/ci/common.yml
check_version_tag:
extends:
- .pre_check_template
- .rules:tag:release
- .before_script:fetch:git_describe
script:
- (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
@ -179,22 +171,22 @@ check_configure_ci_environment_parsing:
- cd tools/ci
- python -m unittest ci_build_apps.py
mr_variables:
pipeline_variables:
extends:
- .pre_check_template
- .rules:mr
- .before_script:minimal
- .before_script:fetch:git_diff
tags:
- build
script:
- echo "MR_MODIFIED_FILES=$(python tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env
- echo "MR_MODIFIED_COMPONENTS=$(python tools/ci/ci_get_mr_info.py components ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env
- >
- MODIFIED_FILES=$(echo $GIT_DIFF_OUTPUT | xargs)
- echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
- echo "MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | xargs)" >> pipeline.env
- |
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then
echo "BUILD_AND_TEST_ALL_APPS=1" >> mr.env
echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
fi
- cat mr.env
- cat pipeline.env
artifacts:
reports:
dotenv: mr.env
dotenv: pipeline.env
expire_in: 4 days

Wyświetl plik

@ -20,6 +20,8 @@ check_pylint:
extends:
- .pre_check_template
- .rules:patterns:python-files
needs:
- pipeline_variables
artifacts:
when: always
reports:
@ -28,7 +30,7 @@ check_pylint:
script:
- |
if [ -n "$CI_MERGE_REQUEST_IID" ]; then
export files=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | grep ".py$");
export files=$(echo $GIT_DIFF_OUTPUT | grep ".py$" | xargs);
else
export files=$(git ls-files "*.py" | xargs);
fi
@ -82,7 +84,7 @@ code_quality_check:
allow_failure: true # since now it's using exit code to indicate the code analysis result,
# we don't want to block ci when critical issues founded
script:
- export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_REF_NAME} | tr '\n' ',')
- export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
# test if this branch have merge request, if not, exit 0
- test -n "$CI_MERGE_REQUEST_IID" || exit 0
- test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0

Wyświetl plik

@ -1,11 +1,10 @@
.target_test_template:
image: $TARGET_TEST_ENV_IMAGE
extends:
- .before_script:fetch:target_test
stage: target_test
timeout: 1 hour
dependencies: []
variables:
GIT_DEPTH: 1
SUBMODULES_TO_FETCH: "none"
cache:
# Usually do not need submodule-cache in target_test
- key: pip-cache

Wyświetl plik

@ -59,9 +59,17 @@ def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
return list(mr.commits())
def get_mr_components(source_branch: str) -> t.List[str]:
def get_mr_components(
source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
) -> t.List[str]:
components: t.Set[str] = set()
for f in get_mr_changed_files(source_branch):
if modified_files is None:
if not source_branch:
raise RuntimeError('--src-branch is required if --modified-files is not provided')
modified_files = get_mr_changed_files(source_branch)
for f in modified_files:
file = Path(f)
if (
file.parts[0] == 'components'
@ -92,10 +100,14 @@ if __name__ == '__main__':
actions = parser.add_subparsers(dest='action', help='info type', required=True)
common_args = argparse.ArgumentParser(add_help=False)
common_args.add_argument('src_branch', help='source branch')
common_args.add_argument('--src-branch', help='source branch')
common_args.add_argument(
'--modified-files',
nargs='+',
help='space-separated list specifies the modified files. will be detected by --src-branch if not provided',
)
actions.add_parser('id', parents=[common_args])
actions.add_parser('files', parents=[common_args])
actions.add_parser('commits', parents=[common_args])
actions.add_parser('components', parents=[common_args])
target = actions.add_parser('target_in_tags')
@ -104,13 +116,15 @@ if __name__ == '__main__':
args = parser.parse_args()
if args.action == 'id':
if not args.src_branch:
raise RuntimeError('--src-branch is required')
print(get_mr_iid(args.src_branch))
elif args.action == 'files':
_print_list(get_mr_changed_files(args.src_branch))
elif args.action == 'commits':
if not args.src_branch:
raise RuntimeError('--src-branch is required')
_print_list([commit.id for commit in get_mr_commits(args.src_branch)])
elif args.action == 'components':
_print_list(get_mr_components(args.src_branch))
_print_list(get_mr_components(args.src_branch, args.modified_files))
elif args.action == 'target_in_tags':
print(get_target_in_tags(args.tags))
else: