ci: Move static code analysis related jobs to static-code-analysis.yml

pull/6718/head
Fu Hanxi 2020-11-30 17:40:24 +08:00
rodzic 7dd7c6c072
commit ffe0b7af32
7 zmienionych plików z 208 dodań i 165 usunięć

Wyświetl plik

@ -118,6 +118,7 @@ before_script:
include:
- '/tools/ci/config/rules.yml'
- '/tools/ci/config/docs.yml'
- '/tools/ci/config/static-code-analysis.yml'
- '/tools/ci/config/pre_check.yml'
- '/tools/ci/config/build.yml'
- '/tools/ci/config/assign-test.yml'

Wyświetl plik

@ -387,92 +387,3 @@ build_template_app:
- job: fast_template_app
artifacts: false
extends: .build_template_app_template
# Sonarqube related jobs put here for this reason:
# Here we have two jobs. code_quality_check and code_quality_report.
#
# code_quality_check will analyze the code changes between your MR and
# code repo stored in sonarqube server. The analysis result is only shown in
# the comments under this MR and won't be transferred to the server.
#
# code_quality_report will analyze and transfer both of the newly added code
# and the analysis result to the server.
#
# Put in the front to ensure that the newly merged code can be stored in
# sonarqube server ASAP, in order to avoid reporting unrelated code issues
.sonar_scan_template:
stage: build
image:
name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
before_script:
- source tools/ci/utils.sh
- export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
- fetch_submodules
# Exclude the submodules, all paths ends with /**
- export SUBMODULES=$(get_all_submodules)
# get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
- export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
# Exclude the report dir
- export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
- python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
variables:
GIT_DEPTH: 0
NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py
REPORT_DIR: examples/get-started/hello_world/tidybuild/report
tags:
- host_test
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- clang_tidy_check_regular
code_quality_check:
extends:
- .sonar_scan_template
- .rules:trigger
allow_failure: true
script:
- export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH})
- export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',')
# test if this branch have merge request, if not, exit 0
- test -n "$CI_MR_IID" || exit 0
- test -n "$CI_MR_COMMITS" || exit 0
- sonar-scanner
-Dsonar.analysis.mode=preview
-Dsonar.host.url=$SONAR_HOST_URL
-Dsonar.login=$SONAR_LOGIN
-Dsonar.sources=$CI_PROJECT_DIR
-Dsonar.sourceEncoding=UTF-8
-Dsonar.projectKey=esp-idf
-Dsonar.projectBaseDir=$CI_PROJECT_DIR
-Dsonar.exclusions=$EXCLUSIONS
-Dsonar.gitlab.project_id=$CI_PROJECT_ID
-Dsonar.gitlab.commit_sha=$CI_MR_COMMITS
-Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-Dsonar.gitlab.failure_notification_mode=exit-code
-Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
-Dsonar.cxx.includeDirectories=components,/usr/include
-Dsonar.python.pylint_config=.pylintrc
-Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID
-Dsonar.gitlab.merge_request_discussion=true
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
code_quality_report:
extends:
- .sonar_scan_template
- .rules:protected-schedule
script:
- sonar-scanner
-Dsonar.host.url=$SONAR_HOST_URL
-Dsonar.login=$SONAR_LOGIN
-Dsonar.sources=$CI_PROJECT_DIR
-Dsonar.sourceEncoding=UTF-8
-Dsonar.projectKey=esp-idf
-Dsonar.projectBaseDir=$CI_PROJECT_DIR
-Dsonar.exclusions=$EXCLUSIONS
-Dsonar.gitlab.project_id=$CI_PROJECT_ID
-Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA
-Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
-Dsonar.cxx.includeDirectories=components,/usr/include
-Dsonar.python.pylint_config=.pylintrc
-Dsonar.branch.name=$CI_COMMIT_REF_NAME

Wyświetl plik

@ -4,42 +4,6 @@
tags:
- deploy
.clang_tidy_deploy_template:
extends: .deploy_job_template
tags:
- deploy
- shiny
script:
- add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
- export GIT_VER=$(git describe --always)
- cd $IDF_PATH/examples/get-started/hello_world/tidybuild
- mv report $GIT_VER
- tar czvf $GIT_VER.tar.gz $GIT_VER
- export STATIC_REPORT_PATH="web/static_analysis/esp-idf/"
- ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy"
- scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy
- ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
# add link to view the report
- echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
- test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
clang_tidy_deploy:
extends: .clang_tidy_deploy_template
# Override default stage to happen before the post_check
stage: test_deploy
needs:
- clang_tidy_check
- clang_tidy_check_all
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
clang_tidy_deploy_regular:
extends:
- .clang_tidy_deploy_template
- .rules:labels:static_analysis-only
needs:
- clang_tidy_check_regular
push_to_github:
extends:
- .deploy_job_template

Wyświetl plik

@ -61,7 +61,9 @@ check_rom_api_header:
- tools/ci/check_rom_apis.sh
check_python_style:
extends: .pre_check_base_template
extends:
- .pre_check_base_template
- .rules:patterns:python-files
artifacts:
when: on_failure
paths:
@ -137,37 +139,6 @@ scan_tests:
- python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
.clang_tidy_check_template:
extends: .pre_check_base_template
image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
artifacts:
reports:
junit: $IDF_PATH/output.xml
when: always
paths:
- $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
expire_in: 1 day
script:
- retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
# Setup parameters of triggered/regular job
- export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
- ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
clang_tidy_check:
extends: .clang_tidy_check_template
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
BOT_LABEL_STATIC_ANALYSIS: 1
clang_tidy_check_regular:
extends: .clang_tidy_check_template
clang_tidy_check_all:
extends: .clang_tidy_check_template
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
BOT_LABEL_STATIC_ANALYSIS_ALL: 1
# For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
# the version returned by 'git describe'
check_version_tag:

Wyświetl plik

@ -8,6 +8,19 @@
- "components/**/sdkconfig*"
- "CONTRIBUTING.rst"
.patterns-c-files: &patterns-c-files
- "**/*.{c,C}"
- "**/*.{h,H}"
.patterns-python-files: &patterns-python-files
- "**/*.py"
.patterns-static-code-analysis: &patterns-static-code-analysis
# - "tools/ci/config/static-code-analysis.yml" # FIXME: after debug
- "**/*.{c,C}"
- "**/*.{h,H}"
- "**/*.py"
# if anchors
.if-ref-master: &if-ref-master
if: '$CI_COMMIT_REF_NAME == "master"'
@ -75,9 +88,6 @@
.if-label-nvs_coverage: &if-label-nvs_coverage
if: '$BOT_LABEL_NVS_COVERAGE'
.if-label-static_analysis: &if-label-static_analysis
if: '$BOT_LABEL_STATIC_ANALYSIS || $BOT_LABEL_STATIC_ANALYSIS_ALL'
.if-label-iperf_stress_test: &if-label-iperf_stress_test
if: '$BOT_LABEL_IPERF_STRESS_TEST'
@ -128,10 +138,6 @@
- <<: *if-schedule
when: always
.rules:labels:static_analysis-only:
rules:
- <<: *if-label-static_analysis
.rules:labels:build:
rules:
- <<: *if-protected-no_label
@ -151,6 +157,28 @@
- <<: *if-dev-push
changes: *patterns-docs
.rules:patterns:clang_tidy:
rules:
- <<: *if-protected-no_label
- <<: *if-dev-push
changes: *patterns-c-files
.rules:patterns:clang_tidy-preview:
rules:
- <<: *if-dev-push
changes: *patterns-c-files
.rules:patterns:python-files:
rules:
- <<: *if-protected-no_label
- <<: *if-dev-push
changes: *patterns-python-files
.rules:patterns:static-code-analysis-preview:
rules:
- <<: *if-dev-push
changes: *patterns-static-code-analysis
.rules:labels:weekend_test-only:
rules:
- <<: *if-label-weekend_test

Wyświetl plik

@ -0,0 +1,163 @@
.clang_tidy_check_template:
extends:
- .pre_check_base_template
- .rules:patterns:clang_tidy
image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
artifacts:
reports:
junit: $IDF_PATH/output.xml
when: always
paths:
- $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
expire_in: 1 day
script:
- retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
# Setup parameters of triggered/regular job
- export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
- ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
# pre_check stage
clang_tidy_check:
extends: .clang_tidy_check_template
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
TRIGGERED_RELATIVE: 1
clang_tidy_check_regular:
extends:
- .clang_tidy_check_template
- .rules:patterns:clang_tidy-preview
clang_tidy_check_all:
extends: .clang_tidy_check_template
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
TRIGGERED_ABSOLUTE: 1
# build stage
# Sonarqube related jobs put here for this reason:
# Here we have two jobs. code_quality_check and code_quality_report.
#
# code_quality_check will analyze the code changes between your MR and
# code repo stored in sonarqube server. The analysis result is only shown in
# the comments under this MR and won't be transferred to the server.
#
# code_quality_report will analyze and transfer both of the newly added code
# and the analysis result to the server.
#
# Put in the front to ensure that the newly merged code can be stored in
# sonarqube server ASAP, in order to avoid reporting unrelated code issues
.sonar_scan_template:
stage: build
image:
name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
before_script:
- source tools/ci/utils.sh
- export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
- fetch_submodules
# Exclude the submodules, all paths ends with /**
- export SUBMODULES=$(get_all_submodules)
# get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
- export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
# Exclude the report dir
- export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
- python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
variables:
GIT_DEPTH: 0
NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py
REPORT_DIR: examples/get-started/hello_world/tidybuild/report
tags:
- host_test
dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
- clang_tidy_check_regular
code_quality_check:
extends:
- .sonar_scan_template
- .rules:patterns:static-code-analysis-preview
allow_failure: true
script:
- export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH})
- export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',')
# test if this branch have merge request, if not, exit 0
- test -n "$CI_MR_IID" || exit 0
- test -n "$CI_MR_COMMITS" || exit 0
- sonar-scanner
-Dsonar.analysis.mode=preview
-Dsonar.host.url=$SONAR_HOST_URL
-Dsonar.login=$SONAR_LOGIN
-Dsonar.sources=$CI_PROJECT_DIR
-Dsonar.sourceEncoding=UTF-8
-Dsonar.projectKey=esp-idf
-Dsonar.projectBaseDir=$CI_PROJECT_DIR
-Dsonar.exclusions=$EXCLUSIONS
-Dsonar.gitlab.project_id=$CI_PROJECT_ID
-Dsonar.gitlab.commit_sha=$CI_MR_COMMITS
-Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-Dsonar.gitlab.failure_notification_mode=exit-code
-Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
-Dsonar.cxx.includeDirectories=components,/usr/include
-Dsonar.python.pylint_config=.pylintrc
-Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID
-Dsonar.gitlab.merge_request_discussion=true
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
code_quality_report:
extends:
- .sonar_scan_template
- .rules:protected-schedule
script:
- sonar-scanner
-Dsonar.host.url=$SONAR_HOST_URL
-Dsonar.login=$SONAR_LOGIN
-Dsonar.sources=$CI_PROJECT_DIR
-Dsonar.sourceEncoding=UTF-8
-Dsonar.projectKey=esp-idf
-Dsonar.projectBaseDir=$CI_PROJECT_DIR
-Dsonar.exclusions=$EXCLUSIONS
-Dsonar.gitlab.project_id=$CI_PROJECT_ID
-Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA
-Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
-Dsonar.cxx.includeDirectories=components,/usr/include
-Dsonar.python.pylint_config=.pylintrc
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
# deploy stage
.clang_tidy_deploy_template:
extends:
- .deploy_job_template
- .rules:patterns:clang_tidy
tags:
- deploy
- shiny
script:
- add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
- export GIT_VER=$(git describe --always)
- cd $IDF_PATH/examples/get-started/hello_world/tidybuild
- mv report $GIT_VER
- tar czvf $GIT_VER.tar.gz $GIT_VER
- export STATIC_REPORT_PATH="web/static_analysis/esp-idf/"
- ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy"
- scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy
- ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
# add link to view the report
- echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
- test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
clang_tidy_deploy:
extends: .clang_tidy_deploy_template
# Override default stage to happen before the post_check
stage: test_deploy
needs:
- clang_tidy_check
- clang_tidy_check_all
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
clang_tidy_deploy_regular:
extends:
- .clang_tidy_deploy_template
- .rules:patterns:clang_tidy-preview
needs:
- clang_tidy_check_regular

Wyświetl plik

@ -1,12 +1,17 @@
#!/usr/bin/env python
import argparse
import re
from os.path import dirname, join, normpath, relpath
from os.path import join, normpath, dirname, relpath, exists
CLANG_TIDY_REGEX = re.compile(r'(.+|[a-zA-Z]:\\\\.+):([0-9]+):([0-9]+): ([^:]+): (.+)')
def normalize_clang_tidy_path(file_path, output_path, base_dir):
if not exists(file_path):
print('Skipping normalizing. This could only happen when skipping clang-tidy check '
'because of no c file modified. Please double check')
return
with open(output_path, 'w') as fw:
for line in open(file_path):
result = CLANG_TIDY_REGEX.match(line)