CI: Do check first in order to save build time

2019-11-12 08:59:11 +01:00 · 2019-11-12 08:59:11 +01:00 · b52b49c8b6
commit b52b49c8b6
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,9 +1,10 @@
 stages:
+  - pre_check
  - build
  - assign_test
  - host_test
  - target_test
-  - check
+  - post_check
  - deploy
  - post_check

@ -120,7 +121,7 @@ after_script:
  - *cleanup_custom_toolchain

 .check_job_template:
-  stage: check
+  stage: pre_check
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - host_test
@ -128,7 +129,7 @@ after_script:
  extends: .before_script_lesser_nofilter

 .check_job_template_with_filter:
-  stage: check
+  stage: pre_check
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - host_test
@ -147,9 +148,10 @@ after_script:
    - source tools/ci/configure_ci_environment.sh

 include:
+  - '/tools/ci/config/pre_check.yml'
  - '/tools/ci/config/build.yml'
  - '/tools/ci/config/assign-test.yml'
  - '/tools/ci/config/host-test.yml'
  - '/tools/ci/config/target-test.yml'
-  - '/tools/ci/config/check.yml'
+  - '/tools/ci/config/post_check.yml'
  - '/tools/ci/config/deploy.yml'
--- a/tools/ci/config/build.yml
+++ b/tools/ci/config/build.yml
@ -22,72 +22,6 @@
      - $BOT_LABEL_UNIT_TEST
      - $BOT_LABEL_REGULAR_TEST

-
-.build_with_make_and_cmake: &build_with_make_and_cmake |
-  echo -e "section_end:"`date +%s`":build_script\r\e[0Ksection_start:"`date +%s`":build_make\r\e[0KBuild with Make"
-  make defconfig
-  make all
-  make clean
-  echo -e "section_end:"`date +%s`":build_make\r\e[0Ksection_start:"`date +%s`":build_cmake\r\e[0KBuild with CMake"
-  rm -rf build sdkconfig
-  idf.py build
-  echo -e "section_end:"`date +%s`":build_cmake\r\e[0Ksection_start:"`date +%s`":build_script\r\e[0K"
-
-build_template_app:
-  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - build
-  variables:
-    BATCH_BUILD: "1"
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-  script:
-    # Set the variable for 'esp-idf-template' testing
-    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
-    - git clone ${ESP_IDF_TEMPLATE_GIT}
-    # Try to use the same branch name for esp-idf-template that we're
-    # using on esp-idf. If it doesn't exist then just stick to the default branch
-    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
-    - cd esp-idf-template
-    - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
-    # CONFIG_COMPILER_OPTIMIZATION_DEFAULT with flag -Og
-    - echo "CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0
-    - echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os
-    - echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2
-    - echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # Same as above, but also disable assertions.
-    - echo "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y" >> sdkconfig.defaults
-    # Don't error out on -Wunused, when assertions are disabled
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
-    - *build_with_make_and_cmake
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
-    # Check if there are any stray printf/ets_printf references in WiFi libs
-    - pushd ../components/esp_wifi/lib
-    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
-    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
-    - popd
-
 build_ssc:
  extends: .build_template
  parallel: 3
@ -279,17 +213,6 @@ build_docs:
    - make html
    - ../check_doc_warnings.sh

-verify_cmake_style:
-  extends: .check_job_template
-  stage: build
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-  script:
-    tools/cmake/run_cmake_lint.sh
-
 test_build_system:
  extends: .build_template
  script:
--- a/tools/ci/config/post_check.yml
+++ b/tools/ci/config/post_check.yml
@ -1,100 +1,10 @@
-
 # copy from .gitlab-ci.yml as anchor is not global
 .show_submodule_urls: &show_submodule_urls |
  git config --get-regexp '^submodule\..*\.url$' || true

-check_line_endings:
-  extends: .check_job_template
-  script:
-    - tools/ci/check-line-endings.sh ${IDF_PATH}
-
-check_commit_msg:
-  extends: .check_job_template
-  script:
-    - git status
-    - git log -n10 --oneline
-    # commit start with "WIP: " need to be squashed before merge
-    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
-
-check_permissions:
-  extends: .check_job_template
-  script:
-    - tools/ci/check-executable.sh
-
-check_version:
-  extends: .check_job_template
-  # Don't run this for feature/bugfix branches, so that it is possible to modify
-  # esp_idf_version.h in a branch before tagging the next version.
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  script:
-    - export IDF_PATH=$PWD
-    - tools/ci/check_idf_version.sh
-
-check_examples_cmake_make:
-  extends: .check_job_template_with_filter
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  script:
-    - tools/ci/check_examples_cmake_make.sh
-
-check_examples_rom_header:
-  extends: .check_job_template_with_filter
-  script:
-    - tools/ci/check_examples_rom_header.sh
-
-check_python_style:
-  extends: .check_job_template_with_filter
-  artifacts:
-    when: on_failure
-    paths:
-      - flake8_output.txt
-    expire_in: 1 week
-  script:
-    # run it only under Python 3 (it is very slow under Python 2)
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
-
-check_kconfigs:
-  extends: .check_job_template_with_filter
-  artifacts:
-    when: on_failure
-    paths:
-      - components/*/Kconfig*.new
-      - examples/*/*/*/Kconfig*.new
-      - examples/*/*/*/*/Kconfig*.new
-      - tools/*/Kconfig*.new
-      - tools/*/*/Kconfig*.new
-      - tools/*/*/*/Kconfig*.new
-    expire_in: 1 week
-  script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
-    - ${IDF_PATH}/tools/check_kconfigs.py
-
-check_deprecated_kconfig_options:
-  extends: .check_job_template_with_filter
-  script:
-    - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
-
-check_ut_cmake_make:
-  extends: .check_job_template_with_filter
-  stage: check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - build
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  dependencies: []
-  script:
-    - tools/ci/check_ut_cmake_make.sh
-
 check_submodule_sync:
  extends: .check_job_template
+  stage: post_check
  tags:
    - github_sync
  retry: 2
@ -125,8 +35,23 @@ check_wifi_lib_md5:
    - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32s2beta $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh

+check_ut_cmake_make:
+  extends: .check_job_template_with_filter
+  stage: post_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  dependencies: []
+  script:
+    - tools/ci/check_ut_cmake_make.sh
+
 check_artifacts_expire_time:
  extends: .check_job_template
+  stage: post_check
  script:
    # check if we have set expire time for all artifacts
    - python tools/ci/check_artifacts_expire_time.py
@ -141,3 +66,12 @@ check_pipeline_triggered_by_label:
    # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
    # We want to make sure some jobs are always executed to detect regression.
    - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }
+
+check_commit_msg:
+  extends: .check_job_template
+  stage: post_check
+  script:
+    - git status
+    - git log -n10 --oneline
+    # commit start with "WIP: " need to be squashed before merge
+    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
--- a/tools/ci/config/pre_check.yml
+++ b/tools/ci/config/pre_check.yml
@ -0,0 +1,142 @@
+check_line_endings:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-line-endings.sh ${IDF_PATH}
+
+check_permissions:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-executable.sh
+
+check_version:
+  extends: .check_job_template
+  # Don't run this for feature/bugfix branches, so that it is possible to modify
+  # esp_idf_version.h in a branch before tagging the next version.
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - export IDF_PATH=$PWD
+    - tools/ci/check_idf_version.sh
+
+check_examples_cmake_make:
+  extends: .check_job_template_with_filter
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - tools/ci/check_examples_cmake_make.sh
+
+check_examples_rom_header:
+  extends: .check_job_template_with_filter
+  script:
+    - tools/ci/check_examples_rom_header.sh
+
+check_python_style:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - flake8_output.txt
+    expire_in: 1 week
+  script:
+    # run it only under Python 3 (it is very slow under Python 2)
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+
+check_kconfigs:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - components/*/Kconfig*.new
+      - examples/*/*/*/Kconfig*.new
+      - examples/*/*/*/*/Kconfig*.new
+      - tools/*/Kconfig*.new
+      - tools/*/*/Kconfig*.new
+      - tools/*/*/*/Kconfig*.new
+    expire_in: 1 week
+  script:
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
+    - ${IDF_PATH}/tools/check_kconfigs.py
+
+check_deprecated_kconfig_options:
+  extends: .check_job_template_with_filter
+  script:
+    - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
+
+check_cmake_style:
+  extends: .check_job_template
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    tools/cmake/run_cmake_lint.sh
+
+.build_with_make_and_cmake: &build_with_make_and_cmake |
+  echo -e "section_end:"`date +%s`":build_script\r\e[0Ksection_start:"`date +%s`":build_make\r\e[0KBuild with Make"
+  make defconfig
+  make all
+  make clean
+  echo -e "section_end:"`date +%s`":build_make\r\e[0Ksection_start:"`date +%s`":build_cmake\r\e[0KBuild with CMake"
+  rm -rf build sdkconfig
+  idf.py build
+  echo -e "section_end:"`date +%s`":build_cmake\r\e[0Ksection_start:"`date +%s`":build_script\r\e[0K"
+
+build_template_app:
+  stage: pre_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  variables:
+    BATCH_BUILD: "1"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - git clone ${ESP_IDF_TEMPLATE_GIT}
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default branch
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - cd esp-idf-template
+    - export PATH="$IDF_PATH/tools:$PATH"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+    # CONFIG_COMPILER_OPTIMIZATION_DEFAULT with flag -Og
+    - echo "CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0
+    - echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os
+    - echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2
+    - echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # Same as above, but also disable assertions.
+    - echo "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y" >> sdkconfig.defaults
+    # Don't error out on -Wunused, when assertions are disabled
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
+    - *build_with_make_and_cmake
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+    # Check if there are any stray printf/ets_printf references in WiFi libs
+    - pushd ../components/esp_wifi/lib
+    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
+    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
+    - popd