ci: split assign tests

2022-01-06 23:21:21 +08:00 · 2022-01-06 23:21:21 +08:00 · 151d8cdcd2
commit 151d8cdcd2
--- a/.gitlab/ci/assign-test.yml
+++ b/.gitlab/ci/assign-test.yml
@ -1,48 +1,107 @@
-assign_test:
-  extends: .rules:test:any_test
-  tags:
-    - assign_test
+.assign_test_template:
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  stage: assign_test
-  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
-  dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
-    - build_ssc_esp32
-    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake_esp32s2
-    - build_esp_idf_tests_cmake_esp32s3
-    - build_esp_idf_tests_cmake_esp32c3
+  tags:
+    - assign_test
  variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
-    CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
-    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
-    INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    SUBMODULES_TO_FETCH: components/esptool_py/esptool
+  artifacts:
+    paths:
+      - ${TEST_DIR}/test_configs
+      - ${BUILD_DIR}/artifact_index.json
+    when: always
+    expire_in: 1 week
+  script:
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_example_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:example_test
+  needs:
+    - job: build_examples_cmake_esp32
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: example_test
+    TEST_DIR: ${CI_PROJECT_DIR}/examples
+    BUILD_DIR: ${CI_PROJECT_DIR}/build_examples
+
+assign_custom_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:custom_test
+  needs:
+    - job: build_test_apps_esp32
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: custom_test
+    TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
+    BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps
+
+assign_unit_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:unit_test
+  needs:
+    - job: build_esp_idf_tests_cmake_esp32
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s2
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32c3
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s3
+      optional: true
+  variables:
+    TEST_TYPE: unit_test
+    TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test
+    BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
+  script:
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_integration_test:
+  extends:
+    - .assign_test_template
+    - .rules:test:integration_test
+  needs:
+    - build_ssc_esp32
+  artifacts:
+    paths:
+      - $TEST_DIR/test_configs
+  variables:
+    TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/integration_test
+    BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin
    INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
    ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    # auto_test_script only supports python 3.7.x
    PYTHON_VER: 3.7.7
-  artifacts:
-    paths:
-      - components/idf_test/*/CIConfigs
-      - $EXAMPLE_TEST_DIR/test_configs
-      - $CUSTOM_TEST_DIR/test_configs
-      - build_examples/artifact_index.json
-      - build_test_apps/artifact_index.json
-      - tools/unit-test-app/builds/artifact_index.json
-    expire_in: 1 week
  script:
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
    # clone test script to assign tests
    # can not retry if downing git lfs files failed, so using empty_branch first.
    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
    # assign integration test cases
-    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
+    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs

 update_test_cases:
  extends: .rules:ref:master-schedule
--- a/.gitlab/ci/dependencies/dependencies.yml
+++ b/.gitlab/ci/dependencies/dependencies.yml
@ -63,6 +63,8 @@
  patterns:
    - build_components
    - build_system
+  included_in:
+    - "build:{0}"

 build:integration_test:
  labels:
@ -85,10 +87,10 @@ build:integration_test:
    - "{0}"
    - "build-{0}"
  included_in: # Parent rules
+    - "build:{0}"
    - "build:{0}-{1}"
    - build:target_test
    - test:target_test
-    - test:any_test

 # -------------
 # Special Cases
@ -105,29 +107,26 @@ build:integration_test:
    - component_ut
    - "build-component_ut-{0}"
  included_in:
+    - build:component_ut
    - "build:component_ut-{0}"
    - build:target_test
    - test:target_test
-    - test:any_test

 "test:integration_test":
  labels:
-    - "integration_test"
+    - integration_test
  patterns:
-    - "integration_test"
+    - integration_test
  included_in:
-    - "build:integration_test"
+    - build:integration_test
    - build:target_test
    - test:target_test
-    - test:any_test

 "test:host_test":
  labels:
    - host_test
  patterns:
    - host_test
-  included_in:
-    - test:any_test

 "test:submodule":
  labels:
@ -142,17 +141,17 @@ build:integration_test:
  labels:
    - iperf_stress_test
  included_in:
+    - build:example_test
    - build:example_test-esp32
    - build:target_test
-    - test:any_test

 "labels:weekend_test": # custom test
  labels:
    - weekend_test
  included_in:
+    - build:custom_test
    - build:custom_test-esp32
    - build:target_test
-    - test:any_test

 "labels:nvs_coverage": # host_test
  labels:
--- a/.gitlab/ci/rules.yml
+++ b/.gitlab/ci/rules.yml
@ -403,6 +403,33 @@
    - <<: *if-dev-push
      changes: *patterns-build_system

+.rules:build:component_ut:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-component_ut
+    - <<: *if-label-component_ut_esp32
+    - <<: *if-label-component_ut_esp32c2
+    - <<: *if-label-component_ut_esp32c3
+    - <<: *if-label-component_ut_esp32h2
+    - <<: *if-label-component_ut_esp32s2
+    - <<: *if-label-component_ut_esp32s3
+    - <<: *if-label-unit_test
+    - <<: *if-label-unit_test_esp32
+    - <<: *if-label-unit_test_esp32c2
+    - <<: *if-label-unit_test_esp32c3
+    - <<: *if-label-unit_test_esp32h2
+    - <<: *if-label-unit_test_esp32s2
+    - <<: *if-label-unit_test_esp32s3
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-component_ut
+
 .rules:build:component_ut-esp32:
  rules:
    - <<: *if-revert-branch
@ -505,6 +532,27 @@
    - <<: *if-dev-push
      changes: *patterns-component_ut

+.rules:build:custom_test:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-custom_test
+    - <<: *if-label-custom_test_esp32
+    - <<: *if-label-custom_test_esp32c2
+    - <<: *if-label-custom_test_esp32c3
+    - <<: *if-label-custom_test_esp32h2
+    - <<: *if-label-custom_test_esp32s2
+    - <<: *if-label-custom_test_esp32s3
+    - <<: *if-label-weekend_test
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-custom_test
+
 .rules:build:custom_test-esp32:
  rules:
    - <<: *if-revert-branch
@ -606,6 +654,29 @@
    - <<: *if-dev-push
      changes: *patterns-docker

+.rules:build:example_test:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-example_test
+    - <<: *if-label-example_test_esp32
+    - <<: *if-label-example_test_esp32c2
+    - <<: *if-label-example_test_esp32c3
+    - <<: *if-label-example_test_esp32h2
+    - <<: *if-label-example_test_esp32s2
+    - <<: *if-label-example_test_esp32s3
+    - <<: *if-label-iperf_stress_test
+    - <<: *if-dev-push
+      changes: *patterns-build-example_test
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-example_test
+
 .rules:build:example_test-esp32:
  rules:
    - <<: *if-revert-branch
@ -788,6 +859,26 @@
    - <<: *if-dev-push
      changes: *patterns-unit_test

+.rules:build:unit_test:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-label-unit_test
+    - <<: *if-label-unit_test_esp32
+    - <<: *if-label-unit_test_esp32c2
+    - <<: *if-label-unit_test_esp32c3
+    - <<: *if-label-unit_test_esp32h2
+    - <<: *if-label-unit_test_esp32s2
+    - <<: *if-label-unit_test_esp32s3
+    - <<: *if-dev-push
+      changes: *patterns-build_components
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-unit_test
+
 .rules:build:unit_test-esp32:
  rules:
    - <<: *if-revert-branch
@ -915,60 +1006,6 @@
      when: never
    - <<: *if-label-weekend_test

-.rules:test:any_test:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-protected
-    - <<: *if-label-build-only
-      when: never
-    - <<: *if-label-component_ut
-    - <<: *if-label-component_ut_esp32
-    - <<: *if-label-component_ut_esp32c2
-    - <<: *if-label-component_ut_esp32c3
-    - <<: *if-label-component_ut_esp32h2
-    - <<: *if-label-component_ut_esp32s2
-    - <<: *if-label-component_ut_esp32s3
-    - <<: *if-label-custom_test
-    - <<: *if-label-custom_test_esp32
-    - <<: *if-label-custom_test_esp32c2
-    - <<: *if-label-custom_test_esp32c3
-    - <<: *if-label-custom_test_esp32h2
-    - <<: *if-label-custom_test_esp32s2
-    - <<: *if-label-custom_test_esp32s3
-    - <<: *if-label-example_test
-    - <<: *if-label-example_test_esp32
-    - <<: *if-label-example_test_esp32c2
-    - <<: *if-label-example_test_esp32c3
-    - <<: *if-label-example_test_esp32h2
-    - <<: *if-label-example_test_esp32s2
-    - <<: *if-label-example_test_esp32s3
-    - <<: *if-label-host_test
-    - <<: *if-label-integration_test
-    - <<: *if-label-iperf_stress_test
-    - <<: *if-label-unit_test
-    - <<: *if-label-unit_test_esp32
-    - <<: *if-label-unit_test_esp32c2
-    - <<: *if-label-unit_test_esp32c3
-    - <<: *if-label-unit_test_esp32h2
-    - <<: *if-label-unit_test_esp32s2
-    - <<: *if-label-unit_test_esp32s3
-    - <<: *if-label-weekend_test
-    - <<: *if-dev-push
-      changes: *patterns-build-example_test
-    - <<: *if-dev-push
-      changes: *patterns-component_ut
-    - <<: *if-dev-push
-      changes: *patterns-custom_test
-    - <<: *if-dev-push
-      changes: *patterns-example_test
-    - <<: *if-dev-push
-      changes: *patterns-host_test
-    - <<: *if-dev-push
-      changes: *patterns-integration_test
-    - <<: *if-dev-push
-      changes: *patterns-unit_test
-
 .rules:test:component_ut-esp32:
  rules:
    - <<: *if-revert-branch
--- a/.gitlab/ci/target-test.yml
+++ b/.gitlab/ci/target-test.yml
@ -173,8 +173,6 @@ component_ut_pytest_esp32c3_generic:

 .target_test_job_template:
  stage: target_test
-  needs:
-    - assign_test
  artifacts:
    when: always
    paths:
@ -204,6 +202,8 @@ component_ut_pytest_esp32c3_generic:

 .example_test_template:
  extends: .target_test_job_template
+  needs:
+    - assign_example_test
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
@ -467,6 +467,8 @@ example_test_ESP32C3_SDSPI:

 .test_app_template:
  extends: .target_test_job_template
+  needs:
+    - assign_custom_test
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
@ -558,9 +560,11 @@ test_app_test_flash_psram_f8r8:

 .unit_test_template:
  extends: .target_test_job_template
+  needs: # the assign already needs all the build jobs
+    - assign_unit_test
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/test_configs"

 .unit_test_esp32_template:
  extends:
@ -867,14 +871,14 @@ UT_S3_FLASH:
    # needn't install idf python env
    - .before_script_minimal
  needs:
-    - assign_test
+    - assign_integration_test
    - build_ssc_esp32
  variables:
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
    LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/test_configs"
    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
    CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
--- a/components/idf_test/integration_test/test_configs/nvs_compatible_test_.yml
+++ b/components/idf_test/integration_test/test_configs/nvs_compatible_test_.yml