diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2be13a169b..ac88ff562d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -97,12 +97,16 @@ variables: CI_PYTHON_TOOL_BRANCH: "" cache: - key: pip-cache - paths: - - .cache/pip # pull only for most of the use cases since it's cache dir. - # Only set "push" policy for "upload_cache" stage jobs since it would install all pypi packages - policy: pull + # Only set "push" policy for "upload_cache" stage jobs + - key: pip-cache + paths: + - .cache/pip + policy: pull + - key: submodule-cache + paths: + - .cache/submodule_archives + policy: pull .setup_tools_unless_target_test: &setup_tools_unless_target_test | if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 5e842a9866..29605b4d86 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -7,11 +7,14 @@ tags: - host_test dependencies: [] + # run host_test jobs immediately, only after upload cache needs: - - job: upload-pip-cache-shiny - optional: true # run host_test jobs immediately, only after upload cache - - job: upload-pip-cache-brew - optional: true # run host_test jobs immediately, only after upload cache + - job: upload-pip-cache + optional: true + artifacts: false + - job: upload-submodules-cache + optional: true + artifacts: false test_nvs_on_host: extends: .host_test_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 377158ba9e..0ae73ec22f 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -194,6 +194,7 @@ - "components/tinyusb/tinyusb" - "components/unity/unity" - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib" + - ".gitmodules" .patterns-example_test-related_changes-ota: &patterns-example_test-related_changes-ota - "examples/system/ota/**/*" @@ -270,6 +271,11 @@ - <<: *if-dev-push changes: *patterns-python-cache +.rules:patterns:submodule: + rules: + - <<: *if-dev-push + changes: *patterns-submodule + .rules:dev: rules: - <<: *if-trigger diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index afbb0e0005..0b2f166195 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -1,17 +1,22 @@ +# pull only for most of the use cases for cache +# only set "push" policy for the jobs under this file. +# The cache would be updated when files matched specified patterns changes. + .upload_cache_template: stage: upload_cache image: $ESP_ENV_IMAGE -.upload_pip_cache_template: +upload-pip-cache: extends: - .upload_cache_template - .rules:patterns:python-cache + tags: + - $GEO + - build cache: key: pip-cache paths: - .cache/pip - # pull only for most of the use cases since it's cache dir. - # Only set "push" policy for "upload_cache" stage jobs since it would install all pypi packages policy: push before_script: [] script: @@ -22,15 +27,29 @@ - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest # TODO: remove this, IDFCI-1207 - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} + parallel: + matrix: + - GEO: [ 'shiny', 'brew' ] -upload-pip-cache-shiny: - extends: .upload_pip_cache_template +upload-submodules-cache: + extends: + - .upload_cache_template + - .rules:patterns:submodule tags: - - shiny - - build - -upload-pip-cache-brew: - extends: .upload_pip_cache_template - tags: - - brew + - $GEO - build + cache: + key: submodule-cache + paths: + - .cache/submodule_archives + policy: push + before_script: [] + script: + - source tools/ci/utils.sh + - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS + - source tools/ci/setup_python.sh + - rm -rf .cache/submodule_archives # clear old submodule archives + - fetch_submodules + parallel: + matrix: + - GEO: [ 'shiny', 'brew' ] diff --git a/.gitmodules b/.gitmodules index 8be23b0f12..802e4b04a7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ [submodule "components/bt/controller/lib_esp32"] path = components/bt/controller/lib_esp32 - url = ../../espressif/esp32-bt-lib.git + url = ../../espressif/esp32-bt-lib.git [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"] path = components/bootloader/subproject/components/micro-ecc/micro-ecc diff --git a/tools/ci/ci_fetch_submodule.py b/tools/ci/ci_fetch_submodule.py index 22879e4391..5961e12522 100644 --- a/tools/ci/ci_fetch_submodule.py +++ b/tools/ci/ci_fetch_submodule.py @@ -19,6 +19,8 @@ PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)') URL_PATTERN = re.compile(r'url\s+=\s+(\S+)') SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive' +# need to match the one defined in CI yaml files for caching purpose +SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives' class SubModule(object): @@ -28,6 +30,7 @@ class SubModule(object): def __init__(self, gitlab_inst, path, url): self.path = path + self.url = url self.gitlab_inst = gitlab_inst self.project_id = self._get_project_id(url) self.commit_id = self._get_commit_id(path) @@ -48,7 +51,7 @@ class SubModule(object): def download_archive(self): print('Update submodule: {}: {}'.format(self.path, self.commit_id)) path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, - self.project_id) + self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR) renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path)) os.rename(path_name, renamed_path) shutil.rmtree(self.path, ignore_errors=True) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 687814375d..3ad20eeb60 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -177,7 +177,7 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str: + def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -191,6 +191,23 @@ class Gitlab(object): else: project = self.gitlab_inst.projects.get(project_id) + if cache_dir: + local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz') + os.makedirs(os.path.dirname(local_archive_file), exist_ok=True) + if os.path.isfile(local_archive_file): + print('Use cached archive file. Skipping download...') + else: + with open(local_archive_file, 'wb') as fw: + try: + project.repository_archive(sha=ref, streamed=True, action=fw.write) + except gitlab.GitlabGetError as e: + print('Failed to archive from project {}'.format(project_id)) + raise e + print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(local_archive_file)) / (1024 * 1024))) + + return self.decompress_archive(local_archive_file, destination) + + # no cache with tempfile.NamedTemporaryFile(delete=False) as temp_file: try: project.repository_archive(sha=ref, streamed=True, action=temp_file.write) @@ -198,9 +215,13 @@ class Gitlab(object): print('Failed to archive from project {}'.format(project_id)) raise e - print('archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) + print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) - with tarfile.open(temp_file.name, 'r') as archive_file: + return self.decompress_archive(temp_file.name, destination) + + @staticmethod + def decompress_archive(path: str, destination: str) -> str: + with tarfile.open(path, 'r') as archive_file: root_name = archive_file.getnames()[0] archive_file.extractall(destination)