kopia lustrzana https://github.com/espressif/esp-idf
ci(rules): add check_tools_file_patterns.py
this file is used to check if all files under `tools` folder are recorded in patterns or in exclude listpull/6718/head
Fu Hanxi
rodzic
b33e344484
commit
8ff6461b4c
|
|
@ -147,3 +147,12 @@ check_version_tag:
|
|||
- .rules:tag:release-no_label
|
||||
script:
|
||||
- (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
|
||||
|
||||
check_tools_file_patterns:
|
||||
extends: .pre_check_job_template
|
||||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||
variables:
|
||||
PYTHON_VER: 3.7.7
|
||||
script:
|
||||
- python tools/ci/check_tools_files_patterns.py
|
||||
allow_failure: true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate
|
||||
|
||||
# Monkey patch starts
|
||||
|
||||
# glob.glob will ignore all files starts with ``.``
|
||||
# don't ignore them here
|
||||
# need to keep the same argument as glob._ishidden
|
||||
|
||||
|
||||
def _ishidden(path): # pylint: disable=W0613
|
||||
return False
|
||||
|
||||
|
||||
fnmatch.translate = translate
|
||||
|
||||
glob.magic_check = magic_check
|
||||
glob.magic_check_bytes = magic_check_bytes
|
||||
glob._ishidden = _ishidden # pylint: disable=W0212
|
||||
# ends here
|
||||
|
||||
|
||||
def check(pattern_yml, exclude_list):
|
||||
rules_dict = yaml.load(open(pattern_yml), Loader=yaml.FullLoader)
|
||||
rules_patterns_set = set()
|
||||
for k, v in rules_dict.items():
|
||||
if k.startswith('.pattern') and isinstance(v, list):
|
||||
rules_patterns_set.update(v)
|
||||
rules_files_set = set()
|
||||
for pat in rules_patterns_set:
|
||||
rules_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True))
|
||||
|
||||
exclude_patterns_set = set()
|
||||
exclude_patterns_set.update([path.split('#')[0].strip() for path in open(exclude_list).readlines() if path])
|
||||
exclude_files_set = set()
|
||||
for pat in exclude_patterns_set:
|
||||
exclude_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True))
|
||||
|
||||
missing_files = set()
|
||||
git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True)
|
||||
for f in git_files:
|
||||
if f in rules_files_set or f in exclude_files_set:
|
||||
continue
|
||||
missing_files.add(os.path.relpath(f, IDF_PATH))
|
||||
|
||||
return missing_files, rules_patterns_set.intersection(exclude_patterns_set)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='check if all tools files are in rules patterns or exclude list')
|
||||
parser.add_argument('-c', '--pattern-yml',
|
||||
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
|
||||
help='yml file path included file patterns')
|
||||
parser.add_argument('-e', '--exclude-list',
|
||||
default=os.path.join(IDF_PATH, 'tools', 'ci', 'exclude_check_tools_files.txt'),
|
||||
help='exclude list path')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
res = 0
|
||||
not_included_files, dup_patterns = check(args.pattern_yml, args.exclude_list)
|
||||
if not_included_files:
|
||||
print('Missing Files: (please add to tools/ci/exclude_check_tools_files.txt')
|
||||
for f in not_included_files:
|
||||
print(f)
|
||||
res = 1
|
||||
if dup_patterns:
|
||||
print('Duplicated Patterns: (please check .gitlab/ci/rules.yml and tools/ci/exclude_check_tools_files.txt')
|
||||
for pat in dup_patterns:
|
||||
print(pat)
|
||||
res = 1
|
||||
|
||||
sys.exit(res)
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
tools/ble/**/*
|
||||
tools/catch/**/*
|
||||
tools/ci/build_template_app.sh
|
||||
tools/ci/check_*.{py,txt,sh} # excluded because run in default pipeline pre-check stage
|
||||
tools/ci/checkout_project_ref.py
|
||||
tools/ci/ci_fetch_submodule.py
|
||||
tools/ci/ci_get_mr_info.py
|
||||
tools/ci/configure_ci_environment.sh
|
||||
tools/ci/deploy_docs.py
|
||||
tools/ci/envsubst.py
|
||||
tools/ci/*exclude*.txt
|
||||
tools/ci/executable-list.txt
|
||||
tools/ci/fix_empty_prototypes.sh
|
||||
tools/ci/get-full-sources.sh
|
||||
tools/ci/idf_ci_utils.py
|
||||
tools/ci/mirror-submodule-update.sh
|
||||
tools/ci/multirun_with_pyenv.sh
|
||||
tools/ci/normalize_clangtidy_path.py
|
||||
tools/ci/push_to_github.sh
|
||||
tools/ci/python_packages/wifi_tools.py
|
||||
tools/ci/setup_python.sh
|
||||
tools/ci/utils.sh
|
||||
tools/eclipse-code-style.xml
|
||||
tools/format-minimal.sh
|
||||
tools/format.sh
|
||||
tools/gen_esp_err_to_name.py
|
||||
tools/kconfig/**/*
|
||||
tools/set-submodules-to-github.sh
|
||||
tools/templates/sample_component/CMakeLists.txt
|
||||
tools/templates/sample_component/include/main.h
|
||||
tools/templates/sample_component/main.c
|
||||
tools/toolchain_versions.mk
|
||||
|
|
@ -46,6 +46,7 @@ tools/ci/check_idf_version.sh
|
|||
tools/ci/check_kconfigs.py
|
||||
tools/ci/check_readme_links.py
|
||||
tools/ci/check_rom_apis.sh
|
||||
tools/ci/check_tools_files_patterns.py
|
||||
tools/ci/check_ut_cmake_make.sh
|
||||
tools/ci/checkout_project_ref.py
|
||||
tools/ci/deploy_docs.py
|
||||
|
|
|
|||
|
|
@ -15,13 +15,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
IDF_PATH = os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))
|
||||
|
||||
|
||||
def get_submodule_dirs(full_path=False): # type: (bool) -> list
|
||||
|
|
@ -41,7 +42,7 @@ def get_submodule_dirs(full_path=False): # type: (bool) -> list
|
|||
dirs.append(os.path.join(IDF_PATH, path))
|
||||
else:
|
||||
dirs.append(path)
|
||||
except Exception as e:
|
||||
except Exception as e: # pylint: disable=W0703
|
||||
logging.warning(str(e))
|
||||
|
||||
return dirs
|
||||
|
|
@ -67,5 +68,132 @@ def is_executable(full_path): # type: (str) -> bool
|
|||
"""
|
||||
if sys.platform == 'win32':
|
||||
return _check_git_filemode(full_path)
|
||||
else:
|
||||
return os.access(full_path, os.X_OK)
|
||||
return os.access(full_path, os.X_OK)
|
||||
|
||||
|
||||
def get_git_files(path=IDF_PATH, full_path=False): # type: (str, bool) -> list[str]
|
||||
"""
|
||||
Get the result of git ls-files
|
||||
:param path: path to run git ls-files
|
||||
:param full_path: return full path if set to True
|
||||
:return: list of file paths
|
||||
"""
|
||||
try:
|
||||
files = subprocess.check_output(['git', 'ls-files'], cwd=path).decode('utf8').strip().split('\n')
|
||||
except Exception as e: # pylint: disable=W0703
|
||||
logging.warning(str(e))
|
||||
files = []
|
||||
return [os.path.join(path, f) for f in files] if full_path else files
|
||||
|
||||
|
||||
# this function is a commit from
|
||||
# https://github.com/python/cpython/pull/6299/commits/bfd63120c18bd055defb338c075550f975e3bec1
|
||||
# In order to solve python https://bugs.python.org/issue9584
|
||||
# glob pattern does not support brace expansion issue
|
||||
def _translate(pat):
|
||||
"""Translate a shell PATTERN to a regular expression.
|
||||
There is no way to quote meta-characters.
|
||||
"""
|
||||
i, n = 0, len(pat)
|
||||
res = ''
|
||||
while i < n:
|
||||
c = pat[i]
|
||||
i = i + 1
|
||||
if c == '*':
|
||||
res = res + '.*'
|
||||
elif c == '?':
|
||||
res = res + '.'
|
||||
elif c == '[':
|
||||
j = i
|
||||
if j < n and pat[j] == '!':
|
||||
j = j + 1
|
||||
if j < n and pat[j] == ']':
|
||||
j = j + 1
|
||||
while j < n and pat[j] != ']':
|
||||
j = j + 1
|
||||
if j >= n:
|
||||
res = res + '\\['
|
||||
else:
|
||||
stuff = pat[i:j]
|
||||
if '--' not in stuff:
|
||||
stuff = stuff.replace('\\', r'\\')
|
||||
else:
|
||||
chunks = []
|
||||
k = i + 2 if pat[i] == '!' else i + 1
|
||||
while True:
|
||||
k = pat.find('-', k, j)
|
||||
if k < 0:
|
||||
break
|
||||
chunks.append(pat[i:k])
|
||||
i = k + 1
|
||||
k = k + 3
|
||||
chunks.append(pat[i:j])
|
||||
# Escape backslashes and hyphens for set difference (--).
|
||||
# Hyphens that create ranges shouldn't be escaped.
|
||||
stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
|
||||
for s in chunks)
|
||||
# Escape set operations (&&, ~~ and ||).
|
||||
stuff = re.sub(r'([&~|])', r'\\\1', stuff)
|
||||
i = j + 1
|
||||
if stuff[0] == '!':
|
||||
stuff = '^' + stuff[1:]
|
||||
elif stuff[0] in ('^', '['):
|
||||
stuff = '\\' + stuff
|
||||
res = '%s[%s]' % (res, stuff)
|
||||
elif c == '{':
|
||||
# Handling of brace expression: '{PATTERN,PATTERN,...}'
|
||||
j = 1
|
||||
while j < n and pat[j] != '}':
|
||||
j = j + 1
|
||||
if j >= n:
|
||||
res = res + '\\{'
|
||||
else:
|
||||
stuff = pat[i:j]
|
||||
i = j + 1
|
||||
|
||||
# Find indices of ',' in pattern excluding r'\,'.
|
||||
# E.g. for r'a\,a,b\b,c' it will be [4, 8]
|
||||
indices = [m.end() for m in re.finditer(r'[^\\],', stuff)]
|
||||
|
||||
# Splitting pattern string based on ',' character.
|
||||
# Also '\,' is translated to ','. E.g. for r'a\,a,b\b,c':
|
||||
# * first_part = 'a,a'
|
||||
# * last_part = 'c'
|
||||
# * middle_part = ['b,b']
|
||||
first_part = stuff[:indices[0] - 1].replace(r'\,', ',')
|
||||
last_part = stuff[indices[-1]:].replace(r'\,', ',')
|
||||
middle_parts = [
|
||||
stuff[st:en - 1].replace(r'\,', ',')
|
||||
for st, en in zip(indices, indices[1:])
|
||||
]
|
||||
|
||||
# creating the regex from splitted pattern. Each part is
|
||||
# recursivelly evaluated.
|
||||
expanded = functools.reduce(
|
||||
lambda a, b: '|'.join((a, b)),
|
||||
(_translate(elem) for elem in [first_part] + middle_parts + [last_part])
|
||||
)
|
||||
res = '%s(%s)' % (res, expanded)
|
||||
else:
|
||||
res = res + re.escape(c)
|
||||
return res
|
||||
|
||||
|
||||
def translate(pat):
|
||||
res = _translate(pat)
|
||||
return r'(?s:%s)\Z' % res
|
||||
|
||||
|
||||
magic_check = re.compile('([*?[{])')
|
||||
magic_check_bytes = re.compile(b'([*?[{])')
|
||||
# cpython github PR 6299 ends here
|
||||
|
||||
# Here's the code block we're going to use to monkey patch ``glob`` module and ``fnmatch`` modules
|
||||
# DO NOT monkey patch here, only patch where you really needs
|
||||
#
|
||||
# import glob
|
||||
# import fnmatch
|
||||
# from idf_ci_utils import magic_check, magic_check_bytes, translate
|
||||
# glob.magic_check = magic_check
|
||||
# glob.magic_check_bytes = magic_check_bytes
|
||||
# fnmatch.translate = translate
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue