From 69e34b6b6bdf45bc1111777c46839a8b5fcb30bd Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 4 Oct 2023 11:20:47 +1100 Subject: [PATCH] all: Switch to new preview build versioning scheme. See https://github.com/micropython/micropython/issues/12127 for details. Previously at the point when a release is made, we update mpconfig.h and set a git tag. i.e. the version increments at the release. Now the version increments immediately after the release. The workflow is: 1. Final commit in the cycle updates mpconfig.h to set (X, Y, 0, 0) (i.e. clear the pre-release state). 2. This commit is tagged "vX.Y.0". 3. First commit for the new cycle updates mpconfig.h to set (X, Y+1, 0, 1) (i.e. increment the minor version, set the pre-release state). 4. This commit is tagged "vX.Y+1.0-preview". The idea is that a nightly build is actually a "preview" of the _next_ release. i.e. any documentation describing the current release may not actually match the nightly build. So we use "preview" as our semver pre-release identifier. Changes in this commit: - Add MICROPY_VERSION_PRERELEASE to mpconfig.h to allow indicating that this is not a release version. - Remove unused MICROPY_VERSION integer. - Append "-preview" to MICROPY_VERSION_STRING when the pre-release state is set. - Update py/makeversionhdr.py to no longer generate MICROPY_GIT_HASH. - Remove the one place MICROPY_GIT_HASH was used (it can use MICROPY_GIT_TAG instead). - Update py/makeversionhdr.py to also understand MICROPY_VERSION_PRERELEASE in mpconfig.h. - Update py/makeversionhdr.py to convert the git-describe output into semver-compatible "X.Y.Z-preview.N.gHASH". - Update autobuild.sh to generate filenames using the new scheme. - Update remove_old_firmware.py to match new scheme. - Update mpremote's pyproject.toml to handle the "-preview" suffix in the tag. setuptools_scm maps to this "rc0" to match PEP440. - Fix docs heading where it incorrectly said "vvX.Y.Z" for release docs. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- docs/conf.py | 4 +- docs/templates/layout.html | 2 +- ports/cc3200/tools/update-wipy.py | 4 +- py/makeversionhdr.py | 93 +++++++++++++------------- py/modsys.c | 20 ++++-- py/mpconfig.h | 28 +++++--- tools/autobuild/autobuild.sh | 14 +++- tools/autobuild/remove_old_firmware.py | 22 +++--- tools/mpremote/pyproject.toml | 7 +- 9 files changed, 116 insertions(+), 78 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0dbbfe0f70..49ddd22be9 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,10 +21,12 @@ import os # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('.')) -# The members of the html_context dict are available inside topindex.html +# The MICROPY_VERSION env var should be "vX.Y.Z" (or unset). micropy_version = os.getenv('MICROPY_VERSION') or 'latest' micropy_all_versions = (os.getenv('MICROPY_ALL_VERSIONS') or 'latest').split(',') url_pattern = '%s/en/%%s' % (os.getenv('MICROPY_URL_PREFIX') or '/',) + +# The members of the html_context dict are available inside topindex.html html_context = { 'cur_version':micropy_version, 'all_versions':[ diff --git a/docs/templates/layout.html b/docs/templates/layout.html index 8563f02af0..21dab5eb42 100644 --- a/docs/templates/layout.html +++ b/docs/templates/layout.html @@ -9,7 +9,7 @@ {% if is_release %}

- This is the v{{ release }} version of the MicroPython + This is the {{ release }} version of the MicroPython documentation. The latest development version of this page may be more current.

diff --git a/ports/cc3200/tools/update-wipy.py b/ports/cc3200/tools/update-wipy.py index 7f5527a52b..e37f3a0b09 100644 --- a/ports/cc3200/tools/update-wipy.py +++ b/ports/cc3200/tools/update-wipy.py @@ -152,9 +152,9 @@ def verify_update(args): with open(tag_file_path) as tag_file: for line in tag_file: bline = bytes(line, "ascii") - if b"MICROPY_GIT_HASH" in bline: + if b"MICROPY_GIT_TAG" in bline: bline = ( - bline.lstrip(b"#define MICROPY_GIT_HASH ") + bline.lstrip(b"#define MICROPY_GIT_TAG ") .replace(b'"', b"") .replace(b"\r", b"") .replace(b"\n", b"") diff --git a/py/makeversionhdr.py b/py/makeversionhdr.py index e682a4e139..2030bb02c4 100644 --- a/py/makeversionhdr.py +++ b/py/makeversionhdr.py @@ -13,6 +13,14 @@ import datetime import subprocess +# The MicroPython repository tags a release commit as "vX.Y.Z", and the commit +# immediately following as "vX.(Y+1).Z-preview". +# This function will return: +# "vX.Y.Z" -- building at the release commit +# "vX.Y.Z-preview" -- building at the first commit in the next cycle +# "vX.Y.Z-preview.N.gHASH" -- building at any subsequent commit in the cycle +# "vX.Y.Z-preview.N.gHASH.dirty" -- building at any subsequent commit in the cycle +# with local changes def get_version_info_from_git(repo_path): # Python 2.6 doesn't have check_output, so check for that try: @@ -29,47 +37,33 @@ def get_version_info_from_git(repo_path): stderr=subprocess.STDOUT, universal_newlines=True, ).strip() - except subprocess.CalledProcessError as er: - if er.returncode == 128: - # git exit code of 128 means no repository found - return None - git_tag = "" - except OSError: - return None - try: - git_hash = subprocess.check_output( - ["git", "rev-parse", "--short", "HEAD"], - cwd=repo_path, - stderr=subprocess.STDOUT, - universal_newlines=True, - ).strip() + # Turn git-describe's output into semver compatible (dot-separated + # identifiers inside the prerelease field). + git_tag = git_tag.split("-", 1) + if len(git_tag) == 1: + return git_tag[0] + else: + return git_tag[0] + "-" + git_tag[1].replace("-", ".") except subprocess.CalledProcessError: - git_hash = "unknown" + return None except OSError: return None - try: - # Check if there are any modified files. - subprocess.check_call( - ["git", "diff", "--no-ext-diff", "--quiet", "--exit-code"], - cwd=repo_path, - stderr=subprocess.STDOUT, - ) - # Check if there are any staged files. - subprocess.check_call( - ["git", "diff-index", "--cached", "--quiet", "HEAD", "--"], - cwd=repo_path, - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError: - git_hash += "-dirty" - except OSError: - return None - - return git_tag, git_hash - +# When building from a source tarball (or any situation where the git repo +# isn't available), this function will use the info in mpconfig.h as a +# fallback. The release commit sets MICROPY_VERSION_PRERELEASE to 0, and the +# commit immediately following increments MICROPY_VERSION_MINOR and sets +# MICROPY_VERSION_PRERELEASE back to 1. +# This function will return: +# "vX.Y.Z" -- building at the release commit +# "vX.Y.Z-preview" -- building at any other commit def get_version_info_from_mpconfig(repo_path): + print( + "makeversionhdr.py: Warning: No git repo or tag info available, falling back to mpconfig.h version info.", + file=sys.stderr, + ) + with open(os.path.join(repo_path, "py", "mpconfig.h")) as f: for line in f: if line.startswith("#define MICROPY_VERSION_MAJOR "): @@ -78,21 +72,30 @@ def get_version_info_from_mpconfig(repo_path): ver_minor = int(line.strip().split()[2]) elif line.startswith("#define MICROPY_VERSION_MICRO "): ver_micro = int(line.strip().split()[2]) - git_tag = "v%d.%d.%d" % (ver_major, ver_minor, ver_micro) - return git_tag, "" + elif line.startswith("#define MICROPY_VERSION_PRERELEASE "): + ver_prerelease = int(line.strip().split()[2]) + git_tag = "v%d.%d.%d%s" % ( + ver_major, + ver_minor, + ver_micro, + "-preview" if ver_prerelease else "", + ) + return git_tag return None def make_version_header(repo_path, filename): - info = None + git_tag = None if "MICROPY_GIT_TAG" in os.environ: - info = [os.environ["MICROPY_GIT_TAG"], os.environ["MICROPY_GIT_HASH"]] - if info is None: - info = get_version_info_from_git(repo_path) - if info is None: - info = get_version_info_from_mpconfig(repo_path) + git_tag = os.environ["MICROPY_GIT_TAG"] + if git_tag is None: + git_tag = get_version_info_from_git(repo_path) + if git_tag is None: + git_tag = get_version_info_from_mpconfig(repo_path) - git_tag, git_hash = info + if not git_tag: + print("makeversionhdr.py: Error: No version information available.") + sys.exit(1) build_date = datetime.date.today() if "SOURCE_DATE_EPOCH" in os.environ: @@ -104,11 +107,9 @@ def make_version_header(repo_path, filename): file_data = """\ // This file was generated by py/makeversionhdr.py #define MICROPY_GIT_TAG "%s" -#define MICROPY_GIT_HASH "%s" #define MICROPY_BUILD_DATE "%s" """ % ( git_tag, - git_hash, build_date.strftime("%Y-%m-%d"), ) diff --git a/py/modsys.c b/py/modsys.c index 38105ee218..e40542467b 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -59,16 +59,24 @@ const mp_print_t mp_sys_stdout_print = {&mp_sys_stdout_obj, mp_stream_write_adap STATIC const MP_DEFINE_STR_OBJ(mp_sys_version_obj, "3.4.0; " MICROPY_BANNER_NAME_AND_VERSION); // version_info - Python language version that this implementation conforms to, as a tuple of ints -#define I(n) MP_OBJ_NEW_SMALL_INT(n) -// TODO: CPython is now at 5-element array, but save 2 els so far... -STATIC const mp_obj_tuple_t mp_sys_version_info_obj = {{&mp_type_tuple}, 3, {I(3), I(4), I(0)}}; +// TODO: CPython is now at 5-element array (major, minor, micro, releaselevel, serial), but save 2 els so far... +STATIC const mp_rom_obj_tuple_t mp_sys_version_info_obj = {{&mp_type_tuple}, 3, {MP_ROM_INT(3), MP_ROM_INT(4), MP_ROM_INT(0)}}; // sys.implementation object // this holds the MicroPython version -STATIC const mp_obj_tuple_t mp_sys_implementation_version_info_obj = { +STATIC const mp_rom_obj_tuple_t mp_sys_implementation_version_info_obj = { {&mp_type_tuple}, - 3, - { I(MICROPY_VERSION_MAJOR), I(MICROPY_VERSION_MINOR), I(MICROPY_VERSION_MICRO) } + 4, + { + MP_ROM_INT(MICROPY_VERSION_MAJOR), + MP_ROM_INT(MICROPY_VERSION_MINOR), + MP_ROM_INT(MICROPY_VERSION_MICRO), + #if MICROPY_VERSION_PRERELEASE + MP_ROM_QSTR(MP_QSTR_preview), + #else + MP_ROM_QSTR(MP_QSTR_), + #endif + } }; STATIC const MP_DEFINE_STR_OBJ(mp_sys_implementation_machine_obj, MICROPY_BANNER_MACHINE); #if MICROPY_PERSISTENT_CODE_LOAD diff --git a/py/mpconfig.h b/py/mpconfig.h index eb3a0eb736..a36f9658fb 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -26,22 +26,32 @@ #ifndef MICROPY_INCLUDED_PY_MPCONFIG_H #define MICROPY_INCLUDED_PY_MPCONFIG_H -// Current version of MicroPython +// Current version of MicroPython. This is used by sys.implementation.version +// as well as a fallback to generate MICROPY_GIT_TAG if the git repo or tags +// are unavailable. #define MICROPY_VERSION_MAJOR 1 -#define MICROPY_VERSION_MINOR 21 +#define MICROPY_VERSION_MINOR 22 #define MICROPY_VERSION_MICRO 0 +#define MICROPY_VERSION_PRERELEASE 1 -// Combined version as a 32-bit number for convenience -#define MICROPY_VERSION ( \ - MICROPY_VERSION_MAJOR << 16 \ - | MICROPY_VERSION_MINOR << 8 \ - | MICROPY_VERSION_MICRO) +// Combined version as a 32-bit number for convenience to allow version +// comparison. Doesn't include prerelease state. +// e.g. #if MICROPY_VERSION < MICROPY_MAKE_VERSION(1, 22, 0) +#define MICROPY_MAKE_VERSION(major, minor, patch) (major << 16 | minor << 8 | patch) +#define MICROPY_VERSION MICROPY_MAKE_VERSION(MICROPY_VERSION_MAJOR, MICROPY_VERSION_MINOR, MICROPY_VERSION_MICRO) -// String version -#define MICROPY_VERSION_STRING \ +// String version. This is only used directly for platform.platform and +// os.uname().release. All other version info available in the firmware (e.g. +// the REPL banner) comes from MICROPY_GIT_TAG. +#define MICROPY_VERSION_STRING_BASE \ MP_STRINGIFY(MICROPY_VERSION_MAJOR) "." \ MP_STRINGIFY(MICROPY_VERSION_MINOR) "." \ MP_STRINGIFY(MICROPY_VERSION_MICRO) +#if MICROPY_VERSION_PRERELEASE +#define MICROPY_VERSION_STRING MICROPY_VERSION_STRING_BASE "-preview" +#else +#define MICROPY_VERSION_STRING MICROPY_VERSION_STRING_BASE +#endif // This file contains default configuration settings for MicroPython. // You can override any of the options below using mpconfigport.h file diff --git a/tools/autobuild/autobuild.sh b/tools/autobuild/autobuild.sh index e1f2287ee5..7854945bbe 100755 --- a/tools/autobuild/autobuild.sh +++ b/tools/autobuild/autobuild.sh @@ -54,9 +54,19 @@ pushd ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} make -C mpy-cross # make the firmware tag +# final filename will be <-VARIANT>--v.ext +# where SEMVER is vX.Y.Z or vX.Y.Z-preview.N.gHASH or vX.Y.Z-preview.N.gHASH.dirty FW_DATE=$(date '+%Y%m%d') -FW_GIT="$(git describe --dirty || echo unknown)" -FW_TAG="-$FW_DATE-unstable-$FW_GIT" +# same logic as makeversionhdr.py, convert git-describe output into semver-compatible +FW_GIT_TAG="$(git describe --tags --dirty --always --match 'v[1-9].*')" +FW_SEMVER_MAJOR_MINOR_PATCH="$(echo $FW_GIT_TAG | cut -d'-' -f1)" +FW_SEMVER_PRERELEASE="$(echo $FW_GIT_TAG | cut -s -d'-' -f2-)" +if [ -z "$FW_SEMVER_PRERELEASE" ]; then + FW_SEMVER="$FW_SEMVER_MAJOR_MINOR_PATCH" +else + FW_SEMVER="$FW_SEMVER_MAJOR_MINOR_PATCH-$(echo $FW_SEMVER_PRERELEASE | tr - .)" +fi +FW_TAG="-$FW_DATE-$FW_SEMVER" # build new firmware cd ports/cc3200 diff --git a/tools/autobuild/remove_old_firmware.py b/tools/autobuild/remove_old_firmware.py index e9d2e8aae8..a6203531c7 100755 --- a/tools/autobuild/remove_old_firmware.py +++ b/tools/autobuild/remove_old_firmware.py @@ -14,7 +14,7 @@ def main(): # SSH to get list of existing files. p = subprocess.run( - ["ssh", ssh_machine, "find", ssh_firmware_dir, "-name", "\\*-unstable-v\\*"], + ["ssh", ssh_machine, "find", ssh_firmware_dir, "-name", "\\*-preview.\\*"], capture_output=True, ) if p.returncode != 0: @@ -26,31 +26,33 @@ def main(): boards = {} for file in all_files: m = re.match( - rb"([a-z/.]+)/([A-Za-z0-9_-]+)-(20[0-9]{6})-unstable-(v[0-9.-]+-g[0-9a-f]+).", + rb"([a-z/.]+)/([A-Za-z0-9_-]+)-(20[0-9]{6})-(v[0-9.]+)-preview.([0-9]+).g[0-9a-f]+.", file, ) if not m: continue - dir, board, date, version = m.groups() + dir, board, date, version, ncommits = m.groups() if board not in boards: boards[board] = {} - if (date, version) not in boards[board]: - boards[board][(date, version)] = [] - boards[board][(date, version)].append(file) + if (date, version, ncommits) not in boards[board]: + boards[board][(date, version, ncommits)] = [] + boards[board][(date, version, ncommits)].append(file) # Collect files to remove based on date and version. remove = [] for board in boards.values(): - filelist = [(date, version, files) for (date, version), files in board.items()] + filelist = [ + (date, version, ncommits, files) for (date, version, ncommits), files in board.items() + ] filelist.sort(reverse=True) keep = [] - for date, version, files in filelist: - if keep and version == keep[-1]: + for date, version, ncommits, files in filelist: + if keep and (version, ncommits) == keep[-1]: remove.extend(files) elif len(keep) >= NUM_KEEP_PER_BOARD: remove.extend(files) else: - keep.append(version) + keep.append((version, ncommits)) if DEBUG: all_files.sort(reverse=True) diff --git a/tools/mpremote/pyproject.toml b/tools/mpremote/pyproject.toml index b01385c3d5..746b40bfdd 100644 --- a/tools/mpremote/pyproject.toml +++ b/tools/mpremote/pyproject.toml @@ -36,9 +36,14 @@ mpremote = "mpremote.main:main" [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] +# This will be PEP-440 normalised into either: +# mpremote-X.Y.Z (on vX.Y.Z release tag) +# mpremote-X.Y.Zrc0 (on vX.Y.Z-preview tag, i.e. first commit in the cycle) +# mpremote-X.Y.Zrc0.postN+gHASH (N commits past vX.Y.Z-preview tag) +# mpremote-X.Y.Zrc0.postN+gHASH.dDATE (N commits past vX.Y.Z-preview tag, dirty) [tool.hatch.version] source = "vcs" -tag-pattern = "(?Pv(\\d+).(\\d+).(\\d+))" +tag-pattern = "(?Pv(\\d+).(\\d+).(\\d+)(-preview)?)" raw-options = { root = "../..", version_scheme = "post-release" } [tool.hatch.build]