From 12ca918eb2ac062f6e6df0772e528eef9d050cb7 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Thu, 29 Sep 2022 00:45:34 +1000 Subject: [PATCH] tools/mpremote: Add `mpremote mip install` to install packages. This supports the same package sources as the new `mip` tool. - micropython-lib (by name) - http(s) & github packages with json description - directly downloading a .py/.mpy file The version is specified with an optional `@version` on the end of the package name. The target dir, index, and mpy/no-mpy can be set through command line args. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- docs/reference/mpremote.rst | 14 +++ docs/reference/packages.rst | 16 +-- tools/mpremote/README.md | 41 ++++--- tools/mpremote/mpremote/main.py | 28 +++++ tools/mpremote/mpremote/mip.py | 191 ++++++++++++++++++++++++++++++++ tools/pyboard.py | 7 ++ 6 files changed, 272 insertions(+), 25 deletions(-) create mode 100644 tools/mpremote/mpremote/mip.py diff --git a/docs/reference/mpremote.rst b/docs/reference/mpremote.rst index e3902f8e5d..bb0686237a 100644 --- a/docs/reference/mpremote.rst +++ b/docs/reference/mpremote.rst @@ -146,6 +146,14 @@ The full list of supported commands are: variable ``$EDITOR``). If the editor exits successfully, the updated file will be copied back to the device. +- install packages from :term:`micropython-lib` (or GitHub) using the ``mip`` tool: + + .. code-block:: bash + + $ mpremote mip install + + See :ref:`packages` for more information. + - mount the local directory on the remote device: .. code-block:: bash @@ -269,3 +277,9 @@ Examples mpremote cp -r dir/ : mpremote cp a.py b.py : + repl + + mpremote mip install aioble + + mpremote mip install github:org/repo@branch + + mpremote mip install --target /flash/third-party functools diff --git a/docs/reference/packages.rst b/docs/reference/packages.rst index 0c049d1fb2..1ddbecb582 100644 --- a/docs/reference/packages.rst +++ b/docs/reference/packages.rst @@ -78,17 +78,17 @@ The :term:`mpremote` tool also includes the same functionality as ``mip`` and can be used from a host PC to install packages to a locally connected device (e.g. via USB or UART):: - $ mpremote install pkgname - $ mpremote install pkgname@x.y - $ mpremote install http://example.com/x/y/foo.py - $ mpremote install github:org/repo - $ mpremote install github:org/repo@branch-or-tag + $ mpremote mip install pkgname + $ mpremote mip install pkgname@x.y + $ mpremote mip install http://example.com/x/y/foo.py + $ mpremote mip install github:org/repo + $ mpremote mip install github:org/repo@branch-or-tag The ``--target=path``, ``--no-mpy``, and ``--index`` arguments can be set:: - $ mpremote install --target=/flash/third-party pkgname - $ mpremote install --no-mpy pkgname - $ mpremote install --index https://host/pi pkgname + $ mpremote mip install --target=/flash/third-party pkgname + $ mpremote mip install --no-mpy pkgname + $ mpremote mip install --index https://host/pi pkgname Installing packages manually ---------------------------- diff --git a/tools/mpremote/README.md b/tools/mpremote/README.md index c294b20811..7f58788fbe 100644 --- a/tools/mpremote/README.md +++ b/tools/mpremote/README.md @@ -11,23 +11,28 @@ This will automatically connect to the device and provide an interactive REPL. The full list of supported commands are: - mpremote connect -- connect to given device - device may be: list, auto, id:x, port:x - or any valid device name/path - mpremote disconnect -- disconnect current device - mpremote mount -- mount local directory on device - mpremote eval -- evaluate and print the string - mpremote exec -- execute the string - mpremote run -- run the given local script - mpremote fs -- execute filesystem commands on the device - command may be: cat, ls, cp, rm, mkdir, rmdir - use ":" as a prefix to specify a file on the device - mpremote repl -- enter REPL - options: - --capture - --inject-code - --inject-file - mpremote help -- print list of commands and exit + mpremote connect -- connect to given device + device may be: list, auto, id:x, port:x + or any valid device name/path + mpremote disconnect -- disconnect current device + mpremote mount -- mount local directory on device + mpremote eval -- evaluate and print the string + mpremote exec -- execute the string + mpremote run -- run the given local script + mpremote fs -- execute filesystem commands on the device + command may be: cat, ls, cp, rm, mkdir, rmdir + use ":" as a prefix to specify a file on the device + mpremote repl -- enter REPL + options: + --capture + --inject-code + --inject-file + mpremote mip install -- Install packages (from micropython-lib or third-party sources) + options: + --target + --index + --no-mpy + mpremote help -- print list of commands and exit Multiple commands can be specified and they will be run sequentially. Connection and disconnection will be done automatically at the start and end of the execution @@ -73,3 +78,5 @@ Examples: mpremote cp :main.py . mpremote cp main.py : mpremote cp -r dir/ : + mpremote mip install aioble + mpremote mip install github:org/repo@branch diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py index 17d2b33738..4f541685a0 100644 --- a/tools/mpremote/mpremote/main.py +++ b/tools/mpremote/mpremote/main.py @@ -36,6 +36,7 @@ from .commands import ( do_resume, do_soft_reset, ) +from .mip import do_mip from .repl import do_repl _PROG = "mpremote" @@ -162,6 +163,29 @@ def argparse_filesystem(): return cmd_parser +def argparse_mip(): + cmd_parser = argparse.ArgumentParser( + description="install packages from micropython-lib or third-party sources" + ) + _bool_flag(cmd_parser, "mpy", "m", True, "download as compiled .mpy files (default)") + cmd_parser.add_argument( + "--target", type=str, required=False, help="destination direction on the device" + ) + cmd_parser.add_argument( + "--index", + type=str, + required=False, + help="package index to use (defaults to micropython-lib)", + ) + cmd_parser.add_argument("command", nargs=1, help="mip command (e.g. install)") + cmd_parser.add_argument( + "packages", + nargs="+", + help="list package specifications, e.g. name, name@version, github:org/repo, github:org/repo@branch", + ) + return cmd_parser + + def argparse_none(description): return lambda: argparse.ArgumentParser(description=description) @@ -216,6 +240,10 @@ _COMMANDS = { do_filesystem, argparse_filesystem, ), + "mip": ( + do_mip, + argparse_mip, + ), "help": ( do_help, argparse_none("print help and exit"), diff --git a/tools/mpremote/mpremote/mip.py b/tools/mpremote/mpremote/mip.py new file mode 100644 index 0000000000..99ca9ff7e3 --- /dev/null +++ b/tools/mpremote/mpremote/mip.py @@ -0,0 +1,191 @@ +# Micropython package installer +# Ported from micropython-lib/micropython/mip/mip.py. +# MIT license; Copyright (c) 2022 Jim Mussared + +import urllib.error +import urllib.request +import json +import tempfile +import os + +from .commands import CommandError, show_progress_bar + + +_PACKAGE_INDEX = "https://micropython.org/pi/v2" +_CHUNK_SIZE = 128 + + +# This implements os.makedirs(os.dirname(path)) +def _ensure_path_exists(pyb, path): + import os + + split = path.split("/") + + # Handle paths starting with "/". + if not split[0]: + split.pop(0) + split[0] = "/" + split[0] + + prefix = "" + for i in range(len(split) - 1): + prefix += split[i] + if not pyb.fs_exists(prefix): + pyb.fs_mkdir(prefix) + prefix += "/" + + +# Copy from src (stream) to dest (function-taking-bytes) +def _chunk(src, dest, length=None, op="downloading"): + buf = memoryview(bytearray(_CHUNK_SIZE)) + total = 0 + if length: + show_progress_bar(0, length, op) + while True: + n = src.readinto(buf) + if n == 0: + break + dest(buf if n == _CHUNK_SIZE else buf[:n]) + total += n + if length: + show_progress_bar(total, length, op) + + +def _rewrite_url(url, branch=None): + if not branch: + branch = "HEAD" + if url.startswith("github:"): + url = url[7:].split("/") + url = ( + "https://raw.githubusercontent.com/" + + url[0] + + "/" + + url[1] + + "/" + + branch + + "/" + + "/".join(url[2:]) + ) + return url + + +def _download_file(pyb, url, dest): + try: + with urllib.request.urlopen(url) as src: + fd, path = tempfile.mkstemp() + try: + print("Installing:", dest) + with os.fdopen(fd, "wb") as f: + _chunk(src, f.write, src.length) + _ensure_path_exists(pyb, dest) + pyb.fs_put(path, dest, progress_callback=show_progress_bar) + finally: + os.unlink(path) + except urllib.error.HTTPError as e: + if e.status == 404: + raise CommandError(f"File not found: {url}") + else: + raise CommandError(f"Error {e.status} requesting {url}") + except urllib.error.URLError as e: + raise CommandError(f"{e.reason} requesting {url}") + + +def _install_json(pyb, package_json_url, index, target, version, mpy): + try: + with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response: + package_json = json.load(response) + except urllib.error.HTTPError as e: + if e.status == 404: + raise CommandError(f"Package not found: {package_json_url}") + else: + raise CommandError(f"Error {e.status} requesting {package_json_url}") + except urllib.error.URLError as e: + raise CommandError(f"{e.reason} requesting {package_json_url}") + for target_path, short_hash in package_json.get("hashes", ()): + fs_target_path = target + "/" + target_path + file_url = f"{index}/file/{short_hash[:2]}/{short_hash}" + _download_file(pyb, file_url, fs_target_path) + for target_path, url in package_json.get("urls", ()): + fs_target_path = target + "/" + target_path + _download_file(pyb, _rewrite_url(url, version), fs_target_path) + for dep, dep_version in package_json.get("deps", ()): + _install_package(pyb, dep, index, target, dep_version, mpy) + + +def _install_package(pyb, package, index, target, version, mpy): + if ( + package.startswith("http://") + or package.startswith("https://") + or package.startswith("github:") + ): + if package.endswith(".py") or package.endswith(".mpy"): + print(f"Downloading {package} to {target}") + _download_file( + pyb, _rewrite_url(package, version), target + "/" + package.rsplit("/")[-1] + ) + return + else: + if not package.endswith(".json"): + if not package.endswith("/"): + package += "/" + package += "package.json" + print(f"Installing {package} to {target}") + else: + if not version: + version = "latest" + print(f"Installing {package} ({version}) from {index} to {target}") + + mpy_version = "py" + if mpy: + pyb.exec("import sys") + mpy_version = ( + int(pyb.eval("getattr(sys.implementation, '_mpy', 0) & 0xFF").decode()) or "py" + ) + + package = f"{index}/package/{mpy_version}/{package}/{version}.json" + + _install_json(pyb, package, index, target, version, mpy) + + +def do_mip(state, args): + state.did_action() + + if args.command[0] == "install": + state.ensure_raw_repl() + + for package in args.packages: + version = None + if "@" in package: + package, version = package.split("@") + + print("Install", package) + + if args.index is None: + args.index = _PACKAGE_INDEX + + if args.target is None: + state.pyb.exec("import sys") + lib_paths = ( + state.pyb.eval("'\\n'.join(p for p in sys.path if p.endswith('/lib'))") + .decode() + .split("\n") + ) + if lib_paths and lib_paths[0]: + args.target = lib_paths[0] + else: + raise CommandError( + "Unable to find lib dir in sys.path, use --target to override" + ) + + if args.mpy is None: + args.mpy = True + + try: + _install_package( + state.pyb, package, args.index.rstrip("/"), args.target, version, args.mpy + ) + except CommandError: + print("Package may be partially installed") + raise + print("Done") + else: + raise CommandError(f"mip: '{args.command[0]}' is not a command") diff --git a/tools/pyboard.py b/tools/pyboard.py index 60cc06508e..043f4f06fb 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -476,6 +476,13 @@ class Pyboard: t = str(self.eval("pyb.RTC().datetime()"), encoding="utf8")[1:-1].split(", ") return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6]) + def fs_exists(self, src): + try: + self.exec_("import uos\nuos.stat(%s)" % (("'%s'" % src) if src else "")) + return True + except PyboardError: + return False + def fs_ls(self, src): cmd = ( "import uos\nfor f in uos.ilistdir(%s):\n"