From 508c7d2b329f2fc98f01d57d5daaa48856312d77 Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Wed, 19 Jun 2019 17:03:48 +0100 Subject: [PATCH] micropip: remove upip_m.py. Update README.md to reflect changes in upip.py --- README.md | 4 +- micropip/README.md | 63 +++----- micropip/upip_m.py | 317 -------------------------------------- micropip/upip_utarfile.py | 94 ----------- 4 files changed, 23 insertions(+), 455 deletions(-) delete mode 100644 micropip/upip_m.py delete mode 100644 micropip/upip_utarfile.py diff --git a/README.md b/README.md index 9850f0c..f40b516 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ Pyboard variants. # Installing MicroPython libraries This is more involved since the advent of the pycopy fork of MicroPython. -[This doc](./micropip/README.md) describes the issues and provides two -utilities for users of official MicroPython firmware to simplify installation. +[This doc](./micropip/README.md) describes the issues and provides a utility +to simplify installation for users of official MicroPython firmware. # Fastbuild diff --git a/micropip/README.md b/micropip/README.md index 4b85a69..e2a9ca4 100644 --- a/micropip/README.md +++ b/micropip/README.md @@ -12,9 +12,9 @@ Libraries may be installed by copying files from the appropriate library repository to the target device. However this requires some attention to detail where there are dependencies or where modules are organised as Python packages. -Each fork has applications for installing library and user contributed modules -modelled on Python's `pip`. These handle dependencies and build the correct -directory structure on the target. +Each version has a tool known as `upip` for installing library and user +contributed modules modelled on Python's `pip`. This handles dependencies and +builds the correct directory structure on the target. Note that `pip` and `pip3` cannot be used for MicroPython modules. This is because the file format is nonstandard. The file format was chosen to enable @@ -26,9 +26,7 @@ the installer to run on targets with minimal resources. 1. [Contents](./README.md#1-contents) 2. [Users of Pycopy firmware](./README.md#2-users-of-pycopy-firmware) 3. [Users of official MicroPython](./README.md#3-users-of-official-micropython) - 3.1 [The installers](./README.md#31-the-installers) - 3.1.1 [upip_m](./README.md#311-upip_m) upip replacement runs on target hardware - 3.1.2 [micropip](./README.md#312-micropip) Runs on a PC + 3.1 [micropip](./README.md#31-micropip) Runs on a PC 4. [Overriding built in library modules](./README.md#4-overriding-built-in-library-modules) ###### [Main README](../README.md) @@ -39,9 +37,8 @@ The library for the `pycopy` fork may be found [here](https://github.com/pfalcon Library modules located on [PyPi](https://pypi.org/) are correct for the `pycopy` firmware. -The preferred installation tool is `upip.py` which may be found in the `tools` -directory of MicroPython. It is installed by default on network enabled -hardware such as Pyboard D, ESP8266 and ESP32. +The `upip` tool may be found in the `tools` directory of `pycopy`. This version +should be used as it installs exclusively from PyPi. For hardware which is not network enabled, `upip` may be run under the Unix build of MicroPython to install to an arbitrary directory on a PC. The @@ -54,43 +51,25 @@ Usage of `upip` is documented in the # 3. Users of official MicroPython The library at [micropython-lib](https://github.com/micropython/micropython-lib) -is compatible with the official firmware. Unfortunately for users of official -firmware its README is misleading, not least because the advocated `upip` -module may produce an incorrect result. This is because it installs from -[PyPi](https://pypi.org/) and some modules there require the `pycopy` firmware. +is compatible with the official firmware. As of version 1.11 the included +version of `upip` will install the correct library module for use with this +firmware, searching for modules in the official library before searching +[PyPi](https://pypi.org/). -Two (unofficial) utilities are provided for users of the official firmware. -Where a library module is to be installed, these will locate a compatible -version. User contributed modules located on PyPi will be handled as normal. - * `upip_m.py` A modified version of `upip.py`. For network enabled targets. - * `micropip.py` Installs modules to a PC for copying to the target device. - This is primarily for non-networked targets and for targets with insufficient - RAM to run `upip_m.py`. Requires CPython 3.2 or later. +Users of non-networked hardware such as the Pyboard 1.x can use `upip` with the +Unix build of MicroPython to install a library module to an arbitrary directory +on a PC, from where the files and directories can be copied to the target +hardware. This approach has the drawback of requiring the Unix build, which has +to be built from source. -## 3.1 The installers +For those unable or unwilling to do this, `micropip.py` in this repo may be +employed. -These have the same invocation details as `upip` and the -[official docs](http://docs.micropython.org/en/latest/reference/packages.html) -should be consulted for usage information. +## 3.1 micropip -### 3.1.1 upip_m - -The file `upip_m.py` should be copied to the target device. If `upip` is not -available on the target `upip_utarfile.py` must also be copied. - -Alternatively and more efficiently these files may be frozen as bytecode. The -method of doing this is [documented here](http://docs.micropython.org/en/latest/reference/packages.html). - -Users of the ESP8266 are unlikely to be able to use `upip_m` unless it is -frozen as bytecode. An alternative is to use `micropip.py` to install to a PC -and then to use [rshell](https://github.com/dhylands/rshell) or other utility -to copy the directory structure to the device. - -### 3.1.2 micropip - -This is a version of `upip_m` which runs under Python 3.2 or above. Library and -user modules are installed to the PC for transfer to the target. It is cross -platform and has been tested under Linux, Windows and OSX. +This runs under Python 3.2 or above. Library and user modules are installed to +the PC for transfer to the target. It is cross-platform and has been tested +under Linux, Windows and OSX. Help may be accessed with diff --git a/micropip/upip_m.py b/micropip/upip_m.py deleted file mode 100644 index be5bee9..0000000 --- a/micropip/upip_m.py +++ /dev/null @@ -1,317 +0,0 @@ -# -# upip_m - Package manager for MicroPython modified for new official repo -# -# Copyright (c) 2015-2018 Paul Sokolovsky -# -# Licensed under the MIT license. -# -import sys -import gc -import uos as os -import uerrno as errno -import ujson as json -import uzlib -import upip_utarfile as tarfile -gc.collect() - - -debug = False -install_path = None -cleanup_files = [] -gzdict_sz = 16 + 15 - -file_buf = bytearray(512) - -class NotFoundError(Exception): - pass - -def op_split(path): - if path == "": - return ("", "") - r = path.rsplit("/", 1) - if len(r) == 1: - return ("", path) - head = r[0] - if not head: - head = "/" - return (head, r[1]) - -def op_basename(path): - return op_split(path)[1] - -# Expects *file* name -def _makedirs(name, mode=0o777): - ret = False - s = "" - comps = name.rstrip("/").split("/")[:-1] - if comps[0] == "": - s = "/" - for c in comps: - if s and s[-1] != "/": - s += "/" - s += c - try: - os.mkdir(s) - ret = True - except OSError as e: - if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR: - raise - ret = False - return ret - - -def save_file(fname, subf): - global file_buf - with open(fname, "wb") as outf: - while True: - sz = subf.readinto(file_buf) - if not sz: - break - outf.write(file_buf, sz) - -def install_tar(f, prefix): - meta = {} - for info in f: - #print(info) - fname = info.name - try: - fname = fname[fname.index("/") + 1:] - except ValueError: - fname = "" - - save = True - for p in ("setup.", "PKG-INFO", "README"): - #print(fname, p) - if fname.startswith(p) or ".egg-info" in fname: - if fname.endswith("/requires.txt"): - meta["deps"] = f.extractfile(info).read() - save = False - if debug: - print("Skipping", fname) - break - - if save: - outfname = prefix + fname - if info.type != tarfile.DIRTYPE: - if debug: - print("Extracting " + outfname) - _makedirs(outfname) - subf = f.extractfile(info) - save_file(outfname, subf) - return meta - -def expandhome(s): - if "~/" in s: - h = os.getenv("HOME") - s = s.replace("~/", h + "/") - return s - -import ussl -import usocket -warn_ussl = True -def url_open(url): - global warn_ussl - - if debug: - print(url) - - proto, _, host, urlpath = url.split('/', 3) - try: - ai = usocket.getaddrinfo(host, 443, 0, usocket.SOCK_STREAM) - except OSError as e: - fatal("Unable to resolve %s (no Internet?)" % host, e) - #print("Address infos:", ai) - ai = ai[0] - - s = usocket.socket(ai[0], ai[1], ai[2]) - try: - #print("Connect address:", addr) - s.connect(ai[-1]) - - if proto == "https:": - s = ussl.wrap_socket(s, server_hostname=host) - if warn_ussl: - print("Warning: %s SSL certificate is not validated" % host) - warn_ussl = False - - # MicroPython rawsocket module supports file interface directly - s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host)) - l = s.readline() - protover, status, msg = l.split(None, 2) - if status != b"200": - if status == b"404" or status == b"301": - raise NotFoundError("Package not found") - raise ValueError(status) - while 1: - l = s.readline() - if not l: - raise ValueError("Unexpected EOF in HTTP headers") - if l == b'\r\n': - break - except Exception as e: - s.close() - raise e - - return s - - -def get_pkg_metadata(name): - try: - f = url_open("https://micropython.org/resources/upi/%s/json" % name) - except: - f = url_open("https://pypi.org/pypi/%s/json" % name) - try: - return json.load(f) - finally: - f.close() - - -def fatal(msg, exc=None): - print("Error:", msg) - if exc and debug: - raise exc - sys.exit(1) - -def install_pkg(pkg_spec, install_path): - data = get_pkg_metadata(pkg_spec) - - latest_ver = data["info"]["version"] - packages = data["releases"][latest_ver] - del data - gc.collect() - assert len(packages) == 1 - package_url = packages[0]["url"] - print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url)) - package_fname = op_basename(package_url) - f1 = url_open(package_url) - try: - f2 = uzlib.DecompIO(f1, gzdict_sz) - f3 = tarfile.TarFile(fileobj=f2) - meta = install_tar(f3, install_path) - finally: - f1.close() - del f3 - del f2 - gc.collect() - return meta - -def install(to_install, install_path=None): - # Calculate gzip dictionary size to use - global gzdict_sz - sz = gc.mem_free() + gc.mem_alloc() - if sz <= 65536: - gzdict_sz = 16 + 12 - - if install_path is None: - install_path = get_install_path() - if install_path[-1] != "/": - install_path += "/" - if not isinstance(to_install, list): - to_install = [to_install] - print("Installing to: " + install_path) - # sets would be perfect here, but don't depend on them - installed = [] - try: - while to_install: - if debug: - print("Queue:", to_install) - pkg_spec = to_install.pop(0) - if pkg_spec in installed: - continue - meta = install_pkg(pkg_spec, install_path) - installed.append(pkg_spec) - if debug: - print(meta) - deps = meta.get("deps", "").rstrip() - if deps: - deps = deps.decode("utf-8").split("\n") - to_install.extend(deps) - except Exception as e: - print("Error installing '{}': {}, packages may be partially installed".format( - pkg_spec, e), - file=sys.stderr) - -def get_install_path(): - global install_path - if install_path is None: - # sys.path[0] is current module's path - install_path = sys.path[1] - install_path = expandhome(install_path) - return install_path - -def cleanup(): - for fname in cleanup_files: - try: - os.unlink(fname) - except OSError: - print("Warning: Cannot delete " + fname) - -def help(): - print("""\ -upip - Simple PyPI package manager for MicroPython -Usage: micropython -m upip install [-p ] ... | -r -import upip; upip.install(package_or_list, []) - -If is not given, packages will be installed into sys.path[1] -(can be set from MICROPYPATH environment variable, if current system -supports that).""") - print("Current value of sys.path[1]:", sys.path[1]) - print("""\ - -Note: only MicroPython packages (usually, named micropython-*) are supported -for installation, upip does not support arbitrary code in setup.py. -""") - -def main(): - global debug - global install_path - install_path = None - - if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": - help() - return - - if sys.argv[1] != "install": - fatal("Only 'install' command supported") - - to_install = [] - - i = 2 - while i < len(sys.argv) and sys.argv[i][0] == "-": - opt = sys.argv[i] - i += 1 - if opt == "-h" or opt == "--help": - help() - return - elif opt == "-p": - install_path = sys.argv[i] - i += 1 - elif opt == "-r": - list_file = sys.argv[i] - i += 1 - with open(list_file) as f: - while True: - l = f.readline() - if not l: - break - if l[0] == "#": - continue - to_install.append(l.rstrip()) - elif opt == "--debug": - debug = True - else: - fatal("Unknown/unsupported option: " + opt) - - to_install.extend(sys.argv[i:]) - if not to_install: - help() - return - - install(to_install) - - if not debug: - cleanup() - - -if __name__ == "__main__": - main() diff --git a/micropip/upip_utarfile.py b/micropip/upip_utarfile.py deleted file mode 100644 index 460ca2c..0000000 --- a/micropip/upip_utarfile.py +++ /dev/null @@ -1,94 +0,0 @@ -import uctypes - -# http://www.gnu.org/software/tar/manual/html_node/Standard.html -TAR_HEADER = { - "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), - "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 11), -} - -DIRTYPE = "dir" -REGTYPE = "file" - -def roundup(val, align): - return (val + align - 1) & ~(align - 1) - -class FileSection: - - def __init__(self, f, content_len, aligned_len): - self.f = f - self.content_len = content_len - self.align = aligned_len - content_len - - def read(self, sz=65536): - if self.content_len == 0: - return b"" - if sz > self.content_len: - sz = self.content_len - data = self.f.read(sz) - sz = len(data) - self.content_len -= sz - return data - - def readinto(self, buf): - if self.content_len == 0: - return 0 - if len(buf) > self.content_len: - buf = memoryview(buf)[:self.content_len] - sz = self.f.readinto(buf) - self.content_len -= sz - return sz - - def skip(self): - sz = self.content_len + self.align - if sz: - buf = bytearray(16) - while sz: - s = min(sz, 16) - self.f.readinto(buf, s) - sz -= s - -class TarInfo: - - def __str__(self): - return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) - -class TarFile: - - def __init__(self, name=None, fileobj=None): - if fileobj: - self.f = fileobj - else: - self.f = open(name, "rb") - self.subf = None - - def next(self): - if self.subf: - self.subf.skip() - buf = self.f.read(512) - if not buf: - return None - - h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) - - # Empty block means end of archive - if h.name[0] == 0: - return None - - d = TarInfo() - d.name = str(h.name, "utf-8").rstrip("\0") - d.size = int(bytes(h.size), 8) - d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] - self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) - return d - - def __iter__(self): - return self - - def __next__(self): - v = self.next() - if v is None: - raise StopIteration - return v - - def extractfile(self, tarinfo): - return tarinfo.subf