Merge 69ce095a64 into 45ead11f96

axtls doesn't define all the CERT_xxx constants, nor the MBEDTLS_VERSION
constant.

This change means that `tls.SSLContext` is imported into the module, but
that's subsequently overridden by the class definition in this module.

Signed-off-by: Damien George <damien@micropython.org>

micropython/mip/manifest.py

@@ -1,4 +1,4 @@
-metadata(version="0.2.0", description="On-device package installer for network-capable boards")
+metadata(version="0.3.0", description="On-device package installer for network-capable boards")
 
 require("requests")
 

micropython/mip/mip/__init__.py

@@ -1,13 +1,25 @@
 # MicroPython package installer
 # MIT license; Copyright (c) 2022 Jim Mussared
+# Modified by Ned Konz to allow relative source URLs in package.json files
 
 from micropython import const
 import requests
 import sys
-
+import gc
+import time
+import os
 
 _PACKAGE_INDEX = const("https://micropython.org/pi/v2")
 _CHUNK_SIZE = 128
+_URL_PREFIXES = const(("http://", "https://", "github:", "file://"))
+
+_top_url = ""
+_wlan = None
+
+
+# Return true if name is a URI that we understand
+def _is_url(name):
+    return any(name.startswith(prefix) for prefix in _URL_PREFIXES)
 
 
 # This implements os.makedirs(os.dirname(path))
@@ -43,8 +55,6 @@ def _chunk(src, dest):
 
 # Check if the specified path exists and matches the hash.
 def _check_exists(path, short_hash):
-    import os
-
     try:
         import binascii
         import hashlib
@@ -58,51 +68,116 @@ def _check_exists(path, short_hash):
         return False
 
 
-def _rewrite_url(url, branch=None):
-    if not branch:
-        branch = "HEAD"
-    if url.startswith("github:"):
-        url = url[7:].split("/")
-        url = (
-            "https://raw.githubusercontent.com/"
-            + url[0]
-            + "/"
-            + url[1]
-            + "/"
-            + branch
-            + "/"
-            + "/".join(url[2:])
+def _rewrite_github_url(url, branch):
+    url = url[7:].split("/")  # user, repo, path...
+    url = "/".join(
+        (
+            "https://raw.githubusercontent.com",
+            url[0],  # user
+            url[1],  # repo
+            branch,
+            "/".join(url[2:]),
         )
+    )
+    return url
+
+
+def _rewrite_url(orig_url, branch=None):
+    global _top_url  # the origin of the package.json URL for re-writing relative URLs
+
+    # rewrite relative URLs as absolute URLs
+    if not _is_url(orig_url):
+        orig_url = _top_url + "/" + orig_url
+
+    url = orig_url
+
+    # now rewrite github: URLs as raw.githubusercontent.com URLs
+    if orig_url.startswith("github:"):
+        if not branch:
+            branch = "HEAD"
+        url = _rewrite_github_url(orig_url, branch)
+
+        # catch URLs that don't start with the same github:user/repo
+        if not url.startswith(_top_url):
+            url = _rewrite_github_url(orig_url, "HEAD")
     return url
 
 
 def _download_file(url, dest):
-    response = requests.get(url)
-    try:
-        if response.status_code != 200:
-            print("Error", response.status_code, "requesting", url)
+    # if url is a file:// url, just copy it
+    if url.startswith("file://"):
+        src_name = url[7:]
+        try:
+            with open(src_name, "rb") as src:
+                print(f"Copying file {src_name} to {dest}")
+                _ensure_path_exists(dest)
+                with open(dest, "wb") as dst:
+                    _chunk(src, dst.write)
+            return True
+        except OSError:
+            print(f"File {src_name} not found")
             return False
 
-        print("Copying:", dest)
-        _ensure_path_exists(dest)
-        with open(dest, "wb") as f:
-            _chunk(response.raw, f.write)
+    retries = 0
+    while retries < 5:
+        gc.collect()
+        try:
+            response = requests.get(url)
 
-        return True
-    finally:
-        response.close()
+            if response.status_code != 200:
+                print("Error", response.status_code, "requesting", url)
+                return False
+
+            print("Copying:", dest)
+            _ensure_path_exists(dest)
+            with open(dest, "wb") as f:
+                _chunk(response.raw, f.write)
+
+            return True
+        except OSError as e:
+            bytes_copied = 0
+            try:
+                bytes_copied = os.stat(dest)[6]
+            except:
+                pass
+            print(f"Exception {e} after copying {bytes_copied} bytes; retrying after 1 second")
+            retries += 1
+            time.sleep(1)
+            response.close()
+            continue
+
+        finally:
+            response.close()
 
 
 def _install_json(package_json_url, index, target, version, mpy):
-    response = requests.get(_rewrite_url(package_json_url, version))
-    try:
-        if response.status_code != 200:
-            print("Package not found:", package_json_url)
-            return False
+    global _top_url
+    # if package_json_url is a file:// url, just download it
+    # and use its json directly
+    if package_json_url.startswith("file://"):
+        import ujson as json
 
-        package_json = response.json()
-    finally:
-        response.close()
+        pkg_name = package_json_url[7:]
+        try:
+            with open(pkg_name) as json_file:
+                package_json = json.load(json_file)
+        except OSError:
+            print(f"File {pkg_name} not found")
+            return False
+    else:
+        pkg_name = _rewrite_url(package_json_url, version)
+        response = requests.get(pkg_name)
+        try:
+            if response.status_code != 200:
+                print(f"Package {package_json_url} not found (tried {pkg_name})")
+                return False
+
+            package_json = response.json()
+        finally:
+            response.close()
+    _top_url = pkg_name.rsplit("/", 1)[0]
+
+    # get mpy files from hashes
     for target_path, short_hash in package_json.get("hashes", ()):
         fs_target_path = target + "/" + target_path
         if _check_exists(fs_target_path, short_hash):
@@ -112,11 +187,13 @@ def _install_json(package_json_url, index, target, version, mpy):
             if not _download_file(file_url, fs_target_path):
                 print("File not found: {} {}".format(target_path, short_hash))
                 return False
+    # get other files from URLs
     for target_path, url in package_json.get("urls", ()):
         fs_target_path = target + "/" + target_path
         if not _download_file(_rewrite_url(url, version), fs_target_path):
             print("File not found: {} {}".format(target_path, url))
             return False
+    # install dependencies
     for dep, dep_version in package_json.get("deps", ()):
         if not _install_package(dep, index, target, dep_version, mpy):
             return False
@@ -124,11 +201,7 @@ def _install_json(package_json_url, index, target, version, mpy):
 
 
 def _install_package(package, index, target, version, mpy):
-    if (
-        package.startswith("http://")
-        or package.startswith("https://")
-        or package.startswith("github:")
-    ):
+    if _is_url(package):
         if package.endswith(".py") or package.endswith(".mpy"):
             print("Downloading {} to {}".format(package, target))
             return _download_file(

micropython/senml/examples/basic_cbor.py

@@ -26,7 +26,7 @@ THE SOFTWARE.
 
 from senml import *
 import time
-from cbor2 import decoder
+import cbor2
 
 pack = SenmlPack("device_name")
 
@@ -38,5 +38,5 @@ while True:
         cbor_val = pack.to_cbor()
         print(cbor_val)
         print(cbor_val.hex())
-        print(decoder.loads(cbor_val))  # convert to string again so we can print it.
+        print(cbor2.loads(cbor_val))  # convert to string again so we can print it.
     time.sleep(1)

micropython/senml/manifest.py

@@ -1,6 +1,6 @@
 metadata(
     description="SenML serialisation for MicroPython.",
-    version="0.1.0",
+    version="0.1.1",
     pypi_publish="micropython-senml",
 )
 

micropython/senml/senml/senml_pack.py

@@ -27,8 +27,7 @@ THE SOFTWARE.
 from senml.senml_record import SenmlRecord
 from senml.senml_base import SenmlBase
 import json
-from cbor2 import encoder
-from cbor2 import decoder
+import cbor2
 
 
 class SenmlPackIterator:
@@ -278,7 +277,7 @@ class SenmlPack(SenmlBase):
         :param data: a byte array.
         :return: None
         """
-        records = decoder.loads(data)  # load the raw senml data
+        records = cbor2.loads(data)  # load the raw senml data
         naming_map = {
             "bn": -2,
             "bt": -3,
@@ -320,7 +319,7 @@ class SenmlPack(SenmlBase):
         }
         converted = []
         self._build_rec_dict(naming_map, converted)
-        return encoder.dumps(converted)
+        return cbor2.dumps(converted)
 
     def add(self, item):
         """

python-ecosys/cbor2/cbor2/__init__.py

@@ -24,5 +24,10 @@ THE SOFTWARE.
 """
 
 
-from . import decoder
-from . import encoder
+from ._decoder import CBORDecoder
+from ._decoder import load
+from ._decoder import loads
+
+from ._encoder import CBOREncoder
+from ._encoder import dump
+from ._encoder import dumps

python-ecosys/cbor2/examples/cbor_test.py

@@ -24,16 +24,15 @@ THE SOFTWARE.
 """
 
 
-from cbor2 import encoder
-from cbor2 import decoder
+import cbor2
 
 input = [
     {"bn": "urn:dev:ow:10e2073a01080063", "u": "Cel", "t": 1.276020076e09, "v": 23.5},
     {"u": "Cel", "t": 1.276020091e09, "v": 23.6},
 ]
 
-data = encoder.dumps(input)
+data = cbor2.dumps(input)
 print(data)
 print(data.hex())
-text = decoder.loads(data)
+text = cbor2.loads(data)
 print(text)

python-ecosys/cbor2/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.1.0", pypi="cbor2")
+metadata(version="1.0.0", pypi="cbor2")
 
 package("cbor2")

python-stdlib/json/manifest.py

@@ -1,3 +0,0 @@
-metadata(version="0.1.0")
-
-package("json")

python-stdlib/ssl/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.2.0")
+metadata(version="0.2.1")
 
 module("ssl.py", opt=3)

python-stdlib/ssl/ssl.py

@@ -1,12 +1,5 @@
 import tls
-from tls import (
-    CERT_NONE,
-    CERT_OPTIONAL,
-    CERT_REQUIRED,
-    MBEDTLS_VERSION,
-    PROTOCOL_TLS_CLIENT,
-    PROTOCOL_TLS_SERVER,
-)
+from tls import *
 
 
 class SSLContext:

unix-ffi/README.md

@@ -19,9 +19,13 @@ replacement for CPython.
 
 ### Usage
 
-To use a unix-specific library, pass `unix_ffi=True` to `require()` in your
-manifest file.
+To use a unix-specific library, a manifest file must add the `unix-ffi`
+library to the library search path using `add_library()`:
 
 ```py
-require("os", unix_ffi=True) # Use the unix-ffi version instead of python-stdlib.
+add_library("unix-ffi", "$(MPY_LIB_DIR)/unix-ffi", prepend=True)
 ```
+
+Prepending the `unix-ffi` library to the path will make it so that the
+`unix-ffi` version of a package will be preferred if that package appears in
+both `unix-ffi` and another library (eg `python-stdlib`).

unix-ffi/_markupbase/manifest.py

@@ -1,5 +1,5 @@
 metadata(version="3.3.4")
 
-require("re", unix_ffi=True)
+require("re")
 
 module("_markupbase.py")

unix-ffi/email.charset/manifest.py

@@ -1,7 +1,7 @@
 metadata(version="0.5.1")
 
 require("functools")
-require("email.encoders", unix_ffi=True)
-require("email.errors", unix_ffi=True)
+require("email.encoders")
+require("email.errors")
 
 package("email")

unix-ffi/email.encoders/manifest.py

@@ -3,7 +3,7 @@ metadata(version="0.5.1")
 require("base64")
 require("binascii")
 require("quopri")
-require("re", unix_ffi=True)
+require("re")
 require("string")
 
 package("email")

unix-ffi/email.feedparser/manifest.py

@@ -1,8 +1,8 @@
 metadata(version="0.5.1")
 
-require("re", unix_ffi=True)
-require("email.errors", unix_ffi=True)
-require("email.message", unix_ffi=True)
-require("email.internal", unix_ffi=True)
+require("re")
+require("email.errors")
+require("email.message")
+require("email.internal")
 
 package("email")

unix-ffi/email.header/manifest.py

@@ -1,9 +1,9 @@
 metadata(version="0.5.2")
 
-require("re", unix_ffi=True)
+require("re")
 require("binascii")
-require("email.encoders", unix_ffi=True)
-require("email.errors", unix_ffi=True)
-require("email.charset", unix_ffi=True)
+require("email.encoders")
+require("email.errors")
+require("email.charset")
 
 package("email")

unix-ffi/email.internal/manifest.py

@@ -1,15 +1,15 @@
 metadata(version="0.5.1")
 
-require("re", unix_ffi=True)
+require("re")
 require("base64")
 require("binascii")
 require("functools")
 require("string")
 # require("calendar") TODO
 require("abc")
-require("email.errors", unix_ffi=True)
-require("email.header", unix_ffi=True)
-require("email.charset", unix_ffi=True)
-require("email.utils", unix_ffi=True)
+require("email.errors")
+require("email.header")
+require("email.charset")
+require("email.utils")
 
 package("email")

unix-ffi/email.message/manifest.py

@@ -1,11 +1,11 @@
 metadata(version="0.5.3")
 
-require("re", unix_ffi=True)
+require("re")
 require("uu")
 require("base64")
 require("binascii")
-require("email.utils", unix_ffi=True)
-require("email.errors", unix_ffi=True)
-require("email.charset", unix_ffi=True)
+require("email.utils")
+require("email.errors")
+require("email.charset")
 
 package("email")

unix-ffi/email.parser/manifest.py

@@ -1,8 +1,8 @@
 metadata(version="0.5.1")
 
 require("warnings")
-require("email.feedparser", unix_ffi=True)
-require("email.message", unix_ffi=True)
-require("email.internal", unix_ffi=True)
+require("email.feedparser")
+require("email.message")
+require("email.internal")
 
 package("email")

unix-ffi/email.utils/manifest.py

@@ -1,13 +1,13 @@
 metadata(version="3.3.4")
 
-require("os", unix_ffi=True)
-require("re", unix_ffi=True)
+require("os")
+require("re")
 require("base64")
 require("random")
 require("datetime")
-require("urllib.parse", unix_ffi=True)
+require("urllib.parse")
 require("warnings")
 require("quopri")
-require("email.charset", unix_ffi=True)
+require("email.charset")
 
 package("email")

unix-ffi/fcntl/manifest.py

@@ -2,6 +2,6 @@ metadata(version="0.0.4")
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("fcntl.py")

unix-ffi/getopt/manifest.py

@@ -1,5 +1,5 @@
 metadata(version="3.3.4")
 
-require("os", unix_ffi=True)
+require("os")
 
 module("getopt.py")

unix-ffi/gettext/manifest.py

@@ -2,6 +2,6 @@ metadata(version="0.1.0")
 
 # Originally written by Riccardo Magliocchetti.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("gettext.py")

unix-ffi/glob/manifest.py

@@ -1,8 +1,8 @@
 metadata(version="0.5.2")
 
-require("os", unix_ffi=True)
+require("os")
 require("os-path")
-require("re", unix_ffi=True)
+require("re")
 require("fnmatch")
 
 module("glob.py")

unix-ffi/html.parser/manifest.py

@@ -1,8 +1,8 @@
 metadata(version="3.3.4")
 
-require("_markupbase", unix_ffi=True)
+require("_markupbase")
 require("warnings")
-require("html.entities", unix_ffi=True)
-require("re", unix_ffi=True)
+require("html.entities")
+require("re")
 
 package("html")

unix-ffi/http.client/manifest.py

@@ -1,10 +1,10 @@
 metadata(version="0.5.1")
 
-require("email.parser", unix_ffi=True)
-require("email.message", unix_ffi=True)
-require("socket", unix_ffi=True)
+require("email.parser")
+require("email.message")
+require("socket")
 require("collections")
-require("urllib.parse", unix_ffi=True)
+require("urllib.parse")
 require("warnings")
 
 package("http")

unix-ffi/json/manifest.py

@@ -0,0 +1,4 @@
+metadata(version="0.2.0")
+
+require("re")
+package("json")

unix-ffi/machine/manifest.py

@@ -2,8 +2,8 @@ metadata(version="0.2.1")
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib", unix_ffi=True)
-require("os", unix_ffi=True)
-require("signal", unix_ffi=True)
+require("ffilib")
+require("os")
+require("signal")
 
 package("machine")

unix-ffi/multiprocessing/manifest.py

@@ -2,8 +2,8 @@ metadata(version="0.1.2")
 
 # Originally written by Paul Sokolovsky.
 
-require("os", unix_ffi=True)
-require("select", unix_ffi=True)
+require("os")
+require("select")
 require("pickle")
 
 module("multiprocessing.py")

unix-ffi/os/manifest.py

@@ -2,7 +2,7 @@ metadata(version="0.6.0")
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 require("errno")
 require("stat")
 

unix-ffi/pwd/manifest.py

@@ -2,6 +2,6 @@ metadata(version="0.1.0")
 
 # Originally written by Riccardo Magliocchetti.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("pwd.py")

unix-ffi/re/manifest.py

@@ -2,6 +2,6 @@ metadata(version="0.2.5")
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("re.py")

unix-ffi/select/manifest.py

@@ -2,7 +2,7 @@ metadata(version="0.3.0")
 
 # Originally written by Paul Sokolovsky.
 
-require("os", unix_ffi=True)
-require("ffilib", unix_ffi=True)
+require("os")
+require("ffilib")
 
 module("select.py")

unix-ffi/signal/manifest.py

@@ -2,6 +2,6 @@ metadata(version="0.3.2")
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("signal.py")

unix-ffi/sqlite3/manifest.py

@@ -2,6 +2,6 @@ metadata(version="0.2.4")
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("sqlite3.py")

unix-ffi/time/manifest.py

@@ -1,5 +1,5 @@
 metadata(version="0.5.0")
 
-require("ffilib", unix_ffi=True)
+require("ffilib")
 
 module("time.py")

unix-ffi/timeit/manifest.py

@@ -1,9 +1,9 @@
 metadata(version="3.3.4")
 
-require("getopt", unix_ffi=True)
+require("getopt")
 require("itertools")
 # require("linecache") TODO
-require("time", unix_ffi=True)
+require("time")
 require("traceback")
 
 module("timeit.py")

unix-ffi/ucurses/manifest.py

@@ -2,8 +2,8 @@ metadata(version="0.1.2")
 
 # Originally written by Paul Sokolovsky.
 
-require("os", unix_ffi=True)
-require("tty", unix_ffi=True)
-require("select", unix_ffi=True)
+require("os")
+require("tty")
+require("select")
 
 package("ucurses")

unix-ffi/urllib.parse/manifest.py

@@ -1,6 +1,6 @@
 metadata(version="0.5.2")
 
-require("re", unix_ffi=True)
+require("re")
 require("collections")
 require("collections-defaultdict")