WIP: Fix alternate github sources; add retry for OSError

micropython/mip/mip/__init__.py

@@ -4,11 +4,16 @@
 
 import urequests as requests
 import sys
+import gc  # TODO: remove
+import network
 
 _PACKAGE_INDEX = const("https://micropython.org/pi/v2")
 _CHUNK_SIZE = 128
 _URL_PREFIXES = const(("http://", "https://", "github:", "file://"))
 
+_top_url = ""
+_wlan = None
+
 
 # Return true if name is a URI that we understand
 def _is_url(name):
@@ -48,8 +53,6 @@ def _chunk(src, dest):
 
 # Check if the specified path exists and matches the hash.
 def _check_exists(path, short_hash):
-    import os
-
     try:
         import binascii
         import hashlib
@@ -63,30 +66,55 @@ def _check_exists(path, short_hash):
         return False
 
 
-def _rewrite_url(url, branch=None, package_json_url=None):
+def _rewrite_github_url(url, branch):
+    url = url[7:].split("/")  # user, repo, path...
+    url = "/".join(
+        (
+            "https://raw.githubusercontent.com",
+            url[0],  # user
+            url[1],  # repo
+            branch,
+            "/".join(url[2:]),
+        )
+    )
+    return url
+
+
+def _rewrite_url(orig_url, branch=None):
+    global _top_url  # the origin of the package.json URL for re-writing relative URLs
+
+    print(f"Rewriting {orig_url} with branch {branch} from {_top_url} to ", end="")  # TODO remove
+
     # rewrite relative URLs as absolute URLs
-    if not _is_url(url):
-        if not package_json_url:
-            raise ValueError("No package.json URL specified")
-        url = package_json_url.rsplit("/", 1)[0] + "/" + url
+    if not _is_url(orig_url):
+        orig_url = _top_url + "/" + orig_url
+
+    url = orig_url
+
     # now rewrite github: URLs as raw.githubusercontent.com URLs
-    if url.startswith("github:"):
+    if orig_url.startswith("github:"):
         if not branch:
             branch = "HEAD"
-        url = url[7:].split("/")
-        url = (
-            "https://raw.githubusercontent.com/"
-            + url[0]
-            + "/"
-            + url[1]
-            + "/"
-            + branch
-            + "/"
-            + "/".join(url[2:])
-        )
+        url = _rewrite_github_url(orig_url, branch)
+
+        # catch URLs that don't start with the same github:user/repo
+        if not url.startswith(_top_url):
+            url = _rewrite_github_url(orig_url, "HEAD")
+
+    print(url)  # TODO remove
     return url
 
 
+def _check_network():
+    global _wlan
+    if not _wlan:
+        _wlan = network.WLAN(network.STA_IF)
+    if not _wlan.isconnected():
+        print("waiting for network...")
+        while not _wlan.isconnected():
+            pass
+
+
 def _download_file(url, dest):
     # if url is a file:// url, just copy it
     if url.startswith("file://"):
@@ -102,45 +130,62 @@ def _download_file(url, dest):
             print(f"File {src_name} not found")
             return False
 
-    response = requests.get(url)
-    try:
-        if response.status_code != 200:
-            print("Error", response.status_code, "requesting", url)
-            return False
+    retries = 0
+    while retries < 5:
+        gc.collect()
+        print(f"Free memory: {gc.mem_free()}")  # TODO: remove
+        try:
+            response = requests.get(url)
 
-        print("Copying:", dest)
-        _ensure_path_exists(dest)
-        with open(dest, "wb") as f:
-            _chunk(response.raw, f.write)
+            if response.status_code != 200:
+                print("Error", response.status_code, "requesting", url)
+                return False
 
-        return True
-    finally:
-        response.close()
+            print("Copying:", dest)
+            _ensure_path_exists(dest)
+            with open(dest, "wb") as f:
+                _chunk(response.raw, f.write)
+
+            return True
+        except OSError as e:
+            print(f"after exc {e}: Free memory: {gc.mem_free()}; waiting")  # TODO: remove
+            retries += 1
+            time.sleep(1)
+            response.close()
+            continue
+
+        finally:
+            response.close()
 
 
 def _install_json(package_json_url, index, target, version, mpy):
+    global _top_url
     # if package_json_url is a file:// url, just download it
     # and use its json directly
     if package_json_url.startswith("file://"):
         import ujson as json
 
-        pkg_file_name = package_json_url[7:]
+        pkg_name = package_json_url[7:]
         try:
-            with open(pkg_file_name) as json_file:
+            with open(pkg_name) as json_file:
                 package_json = json.load(json_file)
         except OSError:
-            print(f"File {pkg_file_name} not found")
+            print(f"File {pkg_name} not found")
             return False
     else:
-        response = requests.get(_rewrite_url(package_json_url, version))
+        pkg_name = _rewrite_url(package_json_url, version)
+        response = requests.get(pkg_name)
         try:
             if response.status_code != 200:
-                print("Package not found:", package_json_url)
+                print(f"Package {package_json_url} not found (tried {pkg_name})")
                 return False
 
             package_json = response.json()
         finally:
             response.close()
+    _top_url = pkg_name.rsplit("/", 1)[0]
+
+    # get mpy files from hashes
     for target_path, short_hash in package_json.get("hashes", ()):
         fs_target_path = target + "/" + target_path
         if _check_exists(fs_target_path, short_hash):
@@ -150,11 +195,13 @@ def _install_json(package_json_url, index, target, version, mpy):
             if not _download_file(file_url, fs_target_path):
                 print("File not found: {} {}".format(target_path, short_hash))
                 return False
+    # get other files from URLs
     for target_path, url in package_json.get("urls", ()):
         fs_target_path = target + "/" + target_path
-        if not _download_file(_rewrite_url(url, version, package_json_url), fs_target_path):
+        if not _download_file(_rewrite_url(url, version), fs_target_path):
             print("File not found: {} {}".format(target_path, url))
             return False
+    # install dependencies
     for dep, dep_version in package_json.get("deps", ()):
         if not _install_package(dep, index, target, dep_version, mpy):
             return False