From 61d74fdef836753cbf1cf0e9ee7b8e50cb80f41d Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Mon, 7 Nov 2016 18:13:56 +0300 Subject: [PATCH] tools, unix: Replace upip tarball with just source files. To make its inclusion as frozen modules in multiple ports less magic. Ports are just expected to symlink 2 files into their scripts/modules subdirs. Unix port updated to use this and in general follow frozen modules setup tested and tried on baremetal ports, where there's "scripts" predefined dir (overridable with FROZEN_DIR make var), and a user just drops Python files there. --- tools/micropython-upip-1.1.4.tar.gz | Bin 4090 -> 0 bytes tools/upip.py | 288 ++++++++++++++++++++++++++++ tools/upip_utarfile.py | 94 +++++++++ unix/Makefile | 31 +-- 4 files changed, 385 insertions(+), 28 deletions(-) delete mode 100644 tools/micropython-upip-1.1.4.tar.gz create mode 100644 tools/upip.py create mode 100644 tools/upip_utarfile.py diff --git a/tools/micropython-upip-1.1.4.tar.gz b/tools/micropython-upip-1.1.4.tar.gz deleted file mode 100644 index 588fc4d62ca715b56c7e5e7e07dfdc561317d6b4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4090 zcmb8xhd&g6!^d%=%*YwVSxGiII_qqa>~(aqoxOMVx;Wd}d&ZTFP}wJOWSteVWsi_` z_IZB4*Yo@X&-eBH8$Pc$M*=y~e|Lvu&&17VRU!@VK%itrZ)ouVo?d_}5)60q4&i3` z^`W*S`xUI&@qov^$E5xC3PH`rnZ!`5x*C<{XP419) z@w09zkoVc?zJhRrE9&37p95y&ARO&sA{xLIBGu`&=q*j}Wx-dI$dbl;ObF$nV!uh# z7^1)SuoG&AR}9(Hwr~AWd9ljydRddNM;)ffMWH=~*DHSelUo1ih@w%9%+O)*n+67#H${8`a zm6XS?pL4LW?WgRVFLgtVX_N6yhquAh%^^ILJSO9yx&+v%v3=URG`mxS{&(u)d4|tU zPeuTx5wM`CbOxeRvUw+iAEU>Q2s9eUJ3{uze(pbwKaSb@;?BRAeWu5YXmJOuFa)G? z)A#ht@jPqDfHWdvopN;@n|jvv3LD=hufRa4k#knI`ns8|Q7^|(G2zXe{8~zOO1j$i z?lE?q+^3}(%f{`41J7mXx#$;XHknOe-#H-EPsk|Z#2sMrs6fC&_E*-%tI`uvarf{Kk!jxZZq2=v~8kV zZ(`V31!2gVPp^IU+0S=#+EiPm-*7GP4M=5ain)z-lM!-Nu$sqx6|Z&0SY8u931^o7 z$Hz2Y{*k>xDokuBIVA;Hd73#cX`cUST}2+Zo$mib`K|#7_LJ7x_M&N(@WJ@?SU1sb zR(2LHqs)Q2#h6a=`EQy4ohLjVU8tM<*)bs(2}{Y4ojBMHtmOAs3Aos6dBp(T%bLKN zqa?mSx#7&%`azdfT79)6v zl#8_L?OiBMWS%_m#7PN@)vX-V&Aq!ellQ2S#!kiJ-!1=Le${igu0<)`Vf7TMtb{)l zEvAxBQ2?@K6Dg-|9o&8>L@tY5pelwwnJF5i7_&UiB>dKBjHFC>cvHaz9eHyK>~ZQ* z5Hvym0%P8s<#Zxm^=D4_9xL;K3Gtd{0kQ^9dF5nHy2%H07~|1rDGq24g@P7BWoT0C zMwCx@G*Li#W(k+6`VqymGiDv%-Q+H)1b5XiiK*;c7AKnh5@o2;Vapbvn#I?t$-T#C zdl3aAA+$XBQ>_1sb7&z(~!3dzA(E)bB|uSQI7|E>*^jt!M#hgG3+p8JD>VOuWC& zcH;ZeR@bxLN>&&o;LmgQJ2$@8^d;!F;0fDedA8zg4OPR{NaWFT!Bu)k#Tbi!QRtMS zD?(a(J5-!|{)z4n<}=4&S_r1{dP(MyREaW35kK4(b%itJ8>bL9vVIB3;>!K-3z_Xt zSBXb9#c0CQ)%%nUcfG zkWD>ik&X$oc70?`T>azGicu@1{nA@wuZz$%m73`7dgbGN@1wwzYBNtztT+g9k2FyE%(zJV_7kWVG5 zvf4+nSOnm{>{BDpsTT)kvF~Y|SHDLqVFp$?O)zwDn^KHd7m0K97FT``AC%{d#+Yo* z)7rV zuJ^N}t+Q}0Nu{BmHg|YQKy>r1RUeiXT65C3a^@iuApVh5aPG3>B=<`@j9ZT5ERr`~ z-9fGGu2KIhjh;H^B!S9Sm4=brWf>!v3JN~$mYgcY*kVxaWMx!LbN7-NTvHaB0iJT| z*`4{Ty#^JB!rYs5?tjHhqxSgo2ZO&{7xYXl3!@2Z*W>vqLDlZo_h;l?oG-Qc7WrQ% z7rR~__s{n<5?2drwRQSWs#Rt+q7FB&hIcK_gk3YtyMGHd#09!>_$$x!N=*ymd9>`G zGBdSerJtNG3zohHF)R4Q80fK#+6maY-9hiua89H>7`>#Je4>fEi`OK+cvLEyuc@^Uvr!pS>Aeas#&CSx7r7p@K`DSl%_A z2jm^r{T>!2+B7_^8D;^uy}Kx}u*^tuo&A?zFdeBaJ*Z#(A%4jxk_#kE#bmcF5+-dX zY4affD`zQ_#q;w^s#5X1vzxEC2V&0A)$)6h+V!oFbgN1UGDf9M#-vn3Uq78RFe|t_ z$)djUE>rdTqAUyNUBxEX?wzlxm&!D5#infZ2TSc0=7gA-os?IL5=O$gH+!KS3N6Gzp*fUmGwtMTEKMh(SIVR()Jgf z^`Q>tqnp92`$1%Y>yBpU$NfDi_dnjj+tcfDCRO;VWsb$MqiTjIG^50PX7$D-%6O-v z9D=r;arTl}z`-}NerohFQ@WdOWIFrg7(Wz7M8OQ*ver3jM2&CI)B_B;Yh7=4cZb|8 zE$SA09#V&wdjQw()?{nEjL8KY7x8{gv`6m})-t4bUKE>`cPWdtn3@+(>oias$|A;p z-dV%(#%+V%jA!&(S9ikW^)sJ)TuEPLX`L*v$uV{wL~&Z(4qrxv3Z5@%M4Ubq(Yc4+ zTxo0VBuT&3>r{fCS2-8OQ9RvdZv!WmPG8Iy7@PtV&Qa*BXA0N{_5(FWl-R~Bj<@6h zE~UIF#r5us{FH2y|1^5B-C@hh7FEH`7sN~#J3$HqT9~%5NinKV`)L}?$G+qE-hgr2 zl8m`TMLZwe_(qfwS)&_%6UF*Mrn;~K`Jzp@%I`!2W>A15`+L`iq)O@^17qDq|kMDqhuRz4S$V7I&-%{-fw24vlEP*i7H*>rvOS>Id879YkX zRdFjlkuDxvK7z3eP6TA;M)(==&UFSr&{u0KOn_1LrRN#~5xeDo_YjX0Ur&WCkTz$- zFU%X;se*;*9_iZ}PFvw6?3=~O{)CP=V{P~*jSUW~;Bg0hHT-TuN@DN``uq?59tcl0 zf7(h%;FHPX-QI1{RJ->wH{23~r#Eafe&*k49?!{QS#;yQGDX7WRO}WWaq*p4ibpt& z1-UZWh0;lut>wFDM76rSP~Iq7dIcYaj}MnQ1-2Pi@3^OG8mv;@-W5NEn$^r`3@e$A zV`q7%8N|b``9swk(Y|_>Uv~lF+{8)dhnPS8jMt9tHX)mHo~=}&Vf_0Nt;|dHGc}LD zwtkUGCW-Gyh|R;xw8V1t@VQ!v+P(tb{)9)}xUr0g|E2zxpMkNkBj(bw5Y!_jBU7@u zb?)h?;Ka&$|IA-4xUa7-M}f1oV~j9%NCPCW-{bSTh^j9PNlpg;AMCaM3wz4_{{R_R zvD$9)ADX9-MWfMZD^!*e4--l5Z~||}0kIZXaabvnWB%bFJgtnIwsx#XtrU4D&)Z!` zL1(eJ5F>Ym4EYzpNGEbP0dh}%6itfeP$(jhlS=%Bl93hTp1EVXemP2=1P5g&7DED( z@H8^`J?l9I*3G?(zI={4oaZ+n_J^N*QK)5Tx?mfN2)Xm?2e=3&IVPS%F7WRzD;HBm zE(5{u@1GRkK(5f!Au>22iVFC__oKWKqn+{X=w{tigB2UGLvGy-NA;kpInuv*+z1Lr zAp2PC68urw{f&&nIm0PqAyZ2?G2ywmVb-+SwXbiD^b#G9F4ngGdEWj^ize#tZQ*s5 z+W&xa-D#~!8u{Cnw6K@NDLupak)MkJ>1=KyuPZM}Vdlm&s_aV`^xuib9-GdZw9lz= zlDboagg0-i?#JLI0LgD}%GeZ1&9g#s<0fr~ihtG49bQ&K940(xMK#{eo&bj+ELKprn1^z>^A6O^f] ... | -r +import upip; upip.install(package_or_list, []) + +If is not given, packages will be installed into sys.path[1] +(can be set from MICROPYPATH environment variable, if current system +supports that).""") + print("Current value of sys.path[1]:", sys.path[1]) + print("""\ + +Note: only MicroPython packages (usually, named micropython-*) are supported +for installation, upip does not support arbitrary code in setup.py. +""") + +def main(): + global debug + global install_path + install_path = None + + if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": + help() + return + + if sys.argv[1] != "install": + fatal("Only 'install' command supported") + + to_install = [] + + i = 2 + while i < len(sys.argv) and sys.argv[i][0] == "-": + opt = sys.argv[i] + i += 1 + if opt == "-h" or opt == "--help": + help() + return + elif opt == "-p": + install_path = sys.argv[i] + i += 1 + elif opt == "-r": + list_file = sys.argv[i] + i += 1 + with open(list_file) as f: + while True: + l = f.readline() + if not l: + break + to_install.append(l.rstrip()) + elif opt == "--debug": + debug = True + else: + fatal("Unknown/unsupported option: " + opt) + + to_install.extend(sys.argv[i:]) + if not to_install: + help() + return + + install(to_install) + + if not debug: + cleanup() + + +if __name__ == "__main__": + main() diff --git a/tools/upip_utarfile.py b/tools/upip_utarfile.py new file mode 100644 index 0000000000..65ce0bdca8 --- /dev/null +++ b/tools/upip_utarfile.py @@ -0,0 +1,94 @@ +import uctypes + +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12), +} + +DIRTYPE = "dir" +REGTYPE = "file" + +def roundup(val, align): + return (val + align - 1) & ~(align - 1) + +class FileSection: + + def __init__(self, f, content_len, aligned_len): + self.f = f + self.content_len = content_len + self.align = aligned_len - content_len + + def read(self, sz=65536): + if self.content_len == 0: + return b"" + if sz > self.content_len: + sz = self.content_len + data = self.f.read(sz) + sz = len(data) + self.content_len -= sz + return data + + def readinto(self, buf): + if self.content_len == 0: + return 0 + if len(buf) > self.content_len: + buf = memoryview(buf)[:self.content_len] + sz = self.f.readinto(buf) + self.content_len -= sz + return sz + + def skip(self): + sz = self.content_len + self.align + if sz: + buf = bytearray(16) + while sz: + s = min(sz, 16) + self.f.readinto(buf, s) + sz -= s + +class TarInfo: + + def __str__(self): + return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) + +class TarFile: + + def __init__(self, name=None, fileobj=None): + if fileobj: + self.f = fileobj + else: + self.f = open(name, "rb") + self.subf = None + + def next(self): + if self.subf: + self.subf.skip() + buf = self.f.read(512) + if not buf: + return None + + h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + d = TarInfo() + d.name = str(h.name, "utf-8").rstrip() + d.size = int(bytes(h.size).rstrip(), 8) + d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] + self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) + return d + + def __iter__(self): + return self + + def __next__(self): + v = self.next() + if v is None: + raise StopIteration + return v + + def extractfile(self, tarinfo): + return tarinfo.subf diff --git a/unix/Makefile b/unix/Makefile index fd98d2ced2..a14ed215c6 100644 --- a/unix/Makefile +++ b/unix/Makefile @@ -1,6 +1,8 @@ -include mpconfigport.mk include ../py/mkenv.mk +FROZEN_DIR = scripts + # define main target PROG = micropython @@ -148,17 +150,6 @@ SRC_C = \ fatfs_port.c \ $(SRC_MOD) -# Include builtin package manager in the standard build (and coverage) -ifeq ($(PROG),micropython) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG),micropython_coverage) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG), micropython_nanbox) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG), micropython_freedos) -SRC_C += $(BUILD)/_frozen_upip.c -endif - LIB_SRC_C = $(addprefix lib/,\ $(LIB_SRC_C_EXTRA) \ timeutils/timeutils.c \ @@ -235,7 +226,7 @@ fast: # build a minimal interpreter minimal: $(MAKE) COPT="-Os -DNDEBUG" CFLAGS_EXTRA='-DMP_CONFIGFILE=""' \ - BUILD=build-minimal PROG=micropython_minimal \ + BUILD=build-minimal PROG=micropython_minimal FROZEN_DIR= \ MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_SOCKET=0 MICROPY_PY_THREAD=0 \ MICROPY_PY_TERMIOS=0 MICROPY_PY_USSL=0 \ MICROPY_USE_READLINE=0 MICROPY_FATFS=0 @@ -272,22 +263,6 @@ coverage_test: coverage gcov -o build-coverage/py ../py/*.c gcov -o build-coverage/extmod ../extmod/*.c -$(BUILD)/_frozen_upip.c: $(BUILD)/frozen_upip/upip.py - $(MAKE_FROZEN) $(dir $^) > $@ - -# Select latest upip version available -UPIP_TARBALL := $(shell ls -1 -v ../tools/micropython-upip-*.tar.gz | tail -n1) - -$(BUILD)/frozen_upip/upip.py: $(UPIP_TARBALL) - $(ECHO) "MISC Preparing upip as frozen module" - $(Q)mkdir -p $(BUILD) - $(Q)rm -rf $(BUILD)/micropython-upip-* - $(Q)tar -C $(BUILD) -xz -f $^ - $(Q)rm -rf $(dir $@) - $(Q)mkdir -p $(dir $@) - $(Q)cp $(BUILD)/micropython-upip-*/upip*.py $(dir $@) - - # Value of configure's --host= option (required for cross-compilation). # Deduce it from CROSS_COMPILE by default, but can be overriden. ifneq ($(CROSS_COMPILE),)