CI/svg-flatten: add wasm builds

2021-09-26 15:45:09 +02:00 · 2021-09-26 15:45:09 +02:00 · 8a64621e8c
commit 8a64621e8c
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -2,41 +2,137 @@ variables:
  GIT_SUBMODULE_STRATEGY: recursive

 stages:
+  - build
  - test
+  - publish

-test_debian_10:
-  stage: test
+include:
+  - local: "/svg-flatten/svg-flatten-wasi-ci.yml"
+
+build:debian_10:
+  stage: build
  image: "registry.gitlab.com/gerbolyze/build-containers/debian:10"
  script:
-    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH CXX=clang++"
+    - "export CXX=clang++"
+    - "make -C svg-flatten"
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-svg-flatten-deb10"
+    paths:
+      - svg-flatten/build/svg-flatten
+      - svg-flatten/build/nopencv-test
+
+test:debian_10:
+  stage: test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  image: "registry.gitlab.com/gerbolyze/build-containers/debian:10"
+  script:
+    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+    - "touch svg-flatten/build/svg-flatten svg-flatten/build/nopencv-test"
    - "python3 setup.py install --user"
    - "gerbolyze --help"
    - "make -C svg-flatten tests"
+  dependencies:
+    - build:debian_10
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-test-deb10"
+    when: on_failure
+    paths:
+      - svg-flatten/testcase-fails/*.png
+      - svg-flatten/testcase-fails/*.svg

-test_ubuntu_2004:
-  stage: test
+build:ubuntu_2004:
+  stage: build
  image: "registry.gitlab.com/gerbolyze/build-containers/ubuntu:20.04"
  script:
-    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH CXX=clang++"
+    - "export CXX=clang++"
+    - "make -C svg-flatten"
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-svg-flatten-ubu20"
+    paths:
+      - svg-flatten/build/svg-flatten
+      - svg-flatten/build/nopencv-test
+
+test:ubuntu_2004:
+  stage: test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  image: "registry.gitlab.com/gerbolyze/build-containers/ubuntu:20.04"
+  script:
+    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+    - "touch svg-flatten/build/svg-flatten svg-flatten/build/nopencv-test"
    - "python3 setup.py install --user"
    - "gerbolyze --help"
    - "make -C svg-flatten tests"
+  dependencies:
+    - build:ubuntu_2004
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-test-ubu20"
+    when: on_failure
+    paths:
+      - svg-flatten/testcase-fails/*.png
+      - svg-flatten/testcase-fails/*.svg

-test_fedora_33:
-  stage: test
+build:fedora_33:
+  stage: build
  image: "registry.gitlab.com/gerbolyze/build-containers/fedora:33"
  script:
-    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH CXX=clang++"
+    - "export CXX=clang++"
+    - "make -C svg-flatten"
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-svg-flatten-fed33"
+    paths:
+      - svg-flatten/build/svg-flatten
+      - svg-flatten/build/nopencv-test
+
+test:fedora_33:
+  stage: test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  image: "registry.gitlab.com/gerbolyze/build-containers/fedora:33"
+  script:
+    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+    - "touch svg-flatten/build/svg-flatten svg-flatten/build/nopencv-test"
    - "python3 setup.py install --user"
    - "gerbolyze --help"
    - "make -C svg-flatten tests"
+  dependencies:
+    - build:fedora_33
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-test-fed33"
+    when: on_failure
+    paths:
+      - svg-flatten/testcase-fails/*.png
+      - svg-flatten/testcase-fails/*.svg

-test_archlinux:
+build:archlinux:
+  stage: build
+  image: "registry.gitlab.com/gerbolyze/build-containers/archlinux:latest"
+  script:
+    - "make -C svg-flatten"
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-svg-flatten-arch"
+    paths:
+      - svg-flatten/build/svg-flatten
+      - svg-flatten/build/nopencv-test
+
+test:archlinux:
  stage: test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
  image: "registry.gitlab.com/gerbolyze/build-containers/archlinux:latest"
  script:
    - "export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+    - "touch svg-flatten/build/svg-flatten svg-flatten/build/nopencv-test"
    - "python setup.py install --user"
    - "gerbolyze --help"
    - "make -C svg-flatten tests"
+  dependencies:
+    - build:archlinux
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-test-arch"
+    when: on_failure
+    paths:
+      - svg-flatten/testcase-fails/*.png
+      - svg-flatten/testcase-fails/*.svg

--- a/.gitmodules
+++ b/.gitmodules
@ -1,6 +1,6 @@
 [submodule "upstream/cpp-base64"]
 	path = upstream/cpp-base64
-	url = https://github.com/ReneNyffenegger/cpp-base64
+    url = https://gitlab.com/gerbolyze/gerbolyze-cpp-base64.git
 [submodule "upstream/voronoi"]
 	path = upstream/voronoi
 	url = https://github.com/JCash/voronoi
@ -9,7 +9,7 @@
 	url = https://github.com/thinks/poisson-disk-sampling
 [submodule "upstream/argagg"]
 	path = upstream/argagg
-	url = https://github.com/vietjtnguyen/argagg
+    url = https://gitlab.com/gerbolyze/gerbolyze-argagg.git
 [submodule "upstream/CavalierContours"]
 	path = upstream/CavalierContours
 	url = https://github.com/jbuckmccready/CavalierContours
--- a/svg-flatten/Makefile
+++ b/svg-flatten/Makefile
@ -74,7 +74,7 @@ WASI_CXXFLAGS ?= -DNOFORK -DNOTHROW -DWASI -DPUGIXML_NO_EXCEPTIONS -fno-exceptio

 BINARY := svg-flatten

-all: $(BUILDDIR)/$(BINARY)
+all: $(BUILDDIR)/$(BINARY) $(BUILDDIR)/nopencv-test

 $(CACHEDIR)/$(WASI_SDK):
 	mkdir -p $(dir $@)
@ -92,19 +92,20 @@ $(BUILDDIR)/host/%.o: %.cpp
 	@mkdir -p $(dir $@) 
 	$(CXX) -c $(HOST_CXXFLAGS) $(HOST_CXXFLAGS) $(HOST_INCLUDES) -o $@ $<

+.INTERMEDIATE: $(HOST_SOURCES:%.cpp=$(BUILDDIR)/host/%.o)
 $(BUILDDIR)/$(BINARY): $(HOST_SOURCES:%.cpp=$(BUILDDIR)/host/%.o)
 	@mkdir -p $(dir $@) 
 	$(CXX) $(HOST_CXXFLAGS) -o $@ $^ $(HOST_LDFLAGS)

 $(BUILDDIR)/nopencv-test: src/test/nopencv_test.cpp src/nopencv.cpp src/util.cpp
 	@mkdir -p $(dir $@) 
-	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LDFLAGS)
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(HOST_LDFLAGS)


 .PHONY: tests
 tests: $(BUILDDIR)/nopencv-test
 	$(BUILDDIR)/nopencv-test
-	$(PYTHON3) src/test/svg_tests.py
+	$(PYTHON3) src/test/svg_tests.py || ( mkdir testcase-fails && cp /tmp/gerbolyze-*.{svg,png} testcase-fails/ && false )

 .PHONY: install
 install:
--- a/svg-flatten/README.rst
+++ b/svg-flatten/README.rst
--- a/svg-flatten/setup.py
+++ b/svg-flatten/setup.py
@ -0,0 +1,48 @@
+import subprocess
+from setuptools import setup, find_packages
+from pathlib import Path
+import re
+import shutil
+
+def version():
+    res = subprocess.run(['git', 'describe', '--tags', '--match', 'v*'], capture_output=True, check=True, text=True)
+    version, _, _rest = res.stdout.strip()[1:].rpartition('-')
+
+def long_description():
+    with open("README.rst") as f:
+        return f.read()
+
+setup(
+    name="svg-flatten-wasi",
+    version=version(),
+    author="jaseg",
+    author_email="pypi@jaseg.de",
+    description="svg-flatten SVG downconverter",
+    long_description=long_description(),
+    long_description_content_type="text/x-rst",
+    license="AGPLv3+",
+    python_requires="~=3.7",
+    setup_requires=["wheel"],
+    install_requires=[
+        "importlib_resources; python_version<'3.9'",
+        "appdirs~=1.4",
+        "wasmtime>=0.28",
+        "click >= 4.0"
+    ],
+    packages=["svg_flatten_wasi"],
+    package_data={"svg_flatten_wasi": [
+        "*.wasm",
+    ]},
+    entry_points={
+        "console_scripts": [
+            "wasi-svg-flatten = svg_flatten_wasi:run_svg_flatten",
+        ],
+    },
+    project_urls={
+        "Source Code": "https://git.jaseg.de/gerbolyze",
+        "Bug Tracker": "https://github.com/jaseg/gerbolyze/issues",
+    },
+    classifiers=[
+        "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+    ],
+)
--- a/svg-flatten/src/out_svg.cpp
+++ b/svg-flatten/src/out_svg.cpp
@ -37,6 +37,7 @@ SimpleSVGOutput::SimpleSVGOutput(ostream &out, bool only_polys, int digits_frac,
 }

 void SimpleSVGOutput::header_impl(d2p origin, d2p size) {
+    cerr << "svg: header" << endl;
    m_offset[0] = origin[0];
    m_offset[1] = origin[1];
    m_out << "<svg width=\"" << size[0] << "mm\" height=\"" << size[1] << "mm\" viewBox=\"0 0 "
@ -44,6 +45,7 @@ void SimpleSVGOutput::header_impl(d2p origin, d2p size) {
 }

 SimpleSVGOutput &SimpleSVGOutput::operator<<(GerberPolarityToken pol) {
+    cerr << "svg: got polarity " << pol << endl;
    if (pol == GRB_POL_DARK) {
        m_current_color = m_dark_color;
    } else if (pol == GRB_POL_CLEAR) {
@ -56,6 +58,7 @@ SimpleSVGOutput &SimpleSVGOutput::operator<<(GerberPolarityToken pol) {
 }

 SimpleSVGOutput &SimpleSVGOutput::operator<<(const Polygon &poly) {
+    cerr << "svg: got poly of size " << poly.size() << endl;
    if (poly.size() < 3) {
        cerr << "Warning: " << poly.size() << "-element polygon passed to SimpleGerberOutput" << endl;
        return *this;
@ -75,6 +78,7 @@ SimpleSVGOutput &SimpleSVGOutput::operator<<(const Polygon &poly) {
 }

 void SimpleSVGOutput::footer_impl() {
+    cerr << "svg: footer" << endl;
    m_out << "</svg>" << endl;
 }

--- a/svg-flatten/src/svg_color.cpp
+++ b/svg-flatten/src/svg_color.cpp
@ -31,6 +31,7 @@ using namespace std;
 * them.
 */
 enum gerber_color gerbolyze::svg_color_to_gerber(string color, string opacity, enum gerber_color default_val, const RenderSettings &rset) {
+    cerr << "resolving svg color spec color=\"" << color << "\", opacity=\"" << opacity << "\", default=" << default_val << endl;
    float alpha = 1.0;
    if (!opacity.empty() && opacity[0] != '\0') {
        char *endptr = nullptr;
--- a/svg-flatten/src/svg_doc.cpp
+++ b/svg-flatten/src/svg_doc.cpp
@ -227,6 +227,7 @@ void gerbolyze::SVGDocument::export_svg_group(RenderContext &ctx, const pugi::xm
 void gerbolyze::SVGDocument::export_svg_path(RenderContext &ctx, const pugi::xml_node &node) {
    enum gerber_color fill_color = gerber_fill_color(node, ctx.settings());
    enum gerber_color stroke_color = gerber_stroke_color(node, ctx.settings());
+    cerr << "path: resolved colors, stroke=" << stroke_color << ", fill=" << fill_color << endl;

    double stroke_width = usvg_double_attr(node, "stroke-width", /* default */ 1.0);
    assert(stroke_width > 0.0);
--- a/svg-flatten/src/test/svg_tests.py
+++ b/svg-flatten/src/test/svg_tests.py
@ -7,6 +7,7 @@ from pathlib import Path
 import subprocess
 import itertools
 import os
+import sys

 from PIL import Image
 import numpy as np
@ -14,6 +15,9 @@ import numpy as np
 def run_svg_flatten(input_file, output_file, *args, **kwargs):
    if 'SVG_FLATTEN' in os.environ:
        svg_flatten = os.environ.get('SVG_FLATTEN')
+        if not hasattr(run_svg_flatten, 'custom_svg_flatten_warned'):
+            print(f'Using svg-flatten from SVG_FLATTEN environment variable: "{svg_flatten}"', file=sys.stderr)
+            run_svg_flatten.custom_svg_flatten_warned = True
    elif (Path(__file__) / '../../build/svg-flatten').is_file():
        svg_flatten = '../../build/svg-flatten'
    elif Path('./build/svg-flatten').is_file():
@ -34,11 +38,12 @@ def run_svg_flatten(input_file, output_file, *args, **kwargs):
    try:
        proc = subprocess.run(args, capture_output=True, check=True)
    except:
-        print('Subprocess stdout:')
-        print(proc.stdout)
-        print('Subprocess stderr:')
-        print(proc.stderr)
        raise
+    finally:
+        print('Subprocess stdout:')
+        print(proc.stdout.decode())
+        print('Subprocess stderr:')
+        print(proc.stderr.decode())

 def run_cargo_cmd(cmd, args, **kwargs):
    if cmd.upper() in os.environ:
@ -84,7 +89,8 @@ class SVGRoundTripTests(unittest.TestCase):
    }

    def compare_images(self, reference, output, test_name, mean=test_mean_default, vectorizer_test=False, rsvg_workaround=False):
-        ref, out = Image.open(reference), Image.open(output)
+        ref = Image.open(reference)
+        out =Image.open(output)

        if vectorizer_test:
            target_size = (100, 100)
@ -184,6 +190,8 @@ class SVGRoundTripTests(unittest.TestCase):
            else:
                run_svg_flatten(test_in_svg, tmp_out_svg.name, format='svg')

+            shutil.copyfile(tmp_out_svg.name, f'/tmp/gerbolyze-intermediate-{test_in_svg.stem}-out.svg')
+
            if not use_rsvg: # default!
                run_cargo_cmd('resvg', [tmp_out_svg.name, tmp_out_png.name], check=True, stdout=subprocess.DEVNULL)
                run_cargo_cmd('resvg', [test_in_svg, tmp_in_png.name], check=True, stdout=subprocess.DEVNULL)
@ -193,7 +201,7 @@ class SVGRoundTripTests(unittest.TestCase):
                subprocess.run(['rsvg-convert', test_in_svg, '-f', 'png', '-o', tmp_in_png.name], check=True, stdout=subprocess.DEVNULL)

            try:
-                self.compare_images(tmp_in_png, tmp_out_png, test_in_svg.stem,
+                self.compare_images(tmp_in_png.name, tmp_out_png.name, test_in_svg.stem,
                        SVGRoundTripTests.test_mean_overrides.get(test_in_svg.stem, SVGRoundTripTests.test_mean_default),
                        vectorizer_test, rsvg_workaround=use_rsvg)

--- a/svg-flatten/svg-flatten-wasi-ci.yml
+++ b/svg-flatten/svg-flatten-wasi-ci.yml
@ -0,0 +1,27 @@
+
+build:wasi-svg-flatten:
+  stage: build
+  image: "registry.gitlab.com/gerbolyze/build-containers/archlinux:latest"
+  script:
+    - cd svg-flatten # we start out in the repo's root
+    - make -j 2 build/svg-flatten.wasm
+    - cp build/svg-flatten.wasm svg_flatten_wasi/
+    - python3 setup.py bdist_wheel
+    - cd ..
+  artifacts:
+    name: "gerbolyze-$CI_COMMIT_REF_NAME-svg-flatten-wasi"
+    paths:
+      - svg-flatten/dist/*.whl
+
+publish:wasi-svg-flatten:
+  stage: publish
+  image: "registry.gitlab.com/gerbolyze/build-containers/archlinux:latest"
+  cache: {}
+  script:
+    - pip install -U --user twine
+    - export TWINE_USERNAME TWINE_PASSWORD
+    - ~/.local/bin/twine upload svg-flatten/dist/*
+  dependencies:
+    - build:wasi-svg-flatten
+  only:
+    - /^v.*$/
--- a/svg-flatten/svg_flatten_wasi/init.py
+++ b/svg-flatten/svg_flatten_wasi/init.py
@ -0,0 +1,82 @@
+import os
+import sys
+import tempfile
+import wasmtime
+import platform
+import click
+import pathlib
+import hashlib
+import appdirs
+import lzma
+from importlib import resources as importlib_resources
+try:
+    importlib_resources.files # py3.9+ stdlib
+except AttributeError:
+    import importlib_resources # py3.8- shim
+
+
+# ==============================
+# Note on wasmtime path handling
+# ==============================
+#
+# Hack: Right now, wasmtime's preopen_dir / --map functionality is completely borked. AFAICT only the first mapping is
+# even considered, and preopening both / and . simply does not work: Either all paths open'ed by the executable must be
+# absolute, or all paths must be relative. I spent some hours trying to track down where exactly this borkage originates
+# from, but I found the code confusing and did not succeed.
+# 
+# FOR NOW we work around this issue the dumb way: We simply have click parse enough of the command line to transform any
+# paths given on the command line to absolute paths. The actual path resolution is done by click because of
+# resolve_path=True.
+# 
+
+
+def _run_wasm_app(wasm_filename, argv, cachedir="svg-flatten-wasi"):
+
+    module_binary = importlib_resources.read_binary(__package__, wasm_filename)
+
+    module_path_digest = hashlib.sha256(__file__.encode()).hexdigest()
+    module_digest = hashlib.sha256(module_binary).hexdigest()
+    cache_path = pathlib.Path(os.getenv("SVG_FLATTEN_WASI_CACHE_DIR", appdirs.user_cache_dir(cachedir)))
+    cache_path.mkdir(parents=True, exist_ok=True)
+    cache_filename = (cache_path / f'{wasm_filename}-{module_path_digest[:8]}-{module_digest[:16]}')
+    
+    wasi_cfg = wasmtime.WasiConfig()
+    wasi_cfg.argv = argv
+    wasi_cfg.preopen_dir('/', '/')
+    wasi_cfg.inherit_stdin()
+    wasi_cfg.inherit_stdout()
+    wasi_cfg.inherit_stderr()
+    engine = wasmtime.Engine()
+
+    import time
+    try:
+        with cache_filename.open("rb") as cache_file:
+            module = wasmtime.Module.deserialize(engine, lzma.decompress(cache_file.read()))
+    except:
+        print("Preparing to run {}. This might take a while...".format(argv[0]), file=sys.stderr)
+        module = wasmtime.Module(engine, module_binary)
+        with cache_filename.open("wb") as cache_file:
+            cache_file.write(lzma.compress(module.serialize(), preset=0))
+
+    linker = wasmtime.Linker(engine)
+    linker.define_wasi()
+    store = wasmtime.Store(engine)
+    store.set_wasi(wasi_cfg)
+    app = linker.instantiate(store, module)
+    linker.define_instance(store, "app", app)
+
+    try:
+        app.exports(store)["_start"](store)
+        return 0
+    except wasmtime.ExitTrap as trap:
+        return trap.code
+
+
+@click.command(context_settings={'ignore_unknown_options': True})
+@click.argument('other_args', nargs=-1, type=click.UNPROCESSED)
+@click.argument('input_file',  type=click.Path(resolve_path=True, dir_okay=False))
+@click.argument('output_file', type=click.Path(resolve_path=True, dir_okay=False, writable=True))
+def run_usvg(input_file, output_file, other_args):
+
+    cmdline = ['svg-flatten', *other_args, input_file, output_file]
+    sys.exit(_run_wasm_app("svg-flatten.wasm", cmdline))