kopia lustrzana https://github.com/deathbeds/ipydrawio
541 wiersze
14 KiB
Plaintext
541 wiersze
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f544e4e0-4968-4de0-b33c-35b94aa89da5",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Schema Updater\n",
|
|
"\n",
|
|
"> This is not a demonstration of functionality, but an iterative notebook for updating the ipydrawio JSON schema, when it changes.\n",
|
|
"> The specific version of the files at play are found in `dodo.py`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b115e4b7-61c2-4baf-b841-c383db1edbca",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import json\n",
|
|
"import re\n",
|
|
"import subprocess\n",
|
|
"import tempfile\n",
|
|
"from copy import deepcopy\n",
|
|
"from pathlib import Path\n",
|
|
"\n",
|
|
"import IPython\n",
|
|
"import lxml.etree as ET\n",
|
|
"import pandas"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e10946b0-d498-4d14-8cd0-da9e7c7b830f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ROOT = Path(\"../..\").resolve()\n",
|
|
"assert ROOT.name == \"ipydrawio\"\n",
|
|
"assert (ROOT / \".git\").exists()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "96c17607-554d-4b50-b506-55bd51497604",
|
|
"metadata": {},
|
|
"source": [
|
|
"The user settings schema in lab is the source of truth for many elements of the "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e035378e-3b3c-4eb1-9fa4-ccc5d0ab83ed",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"SCHEMA_PATH = ROOT / \"packages/ipydrawio/schema/plugin.json\"\n",
|
|
"SCHEMA = json.loads(SCHEMA_PATH.read_text(encoding=\"utf-8\"))\n",
|
|
"IPython.display.JSON(SCHEMA)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "eb44b16b-98c1-4baf-ada3-7caaecfa13ac",
|
|
"metadata": {},
|
|
"source": [
|
|
"This is a continuous check as to whether git is happy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "11c57ef8-70b3-4f7e-bc47-a7fb821dd304",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"SCHEMA_PATH.write_text(json.dumps(SCHEMA, indent=2, sort_keys=True), encoding=\"utf-8\")\n",
|
|
"!jlpm prettier --write {ROOT}/packages/ipydrawio/schema/plugin.json\n",
|
|
"!git diff {ROOT}/packages/ipydrawio/schema/plugin.json || echo 'no change'"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cea18c8c-63ea-476c-bfc9-b586c651e20f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"paths = [*(ROOT / \"build/fetched\").glob(\"*.html\")]\n",
|
|
"paths"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b607c248-f4fb-46a9-811d-611126ca5508",
|
|
"metadata": {},
|
|
"source": [
|
|
"# The URL Params"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a45709f3-de69-4d57-bc63-2e04f14d13ac",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"h_url_params = paths[0]\n",
|
|
"IPython.display.IFrame(\n",
|
|
" f\"/files/{h_url_params}\",\n",
|
|
" height='600px\" style=\"height: 100%;',\n",
|
|
" width=\"100%\",\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2bd5c728-346b-451d-88ab-3f4a0b29de69",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"x_url_params = ET.HTML(h_url_params.read_bytes())\n",
|
|
"x_url_params"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4c59d50d-dd52-4dab-a790-0d7543f9db63",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"li_elements = x_url_params.xpath(\"//li/code[contains(text(), '=')]\")\n",
|
|
"li = li_elements[0]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "42a22e6f-c4af-4430-895b-9a5b22c01190",
|
|
"metadata": {},
|
|
"source": [
|
|
"We're interested in the context around the definition."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "77a3c450-0637-4141-b3e3-660f5a389899",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ET.tostring(li.getparent(), pretty_print=True, encoding=str)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6248c5ec-4fb2-4dc9-9931-93e505eb1dc6",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Clean up the description\n",
|
|
"\n",
|
|
"The raw XML is not very nice to embed in JSON. So many double quotes!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "32a00d47-2a0a-421e-9d06-35a6389c57bb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def one_md_desc(li):\n",
|
|
" with tempfile.TemporaryDirectory() as td:\n",
|
|
" tdp = Path(td)\n",
|
|
" html = tdp / \"test1.html\"\n",
|
|
" md = tdp / \"test1.md\"\n",
|
|
" html.write_text(\n",
|
|
" f\"<ul>{ET.tostring(li.getparent(), encoding=str, pretty_print=True)}</ul>\",\n",
|
|
" )\n",
|
|
" args = [\"pandoc\", html, \"-f\", \"html\", \"-t\", \"commonmark\", \"-s\", \"-o\", md]\n",
|
|
" subprocess.check_call([*map(str, args)])\n",
|
|
" body = \" \".join([line.strip() for line in md.read_text().splitlines()])\n",
|
|
" body = re.sub(r\"\\[(.*)?\\]\\((.*?)\\)\", r\"_\\1_\", body)[2:].strip()\n",
|
|
" print(f\"`{li.text}`\")\n",
|
|
" body = body.replace(f\"`{li.text}`:\", f\"{li.text.split('=')[1]}:\")\n",
|
|
" print(\"\\t\", body)\n",
|
|
" return body\n",
|
|
"\n",
|
|
"\n",
|
|
"IPython.display.Markdown(\n",
|
|
" f\"\"\"> _this is the description of **`{li.text.split(\"=\")[0]}`**_\\n> ```md\\n{one_md_desc(li)}\\n```\"\"\",\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d42e55c7-dd33-4e45-934d-107414387f23",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"url_params = {\n",
|
|
" x.text.split(\"=\")[0]: {\"description\": one_md_desc(x)}\n",
|
|
" for x in li_elements\n",
|
|
" if \"&\" not in x.text\n",
|
|
"}\n",
|
|
"IPython.display.JSON(url_params)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "17de503f-a228-4491-b341-35ea1773961d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"old_url_params = SCHEMA[\"definitions\"][\"drawio-url-params\"][\"properties\"]\n",
|
|
"IPython.display.JSON(old_url_params)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6414f799-914e-42f8-9659-afd6183cb98f",
|
|
"metadata": {},
|
|
"source": [
|
|
"# What is No Longer Documented?"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "33f5e886-1e78-45bd-bf77-30089295937f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"not_documented = {*old_url_params} - {*url_params}\n",
|
|
"not_documented"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a58a22c3-be84-4ee9-8f63-29ef118ac667",
|
|
"metadata": {},
|
|
"source": [
|
|
"# What have we Not Modeled?"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c105ece5-ab6e-4924-a32d-6b481b8a431d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"not_modeled = {*url_params} - {*old_url_params}\n",
|
|
"IPython.display.JSON({k: url_params[k] for k in not_modeled})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "975f301b-5c66-4764-a40f-cfa8ba3e4fd1",
|
|
"metadata": {},
|
|
"source": [
|
|
"Let's get some refs, this is kind of icky."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c19945a4-0ac4-4aa6-ab6a-d4d7e2470d51",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"{\n",
|
|
" url_params[ok].update({attr: ov[attr]})\n",
|
|
" for attr in [\"$ref\", \"format\", \"pattern\", \"type\", \"minimum\"]\n",
|
|
" for ok, ov in old_url_params.items()\n",
|
|
" if ok in url_params and attr in ov\n",
|
|
"}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cb0c3e61-039b-4321-9c8a-f3d6066f9d95",
|
|
"metadata": {},
|
|
"source": [
|
|
"# It's time to treat this like data."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8af96050-0c65-4f20-b3ce-84ca52e0402d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = pandas.DataFrame([{\"id\": k, **v} for k, v in url_params.items()])\n",
|
|
"df = df.fillna(\"\").sort_values(\"id\").set_index([\"id\"])\n",
|
|
"df"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4f1656c5-0a1c-420c-8186-e31b28bf3abf",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Let's update the old with the new."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8e97e69c-760f-410c-beee-0f6cf3d95bcc",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_old = (\n",
|
|
" pandas.DataFrame(\n",
|
|
" [\n",
|
|
" {\"id\": k, **v}\n",
|
|
" for k, v in SCHEMA[\"definitions\"][\"drawio-url-params\"][\"properties\"].items()\n",
|
|
" ],\n",
|
|
" )\n",
|
|
" .sort_values(\"id\")\n",
|
|
" .set_index([\"id\"])\n",
|
|
" .fillna(\"\")\n",
|
|
")\n",
|
|
"df_old"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d1d6a4a0-7c63-4fff-a80a-993fde31e5aa",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def clean_df(df):\n",
|
|
" return {\n",
|
|
" record[\"id\"]: {\n",
|
|
" rk: rv\n",
|
|
" for rk, rv in record.items()\n",
|
|
" if rk not in [\"id\"] and rv and isinstance(rv, str) and rv.strip()\n",
|
|
" }\n",
|
|
" for record in df.reset_index().to_dict(orient=\"records\")\n",
|
|
" }"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d00e2403-c0da-40ec-9b9c-15a56bec89e9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cleaned_df = clean_df(df)\n",
|
|
"cleaned_df_old = clean_df(df_old)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "278e5ca5-3910-417f-ae37-ae8490e3667c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# handle symmetric difference of keys\n",
|
|
"\n",
|
|
"new_df = {}\n",
|
|
"\n",
|
|
"for k, v in cleaned_df_old.items():\n",
|
|
" if k not in cleaned_df:\n",
|
|
" v = {**v}\n",
|
|
" if \"description\" in v:\n",
|
|
" v[\"description\"] += \" **TBD `14.6.10`**\"\n",
|
|
" new_df[k] = v\n",
|
|
"for k, v in cleaned_df.items():\n",
|
|
" if k not in cleaned_df_old:\n",
|
|
" v = {**v}\n",
|
|
" if \"description\" in v:\n",
|
|
" v[\"description\"] += \" **NEW `14.6.10`**\"\n",
|
|
" new_df[k] = v\n",
|
|
"for k in sorted({*cleaned_df_old, *cleaned_df}):\n",
|
|
" if k in new_df:\n",
|
|
" continue\n",
|
|
" new_df[k] = {**cleaned_df_old[k]}\n",
|
|
" new_df[k].update(cleaned_df[k])\n",
|
|
"# new_df"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7315f427-3b89-4c6c-9918-0a9e23c55049",
|
|
"metadata": {},
|
|
"source": [
|
|
"# One serious gotcha"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "69893051-02f6-4940-a3e3-72d49ed24f3c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"before_i18n, after_i18n = new_df[\"lang\"][\"description\"].split(\"*i18n*,\")\n",
|
|
"locales = sorted(\n",
|
|
" [\n",
|
|
" lang.replace(\"`\", \"\").strip().split(\":\")[0].strip()\n",
|
|
" for lang in new_df[\"lang\"][\"description\"].split(\"*i18n*,\")[1].split(\",\")\n",
|
|
" ],\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2284b798-01d7-4a65-92f7-985bdbf36e37",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"new_df[\"lang\"][\"description\"] = f\"\"\"{before_i18n} *i18n* {\", \".join(locales)}\"\"\"\n",
|
|
"new_df[\"lang\"][\"examples\"] = locales"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9b64ecf4-da25-44e2-8014-d2572bcf0cd4",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Fill in some new guesses"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cdda3d64-4b23-480a-9c8c-063b16ed890e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"new_df[\"configure\"][\"$ref\"] = new_df[\"pwa\"][\"$ref\"] = new_df[\"returnbounds\"][\n",
|
|
" \"$ref\"\n",
|
|
"] = new_df[\"svg-warning\"][\"$ref\"] = \"#/definitions/off-switch\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "47e7f2cc-a564-46ba-b5d2-86409ca234b9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"new_df[\"hide-pages\"][\"$ref\"] = \"#/definitions/on-off-switch\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8857785d-75c8-4521-a8b2-db78a4450957",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"new_df[\"math-output\"][\"examples\"] = [\"html\"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5d52fb20-0081-40f0-8cd4-1aa35bacae25",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"new_df[\"ui\"][\"examples\"] = [\"min\", \"atlas\", \"dark\", \"sketch\"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9f4dfe17-2638-45f1-b476-7ce12c59aa28",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"new_df[\"search-shapes\"][\"type\"] = \"string\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "afacc8ae-0f39-41af-a621-78b4a07ff7f1",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Ok, let's ship it."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "983dc291-57b2-4250-a604-8f77e96fa526",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"NEW_SCHEMA = deepcopy(SCHEMA)\n",
|
|
"NEW_SCHEMA[\"definitions\"][\"drawio-url-params\"][\"properties\"] = new_df\n",
|
|
"NEW_SCHEMA_PATH = SCHEMA_PATH.parent / f\"{SCHEMA_PATH.name}.proposed.json\"\n",
|
|
"NEW_SCHEMA_PATH.write_text(\n",
|
|
" json.dumps(NEW_SCHEMA, indent=2, sort_keys=True),\n",
|
|
" encoding=\"utf-8\",\n",
|
|
")\n",
|
|
"!jlpm --silent prettier --write {ROOT}/packages/ipydrawio/schema/plugin.json.proposed.json\n",
|
|
"!git diff --color-words --no-index {ROOT}/packages/ipydrawio/schema/plugin.json {ROOT}/packages/ipydrawio/schema/plugin.json.proposed.json"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.2"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|