Merge 27c2ab76ce into e6faa69a70

Some US roads were not included in tiles, because the `ST_AsMVTGeom` did not transform geometry (result was NULL). This PR is trying to fix the geometry for `ST_AsMVTGeom`, even if the geometry is marked as valid (`ST_IsValid`). The odd geometry was caused by simplification when it could create `LINESTRING(1 1, 2 2, 1 1)` linestring, which in some cases was not converted into MVT. 

Also, for the US (on zoom 4) keeps just the network `us-interstate` and the `highway=motorway` or `osm_national_network(network)` for the rest of the world.

layers/power/README.md

@@ -0,0 +1,9 @@
+## power
+
+### Docs
+
+### Mapping Diagram
+![Mapping diagram for transportation](mapping_diagram.png?raw=true)
+
+### ETL diagram
+![ETL diagram for transportation](etl_diagram.png?raw=true)

layers/power/mapping.yaml

@@ -0,0 +1,99 @@
+tables:
+  # etldoc: imposm3 -> osm_power_point
+  power_point:
+    type: point
+    fields:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: power
+      name: class
+      type: string
+    - key: plant:source
+      name: source
+      type: string
+    - key: plant:method
+      name: method
+      type: string
+    - key: generator:source
+      name: gen_source
+      type: string
+    - key: generator:method
+      name: gen_method
+      type: string  
+    - key: power
+      name: name
+      type: string  
+    mapping:
+      power:
+      - tower
+      - pole
+      - catenary_mast
+      - generator
+      - portal
+      - transformer
+      - switch
+      - insulator
+  # etldoc: imposm3 -> osm_power_linestring
+  power_linestring:
+    type: linestring
+    fields:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: power
+      name: class
+      type: string
+    - key: plant:source
+      name: source
+      type: string
+    - key: plant:method
+      name: method
+      type: string
+    - key: generator:source
+      name: gen_source
+      type: string
+    - key: generator:method
+      name: gen_method
+      type: string  
+    - key: name
+      name: name
+      type: string
+    mapping:
+      power:
+      - line
+      - minor_line
+  # etldoc: imposm3 -> osm_power_polygon
+  power_polygon:
+    type: polygon
+    fields:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: power
+      name: class
+      type: string
+    - key: plant:source
+      name: source
+      type: string
+    - key: plant:method
+      name: method
+      type: string
+    - key: generator:source
+      name: gen_source
+      type: string
+    - key: generator:method
+      name: gen_method
+      type: string  
+    - key: name
+      name: name
+      type: string  
+    mapping:
+      power:
+      - generator
+      - plant
+      - substation
+      - transformer

layers/power/power.sql

@@ -0,0 +1,41 @@
+-- etldoc: layer_power[shape=record fillcolor=lightpink, style="rounded,filled",
+-- etldoc:     label="layer_power | <z13> z13 | <z14> z14+ " ] ;
+
+CREATE OR REPLACE FUNCTION layer_power(bbox geometry, zoom_level int)
+RETURNS TABLE(geometry geometry, class text, source text, method text, name text) AS $$
+SELECT geometry, class, COALESCE(NULLIF(source, ''),NULLIF(gen_source, '')), method, name
+FROM(
+    -- etldoc: osm_power_point -> layer_power:z13
+    SELECT geometry, class, source, gen_source, method, name
+    FROM osm_power_point
+    WHERE zoom_level >= 13 AND class = 'tower' AND geometry && bbox
+    UNION ALL
+    -- etldoc: osm_power_point -> layer_power:z14
+    SELECT geometry, class, source, gen_source, method, name
+    FROM osm_power_point
+    WHERE zoom_level >= 14 AND class = 'pole' AND geometry && bbox
+    UNION ALL
+    -- etldoc: osm_power_linestring -> layer_power:z13
+    SELECT geometry, class, source, gen_source,  method, name
+    FROM osm_power_linestring
+    WHERE zoom_level >= 13 AND class = 'line' AND geometry && bbox
+    UNION ALL
+    -- etldoc: osm_power_linestring -> layer_power:z14
+    SELECT geometry, class, source, gen_source,  method, name
+    FROM osm_power_linestring
+    WHERE zoom_level >= 14 AND class = 'minor_line' AND geometry && bbox
+    UNION ALL
+    -- etldoc: osm_power_polygon -> layer_power:z13    
+    SELECT geometry, class, source, gen_source, method, name
+    FROM osm_power_polygon
+    WHERE zoom_level >= 13 AND class = 'plant' AND geometry && bbox 
+    UNION ALL
+    -- etldoc: osm_power_polygon -> layer_power:z14
+    SELECT geometry, class, source, gen_source, method, name
+    FROM osm_power_polygon
+    WHERE zoom_level >= 14 AND class IN ('substation', 'generator', 'transformer') AND geometry && bbox
+    ) AS zoom_levels
+WHERE geometry && bbox;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/power/power.yaml

@@ -0,0 +1,74 @@
+layer:
+  id: "power"
+  description: |
+      **power** contains power cables and lines.
+  buffer_size: 4
+  fields:
+    name:
+      description: |
+          Name of the feature.
+    class:
+      description: |
+          Class is derived from the value of the
+          [`power`](http://wiki.openstreetmap.org/wiki/Key:power) tag.
+      values:
+      - line
+      - minor_line
+      - tower
+      - pole
+      - catenary_mast
+      - plant
+      - generator
+      - substation
+      - portal
+      - transformer
+      - switch
+      - insulator
+    source:
+      description: |
+          Class is derived from the value of the
+          [`plant:source`](http://wiki.openstreetmap.org/wiki/Key:plant:source) tag.
+      values:
+      - battery
+      - biogas
+      - biomass
+      - coal
+      - combustion
+      - diesel
+      - gas
+      - geothermal
+      - hydro
+      - nuclear
+      - oil
+      - solar
+      - solar_thermal
+      - waste
+      - wind
+      - wind_turbine
+      - wood
+    method:
+      description: |
+          Class is derived from the value of the
+          [`plant:method`](http://wiki.openstreetmap.org/wiki/Key:plant:method) tag.
+      values:
+      - anaerobic_digestion
+      - combustion
+      - dam
+      - fission
+      - gasification
+      - photovoltaic
+      - run-of-the-river
+      - thermal
+      - water-pumpted-storage
+      - water-storage
+      - wind_turbine    
+
+  datasource:
+    geometry_field: geometry
+    query: (SELECT geometry, class, source, method, name FROM layer_power(!bbox!, z(!scale_denominator!))) AS t
+
+schema:
+  - ./power.sql
+datasources:
+  - type: imposm3
+    mapping_file: ./mapping.yaml

layers/transportation/update_transportation_merge.sql

@@ -465,7 +465,7 @@ CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen_z9_geometry_i
 
 -- etldoc: osm_transportation_merge_linestring_gen_z9 -> osm_transportation_merge_linestring_gen_z8
 CREATE TABLE IF NOT EXISTS osm_transportation_merge_linestring_gen_z8(
-    geometry geometry('LineString'),
+    geometry geometry,
     id SERIAL,
     osm_id bigint,
     source_ids int[],
@@ -684,25 +684,68 @@ BEGIN
 
     -- etldoc: osm_transportation_merge_linestring_gen_z8 -> osm_transportation_merge_linestring_gen_z7
     INSERT INTO osm_transportation_merge_linestring_gen_z7
-    SELECT ST_Simplify(geometry, ZRes(9)) AS geometry,
-        id,
-        osm_id,
-        highway,
-        network,
-        construction,
-        -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
-        visible_brunnel(geometry, is_bridge, 8) AS is_bridge,
-        visible_brunnel(geometry, is_tunnel, 8) AS is_tunnel,
-        visible_brunnel(geometry, is_ford, 8) AS is_ford,
-        expressway,
-        z_order
-    FROM osm_transportation_merge_linestring_gen_z8
+    WITH roads_z8 AS (
+        SELECT  id,
+                osm_id,
+                ST_SnapToGrid(
+                    ST_Node(
+                        ST_Collect(
+                            ST_Simplify(geometry, ZRes(9))
+                            )
+                        ), ZRes(9)
+                    ) AS geometry,
+                highway, 
+                NULLIF(network, '') as network,
+                construction,
+                is_bridge, 
+                is_tunnel, 
+                is_ford,
+                expressway,
+                z_order
+        FROM osm_transportation_merge_linestring_gen_z8
+        GROUP BY id, osm_id, highway, construction, network, is_bridge, is_tunnel, is_ford, expressway, z_order
+        ),
+    roads_z8_merge AS (
+        SELECT  id,
+                osm_id,
+                ST_LineMerge(
+                    ST_Union(geometry)
+                    ) AS geometry,
+                highway,
+                network,
+                construction,
+                is_bridge,
+                is_tunnel,
+                is_ford,
+                expressway,
+                z_order
+    FROM roads_z8
+    GROUP BY id, osm_id, highway, network, construction, is_bridge, is_tunnel, is_ford, expressway, z_order
+    )
+
+    SELECT  CASE
+             WHEN ST_StartPoint(geometry) = ST_EndPoint(geometry) 
+                 THEN ST_RemovePoint(geometry, ST_NPoints(geometry) - 1)
+             ELSE geometry
+            END AS geometry,  
+            id,
+            osm_id,
+            highway,
+            network,
+            construction,
+            -- Remove bridge/tunnel/ford attributes from short sections of road so they can be merged
+            visible_brunnel(geometry, is_bridge, 8) AS is_bridge,
+            visible_brunnel(geometry, is_tunnel, 8) AS is_tunnel,
+            visible_brunnel(geometry, is_ford, 8) AS is_ford,
+            expressway,
+            z_order
+    FROM roads_z8_merge
         -- Current view: motorway/trunk/primary
     WHERE
         (full_update IS TRUE OR EXISTS(
             SELECT NULL FROM transportation.changes_z4_z5_z6_z7
             WHERE transportation.changes_z4_z5_z6_z7.is_old IS FALSE AND
-                  transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z8.id
+                  transportation.changes_z4_z5_z6_z7.id = roads_z8_merge.id
         )) AND
         (ST_Length(geometry) > 50)
     ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,
@@ -825,10 +868,12 @@ BEGIN
             WHERE transportation.changes_z4_z5_z6_z7.is_old IS FALSE AND
                   transportation.changes_z4_z5_z6_z7.id = osm_transportation_merge_linestring_gen_z5.id
         )) AND
-        (highway = 'motorway' OR construction = 'motorway'
+        -- All motorways without network (e.g. EU, Asia, South America)
+        ((highway = 'motorway' OR construction = 'motorway') AND (network is NULL or network = '')
         ) OR 
-        (osm_national_network(network) AND network != 'gb-trunk'
-        ) AND
+        -- All roads in network included in osm_national_network except gb-trunk and us-highway
+		( (osm_national_network(network) AND network NOT IN ('gb-trunk', 'us-highway')
+        )) AND
         -- Current view: national-importance motorways and trunks
         ST_Length(geometry) > 1000
     ON CONFLICT (id) DO UPDATE SET osm_id = excluded.osm_id, highway = excluded.highway, network = excluded.network,

openmaptiles.yaml

@@ -1,5 +1,6 @@
 tileset:
   layers:
+    - layers/power/power.yaml
     - layers/water/water.yaml
     - layers/waterway/waterway.yaml
     - layers/landcover/landcover.yaml