kopia lustrzana https://github.com/kartoza/docker-osm
Fix clipping issues by using limito and refactor some code (#87)
* Fix import error due to differences in geom type * fix id missing in clip table and skipping importing shapefiles with geometry errors * fix clip id error in the sql * Change postgresql version from 9.6 to 11 * Added sql to validate geometry and other make commands to run them manually * added pgwatch to docker-compose * Modified importer.py to import clip file if it exists and changed logic for finding if tables are missing * updated readme file * fix logic on first run for importing data * small changes to revert original logic * remove clip functions since they are not needed because of the limitto function * Added logic to check if the geojson should be used if it exist or not, added martin to server vector tiles from the DB * update readmepull/63/merge
mazano
GitHub
rodzic
08069de8f5
commit
dda086905b
45
Makefile
45
Makefile
|
|
@ -80,24 +80,6 @@ live_logs:
|
|||
@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f
|
||||
|
||||
|
||||
###
|
||||
# CLIPPING
|
||||
###
|
||||
|
||||
|
||||
import_clip:
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Importing clip shapefile into the database"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_imposm /usr/bin/ogr2ogr -progress -skipfailures -lco GEOMETRY_NAME=geom -nlt PROMOTE_TO_MULTI -f PostgreSQL PG:"host=db user=docker password=docker dbname=gis" /home/settings/clip/clip.shp
|
||||
|
||||
remove_clip:
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Removing clip shapefile from the database"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'"
|
||||
|
||||
###
|
||||
# STATS
|
||||
|
|
@ -111,33 +93,6 @@ timestamp:
|
|||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt
|
||||
|
||||
###
|
||||
# SQL FILES
|
||||
###
|
||||
|
||||
|
||||
import_sql: import_sql
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Importing SQL files"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/clip/clip.sql gis"
|
||||
|
||||
validate_geom: validate_geom
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Validating geom for all tables"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT validate_geom();'"
|
||||
|
||||
clip_tables: clip_tables
|
||||
@echo
|
||||
@echo "------------------------------------------------------------------"
|
||||
@echo "Clip tables using the clip layer"
|
||||
@echo "------------------------------------------------------------------"
|
||||
@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT clean_tables();'"
|
||||
|
||||
|
||||
|
||||
###
|
||||
# STYLES
|
||||
|
|
|
|||
|
|
@ -6,15 +6,37 @@ version: '2.1'
|
|||
|
||||
services:
|
||||
qgisserver:
|
||||
image: kartoza/qgis-server:2.18
|
||||
image: camptocamp/qgis-server:3.6
|
||||
hostname: dockerosm_qgisserver
|
||||
container_name: dockerosm_qgisserver
|
||||
environment:
|
||||
- QGIS_PROJECT_FILE=/project/project.qgs
|
||||
- GIS_SERVER_LOG_LEVEL=DEBUG
|
||||
- MAX_REQUESTS_PER_PROCESS=100
|
||||
volumes:
|
||||
- ./logs:/var/log/apache2
|
||||
- ./web:/project
|
||||
- ./settings:/web/settings
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
links:
|
||||
- db:db
|
||||
ports:
|
||||
- 8198:80
|
||||
restart: unless-stopped
|
||||
restart: on-failure
|
||||
|
||||
# Server vector tiles from PostgreSQL DB
|
||||
martin:
|
||||
image: urbica/martin
|
||||
hostname: dockerosm_martin
|
||||
container_name: dockerosm_martin
|
||||
restart: on-failure
|
||||
ports:
|
||||
- 3000:3000
|
||||
environment:
|
||||
- WATCH_MODE=true
|
||||
- DATABASE_URL=postgres://docker:docker@db/gis
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -54,10 +54,8 @@ class Importer(object):
|
|||
self.osm_file = None
|
||||
self.mapping_file = None
|
||||
self.post_import_file = None
|
||||
self.clip_shape_file = None
|
||||
self.clip_sql_file = None
|
||||
self.clip_json_file = None
|
||||
self.qgis_style = None
|
||||
|
||||
self.cursor = None
|
||||
self.postgis_uri = None
|
||||
|
||||
|
|
@ -88,14 +86,12 @@ class Importer(object):
|
|||
self.error(msg)
|
||||
else:
|
||||
self.info('Detect SRID: ' + self.default['SRID'])
|
||||
|
||||
# Check valid CLIP.
|
||||
# Check valid CLIP.
|
||||
if self.default['CLIP'] not in ['yes', 'no']:
|
||||
msg = 'CLIP not supported : %s' % self.default['CLIP']
|
||||
self.error(msg)
|
||||
else:
|
||||
self.info('Clip: ' + self.default['CLIP'])
|
||||
|
||||
# Check valid QGIS_STYLE.
|
||||
if self.default['QGIS_STYLE'] not in ['yes', 'no']:
|
||||
msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
|
||||
|
|
@ -131,17 +127,12 @@ class Importer(object):
|
|||
if f == 'post-pbf-import.sql':
|
||||
self.post_import_file = join(self.default['SETTINGS'], f)
|
||||
|
||||
if f == 'clip.geojson':
|
||||
self.clip_json_file = join(self.default['SETTINGS'], f)
|
||||
|
||||
if f == 'qgis_style.sql':
|
||||
self.qgis_style = join(self.default['SETTINGS'], f)
|
||||
|
||||
if f == 'clip':
|
||||
clip_folder = join(self.default['SETTINGS'], f)
|
||||
for clip_file in listdir(clip_folder):
|
||||
if clip_file == 'clip.shp':
|
||||
self.clip_shape_file = join(clip_folder, clip_file)
|
||||
if clip_file == 'clip.sql':
|
||||
self.clip_sql_file = join(clip_folder, clip_file)
|
||||
|
||||
if not self.osm_file:
|
||||
msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS']
|
||||
self.error(msg)
|
||||
|
|
@ -158,6 +149,10 @@ class Importer(object):
|
|||
self.info('No custom SQL files post-pbf-import.sql detected in %s' % self.default['SETTINGS'])
|
||||
else:
|
||||
self.info('SQL Post Import: ' + self.post_import_file)
|
||||
if not self.clip_json_file:
|
||||
self.info('No json files to limit import detected in %s' % self.default['SETTINGS'])
|
||||
else:
|
||||
self.info('Geojson Initial Import Clip: ' + self.clip_json_file)
|
||||
|
||||
if not self.qgis_style and self.default['QGIS_STYLE'] == 'yes':
|
||||
msg = 'qgis_style.sql is missing in %s and QGIS_STYLE = yes.' % self.default['SETTINGS']
|
||||
|
|
@ -167,14 +162,13 @@ class Importer(object):
|
|||
else:
|
||||
self.info('Not using QGIS default styles.')
|
||||
|
||||
if not self.clip_shape_file and self.default['CLIP'] == 'yes':
|
||||
msg = 'clip.shp is missing and CLIP = yes.'
|
||||
if not self.clip_json_file and self.default['CLIP'] == 'yes':
|
||||
msg = 'clip.geojson is missing and CLIP = yes.'
|
||||
self.error(msg)
|
||||
elif self.clip_shape_file and self.default['QGIS_STYLE']:
|
||||
self.info('Shapefile for clipping: ' + self.clip_shape_file)
|
||||
self.info('SQL Clipping function: ' + self.clip_sql_file)
|
||||
elif self.clip_json_file and self.default['QGIS_STYLE']:
|
||||
self.info('Geojson for clipping: ' + self.clip_json_file)
|
||||
else:
|
||||
self.info('No *.shp detected in %s, so no clipping.' % self.default['SETTINGS'])
|
||||
self.info('No *.geojson detected, so no clipping.')
|
||||
|
||||
# In docker-compose, we should wait for the DB is ready.
|
||||
self.info('The checkup is OK.')
|
||||
|
|
@ -232,56 +226,37 @@ class Importer(object):
|
|||
command += ['-f', self.qgis_style]
|
||||
call(command)
|
||||
|
||||
def _import_clip_function(self):
|
||||
"""Create function clean_tables().
|
||||
|
||||
The user must import the clip shapefile to the database!
|
||||
"""
|
||||
self.info('Import clip SQL function.')
|
||||
command = ['psql']
|
||||
command += ['-h', self.default['POSTGRES_HOST']]
|
||||
command += ['-U', self.default['POSTGRES_USER']]
|
||||
command += ['-d', self.default['POSTGRES_DBNAME']]
|
||||
command += ['-f', self.clip_sql_file]
|
||||
call(command)
|
||||
self.info('!! Be sure to run \'make import_clip\' to import the Shapefile into the DB !!')
|
||||
|
||||
def perform_clip_in_db(self):
|
||||
"""Perform clipping if the clip table is here."""
|
||||
if self.count_table('clip') == 1:
|
||||
self.info('Clipping')
|
||||
command = ['psql']
|
||||
command += ['-h', self.default['POSTGRES_HOST']]
|
||||
command += ['-U', self.default['POSTGRES_USER']]
|
||||
command += ['-d', self.default['POSTGRES_DBNAME']]
|
||||
command += ['-c', 'SELECT clean_tables();']
|
||||
call(command)
|
||||
|
||||
def count_table(self, name):
|
||||
"""Check if there is a table starting with name."""
|
||||
sql = 'select count(*) ' \
|
||||
'from information_schema.tables ' \
|
||||
'where table_name like \'%s\';' % name
|
||||
def locate_table(self, name):
|
||||
"""Check for tables in the DB table exists in the DB"""
|
||||
sql = """ SELECT EXISTS (SELECT 1 AS result from information_schema.tables where table_name like 'TEMP_TABLE'); """
|
||||
self.cursor.execute(sql.replace('TEMP_TABLE', '%s' % name))
|
||||
# noinspection PyUnboundLocalVariable
|
||||
self.cursor.execute(sql)
|
||||
return self.cursor.fetchone()[0]
|
||||
|
||||
def run(self):
|
||||
"""First checker."""
|
||||
osm_tables = self.count_table('osm_%')
|
||||
if osm_tables < 1:
|
||||
osm_tables = self.locate_table('osm_%')
|
||||
|
||||
if osm_tables != 1:
|
||||
# It means that the DB is empty. Let's import the PBF file.
|
||||
self._first_pbf_import()
|
||||
|
||||
if self.clip_json_file:
|
||||
self._first_pbf_import(['-limitto', self.clip_json_file])
|
||||
else:
|
||||
self._first_pbf_import([])
|
||||
else:
|
||||
self.info(
|
||||
'The database is not empty. Let\'s import only diff files.')
|
||||
|
||||
if self.default['TIME'] != '0':
|
||||
self._import_diff()
|
||||
if self.clip_json_file:
|
||||
self._import_diff(['-limitto', self.clip_json_file])
|
||||
else:
|
||||
self._import_diff([])
|
||||
else:
|
||||
self.info('No more update to the database. Leaving.')
|
||||
|
||||
def _first_pbf_import(self):
|
||||
def _first_pbf_import(self, args):
|
||||
"""Run the first PBF import into the database."""
|
||||
command = ['imposm', 'import', '-diff', '-deployproduction']
|
||||
command += ['-overwritecache', '-cachedir', self.default['CACHE']]
|
||||
|
|
@ -295,7 +270,8 @@ class Importer(object):
|
|||
command += ['-read', self.osm_file]
|
||||
command += ['-write', '-connection', self.postgis_uri]
|
||||
self.info('The database is empty. Let\'s import the PBF : %s' % self.osm_file)
|
||||
self.info(' '.join(command))
|
||||
|
||||
self.info(command.extend(args))
|
||||
if not call(command) == 0:
|
||||
msg = 'An error occured in imposm with the original file.'
|
||||
self.error(msg)
|
||||
|
|
@ -309,14 +285,10 @@ class Importer(object):
|
|||
if self.post_import_file:
|
||||
self.import_custom_sql()
|
||||
|
||||
if self.clip_shape_file:
|
||||
self._import_clip_function()
|
||||
self.perform_clip_in_db()
|
||||
|
||||
if self.qgis_style:
|
||||
self.import_qgis_styles()
|
||||
|
||||
def _import_diff(self):
|
||||
def _import_diff(self, args):
|
||||
# Finally launch the listening process.
|
||||
while True:
|
||||
import_queue = sorted(listdir(self.default['IMPORT_QUEUE']))
|
||||
|
|
@ -331,10 +303,11 @@ class Importer(object):
|
|||
command += ['-srid', self.default['SRID']]
|
||||
command += ['-diffdir', self.default['SETTINGS']]
|
||||
command += ['-mapping', self.mapping_file]
|
||||
command += ['-limitto', self.clip_json_file]
|
||||
command += ['-connection', self.postgis_uri]
|
||||
command += [join(self.default['IMPORT_QUEUE'], diff)]
|
||||
|
||||
self.info(' '.join(command))
|
||||
self.info(command.extend(args))
|
||||
|
||||
if call(command) == 0:
|
||||
move(
|
||||
|
|
@ -344,11 +317,6 @@ class Importer(object):
|
|||
# Update the timestamp in the file.
|
||||
database_timestamp = diff.split('.')[0].split('->-')[1]
|
||||
self.update_timestamp(database_timestamp)
|
||||
|
||||
if self.clip_shape_file:
|
||||
self.perform_clip_in_db()
|
||||
|
||||
self.info('Import diff successful : %s' % diff)
|
||||
else:
|
||||
msg = 'An error occured in imposm with a diff.'
|
||||
self.error(msg)
|
||||
|
|
|
|||
34
readme.md
34
readme.md
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
A docker compose project to setup an OSM PostGIS database with automatic
|
||||
updates from OSM periodically.
|
||||
The only file you need is a PBF file and run the docker compose project.
|
||||
The only files you need is a PBF file, geojson (if you intent to restrict data download to
|
||||
a smaller extent than the one specified by the PBF) and run the docker compose project.
|
||||
|
||||
## General architecture
|
||||
|
||||
|
|
@ -83,14 +84,19 @@ you don't set a clipping area, you will end with data from all over the world.
|
|||
|
||||
### Clipping
|
||||
|
||||
You can put a shapefile in the clip folder. This shapefile will be
|
||||
used for clipping every features after the import.
|
||||
This file has to be named 'clip.shp' and in the CRS you are using in the database (4326 by default).
|
||||
When the database container is running, import the shapefile in the database using the command :
|
||||
During the initial import or post update imposm uses the flag `-limito` which allows
|
||||
you to define a smaller area that you can work with.
|
||||
This is always desirable to limit the features being imported into the database rather
|
||||
than clipping them.
|
||||
|
||||
`make import_clip`.
|
||||
**NB:** Ensure you add a geojson covering the area you intent to clip into the settings folder.
|
||||
The geojson can be the same extent of the administrative area of your country or it can be a
|
||||
smaller extent. The CRS of the geojson should always be EPSG:4326.
|
||||
|
||||
You can remove the clip file : `make remove_clip`.
|
||||
|
||||
**NB:** It is encouraged to simplify the geometry for the `clip.geojson` as
|
||||
a simplified geometry is easier to process during the import.
|
||||
Rather use the minimum bounding box for the area you intent to clip your dataset with.
|
||||
|
||||
### QGIS Styles
|
||||
|
||||
|
|
@ -239,16 +245,24 @@ With -e, you can add some settings to PostGIS:
|
|||
```bash
|
||||
- ALLOW_IP_RANGE= 0.0.0.0/0
|
||||
```
|
||||
More environment variables for Kartoza/postgis image can be found from https://github.com/kartoza/docker-postgis#environment-variables
|
||||
|
||||
# QGIS Server
|
||||
# QGIS Server and Martin Vector tiles
|
||||
|
||||
You can run a QGIS Server front end to the OSM mirroir by using the provided
|
||||
You can run a QGIS Server front end or martin vector tiles to the OSM mirror by using the provided
|
||||
docker-compose-web.yml file. For example:
|
||||
|
||||
```bash
|
||||
docker-compose -f docker-compose.yml -f docker-compose-web.yml qgisserver up
|
||||
docker-compose -f docker-compose.yml -f docker-compose-web.yml qgisserver up
|
||||
```
|
||||
|
||||
or
|
||||
```bash
|
||||
docker-compose -f docker-compose.yml -f docker-compose-web.yml martin up
|
||||
```
|
||||
For more information about martin configuration and usage can be found from https://github.com/urbica/martin
|
||||
|
||||
|
||||
# Credits
|
||||
|
||||
This application was designed and implemented by:
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,38 +0,0 @@
|
|||
CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
|
||||
$BODY$
|
||||
DECLARE osm_tables CURSOR FOR
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema='public'
|
||||
AND table_type='BASE TABLE'
|
||||
AND table_name LIKE 'osm_%';
|
||||
BEGIN
|
||||
FOR osm_table IN osm_tables LOOP
|
||||
EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
|
||||
SELECT DISTINCT osm_id
|
||||
FROM ' || quote_ident(osm_table.table_name) || '
|
||||
LEFT JOIN clip ON ST_Intersects(geometry, geom) where clip.ogc_fid is NULL)
|
||||
;';
|
||||
END LOOP;
|
||||
END;
|
||||
$BODY$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
-- Function to validate geometry of all tables.
|
||||
-- To run it after creating the function simply run SELECT validate_geom();
|
||||
CREATE OR REPLACE FUNCTION validate_geom() RETURNS void AS
|
||||
$BODY$
|
||||
DECLARE osm_tables CURSOR FOR
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema='public'
|
||||
AND table_type='BASE TABLE'
|
||||
AND table_name LIKE 'osm_%';
|
||||
BEGIN
|
||||
FOR osm_table IN osm_tables LOOP
|
||||
EXECUTE 'UPDATE ' || quote_ident(osm_table.table_name) || ' SET
|
||||
geometry = ST_MakeValid(geometry) where ST_IsValidReason(geometry) = ''F'' ;';
|
||||
END LOOP;
|
||||
END;
|
||||
$BODY$
|
||||
LANGUAGE plpgsql;
|
||||
Ładowanie…
Reference in New Issue