clip with SQL function

.gitignore

@@ -8,4 +8,9 @@ settings/*.state.txt
 settings/*.poly
 settings/last.state.txt
 settings/custom_url_diff.txt
-settings/timestamp.txt
+settings/timestamp.txt
+settings/clip/clip.dbf
+settings/clip/clip.prj
+settings/clip/clip.qpj
+settings/clip/clip.shp
+settings/clip/clip.shx

Makefile

@@ -50,6 +50,20 @@ ipdb:
 	@echo "------------------------------------------------------------------"
 	@docker inspect $(PROJECT_ID)_db | grep '"IPAddress"' | head -1 | cut -d '"' -f 4
 
+import_clip:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Importing clip shapefile"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_db /usr/bin/shp2pgsql -c -I -D -s 4326 /home/settings/clip/clip.shp | docker exec -i $(PROJECT_ID)_db su - postgres -c "psql gis"
+
+remove_clip:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing clip shapefile"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'"
+
 timestamp:
 	@echo
 	@echo "------------------------------------------------------------------"

docker-compose.yml

@@ -20,6 +20,8 @@ db:
   environment:
     - USERNAME=docker
     - PASS=docker
+  volumes_from:
+    - storage
 
 imposm:
   build: docker-imposm3
@@ -56,6 +58,8 @@ imposm:
     - DBSCHEMA_BACKUP=backup
     # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
     - QGIS_STYLE=yes
+    # Use clip in the database
+    - CLIP=no
 
 
 osmupdate:

docker-imposm3/importer.py

@@ -49,11 +49,14 @@ class Importer(object):
             'DBSCHEMA_PRODUCTION': 'public',
             'DBSCHEMA_IMPORT': 'import',
             'DBSCHEMA_BACKUP': 'backup',
+            'CLIP': 'no',
             'QGIS_STYLE': 'yes'
         }
         self.osm_file = None
         self.mapping_file = None
         self.post_import_file = None
+        self.clip_shape_file = None
+        self.clip_sql_file = None
         self.qgis_style = None
 
         self.cursor = None
@@ -82,6 +85,11 @@ class Importer(object):
             msg = 'SRID not supported : %s' % self.default['SRID']
             self.error(msg)
 
+        # Check valid CLIP.
+        if self.default['CLIP'] not in ['yes', 'no']:
+            msg = 'CLIP not supported : %s' % self.default['CLIP']
+            self.error(msg)
+
         # Check valid QGIS_STYLE.
         if self.default['QGIS_STYLE'] not in ['yes', 'no']:
             msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE']
@@ -114,6 +122,14 @@ class Importer(object):
             if f == 'qgis_style.sql':
                 self.qgis_style = join(self.default['SETTINGS'], f)
 
+            if f == 'clip':
+                clip_folder = join(self.default['SETTINGS'], f)
+                for clip_file in listdir(clip_folder):
+                    if clip_file == 'clip.shp':
+                        self.clip_shape_file = join(clip_folder, clip_file)
+                    if clip_file == 'clip.sql':
+                        self.clip_shape_file = join(clip_folder, clip_file)
+
         if not self.osm_file:
             msg = 'OSM file *.pbf is missing in %s' % self.default['SETTINGS']
             self.error(msg)
@@ -135,6 +151,14 @@ class Importer(object):
         else:
             self.info('Not using QGIS default styles.')
 
+        if not self.clip_shape_file and self.default['CLIP'] == 'yes':
+            msg = 'clip.shp is missing and CLIP = yes.'
+            self.error(msg)
+        elif self.clip_shape_file and self.default['QGIS_STYLE']:
+            self.info('%s detected for clipping.' % self.qgis_style)
+        else:
+            self.info('No clipping.')
+
         # In docker-compose, we should wait for the DB is ready.
         self.info('The checkup is OK. The container will continue soon, after the database.')
         sleep(45)
@@ -188,6 +212,26 @@ class Importer(object):
         command += ['-f', self.qgis_style]
         call(command)
 
+    def _import_clip_function(self):
+        """Create function clean_tables()."""
+        command = ['psql']
+        command += ['-h', self.default['HOST']]
+        command += ['-U', self.default['USER']]
+        command += ['-d', self.default['DATABASE']]
+        command += ['-f', self.clip_sql_file]
+        call(command)
+
+    def clip(self):
+        """Perform clipping if the clip table is here."""
+        if self.count_table('clip') == 1:
+            self.info('Clipping')
+            command = ['psql']
+            command += ['-h', self.default['HOST']]
+            command += ['-U', self.default['USER']]
+            command += ['-d', self.default['DATABASE']]
+            command += ['-c', 'SELECT clean_tables();']
+            call(command)
+
     def count_table(self, name):
         """Check if there is a table starting with name."""
         sql = 'select count(*) ' \
@@ -234,6 +278,10 @@ class Importer(object):
         if self.post_import_file:
             self.import_custom_sql()
 
+        if self.clip_shape_file:
+            self._import_clip_function()
+            self.clip()
+
         if self.qgis_style:
             self.import_qgis_styles()
 
@@ -265,6 +313,9 @@ class Importer(object):
                         database_timestamp = diff.split('.')[0].split('->-')[1]
                         self.update_timestamp(database_timestamp)
 
+                        if self.clip_shape_file:
+                            self.clip()
+
                         self.info('Import diff successful : %s' % diff)
                     else:
                         msg = 'An error occured in imposm with a diff.'

settings/clip/clip.sql

@@ -0,0 +1,20 @@
+CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
+$BODY$
+DECLARE osm_tables CURSOR FOR
+    SELECT table_name
+    FROM information_schema.tables
+    WHERE table_schema='public'
+    AND table_type='BASE TABLE'
+    AND table_name LIKE 'osm_%';
+BEGIN
+    FOR osm_table IN osm_tables LOOP
+        EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
+            SELECT DISTINCT osm_id
+            FROM ' || quote_ident(osm_table.table_name) || '
+            LEFT JOIN clip ON ST_Intersects(geometry, geom)
+            WHERE clip.gid IS NULL)
+        ;';
+    END LOOP;
+END;
+$BODY$
+LANGUAGE plpgsql;