Add lockfile to indicate importer has finished and run osmenrich afterwards (#97)

2020-02-02 15:24:04 +02:00 · 2020-02-02 15:24:04 +02:00 · 4d8d4247fb
commit 4d8d4247fb
--- a/.gitignore
+++ b/.gitignore
@ -6,6 +6,7 @@ cache
 import_done
 import_queue
 settings/*.pbf
+settings/*lock
 BACKUP-STYLES.sql
 settings/last.state.txt
 settings/timestamp.txt
--- a/docker-compose-rancher.yml
+++ b/docker-compose-rancher.yml
@ -0,0 +1,162 @@
+version: '2.1'
+
+volumes:
+  osm-postgis-data:
+  import_done:
+  import_queue:
+  cache:
+  osm_settings:
+  pgadmin_data:
+
+services:
+  db:
+    # About the postgresql version, it should match in the dockerfile of docker-imposm3
+    image: kartoza/postgis:12.0
+    hostname: db
+    container_name: dockerosm_db
+    environment:
+      - POSTGRES_USER=docker
+      - POSTGRES_PASS=docker
+      - POSTGRES_DBNAME=gis
+      # Uncomment to expose the postgis database on the network
+      # - ALLOW_IP_RANGE= 0.0.0.0/0
+    volumes:
+      - osm-postgis-data:/var/lib/postgresql
+    # Uncomment to use the postgis database from outside the docker network
+    # ports:
+    #  - "35432:5432"
+    healthcheck:
+      test: "exit 0"
+
+  osm_downloader:
+    image: kartoza/docker-osm:pbf-downloader
+    container_name: dockerosm_pbf_download
+    volumes:
+      # These are sharable to other containers
+      - osm_settings:/home/settings
+    environment:
+      # Read the README in docker-osm-pbf
+      - CONTINENT=africa
+      - COUNTRY=south-africa
+      - BASE_URL=http://download.geofabrik.de
+      - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings
+      - GEOJSON_URL=''
+
+  imposm:
+    image: kartoza/docker-osm:imposm-latest
+    container_name: dockerosm_imposm
+    volumes:
+      # These are sharable to other containers
+      - osm_settings:/home/settings
+      - import_done:/home/import_done
+      - import_queue:/home/import_queue
+      - cache:/home/cache
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      - POSTGRES_USER=docker
+      - POSTGRES_PASS=docker
+      - POSTGRES_DBNAME=gis
+      - POSTGRES_PORT=5432
+      - POSTGRES_HOST=db
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+      # folder for settings (with *.json and *.sql)
+      - SETTINGS=settings
+      # folder for caching
+      - CACHE=cache
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # it can be 3857
+      - SRID=4326
+      # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
+      - OPTIMIZE=false
+      # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_PRODUCTION=public
+      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_IMPORT=import
+      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_BACKUP=backup
+      # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
+      - QGIS_STYLE=yes
+      # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB
+      - CLIP=no
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"
+
+  osmupdate:
+    build: docker-osmupdate
+    image: kartoza/docker-osm:osmupdate-latest
+    container_name: dockerosm_osmupdate
+    volumes:
+      # These are sharable to other containers
+      - osm_settings:/home/settings
+      - import_done:/home/import_done
+      - import_queue:/home/import_queue
+      - cache:/home/cache
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      # These are all currently the defaults but listed here for your
+      # convenience if you want to change them
+      # the maximum time range to assemble a cumulated changefile.
+      - MAX_DAYS=100
+      # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
+      - DIFF=sporadic
+      # argument to determine the maximum number of parallely processed changefiles.
+      - MAX_MERGE=7
+      # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
+      - COMPRESSION_LEVEL=1
+      # change the URL to use a custom URL to fetch regional file updates.
+      - BASE_URL=http://planet.openstreetmap.org/replication/
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
+
+  pgadmin4:
+    image: dpage/pgadmin4:4.16
+    hostname: pgadmin4
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+    environment:
+      - PGADMIN_DEFAULT_EMAIL=docker@gmail.com
+      - PGADMIN_DEFAULT_PASSWORD=docker
+    ports:
+      - 6500:80
+    restart: on-failure
+    depends_on:
+      db:
+        condition: service_healthy
+
+  osmenrich:
+    build: docker-osmenrich
+    container_name: dockerosm_osmenrich
+    volumes:
+      # These are sharable to other containers
+      - ./settings:/home/settings
+      - import_done:/home/import_done
+      - import_queue:/home/import_queue
+      - cache:/home/cache
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      # These are all currently the defaults but listed here for your
+      # convenience if you want to change them
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+    command: bash -c "while [ ! -f /home/settings/importer.lock ] ; do sleep 1; done && python3 -u /home/enrich.py"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -5,7 +5,6 @@ volumes:
  import_done:
  import_queue:
  cache:
-  osm_settings:
  pgadmin_data:

 services:
@ -28,20 +27,6 @@ services:
    healthcheck:
      test: "exit 0"

-  osm_downloader:
-    image: kartoza/docker-osm:pbf-downloader
-    build: docker-osm-pbf
-    container_name: dockerosm_pbf_download
-    volumes:
-      # These are sharable to other containers
-      - osm_settings:/home/settings
-    environment:
-      # Read the README in docker-osm-pbf
-      - CONTINENT=africa
-      - COUNTRY=south-africa
-      - BASE_URL=http://download.geofabrik.de
-      - MAPPING_URL=https://raw.githubusercontent.com/kartoza/docker-osm/develop/settings
-      - GEOJSON_URL=''

  imposm:
    image: kartoza/docker-osm:imposm-latest
@ -49,7 +34,7 @@ services:
    container_name: dockerosm_imposm
    volumes:
      # These are sharable to other containers
-      - osm_settings:/home/settings
+      - ./settings:/home/settings
      - import_done:/home/import_done
      - import_queue:/home/import_queue
      - cache:/home/cache
@ -95,7 +80,7 @@ services:
    container_name: dockerosm_osmupdate
    volumes:
      # These are sharable to other containers
-      - osm_settings:/home/settings
+      - ./settings:/home/settings
      - import_done:/home/import_done
      - import_queue:/home/import_queue
      - cache:/home/cache
@ -161,3 +146,4 @@ services:
      # seconds between 2 executions of the script
      # if 0, then no update will be done, only the first initial import from the PBF
      - TIME=120
+    command: bash -c "while [ ! -f /home/settings/importer.lock ] ; do sleep 1; done && python3 -u /home/enrich.py"
--- a/docker-imposm3/importer.py
+++ b/docker-imposm3/importer.py
@ -20,7 +20,7 @@
 """

 from sys import exit, stderr
-from os import environ, listdir
+from os import environ, listdir, mknod
 from shutil import move
 from os.path import join, exists, abspath, isabs
 from psycopg2 import connect, OperationalError
@ -233,6 +233,14 @@ class Importer(object):
        # noinspection PyUnboundLocalVariable
        return self.cursor.fetchone()[0]

+    def lockfile(self):
+        setup_lockfile = join(self.default['SETTINGS'], 'importer.lock')
+        if not exists(setup_lockfile):
+            mknod(setup_lockfile)
+        else:
+            remove(setup_lockfile)
+            mknod(setup_lockfile)
+
    def run(self):
        """First checker."""
        osm_tables = self.locate_table('osm_%')
@ -278,6 +286,7 @@ class Importer(object):
            self.error(msg)
        else:
            self.info('Import PBF successful : %s' % self.osm_file)
+            self.lockfile()

        if self.post_import_file or self.qgis_style:
            # Set the password for psql
--- a/readme.md
+++ b/readme.md
@ -12,8 +12,19 @@ a smaller extent than the one specified by the PBF) and run the docker compose p
 ## Quick setup

 As a quick example, we are going to setup Docker-OSM with default values everywhere:
-* Run the docker-compose file and make sure the environment variables are setup properly for
-osm_downloader to download the correct pbf file.
+* Download a PBF file from http://download.geofabrik.de/
+* Put the file in the `settings` folder. 
+
+Alternatively you can execute the `settings_downloader.sh` script to download the pbf and the clip file
+``
+./settings_downloader.sh GEOJSON_URL CONTINENT COUNTRY ie
+./settings_downloader.sh https://github.com/kartoza/docker-osm/blob/develop/settings/clip.geojson africa south-africa
+`` 
+
+If you use rancher for container management you can use the provided docker-compose.yml which allows you to automatically
+download the settings onto the host machine by using environment variables.
+* Run the docker-compose-rancher file and make sure the environment variables are setup properly for
+osm_downloader to download the correct pbf file

 * If you want to connect from your local QGIS Desktop:
  * In the file `docker-compose.yml`, uncomment the block:
--- a/settings_downloader.sh
+++ b/settings_downloader.sh
@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+
+
+GEOJSON_URL=
+
+if [ -n "$1" ]
+then
+    GEOJSON_URL=$1
+fi
+
+CONTINENT=africa
+if [ -n "$2" ]
+then
+    CONTINENT=$2
+fi
+
+COUNTRY=south-africa
+
+if [ -n "$3" ]
+then
+    COUNTRY=$3
+fi
+
+CONTINENT_LOCKFILE=./settings/.${CONTINENT}_lock
+COUNTRY_LOCKFILE=./settings/.${COUNTRY}_lock
+BASE_URL=http://download.geofabrik.de
+
+# Download OSM Mapping file and Geojson data
+
+if [[ ! -f ./settings/clip.geojson && -z ${GEOJSON_URL} ]]; then \
+  echo "We are not downloading any Geojson"
+else
+  wget -c ${GEOJSON_URL} -O ./settings/clip.geojson
+fi
+
+# Download OSM PBF
+if [[ ! -f ${CONTINENT_LOCKFILE} && -z ${COUNTRY} ]]; then \
+
+  echo "${BASE_URL}/${CONTINENT}-latest.osm.pbf"
+  wget  -c --no-check-certificate ${BASE_URL}/${CONTINENT}-latest.osm.pbf -O ./settings/country.pbf
+  touch ${CONTINENT_LOCKFILE}
+
+
+elif [[ ! -f ${COUNTRY_LOCKFILE} ]]; then
+
+  echo "${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf"
+  wget  -c --no-check-certificate ${BASE_URL}/${CONTINENT}/${COUNTRY}-latest.osm.pbf -O ./settings/country.pbf
+  touch ${COUNTRY_LOCKFILE}
+fi
+