diff --git a/.env b/.example.env similarity index 86% rename from .env rename to .example.env index 53a6083..71d815d 100644 --- a/.env +++ b/.example.env @@ -1,4 +1,8 @@ -COMPOSE_PROJECT_NAME=kartozadockerosm +COMPOSE_PROJECT_NAME=dockerosm +# Change the following variable if you want to merge multiple compose files +# docker-compose will automatically merged docker-compose.yml and docker-compose.override.yml if exists. +# COMPOSE_FILE=docker-compose.yml:docker-compose.develop.yml:docker-compose-web.yml +COMPOSE_FILE=docker-compose.yml:docker-compose.develop.yml POSTGRES_USER=docker POSTGRES_PASS=docker POSTGRES_DBNAME=gis diff --git a/.github/workflows/build-image-latest.yaml b/.github/workflows/build-image-latest.yaml new file mode 100644 index 0000000..1ad5ab2 --- /dev/null +++ b/.github/workflows/build-image-latest.yaml @@ -0,0 +1,56 @@ +name: build-image-latest +on: + workflow_dispatch: + push: + branches: + - develop +jobs: + docker-builder: + strategy: + matrix: + component: + - imposm + - osmupdate + - osmenrich + # Only push if PR happens in the same repo + # if: github.event.pull_request.base.repo.url == github.event.pull_request.head.repo.url + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + # We can't use Docker meta GitHub Action because + # our tag name is unconventional + # - name: Docker meta + # id: docker_meta + # uses: docker/metadata-action@v3 + # with: + # images: ${{ secrets.DOCKERHUB_REPO }}/docker-osm + # tags: | + # type=semver,pattern=\d.\d.\d + # type=ref,event=branch + # type=ref,event=pr + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + file: docker-${{ matrix.component }} + push: true + target: prod + tags: ${{ secrets.DOCKERHUB_REPO }}/docker-osm:${{ matrix.component }}-latest + cache-from: | + type=gha,scope=prod + cache-to: type=gha,scope=prod + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1649511..b4e351e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ settings/clip/clip.prj settings/clip/clip.qpj settings/clip/clip.shp settings/clip/clip.shx + +.venv +.env \ No newline at end of file diff --git a/Makefile b/Makefile index 683b027..746e8fc 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,3 @@ -PROJECT_ID := dockerosm -COMPOSE_FILE := docker-compose.yml -# Uncomment the next line if you want to display data with Leaflet. -# COMPOSE_FILE := docker-compose-web.yml - .PHONY: logs ### @@ -14,70 +9,77 @@ status: @echo "------------------------------------------------------------------" @echo "Status in production mode" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) ps + @docker-compose ps build: @echo @echo "------------------------------------------------------------------" @echo "Building in production mode" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) build + @docker-compose build + +redeploy: + @echo + @echo "------------------------------------------------------------------" + @echo "Recreate containers" + @echo "------------------------------------------------------------------" + @docker-compose up -d run: @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up -d --no-recreate + @docker-compose up -d --no-recreate rundev: @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up + @docker-compose up stop: @echo @echo "------------------------------------------------------------------" @echo "Stopping in production mode" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) stop + @docker-compose stop kill: @echo @echo "------------------------------------------------------------------" @echo "Killing in production mode" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) kill + @docker-compose kill rm: kill rm-volumes @echo @echo "------------------------------------------------------------------" @echo "Removing production instance and all volumes!!! " @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) rm + @docker-compose rm rm-volumes: @echo @echo "------------------------------------------------------------------" @echo "Removing all volumes!!!! " @echo "------------------------------------------------------------------" - @docker volume rm $(PROJECT_ID)_osm-postgis-data $(PROJECT_ID)_import_queue $(PROJECT_ID)_import_done $(PROJECT_ID)_cache + @docker-compose down --volumes logs: @echo @echo "------------------------------------------------------------------" @echo "Logs" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs + @docker-compose logs live_logs: @echo @echo "------------------------------------------------------------------" @echo "Live Logs" @echo "------------------------------------------------------------------" - @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f + @docker-compose logs -f @@ -91,7 +93,7 @@ timestamp: @echo "------------------------------------------------------------------" @echo "Timestamp" @echo "------------------------------------------------------------------" - @docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt + @docker-compose exec -T imposm cat /home/settings/timestamp.txt ### @@ -104,14 +106,14 @@ import_styles: import_styles @echo "------------------------------------------------------------------" @echo "Importing QGIS styles" @echo "------------------------------------------------------------------" - @docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/qgis_style.sql gis" + @docker-compose exec -T db su - postgres -c "psql -f /home/settings/qgis_style.sql gis" remove_styles: @echo @echo "------------------------------------------------------------------" @echo "Removing QGIS styles" @echo "------------------------------------------------------------------" - @docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS layer_styles;'" + @docker-compose exec -T db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS layer_styles;'" backup_styles: @echo @@ -119,4 +121,4 @@ backup_styles: @echo "Backup QGIS styles to BACKUP.sql" @echo "------------------------------------------------------------------" @echo "SET XML OPTION DOCUMENT;" > BACKUP-STYLES.sql - @ docker exec -t $(PROJECT_ID)_db su - postgres -c "/usr/bin/pg_dump --format plain --inserts --table public.layer_styles gis" >> BACKUP-STYLES.sql + @docker-compose exec -T db su - postgres -c "/usr/bin/pg_dump --format plain --inserts --table public.layer_styles gis" >> BACKUP-STYLES.sql diff --git a/docker-compose-web.yml b/docker-compose-web.yml deleted file mode 100644 index 30a9794..0000000 --- a/docker-compose-web.yml +++ /dev/null @@ -1,135 +0,0 @@ -version: '2.1' - -volumes: - osm-postgis-data: - import_done: - import_queue: - cache: - pgadmin_data: - -services: - db: - image: kartoza/postgis:${POSTGRES_VERSION} - hostname: db - container_name: dockerosm_db - environment: - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASS=${POSTGRES_PASS} - - POSTGRES_DBNAME=${POSTGRES_DBNAME} - - ALLOW_IP_RANGE=${ALLOW_IP_RANGE} - volumes: - - osm-postgis-data:/var/lib/postgresql - ports: - - ${DB_PORT}:5432 - healthcheck: - test: "exit 0" - - imposm: - image: kartoza/docker-osm:imposm-latest - build: docker-imposm3 - container_name: dockerosm_imposm - volumes: - - ./settings:/home/settings - - import_done:/home/import_done - - import_queue:/home/import_queue - - cache:/home/cache - depends_on: - db: - condition: service_healthy - environment: - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASS=${POSTGRES_PASS} - - POSTGRES_DBNAME=${POSTGRES_DBNAME} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_HOST=${POSTGRES_HOST} - - TIME=${TIME} - - SETTINGS=${SETTINGS} - - CACHE=${CACHE} - - IMPORT_DONE=${IMPORT_DONE} - - IMPORT_QUEUE=${IMPORT_QUEUE} - - SRID=${SRID} - - OPTIMIZE=${OPTIMIZE} - - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION} - - DBSCHEMA_IMPORT=${DBSCHEMA_IMPORT} - - DBSCHEMA_BACKUP=${DBSCHEMA_BACKUP} - - QGIS_STYLE=${QGIS_STYLE} - - CLIP=${CLIP} - command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py" - - osmupdate: - build: docker-osmupdate - image: kartoza/docker-osm:osmupdate-latest - container_name: dockerosm_osmupdate - volumes_from: - - imposm - depends_on: - db: - condition: service_healthy - environment: - - MAX_DAYS=${MAX_DAYS} - - DIFF=${DIFF} - - MAX_MERGE=${MAX_MERGE} - - COMPRESSION_LEVEL=${COMPRESSION_LEVEL} - - BASE_URL=${BASE_URL} - - IMPORT_QUEUE=${IMPORT_QUEUE} - - IMPORT_DONE=${IMPORT_DONE} - - TIME=${TIME} - command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py" - - pgadmin4: - image: dpage/pgadmin4:4.16 - hostname: pgadmin4 - volumes: - - pgadmin_data:/var/lib/pgadmin - environment: - - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL} - - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD} - ports: - - ${PGDB_PORT}:80 - restart: on-failure - depends_on: - db: - condition: service_healthy - - osmenrich: - build: docker-osmenrich - volumes_from: - - imposm - depends_on: - db: - condition: service_healthy - environment: - - IMPORT_QUEUE=${IMPORT_QUEUE} - - IMPORT_DONE=${IMPORT_DONE} - - TIME=${TIME} - - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION} - - martin: - image: urbica/martin - restart: on-failure - ports: - - ${MARTIN_PORT}:3000 - environment: - - WATCH_MODE=${WATCH_MODE} - - DATABASE_URL=${DATABASE_URL} - depends_on: - db: - condition: service_healthy - - qgisserver: - image: openquake/qgis-server:ltr - - environment: - - QGIS_PROJECT_FILE=/project/project.qgs - - QGIS_SERVER_LOG_LEVEL=DEBUG - - QGIS_SERVER_PARALLEL_RENDERING=1 - volumes: - - ./logs:/var/log/apache2 - - ./web:/project - - ./settings:/web/settings - depends_on: - db: - condition: service_healthy - ports: - - 8198:80 - restart: on-failure diff --git a/docker-compose.build.yml b/docker-compose.build.yml new file mode 100644 index 0000000..f14210f --- /dev/null +++ b/docker-compose.build.yml @@ -0,0 +1,10 @@ +version: '3.4' +services: + imposm: + build: docker-imposm3 + + osmupdate: + build: docker-osmupdate + + osmenrich: + build: docker-osmenrich diff --git a/docker-compose.develop.yml b/docker-compose.develop.yml new file mode 100644 index 0000000..b1040f2 --- /dev/null +++ b/docker-compose.develop.yml @@ -0,0 +1,19 @@ +version: '3.4' +services: + db: + volumes: + - ./settings:/home/settings + ports: + - ${DB_PORT}:5432 + + imposm: + volumes: + - ./settings:/home/settings + + osmupdate: + volumes: + - ./settings:/home/settings + + osmenrich: + volumes: + - ./settings:/home/settings diff --git a/docker-compose.pgadmin.yml b/docker-compose.pgadmin.yml new file mode 100644 index 0000000..8f5723b --- /dev/null +++ b/docker-compose.pgadmin.yml @@ -0,0 +1,17 @@ +version: '3.4' +volumes: + pgadmin_data: +services: + pgadmin4: + image: dpage/pgadmin4:latest + volumes: + - pgadmin_data:/var/lib/pgadmin + environment: + - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL} + - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD} + ports: + - ${PGDB_PORT}:80 + restart: on-failure + depends_on: + db: + condition: service_healthy \ No newline at end of file diff --git a/docker-compose.web.yml b/docker-compose.web.yml new file mode 100644 index 0000000..4853037 --- /dev/null +++ b/docker-compose.web.yml @@ -0,0 +1,30 @@ +version: '3.4' +services: + martin: + image: urbica/martin + restart: on-failure + ports: + - ${MARTIN_PORT}:3000 + environment: + - WATCH_MODE=${WATCH_MODE} + - DATABASE_URL=${DATABASE_URL} + depends_on: + db: + condition: service_healthy + + qgisserver: + image: openquake/qgis-server:ltr + environment: + - QGIS_PROJECT_FILE=/project/project.qgs + - QGIS_SERVER_LOG_LEVEL=DEBUG + - QGIS_SERVER_PARALLEL_RENDERING=1 + volumes: + - ./logs:/var/log/apache2 + - ./web:/project + - ./settings:/web/settings + depends_on: + db: + condition: service_healthy + ports: + - 8198:80 + restart: on-failure diff --git a/docker-compose.yml b/docker-compose.yml index 8de89a3..fd84f76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,15 @@ -version: '2.1' +version: '3.4' volumes: osm-postgis-data: + settings-data: import_done: import_queue: cache: - pgadmin_data: services: db: image: kartoza/postgis:${POSTGRES_VERSION} - hostname: db - container_name: dockerosm_db environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASS=${POSTGRES_PASS} @@ -20,18 +18,14 @@ services: - FORCE_SSL=${FORCE_SSL} volumes: - osm-postgis-data:/var/lib/postgresql - ports: - - ${DB_PORT}:5432 + - settings-data:/home/settings healthcheck: - test: "exit 0" - + test: "pg_isready -d ${POSTGRES_DBNAME}" imposm: image: kartoza/docker-osm:imposm-latest - build: docker-imposm3 - container_name: dockerosm_imposm volumes: - - ./settings:/home/settings + - settings-data:/home/settings - import_done:/home/import_done - import_queue:/home/import_queue - cache:/home/cache @@ -57,14 +51,14 @@ services: - QGIS_STYLE=${QGIS_STYLE} - CLIP=${CLIP} - SSL_MODE=${SSL_MODE} - command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py" osmupdate: - build: docker-osmupdate image: kartoza/docker-osm:osmupdate-latest - container_name: dockerosm_osmupdate - volumes_from: - - imposm + volumes: + - settings-data:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache depends_on: db: condition: service_healthy @@ -78,26 +72,13 @@ services: - IMPORT_DONE=${IMPORT_DONE} - TIME=${TIME} - pgadmin4: - image: dpage/pgadmin4:latest - hostname: pgadmin4 - volumes: - - pgadmin_data:/var/lib/pgadmin - environment: - - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL} - - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD} - ports: - - ${PGDB_PORT}:80 - restart: on-failure - depends_on: - db: - condition: service_healthy - osmenrich: - build: docker-osmenrich - container_name: dockerosm_osmenrich - volumes_from: - - imposm + image: kartoza/docker-osm:osmenrich-latest + volumes: + - settings-data:/home/settings + - import_done:/home/import_done + - import_queue:/home/import_queue + - cache:/home/cache depends_on: db: condition: service_healthy diff --git a/docker-imposm3/Dockerfile b/docker-imposm/Dockerfile similarity index 100% rename from docker-imposm3/Dockerfile rename to docker-imposm/Dockerfile diff --git a/docker-imposm3/importer.py b/docker-imposm/importer.py similarity index 100% rename from docker-imposm3/importer.py rename to docker-imposm/importer.py diff --git a/docker-imposm3/requirements.txt b/docker-imposm/requirements.txt similarity index 100% rename from docker-imposm3/requirements.txt rename to docker-imposm/requirements.txt diff --git a/readme.md b/readme.md index 51e56c5..911aa51 100644 --- a/readme.md +++ b/readme.md @@ -17,13 +17,56 @@ As a quick example, we are going to setup Docker-OSM with default values everywh Alternatively you can execute the `settings_downloader.sh` script to download the pbf and the clip file ```bash -./settings_downloader.sh GEOJSON_URL CONTINENT COUNTRY ie -./settings_downloader.sh https://github.com/kartoza/docker-osm/raw/develop/settings/clip.geojson africa south-africa +bash ./settings_downloader.sh GEOJSON_URL CONTINENT COUNTRY ie +bash ./settings_downloader.sh https://github.com/kartoza/docker-osm/raw/develop/settings/clip.geojson africa south-africa ``` For a full list of allowed file names read json file `countries.json` Alternatively you can use the python script `pbf_downloader.py` +For local usage, the containers are set up using docker-compose configuration. The configuration files consists of +two sets of config file. The first one is `.env` which contains lists of environment variables. +Copy the `.env` file from the `.example.env` in this repo. + +```bash +cp .example.env .env +``` + +For subsequent configuration, edit the `.env` files to tweak your options. + +The second set of configuration is using a `docker-compose.yml` files, which is a compose files docker-compose is using. + +For minimum set of production environment, the configuration file is described in `docker-compose.yml` file. +The other YAML files with prefix `docker-compose` is a configuration file that you can merge with the basic `docker-compose.yml` file. To use more than one configuration file, you edit `.env` file and change the `COMPOSE_FILE` variable to include all +the compose file you desired, separated by a colon `:` for each file. + +For example, by default, the example file is using both `docker-compose.yml` and `docker-compose.develop.yml` because +we expect you to provide the necessary settings in `settings` folder. Thus the `COMPOSE_FILE` variable looks like this: + +``` +COMPOSE_FILE=docker-compose.yml:docker-compose.develop.yml +``` + +In production environment, normally all the persistent data is stored in a volume, instead of bind mounted. +In this case only the `docker-compose.yml` is enough, and you need to provide the settings inside the volume itself. + +To build the image yourself, include the `docker-compose.build.yml` file. + +To use helper services such as pgadmin, include `docker-compose.pgadmin.yml`, and `docker-compose.web.yml` for web demo. + +If you are familiar with how docker-compose work, you can also use a standard convention by putting `docker-compose.override.yml` +file and include it in the `COMPOSE_FILE` variable. + +To store the configuration for long-term use (for archiving or diffing of different configuration). You can interpolate the +current variables in `.env` and generate a full config files: + +```bash +docker-compose config > docker-compose.production.yml +``` + +The above command will include and merged all your config files specified in `COMPOSE_FILE` variable and also fill out the +variables parameterized in the docker-compose file. + * If you want to connect from your local QGIS Desktop: * In the file `docker-compose.yml`, uncomment the block: