From 2bebc32b1c48df6d216a7c00c30eaee73553fea2 Mon Sep 17 00:00:00 2001 From: "a.tselichshev" Date: Tue, 8 Nov 2022 09:42:13 +0600 Subject: [PATCH 1/3] The algorithm of safe import and subsequent updating of data with images and containers --- 4.1/README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4.1/init.sh | 6 +++++- 4.1/start.sh | 2 +- 3 files changed, 59 insertions(+), 2 deletions(-) mode change 100755 => 100644 4.1/init.sh mode change 100755 => 100644 4.1/start.sh diff --git a/4.1/README.md b/4.1/README.md index 575ab9fd..3eeed4c0 100644 --- a/4.1/README.md +++ b/4.1/README.md @@ -55,6 +55,8 @@ The following environment variables are available for configuration: - `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) - `THREADS`: How many threads should be used to import (default: `16`) - `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) +- `PGDATABASE`: Name of the database (default: `nominatim`) +- `SKIP_IMPORT`: If it `yes` it can skip import and create new database. It is used when the container is naive, i.e. it has not imported data, but is used only to update existing data. The following run parameters are available for configuration: @@ -206,6 +208,57 @@ docker run -it \ nominatim ``` +## Using an external PostgreSQL database and setup with updating data container + +If you want to safely update your data and set up regular updates, then you can do the following. + +First of all you need build new container from Dockerfile from 4.1 folder: + +``` +cd nominatim-docker/4.1 +docker build -t nominatim . +``` + +Then you need to import data into the new `nominatim_new` database so as not to touch the `nominatim` database. + +``` +sudo docker run -d --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e NOMINATIM_TOKENIZER=icu \ + -e NOMINATIM_DATABASE_DSN="pgsql:dbname=nominatim_new;hostaddr=192.168.0.1;user=postgres;password=SUPER_PASSWORD" \ + -e PGHOSTADDR=192.168.0.1 \ + -e PGDATABASE=nominatim_new \ + -e PGUSER=postgres \ + -e PGPASSWORD=SUPER_PASSWORD \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -p 8080:8080 \ + --name nominatim-import \ + nominatim +``` + +Next, you make any adjustments and checks in `nominatim_new`. After making sure that everything is in order, we can rename the database to `nominatim`. And then safely update as follows: + +``` +sudo docker run -it \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates/ \ + -e UPDATE_MODE=once \ + -e SKIP_IMPORT=true \ + -e NOMINATIM_DATABASE_DSN="pgsql:dbname=nominatim;hostaddr=192.168.0.1;user=postgres;password=SUPER_PASSWORD" \ + -e PGHOSTADDR=192.168.0.1 \ + -e PGDATABASE=nominatim \ + -e PGUSER=postgres \ + -e PGPASSWORD=SUPER_PASSWORD \ + -p 8080:8080 \ + --name nominatim-update \ + nominatim +``` + +After all, we can just restart the container for daily updates +``` +sudo docker start -i nominatim-update +``` + ## Docker Compose In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. diff --git a/4.1/init.sh b/4.1/init.sh old mode 100755 new mode 100644 index 7815f04e..65a08a44 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -49,6 +49,10 @@ if [ "$PBF_PATH" != "" ]; then OSMFILE=$PBF_PATH fi +if [ !-z $PGDATABASE ]; then + echo Make default PGDATABASE=nominatim + PGDATABASE=nominatim +fi # if we use a bind mount then the PG directory is empty and we have to create it if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then @@ -66,7 +70,7 @@ sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='ww sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ -sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS $PGDATABASE" chown -R nominatim:nominatim ${PROJECT_DIR} diff --git a/4.1/start.sh b/4.1/start.sh old mode 100755 new mode 100644 index 5acb9473..6b893217 --- a/4.1/start.sh +++ b/4.1/start.sh @@ -21,7 +21,7 @@ fi IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished -if [ ! -f ${IMPORT_FINISHED} ]; then +if [ ! -f ${IMPORT_FINISHED} ] && [ "$SKIP_IMPORT" != "true" ]; then /app/init.sh touch ${IMPORT_FINISHED} else From 896ba88baf20ebbf7867cfe8ef70a248ea517f11 Mon Sep 17 00:00:00 2001 From: "a.tselichshev" Date: Tue, 8 Nov 2022 17:58:15 +0600 Subject: [PATCH 2/3] fix rights for exec files --- 4.1/init.sh | 0 4.1/start.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 4.1/init.sh mode change 100644 => 100755 4.1/start.sh diff --git a/4.1/init.sh b/4.1/init.sh old mode 100644 new mode 100755 diff --git a/4.1/start.sh b/4.1/start.sh old mode 100644 new mode 100755 From a36ae384090c4ec13a1666854430f6828799fb3d Mon Sep 17 00:00:00 2001 From: "a.tselichshev" Date: Tue, 8 Nov 2022 18:16:30 +0600 Subject: [PATCH 3/3] fix default nominatim to $PGDATABASE --- 4.1/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.1/init.sh b/4.1/init.sh index 65a08a44..8df7474a 100755 --- a/4.1/init.sh +++ b/4.1/init.sh @@ -106,7 +106,7 @@ sudo -E -u nominatim nominatim admin --warm # gather statistics for query planner to potentially improve query performance # see, https://github.com/osm-search/Nominatim/issues/1023 # and https://github.com/osm-search/Nominatim/issues/1139 -sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" +sudo -E -u nominatim psql -d $PGDATABASE -c "ANALYZE VERBOSE" sudo service postgresql stop