From 377ca375a0d5dd188614593e4f488b72912c06dc Mon Sep 17 00:00:00 2001
From: Nicolas Pernoud <github@ninico.fr>
Date: Fri, 12 Oct 2018 14:42:50 +0200
Subject: [PATCH] First release candidate
---
.gitignore | 2 --
README.md | 7 +++-
docker-compose.yml | 2 +-
nominatim-3.1/Dockerfile | 56 +++++++++++++++++++++++++++++
nominatim-3.1/README.md | 66 ++++++++++++++++++++++++++++++++++
nominatim-3.1/init.sh | 18 ++++++++++
nominatim-3.1/local.php | 15 ++++++++
nominatim-3.1/nominatim.conf | 13 +++++++
nominatim-3.1/start.sh | 4 +++
nominatim-3.1/startapache.sh | 3 ++
nominatim-3.1/startpostgres.sh | 4 +++
settings/local.php | 3 --
setup.sh | 18 +++++-----
start.sh | 7 +++-
14 files changed, 200 insertions(+), 18 deletions(-)
create mode 100755 nominatim-3.1/Dockerfile
create mode 100755 nominatim-3.1/README.md
create mode 100755 nominatim-3.1/init.sh
create mode 100755 nominatim-3.1/local.php
create mode 100755 nominatim-3.1/nominatim.conf
create mode 100755 nominatim-3.1/start.sh
create mode 100755 nominatim-3.1/startapache.sh
create mode 100755 nominatim-3.1/startpostgres.sh
delete mode 100755 settings/local.php
diff --git a/.gitignore b/.gitignore
index 105fe30..6120e1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,2 @@
-/nominatim-docker
/photondata
-/postgresdata
/nominatimdata
\ No newline at end of file
diff --git a/README.md b/README.md
index 5156d4b..1cacf4e 100755
--- a/README.md
+++ b/README.md
@@ -6,4 +6,9 @@ Run `./setup.sh`
## Run (with continuous updates)
-Run `./start.sh`
\ No newline at end of file
+Run `./start.sh`
+
+## Resources
+
+Photon : https://github.com/komoot/photon
+Nominatim-docker : https://github.com/mediagis/nominatim-docker (this project uses it)
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 65f99a0..1b70439 100755
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
restart: unless-stopped
volumes:
- ./nominatimdata/postgresdata:/var/lib/postgresql/9.5/main
- - ./settings/local.php:/data/settings/local.php
+ - ./nominatim-3.1/local.php:/app/src/build/settings/local.php
ports:
- 6432:5432
- 7070:8080
diff --git a/nominatim-3.1/Dockerfile b/nominatim-3.1/Dockerfile
new file mode 100755
index 0000000..faa71cd
--- /dev/null
+++ b/nominatim-3.1/Dockerfile
@@ -0,0 +1,56 @@
+FROM ubuntu:xenial
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+
+RUN apt-get -y update -qq && \
+ apt-get -y install locales && \
+ locale-gen en_US.UTF-8 && \
+ update-locale LANG=en_US.UTF-8 && \
+ apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
+ libboost-filesystem-dev libexpat1-dev zlib1g-dev libxml2-dev\
+ libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev \
+ postgresql-server-dev-9.5 postgresql-9.5-postgis-2.2 postgresql-contrib-9.5 \
+ apache2 php php-pgsql libapache2-mod-php php-pear php-db \
+ php-intl git curl sudo \
+ python-pip libboost-python-dev \
+ osmosis && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/* && \
+ rm -rf /tmp/* /var/tmp/*
+
+WORKDIR /app
+
+# Configure postgres
+RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf && \
+ echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
+
+# Nominatim install
+ENV NOMINATIM_VERSION v3.1.0
+RUN git clone --recursive https://github.com/openstreetmap/Nominatim ./src
+RUN cd ./src && git checkout tags/$NOMINATIM_VERSION && git submodule update --recursive --init && \
+ mkdir build && cd build && cmake .. && make
+
+# Osmium install to run continuous updates
+RUN pip install osmium
+
+# Apache configure
+COPY local.php /app/src/build/settings/local.php
+COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf
+
+# Load initial data
+RUN curl http://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz
+
+# Create nominatim user
+RUN useradd -m -p password1234 nominatim && \
+ chown -R nominatim:nominatim ./src
+
+EXPOSE 5432
+EXPOSE 8080
+
+COPY start.sh /app/start.sh
+COPY startapache.sh /app/startapache.sh
+COPY startpostgres.sh /app/startpostgres.sh
+COPY init.sh /app/init.sh
+
+
diff --git a/nominatim-3.1/README.md b/nominatim-3.1/README.md
new file mode 100755
index 0000000..8e58bf9
--- /dev/null
+++ b/nominatim-3.1/README.md
@@ -0,0 +1,66 @@
+# Nominatim Docker (Nominatim version 3.1)
+
+1. Build
+ ```
+ docker build -t nominatim .
+ ```
+2. Copy <your_country>.osm.pbf to a local directory (i.e. /home/me/nominatimdata)
+
+3. Initialize Nominatim Database
+ ```
+ docker run -t -v /home/me/nominatimdata:/data nominatim sh /app/init.sh /data/merged.osm.pbf postgresdata 4
+ ```
+ Where 4 is the number of threads to use during import. In general the import of data in postgres is a very time consuming
+ process that may take hours or days. If you run this process on a multiprocessor system make sure that it makes the best use
+ of it. You can delete the /home/me/nominatimdata/merged.osm.pbf once the import is finished.
+
+
+4. After the import is finished the /home/me/nominatimdata/postgresdata folder will contain the full postgress binaries of
+ a postgis/nominatim database. The easiest way to start the nominatim as a single node is the following:
+ ```
+ docker run --restart=always -p 6432:5432 -p 7070:8080 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/9.5/main nominatim sh /app/start.sh
+ ```
+
+5. Advanced configuration. If necessary you can split the osm installation into a database and restservice layer
+
+ In order to set the nominatib-db only node:
+
+ ```
+ docker run --restart=always -p 6432:5432 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/9.5/main nominatim sh /app/startpostgres.sh
+ ```
+ After doing this create the /home/me/nominatimdata/conf folder and copy there the docker/local.php file. Then uncomment the following line:
+
+ ```
+ @define('CONST_Database_DSN', 'pgsql://nominatim:password1234@192.168.1.128:6432/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database>
+ ```
+
+ You can start the nominatib-rest only node with the following command:
+
+ ```
+ docker run --restart=always -p 7070:8080 -d -v /home/me/nominatimdata/conf:/data nominatim sh /app/startapache.sh
+ ```
+
+6. Configure incremental update. By default CONST_Replication_Url configured for Monaco.
+If you want a different update source, you will need to declare `CONST_Replication_Url` in local.php. Documentation [here] (https://github.com/openstreetmap/Nominatim/blob/master/docs/Import-and-Update.md#updates). For example, to use the daily country extracts diffs for Gemany from geofabrik add the following:
+ ```
+ @define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/germany-updates');
+ ```
+
+ Now you will have a fully functioning nominatim instance available at : [http://localhost:7070/](http://localhost:7070). Unlike the previous versions
+ this one does not store data in the docker context and this results to a much slimmer docker image.
+
+
+# Update
+
+Full documentation for Nominatim update available [here](https://github.com/openstreetmap/Nominatim/blob/master/docs/Import-and-Update.md#updates). For a list of other methods see the output of:
+ ```
+ docker exec -it nominatim sudo -u nominatim ./src/build/utils/update.php --help
+ ```
+
+The following command will keep your database constantly up to date:
+ ```
+ docker exec -it nominatim sudo -u nominatim ./src/build/utils/update.php --import-osmosis-all
+ ```
+If you have imported multiple country extracts and want to keep them
+up-to-date, have a look at the script in
+[issue #60](https://github.com/openstreetmap/Nominatim/issues/60).
diff --git a/nominatim-3.1/init.sh b/nominatim-3.1/init.sh
new file mode 100755
index 0000000..032c421
--- /dev/null
+++ b/nominatim-3.1/init.sh
@@ -0,0 +1,18 @@
+OSMFILE=$1
+PGDIR=$2
+THREADS=$3
+
+mkdir -p /data/$PGDIR && \
+
+chown postgres:postgres /data/$PGDIR && \
+
+export PGDATA=/data/$PGDIR && \
+sudo -u postgres /usr/lib/postgresql/9.5/bin/initdb -D /data/$PGDIR && \
+sudo -u postgres /usr/lib/postgresql/9.5/bin/pg_ctl -D /data/$PGDIR start && \
+sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -u postgres createuser -s nominatim && \
+sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -u postgres createuser -SDR www-data && \
+sudo -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" && \
+sudo -u nominatim ./src/build/utils/setup.php --osm-file $OSMFILE --all --threads $THREADS && \
+sudo -u postgres psql postgres -tAc "CREATE INDEX nodes_index ON public.planet_osm_ways USING gin (nodes);"
+sudo -u postgres /usr/lib/postgresql/9.5/bin/pg_ctl -D /data/$PGDIR stop && \
+sudo chown -R postgres:postgres /data/$PGDIR
\ No newline at end of file
diff --git a/nominatim-3.1/local.php b/nominatim-3.1/local.php
new file mode 100755
index 0000000..12365e7
--- /dev/null
+++ b/nominatim-3.1/local.php
@@ -0,0 +1,15 @@
+<?php
+ // Paths
+ @define('CONST_Postgresql_Version', '9.5');
+ @define('CONST_Postgis_Version', '2.2');
+ // Website settings
+ @define('CONST_Website_BaseURL', '/');
+ @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/france-updates/');
+ @define('CONST_Replication_MaxInterval', '86400'); // Process each update separately, osmosis cannot merge multiple updates
+ @define('CONST_Replication_Update_Interval', '86400'); // How often upstream publishes diffs
+ @define('CONST_Replication_Recheck_Interval', '900'); // How long to sleep if no update found yet
+ @define('CONST_Pyosmium_Binary', '/usr/local/bin/pyosmium-get-changes');
+
+ //@define('CONST_Database_DSN', 'pgsql://nominatim:password1234@192.168.1.128:6432/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database>
+
+?>
diff --git a/nominatim-3.1/nominatim.conf b/nominatim-3.1/nominatim.conf
new file mode 100755
index 0000000..9031c6f
--- /dev/null
+++ b/nominatim-3.1/nominatim.conf
@@ -0,0 +1,13 @@
+Listen 8080
+<VirtualHost *:8080>
+ DocumentRoot /app/src/build/website
+ CustomLog /var/log/apache2/access.log combined
+ ErrorLog /var/log/apache2/error.log
+ LogLevel debug
+ <Directory /app/src/build/website>
+ Options FollowSymLinks MultiViews
+ DirectoryIndex search.php
+ Require all granted
+ </Directory>
+ AddType text/html .php
+</VirtualHost>
diff --git a/nominatim-3.1/start.sh b/nominatim-3.1/start.sh
new file mode 100755
index 0000000..8120aea
--- /dev/null
+++ b/nominatim-3.1/start.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+service postgresql start
+/usr/sbin/apache2ctl -D FOREGROUND
+tail -f /var/log/postgresql/postgresql-9.5-main.log
diff --git a/nominatim-3.1/startapache.sh b/nominatim-3.1/startapache.sh
new file mode 100755
index 0000000..a204587
--- /dev/null
+++ b/nominatim-3.1/startapache.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+/usr/sbin/apache2ctl -D FOREGROUND
+tail -f /var/log/apache2/error.log
\ No newline at end of file
diff --git a/nominatim-3.1/startpostgres.sh b/nominatim-3.1/startpostgres.sh
new file mode 100755
index 0000000..320fefe
--- /dev/null
+++ b/nominatim-3.1/startpostgres.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+service postgresql start
+tail -f /var/log/postgresql/postgresql-9.5-main.log
\ No newline at end of file
diff --git a/settings/local.php b/settings/local.php
deleted file mode 100755
index d3dd08f..0000000
--- a/settings/local.php
+++ /dev/null
@@ -1,3 +0,0 @@
-<?php
-// base URL of the replication service
-@define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/france-updates/');
\ No newline at end of file
diff --git a/setup.sh b/setup.sh
index 2f53369..aad3497 100755
--- a/setup.sh
+++ b/setup.sh
@@ -1,25 +1,23 @@
#!/bin/bash
-# Store current directory
-WORKDIR=`pwd`
-echo $WORKDIR
-
### Nominatim ###
# Stop existing containers
docker-compose down
+docker-compose rm -f
# Clear existing data
-sudo rm -Rf ./nominatimdata/*
+sudo rm -Rf ./nominatimdata
+mkdir nominatimdata
+sudo rm -Rf ./photondata
+mkdir photondata
-# Get the container image source
-git clone https://github.com/mediagis/nominatim-docker.git
# Build the image
-cd nominatim-docker/3.1
+cd nominatim-3.1
docker build -t nominatim .
-cd WORKDIR
+cd ..
# Get the Rhône-Alpes database
wget --directory-prefix=./nominatimdata https://download.geofabrik.de/europe/france/rhone-alpes-latest.osm.pbf
-# Init the database
+# Initialize the database
docker run -t \
-v `pwd`/nominatimdata:/data \
nominatim \
diff --git a/start.sh b/start.sh
index ba4a144..09aab7f 100755
--- a/start.sh
+++ b/start.sh
@@ -5,11 +5,16 @@
docker-compose down
docker-compose up -d
+## Set up nominatim updates ###
+
+docker exec -it photon-deploy_nominatim_1 sudo -u nominatim ./src/build/utils/update.php --init-updates
+
### Start continuous update ###
+
while true; do
starttime=$(date +%s)
- docker exec -it photon-deploy_nominatim_1 sudo -u nominatim ./src/build/utils/update.php --no-npi --import-osmosis --no-index
+ docker exec -it photon-deploy_nominatim_1 sudo -u nominatim ./src/build/utils/update.php --import-osmosis
curl http://localhost:2322/nominatim-update
--
GitLab