diff --git a/.gitignore b/.gitignore index 105fe30e8b19e640a8cc583f4d29183de7bb8df3..6120e1d2560792219764ba76cd0ca3a6a0ab19fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ -/nominatim-docker /photondata -/postgresdata /nominatimdata \ No newline at end of file diff --git a/README.md b/README.md index 5156d4b90b903096e0709acb7adad323fd456e62..1cacf4e17e48fe171034f10e9fae4319a5dfd9fd 100755 --- a/README.md +++ b/README.md @@ -6,4 +6,9 @@ Run `./setup.sh` ## Run (with continuous updates) -Run `./start.sh` \ No newline at end of file +Run `./start.sh` + +## Resources + +Photon : https://github.com/komoot/photon +Nominatim-docker : https://github.com/mediagis/nominatim-docker (this project uses it) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 65f99a0446a4d67cd2b59f9328518afa05bf29f1..1b7043983007301ea21f3ac4a696866342493e38 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: unless-stopped volumes: - ./nominatimdata/postgresdata:/var/lib/postgresql/9.5/main - - ./settings/local.php:/data/settings/local.php + - ./nominatim-3.1/local.php:/app/src/build/settings/local.php ports: - 6432:5432 - 7070:8080 diff --git a/nominatim-3.1/Dockerfile b/nominatim-3.1/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..faa71cd6b3b80fed7918216ed222e0acc222d26f --- /dev/null +++ b/nominatim-3.1/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:xenial + +ENV DEBIAN_FRONTEND noninteractive +ENV LANG C.UTF-8 + +RUN apt-get -y update -qq && \ + apt-get -y install locales && \ + locale-gen en_US.UTF-8 && \ + update-locale LANG=en_US.UTF-8 && \ + apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \ + libboost-filesystem-dev libexpat1-dev zlib1g-dev libxml2-dev\ + libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev \ + postgresql-server-dev-9.5 postgresql-9.5-postgis-2.2 postgresql-contrib-9.5 \ + apache2 php php-pgsql libapache2-mod-php php-pear php-db \ + php-intl git curl sudo \ + python-pip libboost-python-dev \ + osmosis && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/* /var/tmp/* + +WORKDIR /app + +# Configure postgres +RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf && \ + echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf + +# Nominatim install +ENV NOMINATIM_VERSION v3.1.0 +RUN git clone --recursive https://github.com/openstreetmap/Nominatim ./src +RUN cd ./src && git checkout tags/$NOMINATIM_VERSION && git submodule update --recursive --init && \ + mkdir build && cd build && cmake .. && make + +# Osmium install to run continuous updates +RUN pip install osmium + +# Apache configure +COPY local.php /app/src/build/settings/local.php +COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf + +# Load initial data +RUN curl http://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz + +# Create nominatim user +RUN useradd -m -p password1234 nominatim && \ + chown -R nominatim:nominatim ./src + +EXPOSE 5432 +EXPOSE 8080 + +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh +COPY init.sh /app/init.sh + + diff --git a/nominatim-3.1/README.md b/nominatim-3.1/README.md new file mode 100755 index 0000000000000000000000000000000000000000..8e58bf975468068a46e74b7735c45e5d23bc1632 --- /dev/null +++ b/nominatim-3.1/README.md @@ -0,0 +1,66 @@ +# Nominatim Docker (Nominatim version 3.1) + +1. Build + ``` + docker build -t nominatim . + ``` +2. Copy <your_country>.osm.pbf to a local directory (i.e. /home/me/nominatimdata) + +3. Initialize Nominatim Database + ``` + docker run -t -v /home/me/nominatimdata:/data nominatim sh /app/init.sh /data/merged.osm.pbf postgresdata 4 + ``` + Where 4 is the number of threads to use during import. In general the import of data in postgres is a very time consuming + process that may take hours or days. If you run this process on a multiprocessor system make sure that it makes the best use + of it. You can delete the /home/me/nominatimdata/merged.osm.pbf once the import is finished. + + +4. After the import is finished the /home/me/nominatimdata/postgresdata folder will contain the full postgress binaries of + a postgis/nominatim database. The easiest way to start the nominatim as a single node is the following: + ``` + docker run --restart=always -p 6432:5432 -p 7070:8080 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/9.5/main nominatim sh /app/start.sh + ``` + +5. Advanced configuration. If necessary you can split the osm installation into a database and restservice layer + + In order to set the nominatib-db only node: + + ``` + docker run --restart=always -p 6432:5432 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/9.5/main nominatim sh /app/startpostgres.sh + ``` + After doing this create the /home/me/nominatimdata/conf folder and copy there the docker/local.php file. Then uncomment the following line: + + ``` + @define('CONST_Database_DSN', 'pgsql://nominatim:password1234@192.168.1.128:6432/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database> + ``` + + You can start the nominatib-rest only node with the following command: + + ``` + docker run --restart=always -p 7070:8080 -d -v /home/me/nominatimdata/conf:/data nominatim sh /app/startapache.sh + ``` + +6. Configure incremental update. By default CONST_Replication_Url configured for Monaco. +If you want a different update source, you will need to declare `CONST_Replication_Url` in local.php. Documentation [here] (https://github.com/openstreetmap/Nominatim/blob/master/docs/Import-and-Update.md#updates). For example, to use the daily country extracts diffs for Gemany from geofabrik add the following: + ``` + @define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/germany-updates'); + ``` + + Now you will have a fully functioning nominatim instance available at : [http://localhost:7070/](http://localhost:7070). Unlike the previous versions + this one does not store data in the docker context and this results to a much slimmer docker image. + + +# Update + +Full documentation for Nominatim update available [here](https://github.com/openstreetmap/Nominatim/blob/master/docs/Import-and-Update.md#updates). For a list of other methods see the output of: + ``` + docker exec -it nominatim sudo -u nominatim ./src/build/utils/update.php --help + ``` + +The following command will keep your database constantly up to date: + ``` + docker exec -it nominatim sudo -u nominatim ./src/build/utils/update.php --import-osmosis-all + ``` +If you have imported multiple country extracts and want to keep them +up-to-date, have a look at the script in +[issue #60](https://github.com/openstreetmap/Nominatim/issues/60). diff --git a/nominatim-3.1/init.sh b/nominatim-3.1/init.sh new file mode 100755 index 0000000000000000000000000000000000000000..032c421b92db04482ff7e0552bb0b148461edb4f --- /dev/null +++ b/nominatim-3.1/init.sh @@ -0,0 +1,18 @@ +OSMFILE=$1 +PGDIR=$2 +THREADS=$3 + +mkdir -p /data/$PGDIR && \ + +chown postgres:postgres /data/$PGDIR && \ + +export PGDATA=/data/$PGDIR && \ +sudo -u postgres /usr/lib/postgresql/9.5/bin/initdb -D /data/$PGDIR && \ +sudo -u postgres /usr/lib/postgresql/9.5/bin/pg_ctl -D /data/$PGDIR start && \ +sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -u postgres createuser -s nominatim && \ +sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -u postgres createuser -SDR www-data && \ +sudo -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" && \ +sudo -u nominatim ./src/build/utils/setup.php --osm-file $OSMFILE --all --threads $THREADS && \ +sudo -u postgres psql postgres -tAc "CREATE INDEX nodes_index ON public.planet_osm_ways USING gin (nodes);" +sudo -u postgres /usr/lib/postgresql/9.5/bin/pg_ctl -D /data/$PGDIR stop && \ +sudo chown -R postgres:postgres /data/$PGDIR \ No newline at end of file diff --git a/nominatim-3.1/local.php b/nominatim-3.1/local.php new file mode 100755 index 0000000000000000000000000000000000000000..12365e7453e623df8615d25aa060b77f4f1c55aa --- /dev/null +++ b/nominatim-3.1/local.php @@ -0,0 +1,15 @@ +<?php + // Paths + @define('CONST_Postgresql_Version', '9.5'); + @define('CONST_Postgis_Version', '2.2'); + // Website settings + @define('CONST_Website_BaseURL', '/'); + @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/france-updates/'); + @define('CONST_Replication_MaxInterval', '86400'); // Process each update separately, osmosis cannot merge multiple updates + @define('CONST_Replication_Update_Interval', '86400'); // How often upstream publishes diffs + @define('CONST_Replication_Recheck_Interval', '900'); // How long to sleep if no update found yet + @define('CONST_Pyosmium_Binary', '/usr/local/bin/pyosmium-get-changes'); + + //@define('CONST_Database_DSN', 'pgsql://nominatim:password1234@192.168.1.128:6432/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database> + +?> diff --git a/nominatim-3.1/nominatim.conf b/nominatim-3.1/nominatim.conf new file mode 100755 index 0000000000000000000000000000000000000000..9031c6f009e88ea19f94b99475e4d74ffeb51de9 --- /dev/null +++ b/nominatim-3.1/nominatim.conf @@ -0,0 +1,13 @@ +Listen 8080 +<VirtualHost *:8080> + DocumentRoot /app/src/build/website + CustomLog /var/log/apache2/access.log combined + ErrorLog /var/log/apache2/error.log + LogLevel debug + <Directory /app/src/build/website> + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + </Directory> + AddType text/html .php +</VirtualHost> diff --git a/nominatim-3.1/start.sh b/nominatim-3.1/start.sh new file mode 100755 index 0000000000000000000000000000000000000000..8120aea1de15d89398f0cd460edd8f70198264d7 --- /dev/null +++ b/nominatim-3.1/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +service postgresql start +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/postgresql/postgresql-9.5-main.log diff --git a/nominatim-3.1/startapache.sh b/nominatim-3.1/startapache.sh new file mode 100755 index 0000000000000000000000000000000000000000..a204587f251ff1c2f297e21bd01e81fb08a4c1c6 --- /dev/null +++ b/nominatim-3.1/startapache.sh @@ -0,0 +1,3 @@ +#!/bin/bash +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log \ No newline at end of file diff --git a/nominatim-3.1/startpostgres.sh b/nominatim-3.1/startpostgres.sh new file mode 100755 index 0000000000000000000000000000000000000000..320fefea0ecfd5aafae6b56bc614dbe0a4d59313 --- /dev/null +++ b/nominatim-3.1/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-9.5-main.log \ No newline at end of file diff --git a/settings/local.php b/settings/local.php deleted file mode 100755 index d3dd08f9fee420c958219aec0979c6550dc62fca..0000000000000000000000000000000000000000 --- a/settings/local.php +++ /dev/null @@ -1,3 +0,0 @@ -<?php -// base URL of the replication service -@define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/france-updates/'); \ No newline at end of file diff --git a/setup.sh b/setup.sh index 2f533696e38322bbbfd90cc14eea53a8c61b6b9c..aad3497143777d878a62eac7f0b00ac0e1c5c888 100755 --- a/setup.sh +++ b/setup.sh @@ -1,25 +1,23 @@ #!/bin/bash -# Store current directory -WORKDIR=`pwd` -echo $WORKDIR - ### Nominatim ### # Stop existing containers docker-compose down +docker-compose rm -f # Clear existing data -sudo rm -Rf ./nominatimdata/* +sudo rm -Rf ./nominatimdata +mkdir nominatimdata +sudo rm -Rf ./photondata +mkdir photondata -# Get the container image source -git clone https://github.com/mediagis/nominatim-docker.git # Build the image -cd nominatim-docker/3.1 +cd nominatim-3.1 docker build -t nominatim . -cd WORKDIR +cd .. # Get the Rhône-Alpes database wget --directory-prefix=./nominatimdata https://download.geofabrik.de/europe/france/rhone-alpes-latest.osm.pbf -# Init the database +# Initialize the database docker run -t \ -v `pwd`/nominatimdata:/data \ nominatim \ diff --git a/start.sh b/start.sh index ba4a1444ce9987500cfe96c3f690656108ba2b13..09aab7f2d8bd6f364af371d039b6b06e971deeee 100755 --- a/start.sh +++ b/start.sh @@ -5,11 +5,16 @@ docker-compose down docker-compose up -d +## Set up nominatim updates ### + +docker exec -it photon-deploy_nominatim_1 sudo -u nominatim ./src/build/utils/update.php --init-updates + ### Start continuous update ### + while true; do starttime=$(date +%s) - docker exec -it photon-deploy_nominatim_1 sudo -u nominatim ./src/build/utils/update.php --no-npi --import-osmosis --no-index + docker exec -it photon-deploy_nominatim_1 sudo -u nominatim ./src/build/utils/update.php --import-osmosis curl http://localhost:2322/nominatim-update